bulltrackers-module 1.0.246 → 1.0.248
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/WorkflowOrchestrator.js +22 -127
- package/functions/computation-system/data/AvailabilityChecker.js +41 -70
- package/functions/computation-system/executors/MetaExecutor.js +5 -0
- package/functions/computation-system/executors/StandardExecutor.js +9 -3
- package/functions/computation-system/tools/BuildReporter.js +21 -16
- package/index.js +5 -0
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Main Orchestrator. Coordinates the topological execution.
|
|
3
|
-
* UPDATED: Implements State Simulation
|
|
3
|
+
* UPDATED: Implements State Simulation and handles Indexer-based null references.
|
|
4
4
|
*/
|
|
5
5
|
const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('./utils/utils');
|
|
6
6
|
const { checkRootDataAvailability } = require('./data/AvailabilityChecker');
|
|
@@ -21,76 +21,49 @@ function groupByPass(manifest) {
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Analyzes whether calculations should run, be skipped, or are blocked.
|
|
24
|
-
* NOW WITH SIMULATION: Updates a local status map as it progresses to ensure
|
|
25
|
-
* downstream dependencies 'see' the decisions made by upstream calculations.
|
|
26
24
|
*/
|
|
27
25
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
28
|
-
const report = {
|
|
29
|
-
runnable: [],
|
|
30
|
-
blocked: [],
|
|
31
|
-
impossible: [],
|
|
32
|
-
failedDependency: [],
|
|
33
|
-
reRuns: [],
|
|
34
|
-
skipped: []
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
// [SIMULATION STATE] Clone the initial DB status.
|
|
38
|
-
// We will update this locally as we make decisions, allowing 'future' calcs
|
|
39
|
-
// in this list to see the predicted state of their dependencies.
|
|
26
|
+
const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
|
|
40
27
|
const simulationStatus = { ...dailyStatus };
|
|
41
|
-
|
|
42
28
|
const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
|
|
43
29
|
|
|
44
30
|
const isDepSatisfied = (depName, currentStatusMap, manifestMap) => {
|
|
45
31
|
const norm = normalizeName(depName);
|
|
46
32
|
const stored = currentStatusMap[norm];
|
|
47
33
|
const depManifest = manifestMap.get(norm);
|
|
48
|
-
|
|
49
34
|
if (!stored) return false;
|
|
50
35
|
if (stored.hash === STATUS_IMPOSSIBLE) return false;
|
|
51
36
|
if (!depManifest) return false;
|
|
52
37
|
if (stored.hash !== depManifest.hash) return false;
|
|
53
|
-
|
|
54
38
|
return true;
|
|
55
39
|
};
|
|
56
40
|
|
|
57
41
|
for (const calc of calcsInPass) {
|
|
58
42
|
const cName = normalizeName(calc.name);
|
|
59
|
-
|
|
60
|
-
// Use simulationStatus instead of dailyStatus
|
|
61
43
|
const stored = simulationStatus[cName];
|
|
62
|
-
|
|
63
44
|
const storedHash = stored ? stored.hash : null;
|
|
64
45
|
const storedCategory = stored ? stored.category : null;
|
|
65
46
|
const currentHash = calc.hash;
|
|
66
47
|
|
|
67
|
-
// Decision Helpers
|
|
68
48
|
const markImpossible = (reason) => {
|
|
69
49
|
report.impossible.push({ name: cName, reason });
|
|
70
|
-
// UPDATE SIMULATION: Downstream deps will now see this as IMPOSSIBLE
|
|
71
50
|
simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category };
|
|
72
51
|
};
|
|
73
52
|
|
|
74
53
|
const markRunnable = (isReRun = false, reRunDetails = null) => {
|
|
75
54
|
if (isReRun) report.reRuns.push(reRunDetails);
|
|
76
55
|
else report.runnable.push(calc);
|
|
77
|
-
// UPDATE SIMULATION: Downstream deps will see this as SUCCESS (matching hash)
|
|
78
56
|
simulationStatus[cName] = { hash: currentHash, category: calc.category };
|
|
79
57
|
};
|
|
80
58
|
|
|
81
59
|
let migrationOldCategory = null;
|
|
82
|
-
if (storedCategory && storedCategory !== calc.category) {
|
|
83
|
-
migrationOldCategory = storedCategory;
|
|
84
|
-
}
|
|
60
|
+
if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
|
|
85
61
|
|
|
86
|
-
// 1. Check Impossible (Previously recorded)
|
|
87
62
|
if (storedHash === STATUS_IMPOSSIBLE) {
|
|
88
63
|
report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
|
|
89
|
-
// Simulation state remains IMPOSSIBLE
|
|
90
64
|
continue;
|
|
91
65
|
}
|
|
92
66
|
|
|
93
|
-
// 2. Root Data Check
|
|
94
67
|
const missingRoots = [];
|
|
95
68
|
if (calc.rootDataDependencies) {
|
|
96
69
|
for (const dep of calc.rootDataDependencies) {
|
|
@@ -104,91 +77,44 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
104
77
|
|
|
105
78
|
if (missingRoots.length > 0) {
|
|
106
79
|
if (!isTargetToday) {
|
|
107
|
-
// If it's a past date and root data is missing, it's permanently impossible.
|
|
108
80
|
markImpossible(`Missing Root Data: ${missingRoots.join(', ')} (Historical)`);
|
|
109
81
|
} else {
|
|
110
|
-
// If it's today, we might just be early. Block, don't Impossible.
|
|
111
82
|
report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Waiting)` });
|
|
112
|
-
// We DO NOT update simulationStatus here because it's not permanently dead, just waiting.
|
|
113
83
|
}
|
|
114
84
|
continue;
|
|
115
85
|
}
|
|
116
86
|
|
|
117
|
-
// 3. Dependency Check (Using Simulation Status)
|
|
118
87
|
let dependencyIsImpossible = false;
|
|
119
88
|
const missingDeps = [];
|
|
120
|
-
|
|
121
89
|
if (calc.dependencies) {
|
|
122
90
|
for (const dep of calc.dependencies) {
|
|
123
91
|
const normDep = normalizeName(dep);
|
|
124
|
-
|
|
125
|
-
// LOOK AT SIMULATION STATUS, NOT DB SNAPSHOT
|
|
126
92
|
const depStored = simulationStatus[normDep];
|
|
127
|
-
|
|
128
|
-
if (
|
|
129
|
-
dependencyIsImpossible = true;
|
|
130
|
-
break;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
if (!isDepSatisfied(dep, simulationStatus, manifestMap)) {
|
|
134
|
-
missingDeps.push(dep);
|
|
135
|
-
}
|
|
93
|
+
if (depStored && depStored.hash === STATUS_IMPOSSIBLE) { dependencyIsImpossible = true; break; }
|
|
94
|
+
if (!isDepSatisfied(dep, simulationStatus, manifestMap)) { missingDeps.push(dep); }
|
|
136
95
|
}
|
|
137
96
|
}
|
|
138
97
|
|
|
139
|
-
if (dependencyIsImpossible) {
|
|
140
|
-
|
|
141
|
-
continue;
|
|
142
|
-
}
|
|
98
|
+
if (dependencyIsImpossible) { markImpossible('Dependency is Impossible'); continue; }
|
|
99
|
+
if (missingDeps.length > 0) { report.failedDependency.push({ name: cName, missing: missingDeps }); continue; }
|
|
143
100
|
|
|
144
|
-
if (missingDeps.length > 0) {
|
|
145
|
-
report.failedDependency.push({ name: cName, missing: missingDeps });
|
|
146
|
-
// Do not update simulation status; downstream will see this as 'missing' (Blocked)
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// 4. Strict Historical Consistency
|
|
151
101
|
if (calc.isHistorical && prevDailyStatus) {
|
|
152
102
|
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
153
103
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
154
|
-
|
|
155
104
|
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
156
105
|
const prevStored = prevDailyStatus[cName];
|
|
157
|
-
|
|
158
106
|
if (!prevStored || prevStored.hash !== currentHash) {
|
|
159
|
-
report.blocked.push({
|
|
160
|
-
name: cName,
|
|
161
|
-
reason: `Waiting for historical continuity (Yesterday ${!prevStored ? 'Missing' : 'Hash Mismatch'})`
|
|
162
|
-
});
|
|
107
|
+
report.blocked.push({ name: cName, reason: `Waiting for historical continuity (Yesterday ${!prevStored ? 'Missing' : 'Hash Mismatch'})` });
|
|
163
108
|
continue;
|
|
164
109
|
}
|
|
165
110
|
}
|
|
166
111
|
}
|
|
167
112
|
|
|
168
|
-
|
|
169
|
-
if (
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
markRunnable(true, {
|
|
173
|
-
name: cName,
|
|
174
|
-
oldHash: storedHash,
|
|
175
|
-
newHash: currentHash,
|
|
176
|
-
previousCategory: migrationOldCategory
|
|
177
|
-
});
|
|
178
|
-
} else if (migrationOldCategory) {
|
|
179
|
-
markRunnable(true, {
|
|
180
|
-
name: cName,
|
|
181
|
-
reason: 'Category Migration',
|
|
182
|
-
previousCategory: migrationOldCategory,
|
|
183
|
-
newCategory: calc.category
|
|
184
|
-
});
|
|
185
|
-
} else {
|
|
186
|
-
report.skipped.push({ name: cName });
|
|
187
|
-
// Even if skipped, ensure simulation status is fresh/set (it usually is from clone)
|
|
188
|
-
simulationStatus[cName] = { hash: currentHash, category: calc.category };
|
|
189
|
-
}
|
|
113
|
+
if (!storedHash) { markRunnable(); }
|
|
114
|
+
else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); }
|
|
115
|
+
else if (migrationOldCategory) { markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); }
|
|
116
|
+
else { report.skipped.push({ name: cName }); simulationStatus[cName] = { hash: currentHash, category: calc.category }; }
|
|
190
117
|
}
|
|
191
|
-
|
|
192
118
|
return report;
|
|
193
119
|
}
|
|
194
120
|
|
|
@@ -204,15 +130,14 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
204
130
|
const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
|
|
205
131
|
const calcManifest = manifestMap.get(normalizeName(targetComputation));
|
|
206
132
|
|
|
207
|
-
if (!calcManifest) {
|
|
208
|
-
throw new Error(`Calculation '${targetComputation}' not found in manifest.`);
|
|
209
|
-
}
|
|
133
|
+
if (!calcManifest) { throw new Error(`Calculation '${targetComputation}' not found in manifest.`); }
|
|
210
134
|
|
|
211
|
-
// 2. Fetch Root Data
|
|
135
|
+
// 2. Fetch Root Data Availability
|
|
136
|
+
// Note: this returns { status: {...}, portfolioRefs: null, historyRefs: null, ... }
|
|
212
137
|
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
213
138
|
|
|
214
139
|
if (!rootData) {
|
|
215
|
-
logger.log('ERROR', `[Executor] FATAL: Root data
|
|
140
|
+
logger.log('ERROR', `[Executor] FATAL: Root data check failed for ${targetComputation} on ${dateStr}. Index might be missing.`);
|
|
216
141
|
return;
|
|
217
142
|
}
|
|
218
143
|
|
|
@@ -230,53 +155,23 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
230
155
|
|
|
231
156
|
// 4. Execute
|
|
232
157
|
logger.log('INFO', `[Executor] Running ${calcManifest.name} for ${dateStr}`, { processId: pid });
|
|
233
|
-
|
|
234
158
|
let resultUpdates = {};
|
|
235
159
|
|
|
236
160
|
try {
|
|
237
161
|
if (calcManifest.type === 'standard') {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
[calcManifest],
|
|
241
|
-
`Pass ${pass}`,
|
|
242
|
-
config,
|
|
243
|
-
dependencies,
|
|
244
|
-
rootData,
|
|
245
|
-
existingResults,
|
|
246
|
-
previousResults
|
|
247
|
-
);
|
|
162
|
+
// StandardExecutor handles the null refs in rootData by fetching on demand
|
|
163
|
+
resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
|
|
248
164
|
} else if (calcManifest.type === 'meta') {
|
|
249
|
-
resultUpdates = await MetaExecutor.run(
|
|
250
|
-
new Date(dateStr + 'T00:00:00Z'),
|
|
251
|
-
[calcManifest],
|
|
252
|
-
`Pass ${pass}`,
|
|
253
|
-
config,
|
|
254
|
-
dependencies,
|
|
255
|
-
existingResults,
|
|
256
|
-
previousResults,
|
|
257
|
-
rootData
|
|
258
|
-
);
|
|
165
|
+
resultUpdates = await MetaExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
|
|
259
166
|
}
|
|
260
|
-
|
|
261
167
|
logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
|
|
262
168
|
return { date: dateStr, updates: resultUpdates };
|
|
263
|
-
|
|
264
169
|
} catch (err) {
|
|
265
170
|
logger.log('ERROR', `[Executor] Failed ${calcManifest.name}: ${err.message}`, { processId: pid, stack: err.stack });
|
|
266
|
-
throw err;
|
|
171
|
+
throw err;
|
|
267
172
|
}
|
|
268
173
|
}
|
|
269
174
|
|
|
270
|
-
|
|
271
|
-
* Legacy/Orchestrator Mode execution (Performs analysis).
|
|
272
|
-
*/
|
|
273
|
-
async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
|
|
274
|
-
// Legacy support logic...
|
|
275
|
-
}
|
|
175
|
+
async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) { /* Legacy support stub */ }
|
|
276
176
|
|
|
277
|
-
module.exports = {
|
|
278
|
-
runDateComputation,
|
|
279
|
-
executeDispatchTask,
|
|
280
|
-
groupByPass,
|
|
281
|
-
analyzeDateExecution
|
|
282
|
-
};
|
|
177
|
+
module.exports = { runDateComputation, executeDispatchTask, groupByPass, analyzeDateExecution };
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview Checks availability of root data
|
|
2
|
+
* @fileoverview Checks availability of root data via the Root Data Index.
|
|
3
|
+
* REFACTORED: Now relies on the centralized 'system_root_data_index' map.
|
|
3
4
|
*/
|
|
4
|
-
const {
|
|
5
|
-
getPortfolioPartRefs,
|
|
6
|
-
loadDailyInsights,
|
|
7
|
-
loadDailySocialPostInsights,
|
|
8
|
-
getHistoryPartRefs
|
|
9
|
-
} = require('../utils/data_loader');
|
|
10
5
|
const { normalizeName } = require('../utils/utils');
|
|
11
6
|
|
|
7
|
+
// Config for the index location (matches rootDataIndexer_config)
|
|
8
|
+
const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
|
|
9
|
+
|
|
12
10
|
function checkRootDependencies(calcManifest, rootDataStatus) {
|
|
13
11
|
const missing = [];
|
|
14
12
|
if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
|
|
@@ -22,28 +20,14 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
|
|
|
22
20
|
return { canRun: missing.length === 0, missing };
|
|
23
21
|
}
|
|
24
22
|
|
|
25
|
-
/**
|
|
26
|
-
* Filters candidates to only those that are strictly "viable" to run.
|
|
27
|
-
* A calculation is Viable if:
|
|
28
|
-
* 1. All required Root Data is present.
|
|
29
|
-
* 2. All required Dependencies are present AND their stored hash matches their current code hash.
|
|
30
|
-
* * @param {Array} candidates - Calculations attempting to run in this pass.
|
|
31
|
-
* @param {Array} fullManifest - The complete manifest (to lookup dependency current hashes).
|
|
32
|
-
* @param {Object} rootDataStatus - { hasPortfolio: bool, hasPrices: bool... }
|
|
33
|
-
* @param {Object} dailyStatus - Map of { "calc-name": "hash" } for completed items.
|
|
34
|
-
*/
|
|
35
23
|
function getViableCalculations(candidates, fullManifest, rootDataStatus, dailyStatus) {
|
|
36
24
|
const viable = [];
|
|
37
25
|
const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
|
|
38
26
|
|
|
39
27
|
for (const calc of candidates) {
|
|
40
|
-
// 1. Check Root Data
|
|
41
28
|
const rootCheck = checkRootDependencies(calc, rootDataStatus);
|
|
42
|
-
if (!rootCheck.canRun)
|
|
43
|
-
continue; // Root data missing -> Impossible.
|
|
44
|
-
}
|
|
29
|
+
if (!rootCheck.canRun) continue;
|
|
45
30
|
|
|
46
|
-
// 2. Check Dependencies (Strict Hash Verification)
|
|
47
31
|
let dependenciesMet = true;
|
|
48
32
|
if (calc.dependencies && calc.dependencies.length > 0) {
|
|
49
33
|
for (const depName of calc.dependencies) {
|
|
@@ -51,15 +35,9 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
|
|
|
51
35
|
const storedHash = dailyStatus[normDep];
|
|
52
36
|
const depManifest = manifestMap.get(normDep);
|
|
53
37
|
|
|
54
|
-
// If dependency is missing from manifest, we can't verify it (shouldn't happen)
|
|
55
38
|
if (!depManifest) { dependenciesMet = false; break; }
|
|
56
|
-
|
|
57
|
-
// CHECK: Does the dependency exist in DB?
|
|
58
39
|
if (!storedHash) { dependenciesMet = false; break; }
|
|
59
|
-
|
|
60
|
-
// CHECK: Does the stored hash match the current code hash?
|
|
61
|
-
// This prevents running on stale data if a dependency failed to update.
|
|
62
|
-
if (storedHash !== depManifest.hash) { dependenciesMet = false; break; }
|
|
40
|
+
if (storedHash.hash !== depManifest.hash) { dependenciesMet = false; break; }
|
|
63
41
|
}
|
|
64
42
|
}
|
|
65
43
|
|
|
@@ -69,56 +47,49 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
|
|
|
69
47
|
return viable;
|
|
70
48
|
}
|
|
71
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Checks data availability by reading the centralized index.
|
|
52
|
+
* Only falls back to raw checks if explicitly configured or index is missing.
|
|
53
|
+
*/
|
|
72
54
|
async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
|
|
73
55
|
const { logger, db } = dependencies;
|
|
74
|
-
const dateToProcess = new Date(dateStr + 'T00:00:00Z');
|
|
75
56
|
|
|
76
|
-
let portfolioRefs = [], historyRefs = [];
|
|
77
|
-
let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
|
|
78
|
-
let insightsData = null, socialData = null;
|
|
79
|
-
|
|
80
57
|
try {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
58
|
+
// 1. Try reading the Index
|
|
59
|
+
const indexDoc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
|
|
60
|
+
|
|
61
|
+
if (indexDoc.exists) {
|
|
62
|
+
const data = indexDoc.data();
|
|
63
|
+
// Return status based on the map
|
|
64
|
+
// Note: We return null references. The data loaders in streamPortfolioData
|
|
65
|
+
// have logic to fetch refs if providedRefs is null (which they are here).
|
|
66
|
+
return {
|
|
67
|
+
status: {
|
|
68
|
+
hasPortfolio: !!data.hasPortfolio,
|
|
69
|
+
hasHistory: !!data.hasHistory,
|
|
70
|
+
hasSocial: !!data.hasSocial,
|
|
71
|
+
hasInsights: !!data.hasInsights,
|
|
72
|
+
hasPrices: !!data.hasPrices
|
|
73
|
+
},
|
|
74
|
+
portfolioRefs: null,
|
|
75
|
+
historyRefs: null,
|
|
76
|
+
todayInsights: null,
|
|
77
|
+
todaySocialPostInsights: null,
|
|
78
|
+
yesterdayPortfolioRefs: null
|
|
79
|
+
};
|
|
80
|
+
} else {
|
|
81
|
+
// Index missing: implies data hasn't been indexed yet or doesn't exist.
|
|
82
|
+
// For safety in this strict model, we assume MISSING.
|
|
83
|
+
logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
|
|
84
|
+
return {
|
|
85
|
+
status: { hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false }
|
|
86
|
+
};
|
|
96
87
|
}
|
|
97
88
|
|
|
98
|
-
await Promise.all(tasks);
|
|
99
|
-
|
|
100
|
-
if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
|
|
101
|
-
|
|
102
|
-
return {
|
|
103
|
-
portfolioRefs,
|
|
104
|
-
historyRefs,
|
|
105
|
-
todayInsights: insightsData,
|
|
106
|
-
todaySocialPostInsights: socialData,
|
|
107
|
-
status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices },
|
|
108
|
-
yesterdayPortfolioRefs: null
|
|
109
|
-
};
|
|
110
89
|
} catch (err) {
|
|
111
|
-
logger.log('ERROR', `Error checking
|
|
90
|
+
logger.log('ERROR', `Error checking availability index: ${err.message}`);
|
|
112
91
|
return null;
|
|
113
92
|
}
|
|
114
93
|
}
|
|
115
94
|
|
|
116
|
-
async function checkPriceAvailability(config, db) {
|
|
117
|
-
try {
|
|
118
|
-
const collection = config.priceCollection || 'asset_prices';
|
|
119
|
-
const snapshot = await db.collection(collection).limit(1).get();
|
|
120
|
-
return !snapshot.empty;
|
|
121
|
-
} catch (e) { return false; }
|
|
122
|
-
}
|
|
123
|
-
|
|
124
95
|
module.exports = { checkRootDependencies, checkRootDataAvailability, getViableCalculations };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Executor for "Meta" (global) calculations.
|
|
3
|
+
* UPDATED: Uses CachedDataLoader for all data access.
|
|
3
4
|
*/
|
|
4
5
|
const { normalizeName } = require('../utils/utils');
|
|
5
6
|
const { CachedDataLoader } = require('../data/CachedDataLoader');
|
|
@@ -18,6 +19,8 @@ class MetaExecutor {
|
|
|
18
19
|
const inst = new mCalc.class();
|
|
19
20
|
inst.manifest = mCalc;
|
|
20
21
|
|
|
22
|
+
// We do not pass 'rootData' (which has null refs) to execution.
|
|
23
|
+
// The Executor fetches its own data via loader.
|
|
21
24
|
await MetaExecutor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader);
|
|
22
25
|
state[normalizeName(mCalc.name)] = inst;
|
|
23
26
|
} catch (e) {
|
|
@@ -30,6 +33,8 @@ class MetaExecutor {
|
|
|
30
33
|
static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
|
|
31
34
|
const mappings = await loader.loadMappings();
|
|
32
35
|
const { logger } = deps;
|
|
36
|
+
|
|
37
|
+
// Lazy fetch insights/social using the loader
|
|
33
38
|
const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
|
|
34
39
|
const social = metadata.rootDataDependencies?.includes('social') ? { today: await loader.loadSocial(dateStr) } : null;
|
|
35
40
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Executor for "Standard" (per-user) calculations.
|
|
3
|
+
* UPDATED: Handles lazy loading of data references (accepts null refs from Indexer).
|
|
3
4
|
*/
|
|
4
5
|
const { normalizeName } = require('../utils/utils');
|
|
5
6
|
const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
|
|
@@ -18,6 +19,7 @@ class StandardExecutor {
|
|
|
18
19
|
if (calcs.some(c => c.isHistorical)) {
|
|
19
20
|
const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
|
|
20
21
|
const prevStr = prev.toISOString().slice(0, 10);
|
|
22
|
+
// Explicitly fetch yesterday's refs as they aren't provided by the daily indexer
|
|
21
23
|
fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
|
|
22
24
|
}
|
|
23
25
|
|
|
@@ -56,12 +58,16 @@ class StandardExecutor {
|
|
|
56
58
|
const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
|
|
57
59
|
const prevDateStr = prevDate.toISOString().slice(0, 10);
|
|
58
60
|
|
|
59
|
-
|
|
61
|
+
// [FIX] pass null if portfolioRefs is null; streamPortfolioData handles the fetch
|
|
62
|
+
const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
|
|
63
|
+
|
|
60
64
|
const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
|
|
61
|
-
|
|
65
|
+
// yesterdayPortfolioRefs are manually fetched in run(), so they are usually populated
|
|
66
|
+
const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
|
|
62
67
|
|
|
63
68
|
const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
|
|
64
|
-
|
|
69
|
+
// [FIX] Removed '&& historyRefs' check. We pass null to streamHistoryData if refs are missing, allowing it to fetch them.
|
|
70
|
+
const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
|
|
65
71
|
|
|
66
72
|
let yP_chunk = {}, tH_chunk = {};
|
|
67
73
|
|
|
@@ -11,38 +11,43 @@ const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
|
|
|
11
11
|
const { FieldValue } = require('@google-cloud/firestore');
|
|
12
12
|
const pLimit = require('p-limit');
|
|
13
13
|
const path = require('path');
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
// Attempt to load package.json to get version. Path depends on where this is invoked.
|
|
17
14
|
const packageJson = require(path.join(__dirname, '..', '..', 'package.json'));
|
|
18
|
-
|
|
19
15
|
const packageVersion = packageJson.version;
|
|
20
16
|
|
|
21
|
-
|
|
22
17
|
/**
|
|
23
18
|
* AUTO-RUN ENTRY POINT
|
|
24
19
|
* Checks if a report for the current version exists. If not, runs it.
|
|
25
|
-
* Designed to be called "Fire-and-Forget" at system init.
|
|
26
20
|
*/
|
|
27
21
|
async function ensureBuildReport(config, dependencies, manifest) {
|
|
28
22
|
const { db, logger } = dependencies;
|
|
29
23
|
const now = new Date();
|
|
24
|
+
|
|
25
|
+
// BuildId still includes timestamp for uniqueness
|
|
30
26
|
const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}-${String(now.getMinutes()).padStart(2,'0')}-${String(now.getSeconds()).padStart(2,'0')}`;
|
|
31
|
-
|
|
27
|
+
|
|
28
|
+
// Reference to "latest" doc
|
|
29
|
+
const latestRef = db.collection('computation_build_records').doc('latest');
|
|
32
30
|
|
|
33
31
|
try {
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
const latestDoc = await latestRef.get();
|
|
33
|
+
const priorVersion = latestDoc.exists ? latestDoc.data().packageVersion : null;
|
|
34
|
+
|
|
35
|
+
if (priorVersion === packageVersion) {
|
|
36
|
+
logger.log('INFO', `[BuildReporter] ✅ Version ${packageVersion} already has a report. Skipping.`);
|
|
37
37
|
return;
|
|
38
38
|
}
|
|
39
|
-
|
|
40
|
-
logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${
|
|
41
|
-
|
|
42
|
-
// but here we are in an async function so we proceed.
|
|
43
|
-
// Defaulting to 90 days for the auto-run to capture full historical impact
|
|
39
|
+
|
|
40
|
+
logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${packageVersion}). Auto-running Pre-flight Report...`);
|
|
41
|
+
|
|
44
42
|
await generateBuildReport(config, dependencies, manifest, 90, buildId);
|
|
45
|
-
|
|
43
|
+
|
|
44
|
+
// Update "latest" pointer
|
|
45
|
+
await latestRef.set({
|
|
46
|
+
packageVersion,
|
|
47
|
+
buildId,
|
|
48
|
+
generatedAt: now.toISOString()
|
|
49
|
+
});
|
|
50
|
+
|
|
46
51
|
} catch (e) {
|
|
47
52
|
logger.log('ERROR', `[BuildReporter] Auto-run check failed: ${e.message}`);
|
|
48
53
|
}
|
package/index.js
CHANGED
|
@@ -52,6 +52,10 @@ const { runBackfillAssetPrices } = require('./functions
|
|
|
52
52
|
// Proxy
|
|
53
53
|
const { handlePost } = require('./functions/appscript-api/index');
|
|
54
54
|
|
|
55
|
+
// NEW
|
|
56
|
+
|
|
57
|
+
const { runRootDataIndexer } = require('./functions/root-data-indexer/index'); // <--- IMPORT
|
|
58
|
+
|
|
55
59
|
const core = {
|
|
56
60
|
IntelligentHeaderManager,
|
|
57
61
|
IntelligentProxyManager,
|
|
@@ -108,6 +112,7 @@ const maintenance = {
|
|
|
108
112
|
runSocialOrchestrator,
|
|
109
113
|
handleSocialTask,
|
|
110
114
|
runBackfillAssetPrices,
|
|
115
|
+
runRootDataIndexer, // <--- EXPORT
|
|
111
116
|
};
|
|
112
117
|
|
|
113
118
|
const proxy = { handlePost };
|