bulltrackers-module 1.0.87 → 1.0.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -14,12 +14,16 @@ const {
|
|
|
14
14
|
loadDailyInsights,
|
|
15
15
|
loadDailySocialPostInsights // <-- NEW
|
|
16
16
|
} = require('../utils/data_loader.js');
|
|
17
|
+
|
|
18
|
+
// --- MODIFIED: Import new meta calculation objects ---
|
|
17
19
|
const {
|
|
18
|
-
historicalCalculations, dailyCalculations,
|
|
20
|
+
historicalCalculations, dailyCalculations, metaCalculations,
|
|
21
|
+
HISTORICAL_CALC_NAMES, META_CALC_NAMES,
|
|
19
22
|
withRetry,
|
|
20
23
|
getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
|
|
21
24
|
commitBatchInChunks, unifiedUtils
|
|
22
25
|
} = require('../utils/utils.js');
|
|
26
|
+
// --- END MODIFIED ---
|
|
23
27
|
|
|
24
28
|
/**
|
|
25
29
|
* Main pipe: pipe.computationSystem.runOrchestration
|
|
@@ -29,7 +33,9 @@ const {
|
|
|
29
33
|
*/
|
|
30
34
|
async function runComputationOrchestrator(config, dependencies) {
|
|
31
35
|
const { logger, db } = dependencies; // Added db here
|
|
32
|
-
|
|
36
|
+
// --- MODIFIED: Add pass3_results to summary ---
|
|
37
|
+
const summary = { pass1_results: [], pass2_results: [], pass3_results: [] };
|
|
38
|
+
// --- END MODIFIED ---
|
|
33
39
|
const yesterday = new Date();
|
|
34
40
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
35
41
|
const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
|
|
@@ -40,20 +46,23 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
40
46
|
const masterDailyList = Object.entries(dailyCalculations).flatMap(([cat, calcs]) =>
|
|
41
47
|
Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
|
|
42
48
|
);
|
|
43
|
-
// --- (EXISTING) insights calculations ---
|
|
44
49
|
const insightsCalculations = require('aiden-shared-calculations-unified').calculations.insights || {};
|
|
45
50
|
const masterInsightsList = Object.keys(insightsCalculations).map(name => ({ category: 'insights', calcName: name }));
|
|
46
|
-
|
|
47
|
-
// --- NEW: Add social post calculations to the master lists ---
|
|
48
51
|
const socialPostCalculations = require('aiden-shared-calculations-unified').calculations.socialPosts || {};
|
|
49
52
|
const masterSocialPostList = Object.keys(socialPostCalculations).map(name => ({ category: 'socialPosts', calcName: name }));
|
|
53
|
+
|
|
54
|
+
// --- NEW: Create master list for meta calcs ---
|
|
55
|
+
const masterMetaList = Object.entries(metaCalculations).flatMap(([cat, calcs]) =>
|
|
56
|
+
Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
|
|
57
|
+
);
|
|
50
58
|
// --- END NEW ---
|
|
51
59
|
|
|
52
60
|
const masterFullList = [
|
|
53
61
|
...masterHistoricalList,
|
|
54
62
|
...masterDailyList,
|
|
55
63
|
...masterInsightsList,
|
|
56
|
-
...masterSocialPostList
|
|
64
|
+
...masterSocialPostList,
|
|
65
|
+
...masterMetaList // <-- NEW
|
|
57
66
|
];
|
|
58
67
|
|
|
59
68
|
// Pass dependencies to sub-pipe
|
|
@@ -69,21 +78,17 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
69
78
|
const existingDateIds = new Set(insightDocs.docs.map(d => d.id).filter(id => /^\d{4}-\d{2}-\d{2}$/.test(id)));
|
|
70
79
|
|
|
71
80
|
// --- PASS 1 ---
|
|
72
|
-
//
|
|
81
|
+
// (Existing Pass 1 code ... no changes here)
|
|
73
82
|
const missingDates = allExpectedDates.filter(dateStr => !existingDateIds.has(dateStr));
|
|
74
83
|
const pass1Jobs = missingDates.map(date => ({
|
|
75
84
|
date,
|
|
76
|
-
missing: [...masterDailyList, ...masterInsightsList, ...masterSocialPostList]
|
|
85
|
+
missing: [...masterDailyList, ...masterInsightsList, ...masterSocialPostList]
|
|
77
86
|
}));
|
|
78
|
-
// --- END MODIFIED ---
|
|
79
|
-
|
|
80
87
|
logger.log('INFO', `[Orchestrator] Pass 1: Found ${pass1Jobs.length} missing dates to process (daily, insights & social calcs).`);
|
|
81
|
-
// Pass dependencies to sub-pipe
|
|
82
|
-
// --- MODIFIED: Combine daily, insights, and social calcs for Pass 1 run ---
|
|
83
88
|
const pass1SourcePackage = {
|
|
84
89
|
...dailyCalculations,
|
|
85
90
|
insights: insightsCalculations,
|
|
86
|
-
socialPosts: socialPostCalculations
|
|
91
|
+
socialPosts: socialPostCalculations
|
|
87
92
|
};
|
|
88
93
|
const pass1Results = await processJobsInParallel(
|
|
89
94
|
pass1Jobs,
|
|
@@ -91,36 +96,33 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
91
96
|
'Pass 1',
|
|
92
97
|
config
|
|
93
98
|
);
|
|
94
|
-
// --- END MODIFIED ---
|
|
95
99
|
summary.pass1_results = pass1Results.map((r, i) => r.status === 'fulfilled' ? r.value : { success: false, date: pass1Jobs[i].date, error: r.reason?.message });
|
|
96
100
|
|
|
97
101
|
|
|
98
102
|
// --- PASS 2 ---
|
|
103
|
+
// (Existing Pass 2 code ... no changes here)
|
|
99
104
|
const pass2Jobs = [];
|
|
100
|
-
// Use db from dependencies
|
|
101
105
|
const updatedInsightDocs = await withRetry(() => dependencies.db.collection(config.resultsCollection).get(), "ListAllInsightDocs-Pass2");
|
|
102
|
-
|
|
103
106
|
updatedInsightDocs.forEach(doc => {
|
|
104
107
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(doc.id)) return;
|
|
105
108
|
const data = doc.data();
|
|
106
|
-
const missingCalcs = masterFullList.filter(({ category, calcName }) =>
|
|
109
|
+
const missingCalcs = masterFullList.filter(({ category, calcName }) =>
|
|
110
|
+
!META_CALC_NAMES.has(calcName) && // <-- Exclude meta calcs from Pass 2
|
|
111
|
+
!data?.[category]?.[calcName]
|
|
112
|
+
);
|
|
107
113
|
|
|
108
114
|
if (missingCalcs.length > 0) {
|
|
109
115
|
pass2Jobs.push({ date: doc.id, missing: missingCalcs });
|
|
110
116
|
}
|
|
111
117
|
});
|
|
112
|
-
|
|
113
118
|
logger.log('INFO', `[Orchestrator] Pass 2: Found ${pass2Jobs.length} incomplete dates to process.`);
|
|
114
|
-
// Pass dependencies to sub-pipe
|
|
115
119
|
const pass2Results = await processJobsInParallel(
|
|
116
120
|
pass2Jobs,
|
|
117
121
|
async (date, missing) => {
|
|
118
122
|
const historicalMissing = missing.filter(c => HISTORICAL_CALC_NAMES.has(c.calcName));
|
|
119
|
-
// --- MODIFIED: Include insights & social in daily missing ---
|
|
120
123
|
const dailyMissing = missing.filter(c => !HISTORICAL_CALC_NAMES.has(c.calcName) && c.category !== 'insights' && c.category !== 'socialPosts');
|
|
121
124
|
const insightsMissing = missing.filter(c => c.category === 'insights');
|
|
122
|
-
const socialPostsMissing = missing.filter(c => c.category === 'socialPosts');
|
|
123
|
-
// --- END MODIFIED ---
|
|
125
|
+
const socialPostsMissing = missing.filter(c => c.category === 'socialPosts');
|
|
124
126
|
const results = [];
|
|
125
127
|
|
|
126
128
|
if (historicalMissing.length > 0) {
|
|
@@ -128,19 +130,17 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
128
130
|
const histResult = await runUnifiedComputation(date, historicalMissing, 'Pass 2 (Historical)', historicalCalculations, config, dependencies);
|
|
129
131
|
results.push(histResult);
|
|
130
132
|
}
|
|
131
|
-
|
|
132
|
-
const dailyAndInsightsAndSocialMissing = [...dailyMissing, ...insightsMissing, ...socialPostsMissing]; // <-- NEW
|
|
133
|
+
const dailyAndInsightsAndSocialMissing = [...dailyMissing, ...insightsMissing, ...socialPostsMissing];
|
|
133
134
|
if (dailyAndInsightsAndSocialMissing.length > 0) {
|
|
134
135
|
logger.log('INFO', `[Pass 2] Running ${dailyAndInsightsAndSocialMissing.length} daily/insights/social calcs for ${date.toISOString().slice(0, 10)}.`);
|
|
135
136
|
const dailyAndInsightsSourcePackage = {
|
|
136
137
|
...dailyCalculations,
|
|
137
138
|
insights: insightsCalculations,
|
|
138
|
-
socialPosts: socialPostCalculations
|
|
139
|
+
socialPosts: socialPostCalculations
|
|
139
140
|
};
|
|
140
141
|
const dailyResult = await runUnifiedComputation(date, dailyAndInsightsAndSocialMissing, 'Pass 2 (Daily/Insights/Social)', dailyAndInsightsSourcePackage, config, dependencies);
|
|
141
142
|
results.push(dailyResult);
|
|
142
143
|
}
|
|
143
|
-
// --- END MODIFIED ---
|
|
144
144
|
return results.length === 1 ? results[0] : { date: date.toISOString().slice(0,10), results };
|
|
145
145
|
},
|
|
146
146
|
'Pass 2',
|
|
@@ -148,6 +148,49 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
148
148
|
);
|
|
149
149
|
summary.pass2_results = pass2Results.map((r, i) => r.status === 'fulfilled' ? r.value : { success: false, date: pass2Jobs[i].date, error: r.reason?.message });
|
|
150
150
|
|
|
151
|
+
|
|
152
|
+
// --- NEW: PASS 3 (Meta) ---
|
|
153
|
+
// This pass runs *after* Pass 1 & 2, checking for missing *meta* calculations.
|
|
154
|
+
const pass3Jobs = [];
|
|
155
|
+
const finalInsightDocs = await withRetry(() => dependencies.db.collection(config.resultsCollection).get(), "ListAllInsightDocs-Pass3");
|
|
156
|
+
|
|
157
|
+
finalInsightDocs.forEach(doc => {
|
|
158
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(doc.id)) return;
|
|
159
|
+
|
|
160
|
+
// Only check dates that are expected to be processed
|
|
161
|
+
if (!allExpectedDates.includes(doc.id)) return;
|
|
162
|
+
|
|
163
|
+
const data = doc.data();
|
|
164
|
+
// Find meta calcs that are missing from the summary doc
|
|
165
|
+
const missingMetaCalcs = masterMetaList.filter(({ category, calcName }) =>
|
|
166
|
+
!data?.[category]?.[calcName]
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
if (missingMetaCalcs.length > 0) {
|
|
170
|
+
pass3Jobs.push({ date: doc.id, missing: missingMetaCalcs });
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
logger.log('INFO', `[Orchestrator] Pass 3: Found ${pass3Jobs.length} dates to process for meta-calculations.`);
|
|
175
|
+
|
|
176
|
+
// Pass dependencies to the new sub-pipe `runMetaComputation`
|
|
177
|
+
const pass3Results = await processJobsInParallel(
|
|
178
|
+
pass3Jobs,
|
|
179
|
+
(date, missing) => runMetaComputation(
|
|
180
|
+
date,
|
|
181
|
+
missing,
|
|
182
|
+
'Pass 3 (Meta)',
|
|
183
|
+
metaCalculations, // Pass the meta calcs package
|
|
184
|
+
config,
|
|
185
|
+
dependencies
|
|
186
|
+
),
|
|
187
|
+
'Pass 3',
|
|
188
|
+
config
|
|
189
|
+
);
|
|
190
|
+
summary.pass3_results = pass3Results.map((r, i) => r.status === 'fulfilled' ? r.value : { success: false, date: pass3Jobs[i].date, error: r.reason?.message });
|
|
191
|
+
// --- END NEW PASS 3 ---
|
|
192
|
+
|
|
193
|
+
|
|
151
194
|
logger.log('INFO', '[Orchestrator] Computation orchestration finished.');
|
|
152
195
|
return summary;
|
|
153
196
|
}
|
|
@@ -156,6 +199,7 @@ async function runComputationOrchestrator(config, dependencies) {
|
|
|
156
199
|
* Internal sub-pipe: Initializes calculator instances.
|
|
157
200
|
*/
|
|
158
201
|
function initializeCalculators(calculationsToRun, sourcePackage, logger) {
|
|
202
|
+
// ... (existing code unchanged) ...
|
|
159
203
|
const state = {};
|
|
160
204
|
for (const { category, calcName } of calculationsToRun) {
|
|
161
205
|
const CalculationClass = sourcePackage[category]?.[calcName];
|
|
@@ -186,10 +230,10 @@ async function streamAndProcess(
|
|
|
186
230
|
todaySocialPostInsights = null,
|
|
187
231
|
yesterdaySocialPostInsights = null
|
|
188
232
|
) {
|
|
233
|
+
// ... (existing code unchanged) ...
|
|
189
234
|
const { db, logger } = dependencies;
|
|
190
235
|
logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
|
|
191
236
|
|
|
192
|
-
// --- START MODIFICATION ---
|
|
193
237
|
// Calculate yesterday's date string
|
|
194
238
|
const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
|
|
195
239
|
yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
|
|
@@ -203,7 +247,6 @@ async function streamAndProcess(
|
|
|
203
247
|
todayDateStr: dateStr,
|
|
204
248
|
yesterdayDateStr: yesterdayStr
|
|
205
249
|
};
|
|
206
|
-
// --- END MODIFICATION ---
|
|
207
250
|
|
|
208
251
|
const batchSize = config.partRefBatchSize || 10;
|
|
209
252
|
let isFirstUser = true; // Flag for insights/social calculations
|
|
@@ -234,7 +277,6 @@ async function streamAndProcess(
|
|
|
234
277
|
yesterdaySocialPostInsights // <-- NEW
|
|
235
278
|
];
|
|
236
279
|
|
|
237
|
-
// --- MODIFIED: Handle insights & social calculations ---
|
|
238
280
|
if (category === 'insights' || category === 'socialPosts') {
|
|
239
281
|
// Only process these once per day, using the first user as a trigger
|
|
240
282
|
if (isFirstUser) {
|
|
@@ -243,7 +285,6 @@ async function streamAndProcess(
|
|
|
243
285
|
continue; // Skip processing for subsequent users
|
|
244
286
|
}
|
|
245
287
|
} else if (HISTORICAL_CALC_NAMES.has(calcName)) {
|
|
246
|
-
// --- MODIFIED: Handle missing yesterday's portfolio gracefully ---
|
|
247
288
|
const pYesterday = yesterdayPortfolios[uid];
|
|
248
289
|
// Skip if yesterday's portfolio is required but missing
|
|
249
290
|
if (!pYesterday && calc.constructor.prototype.process.length >= 3) {
|
|
@@ -259,7 +300,6 @@ async function streamAndProcess(
|
|
|
259
300
|
}
|
|
260
301
|
processArgs = [p, null, uid, ...allContextArgs]; // Pass null for yesterdayPortfolio, add all context
|
|
261
302
|
}
|
|
262
|
-
// --- END MODIFIED ---
|
|
263
303
|
|
|
264
304
|
try {
|
|
265
305
|
await Promise.resolve(calc.process(...processArgs));
|
|
@@ -271,7 +311,7 @@ async function streamAndProcess(
|
|
|
271
311
|
}
|
|
272
312
|
}
|
|
273
313
|
|
|
274
|
-
//
|
|
314
|
+
// Handle case where there are no users but we still need to run insights/social calcs
|
|
275
315
|
if (todayRefs.length === 0 && isFirstUser) {
|
|
276
316
|
logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
|
|
277
317
|
const allContextArgs = [
|
|
@@ -296,49 +336,40 @@ async function streamAndProcess(
|
|
|
296
336
|
}
|
|
297
337
|
}
|
|
298
338
|
}
|
|
299
|
-
// --- END NEW ---
|
|
300
339
|
}
|
|
301
340
|
|
|
302
341
|
|
|
303
342
|
/**
|
|
304
343
|
* Internal sub-pipe: Runs computations for a single date.
|
|
305
|
-
*
|
|
344
|
+
* (Existing code ... no changes here)
|
|
306
345
|
*/
|
|
307
346
|
async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, sourcePackage, config, dependencies) {
|
|
347
|
+
// ... (existing code unchanged) ...
|
|
308
348
|
const { db, logger } = dependencies;
|
|
309
349
|
const dateStr = dateToProcess.toISOString().slice(0, 10);
|
|
310
350
|
logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
|
|
311
351
|
|
|
312
352
|
try {
|
|
313
|
-
// --- (EXISTING) Load today's instrument insights ---
|
|
314
353
|
const todayInsightsData = await loadDailyInsights(config, dependencies, dateStr);
|
|
315
|
-
// --- NEW: Load today's social post insights ---
|
|
316
354
|
const todaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, dateStr);
|
|
317
|
-
// --- END NEW ---
|
|
318
355
|
|
|
319
|
-
// Pass dependencies to sub-pipe
|
|
320
356
|
const todayRefs = await getPortfolioPartRefs(config, dependencies, dateStr);
|
|
321
|
-
// --- MODIFIED: Check if *any* data exists (portfolio OR insights OR social) ---
|
|
322
357
|
if (todayRefs.length === 0 && !todayInsightsData && !todaySocialPostInsightsData) {
|
|
323
358
|
logger.log('WARN', `[${passName}] No portfolio, instrument insights, OR social post data found for ${dateStr}. Skipping.`);
|
|
324
359
|
return { success: true, date: dateStr, message: "No source data for today." };
|
|
325
360
|
}
|
|
326
|
-
// --- END MODIFIED ---
|
|
327
361
|
|
|
328
362
|
|
|
329
363
|
let yesterdayPortfolios = {};
|
|
330
364
|
let yesterdayInsightsData = null;
|
|
331
|
-
let yesterdaySocialPostInsightsData = null;
|
|
365
|
+
let yesterdaySocialPostInsightsData = null;
|
|
332
366
|
|
|
333
|
-
// --- MODIFIED: Check if *any* calc requires yesterday data ---
|
|
334
367
|
const requiresYesterdayPortfolio = calculationsToRun.some(c => sourcePackage[c.category]?.[c.calcName]?.prototype?.process.length >= 3);
|
|
335
368
|
const requiresYesterdayInsights = calculationsToRun.some(c => c.category === 'insights');
|
|
336
|
-
const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.category === 'socialPosts');
|
|
337
|
-
// --- END MODIFIED ---
|
|
369
|
+
const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.category === 'socialPosts');
|
|
338
370
|
|
|
339
|
-
if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) {
|
|
371
|
+
if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) {
|
|
340
372
|
|
|
341
|
-
// --- (EXISTING) Load yesterday's instrument insights ---
|
|
342
373
|
if(requiresYesterdayInsights) {
|
|
343
374
|
let daysAgo = 1;
|
|
344
375
|
const maxLookback = 30; // Or from config
|
|
@@ -360,9 +391,7 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
360
391
|
logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
|
|
361
392
|
}
|
|
362
393
|
}
|
|
363
|
-
// --- END (EXISTING) ---
|
|
364
394
|
|
|
365
|
-
// --- NEW: Load yesterday's social post insights ---
|
|
366
395
|
if(requiresYesterdaySocialPosts) {
|
|
367
396
|
let daysAgo = 1;
|
|
368
397
|
const maxLookback = 30;
|
|
@@ -384,9 +413,7 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
384
413
|
logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
|
|
385
414
|
}
|
|
386
415
|
}
|
|
387
|
-
// --- END NEW ---
|
|
388
416
|
|
|
389
|
-
// --- (EXISTING) Load yesterday's portfolio data ---
|
|
390
417
|
if (requiresYesterdayPortfolio) {
|
|
391
418
|
const prev = new Date(dateToProcess);
|
|
392
419
|
prev.setUTCDate(prev.getUTCDate() - 1);
|
|
@@ -400,7 +427,6 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
400
427
|
logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
|
|
401
428
|
}
|
|
402
429
|
}
|
|
403
|
-
// --- END (EXISTING) ---
|
|
404
430
|
}
|
|
405
431
|
|
|
406
432
|
|
|
@@ -416,8 +442,8 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
416
442
|
yesterdayPortfolios,
|
|
417
443
|
todayInsightsData,
|
|
418
444
|
yesterdayInsightsData,
|
|
419
|
-
todaySocialPostInsightsData,
|
|
420
|
-
yesterdaySocialPostInsightsData
|
|
445
|
+
todaySocialPostInsightsData,
|
|
446
|
+
yesterdaySocialPostInsightsData
|
|
421
447
|
);
|
|
422
448
|
|
|
423
449
|
let successCount = 0;
|
|
@@ -471,13 +497,7 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
471
497
|
successCount++;
|
|
472
498
|
}
|
|
473
499
|
} else {
|
|
474
|
-
// --- START MODIFICATION ---
|
|
475
|
-
// REASON: Do not count a null/empty result as a success.
|
|
476
|
-
// This makes the failure visible in the summary log and ensures
|
|
477
|
-
// Pass 2 will detect the missing calculation for backfilling.
|
|
478
500
|
logger.log('WARN', `[${passName}] Calculation ${key} produced no results for ${dateStr}. Skipping write to allow backfill.`);
|
|
479
|
-
// DO NOT increment successCount here.
|
|
480
|
-
// --- END MODIFICATION ---
|
|
481
501
|
}
|
|
482
502
|
} catch (e) {
|
|
483
503
|
logger.log('ERROR', `[${passName}] getResult/Commit failed for ${key} on ${dateStr}`, { err: e.message });
|
|
@@ -496,6 +516,83 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
|
|
|
496
516
|
}
|
|
497
517
|
|
|
498
518
|
|
|
519
|
+
// --- NEW: Add the handler for meta-calculations ---
|
|
520
|
+
/**
|
|
521
|
+
* Internal sub-pipe: Runs "meta" computations for a single date.
|
|
522
|
+
* These calculations do NOT stream user data, but rather use the
|
|
523
|
+
* special `process(dateStr, dependencies, config)` signature.
|
|
524
|
+
*/
|
|
525
|
+
async function runMetaComputation(dateToProcess, calculationsToRun, passName, sourcePackage, config, dependencies) {
|
|
526
|
+
const { db, logger } = dependencies;
|
|
527
|
+
const dateStr = dateToProcess.toISOString().slice(0, 10);
|
|
528
|
+
logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} meta-calcs.`);
|
|
529
|
+
|
|
530
|
+
try {
|
|
531
|
+
const state = initializeCalculators(calculationsToRun, sourcePackage, logger);
|
|
532
|
+
let successCount = 0;
|
|
533
|
+
const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
|
|
534
|
+
|
|
535
|
+
for (const key in state) {
|
|
536
|
+
const calc = state[key];
|
|
537
|
+
// Check for the unique meta-calc signature
|
|
538
|
+
if (!calc || typeof calc.process !== 'function') continue;
|
|
539
|
+
|
|
540
|
+
const [category, calcName] = key.split('/');
|
|
541
|
+
|
|
542
|
+
try {
|
|
543
|
+
// Call the special process method that returns the result directly
|
|
544
|
+
const result = await Promise.resolve(calc.process(dateStr, dependencies, config));
|
|
545
|
+
|
|
546
|
+
const pendingWrites = [];
|
|
547
|
+
const summaryData = {};
|
|
548
|
+
|
|
549
|
+
// Check if the result is valid
|
|
550
|
+
if (result && Object.keys(result).length > 0) {
|
|
551
|
+
// Meta calcs are not expected to be sharded, but we handle the standard case
|
|
552
|
+
const computationDocRef = resultsCollectionRef.doc(category)
|
|
553
|
+
.collection(config.computationsSubcollection)
|
|
554
|
+
.doc(calcName);
|
|
555
|
+
pendingWrites.push({ ref: computationDocRef, data: result });
|
|
556
|
+
|
|
557
|
+
if (!summaryData[category]) summaryData[category] = {};
|
|
558
|
+
summaryData[category][calcName] = true;
|
|
559
|
+
|
|
560
|
+
// Add the summary flag to the top-level date doc
|
|
561
|
+
if (Object.keys(summaryData).length > 0) {
|
|
562
|
+
const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
|
|
563
|
+
pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// Commit the writes
|
|
567
|
+
if (pendingWrites.length > 0) {
|
|
568
|
+
await commitBatchInChunks(
|
|
569
|
+
config,
|
|
570
|
+
dependencies,
|
|
571
|
+
pendingWrites,
|
|
572
|
+
`Commit ${passName} ${dateStr} [${key}]`
|
|
573
|
+
);
|
|
574
|
+
successCount++;
|
|
575
|
+
}
|
|
576
|
+
} else {
|
|
577
|
+
logger.log('WARN', `[${passName}] Meta-calculation ${key} produced no results for ${dateStr}. Skipping write.`);
|
|
578
|
+
}
|
|
579
|
+
} catch (e) {
|
|
580
|
+
logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${key} on ${dateStr}`, { err: e.message, stack: e.stack });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
|
|
585
|
+
logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
|
|
586
|
+
return { success: true, date: dateStr, successful: successCount, failed: calculationsToRun.length - successCount };
|
|
587
|
+
|
|
588
|
+
} catch (err) {
|
|
589
|
+
logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
|
|
590
|
+
return { success: false, date: dateStr, error: err.message };
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
// --- END NEW FUNCTION ---
|
|
594
|
+
|
|
595
|
+
|
|
499
596
|
module.exports = {
|
|
500
597
|
runComputationOrchestrator,
|
|
501
|
-
};
|
|
598
|
+
};
|
|
@@ -14,19 +14,30 @@ const HISTORICAL_CALC_NAMES = new Set([
|
|
|
14
14
|
'risk-appetite-change', 'drawdown-response', 'gain-response',
|
|
15
15
|
'tsl-effectiveness', 'position-count-pnl', 'diversification-pnl',
|
|
16
16
|
|
|
17
|
-
// --- ADD THESE THREE LINES ---
|
|
18
17
|
'deposit-withdrawal-percentage',
|
|
19
18
|
'new-allocation-percentage',
|
|
20
19
|
'reallocation-increase-percentage',
|
|
21
|
-
'asset-crowd-flow'
|
|
20
|
+
'asset-crowd-flow'
|
|
22
21
|
]);
|
|
22
|
+
|
|
23
|
+
// --- NEW: Add a set for meta calculation names ---
|
|
24
|
+
const META_CALC_NAMES = new Set([
|
|
25
|
+
'cash-flow-deployment'
|
|
26
|
+
]);
|
|
27
|
+
// --- END NEW ---
|
|
28
|
+
|
|
23
29
|
const historicalCalculations = {};
|
|
24
30
|
const dailyCalculations = {};
|
|
31
|
+
const metaCalculations = {}; // <-- NEW: Add object for meta calcs
|
|
32
|
+
|
|
25
33
|
for (const category in calculations) {
|
|
26
34
|
for (const calcName in calculations[category]) {
|
|
27
35
|
if (HISTORICAL_CALC_NAMES.has(calcName)) {
|
|
28
36
|
if (!historicalCalculations[category]) historicalCalculations[category] = {};
|
|
29
37
|
historicalCalculations[category][calcName] = calculations[category][calcName];
|
|
38
|
+
} else if (META_CALC_NAMES.has(calcName)) { // <-- NEW: Add else-if block
|
|
39
|
+
if (!metaCalculations[category]) metaCalculations[category] = {};
|
|
40
|
+
metaCalculations[category][calcName] = calculations[category][calcName];
|
|
30
41
|
} else {
|
|
31
42
|
if (!dailyCalculations[category]) dailyCalculations[category] = {};
|
|
32
43
|
dailyCalculations[category][calcName] = calculations[category][calcName];
|
|
@@ -44,6 +55,7 @@ for (const category in calculations) {
|
|
|
44
55
|
* @param {string} operationName - Name for logging.
|
|
45
56
|
*/
|
|
46
57
|
async function commitBatchInChunks(config, dependencies, writes, operationName) {
|
|
58
|
+
// ... (existing code unchanged) ...
|
|
47
59
|
const { db, logger } = dependencies;
|
|
48
60
|
const batchSizeLimit = config.batchSizeLimit || 450;
|
|
49
61
|
|
|
@@ -71,6 +83,7 @@ async function commitBatchInChunks(config, dependencies, writes, operationName)
|
|
|
71
83
|
* (Stateless)
|
|
72
84
|
*/
|
|
73
85
|
function getExpectedDateStrings(startDate, endDate) {
|
|
86
|
+
// ... (existing code unchanged) ...
|
|
74
87
|
const dateStrings = [];
|
|
75
88
|
if (startDate <= endDate) {
|
|
76
89
|
const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
|
|
@@ -88,6 +101,7 @@ function getExpectedDateStrings(startDate, endDate) {
|
|
|
88
101
|
* (Stateless, as taskFunction will receive dependencies)
|
|
89
102
|
*/
|
|
90
103
|
async function processJobsInParallel(jobs, taskFunction, passName, config) {
|
|
104
|
+
// ... (existing code unchanged) ...
|
|
91
105
|
// This function itself doesn't need dependencies,
|
|
92
106
|
// but the 'taskFunction' it calls *will* receive them from its caller.
|
|
93
107
|
const { logger } = require("sharedsetup")(__filename); // Use local logger for this static util
|
|
@@ -111,6 +125,7 @@ async function processJobsInParallel(jobs, taskFunction, passName, config) {
|
|
|
111
125
|
* Internal helper: Finds the earliest date document in a collection.
|
|
112
126
|
*/
|
|
113
127
|
async function getFirstDateFromCollection(config, dependencies, collectionName) {
|
|
128
|
+
// ... (existing code unchanged) ...
|
|
114
129
|
const { db, logger } = dependencies;
|
|
115
130
|
let earliestDate = null;
|
|
116
131
|
try {
|
|
@@ -156,6 +171,7 @@ async function getFirstDateFromCollection(config, dependencies, collectionName)
|
|
|
156
171
|
* @returns {Promise<Date>} The earliest date found or a default fallback date.
|
|
157
172
|
*/
|
|
158
173
|
async function getFirstDateFromSourceData(config, dependencies) {
|
|
174
|
+
// ... (existing code unchanged) ...
|
|
159
175
|
const { logger } = dependencies;
|
|
160
176
|
logger.log('INFO', 'Querying for the earliest date from source portfolio data...');
|
|
161
177
|
|
|
@@ -182,7 +198,11 @@ async function getFirstDateFromSourceData(config, dependencies) {
|
|
|
182
198
|
|
|
183
199
|
module.exports = {
|
|
184
200
|
FieldValue, FieldPath,
|
|
185
|
-
|
|
201
|
+
// --- MODIFIED: Export meta calculations ---
|
|
202
|
+
historicalCalculations, dailyCalculations, metaCalculations,
|
|
203
|
+
unifiedUtils: utils,
|
|
204
|
+
HISTORICAL_CALC_NAMES, META_CALC_NAMES,
|
|
205
|
+
// --- END MODIFIED ---
|
|
186
206
|
withRetry, commitBatchInChunks,
|
|
187
207
|
getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
|
|
188
|
-
};
|
|
208
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bulltrackers-module",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.88",
|
|
4
4
|
"description": "Helper Functions for Bulltrackers.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"files": [
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"@google-cloud/firestore": "^7.11.3",
|
|
33
33
|
"sharedsetup": "latest",
|
|
34
34
|
"require-all": "^3.0.0",
|
|
35
|
-
"aiden-shared-calculations-unified": "1.0.
|
|
35
|
+
"aiden-shared-calculations-unified": "1.0.13",
|
|
36
36
|
"@google-cloud/pubsub": "latest",
|
|
37
37
|
"express": "^4.19.2",
|
|
38
38
|
"cors": "^2.8.5",
|