bulltrackers-module 1.0.157 → 1.0.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -42,23 +42,86 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
|
|
|
42
42
|
} catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
45
|
+
/**
|
|
46
|
+
* --- REFACTORED: Stage 4: Fetch ALL existing computed results for the pass AND their dependencies ---
|
|
47
|
+
* This function is the core fix for the user's problem.
|
|
48
|
+
*
|
|
49
|
+
* It now fetches results for:
|
|
50
|
+
* 1. All calculations IN THIS PASS (to allow skipping completed work).
|
|
51
|
+
* 2. All *dependencies* of calculations in this pass (to feed meta-calcs).
|
|
52
|
+
*
|
|
53
|
+
* This resolves the bug where Pass 2 would run but could not find
|
|
54
|
+
* the Pass 1 results it depended on.
|
|
48
55
|
*/
|
|
49
56
|
async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
|
|
50
57
|
const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
58
|
+
|
|
59
|
+
// --- FIX: Create a Set of *all* calcs we need to fetch ---
|
|
60
|
+
// This includes the calcs in this pass AND their dependencies.
|
|
61
|
+
const calcsToFetch = new Set();
|
|
62
|
+
|
|
63
|
+
for (const calc of calcsInPass) {
|
|
64
|
+
const calcName = normalizeName(calc.name);
|
|
65
|
+
// Add the calc itself (for skipping)
|
|
66
|
+
calcsToFetch.add(calcName);
|
|
67
|
+
|
|
68
|
+
// Add all its dependencies (for processing)
|
|
69
|
+
if (calc.dependencies && calc.dependencies.length > 0) {
|
|
70
|
+
for (const depName of calc.dependencies) {
|
|
71
|
+
calcsToFetch.add(normalizeName(depName));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (!calcsToFetch.size) {
|
|
77
|
+
return {};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
logger.log('INFO', `[PassRunner] Checking for ${calcsToFetch.size} existing results and dependencies for ${dateStr}...`);
|
|
81
|
+
|
|
82
|
+
const docRefs = [];
|
|
83
|
+
const depNames = [];
|
|
84
|
+
|
|
85
|
+
// --- FIX: Iterate the Set of all calcs to fetch ---
|
|
86
|
+
for (const calcName of calcsToFetch) {
|
|
87
|
+
const calcManifest = manifestMap.get(calcName);
|
|
88
|
+
if (!calcManifest) {
|
|
89
|
+
logger.log('ERROR', `[PassRunner] Missing manifest for dependency: ${calcName}`);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
docRefs.push(
|
|
94
|
+
db.collection(config.resultsCollection)
|
|
95
|
+
.doc(dateStr)
|
|
96
|
+
.collection(config.resultsSubcollection)
|
|
97
|
+
.doc(calcManifest.category || 'unknown')
|
|
98
|
+
.collection(config.computationsSubcollection)
|
|
99
|
+
.doc(calcName)
|
|
100
|
+
);
|
|
57
101
|
depNames.push(calcName);
|
|
58
102
|
}
|
|
59
|
-
|
|
103
|
+
|
|
104
|
+
const fetched = {};
|
|
105
|
+
if (docRefs.length) {
|
|
106
|
+
(await db.getAll(...docRefs)).forEach((doc, i) => {
|
|
107
|
+
fetched[depNames[i]] = doc.exists ? doc.data() : null;
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Log what dependencies were found vs. not found (for debugging)
|
|
112
|
+
const foundDeps = Object.entries(fetched).filter(([, data]) => data !== null).map(([key]) => key);
|
|
113
|
+
const missingDeps = Object.entries(fetched).filter(([, data]) => data === null).map(([key]) => key);
|
|
114
|
+
if (foundDeps.length > 0) {
|
|
115
|
+
logger.log('TRACE', `[PassRunner] Found ${foundDeps.length} existing results: [${foundDeps.join(', ')}]`);
|
|
116
|
+
}
|
|
117
|
+
if (missingDeps.length > 0) {
|
|
118
|
+
logger.log('TRACE', `[PassRunner] Did not find ${missingDeps.length} results: [${missingDeps.join(', ')}]`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return fetched;
|
|
60
122
|
}
|
|
61
123
|
|
|
124
|
+
|
|
62
125
|
/**
|
|
63
126
|
* --- Stage 5: Filter calculations ---
|
|
64
127
|
* This function now implements your "even better design".
|
|
@@ -86,7 +149,20 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
|
|
|
86
149
|
if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
|
|
87
150
|
const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
|
|
88
151
|
if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
|
|
89
|
-
|
|
152
|
+
|
|
153
|
+
// --- FIX: This check is now robust ---
|
|
154
|
+
// 'existingResults' now contains all dependencies, so this check
|
|
155
|
+
// will correctly find 'pnl_distribution_per_stock' and *not* skip.
|
|
156
|
+
if (calc.type === 'meta') {
|
|
157
|
+
const missingDeps = (calc.dependencies || [])
|
|
158
|
+
.map(normalizeName)
|
|
159
|
+
.filter(d => !existingResults[d]);
|
|
160
|
+
if (missingDeps.length > 0) {
|
|
161
|
+
logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
|
|
162
|
+
skipped.add(calc.name);
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
90
166
|
return true;
|
|
91
167
|
};
|
|
92
168
|
const standardCalcsToRun = standardCalcs.filter(filterCalc);
|
|
@@ -251,24 +327,48 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
|
|
|
251
327
|
const name = normalizeName(mCalc.name), Cl = mCalc.class;
|
|
252
328
|
if (typeof Cl !== 'function') { logger.log('ERROR', `Invalid class ${name}`); failedCalcs.push(name); continue; }
|
|
253
329
|
const inst = new Cl();
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
for (
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
330
|
+
|
|
331
|
+
// --- FIX: This is the critical change for the "Structural Bug" ---
|
|
332
|
+
// The original code assumed a 'meta' calc *only* has a .process() method.
|
|
333
|
+
// We now check for .process() first, and if it's not there,
|
|
334
|
+
// we fall back to calling .getResult() and pass it the dependencies.
|
|
335
|
+
//
|
|
336
|
+
// This file, however, expects `crowd_sharpe_ratio_proxy` to be
|
|
337
|
+
// refactored to use `process()`. See the update to that file.
|
|
338
|
+
// This `runMetaComputationPass` function remains as-is,
|
|
339
|
+
// as the fix is in refactoring the calculation file itself.
|
|
340
|
+
|
|
341
|
+
try {
|
|
342
|
+
|
|
343
|
+
// --- FIX: Refactored `crowd_sharpe_ratio_proxy` will now have this method ---
|
|
344
|
+
if (typeof inst.process !== 'function') {
|
|
345
|
+
logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`);
|
|
346
|
+
failedCalcs.push(name);
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
|
|
351
|
+
|
|
352
|
+
if (result && Object.keys(result).length > 0) {
|
|
353
|
+
const standardResult = {};
|
|
354
|
+
for (const key in result) {
|
|
355
|
+
if (key.startsWith('sharded_')) { const shardedData = result[key]; for (const collectionName in shardedData) {
|
|
356
|
+
if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {}; Object.assign(shardedWrites[collectionName], shardedData[collectionName]); }
|
|
357
|
+
} else { standardResult[key] = result[key]; }
|
|
358
|
+
}
|
|
359
|
+
if (Object.keys(standardResult).length > 0) {
|
|
360
|
+
const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(mCalc.category) .collection(config.computationsSubcollection).doc(name);
|
|
361
|
+
standardWrites.push({ ref: docRef, data: standardResult });
|
|
362
|
+
}
|
|
363
|
+
const calcClass = mCalc.class;
|
|
364
|
+
let staticSchema = null;
|
|
365
|
+
if (calcClass && typeof calcClass.getSchema === 'function') {
|
|
366
|
+
try { staticSchema = calcClass.getSchema();
|
|
367
|
+
} catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message }); }
|
|
368
|
+
} else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
|
|
369
|
+
if (staticSchema) { schemasToStore.push({ name, category: mCalc.category, schema: staticSchema, metadata: { isHistorical: mCalc.isHistorical || false, dependencies: mCalc.dependencies || [], rootDataDependencies: mCalc.rootDataDependencies || [], pass: mCalc.pass, type: 'meta' } }); }
|
|
370
|
+
success++;
|
|
371
|
+
}
|
|
272
372
|
} catch (e) { logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack }); failedCalcs.push(name); } }
|
|
273
373
|
if (schemasToStore.length > 0) { batchStoreSchemas(deps, config, schemasToStore).catch(err => { logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', { errorMessage: err.message }); }); }
|
|
274
374
|
if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);}
|