bulltrackers-module 1.0.257 → 1.0.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/WorkflowOrchestrator.js +47 -43
- package/functions/computation-system/config/validation_overrides.js +6 -0
- package/functions/computation-system/helpers/computation_dispatcher.js +1 -0
- package/functions/computation-system/helpers/computation_worker.js +5 -2
- package/functions/computation-system/persistence/ResultCommitter.js +28 -39
- package/functions/computation-system/persistence/ResultsValidator.js +136 -0
- package/index.js +2 -2
- package/package.json +1 -1
|
@@ -12,43 +12,35 @@ const { generateProcessId, PROCESS_TYPES } = require('./logger/l
|
|
|
12
12
|
|
|
13
13
|
const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
|
|
14
14
|
|
|
15
|
-
function groupByPass(manifest) {
|
|
16
|
-
return manifest.reduce((acc, calc) => {
|
|
17
|
-
(acc[calc.pass] = acc[calc.pass] || []).push(calc);
|
|
18
|
-
return acc;
|
|
19
|
-
}, {});
|
|
20
|
-
}
|
|
15
|
+
function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
|
|
21
16
|
|
|
22
17
|
/**
|
|
23
18
|
* Analyzes whether calculations should run, be skipped, or are blocked.
|
|
24
19
|
*/
|
|
25
20
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
26
|
-
const report
|
|
21
|
+
const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
|
|
27
22
|
const simulationStatus = { ...dailyStatus };
|
|
28
|
-
const isTargetToday
|
|
23
|
+
const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
|
|
29
24
|
|
|
30
25
|
const isDepSatisfied = (depName, currentStatusMap, manifestMap) => {
|
|
31
|
-
const norm
|
|
32
|
-
const stored
|
|
26
|
+
const norm = normalizeName(depName);
|
|
27
|
+
const stored = currentStatusMap[norm];
|
|
33
28
|
const depManifest = manifestMap.get(norm);
|
|
34
|
-
if (!stored)
|
|
29
|
+
if (!stored) return false;
|
|
35
30
|
if (stored.hash === STATUS_IMPOSSIBLE) return false;
|
|
36
|
-
if (!depManifest)
|
|
37
|
-
if (stored.hash !== depManifest.hash)
|
|
31
|
+
if (!depManifest) return false;
|
|
32
|
+
if (stored.hash !== depManifest.hash) return false;
|
|
38
33
|
return true;
|
|
39
34
|
};
|
|
40
35
|
|
|
41
36
|
for (const calc of calcsInPass) {
|
|
42
|
-
const cName
|
|
43
|
-
const stored
|
|
44
|
-
const storedHash
|
|
37
|
+
const cName = normalizeName(calc.name);
|
|
38
|
+
const stored = simulationStatus[cName];
|
|
39
|
+
const storedHash = stored ? stored.hash : null;
|
|
45
40
|
const storedCategory = stored ? stored.category : null;
|
|
46
|
-
const currentHash
|
|
41
|
+
const currentHash = calc.hash;
|
|
47
42
|
|
|
48
|
-
const markImpossible = (reason) => {
|
|
49
|
-
report.impossible.push({ name: cName, reason });
|
|
50
|
-
simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category };
|
|
51
|
-
};
|
|
43
|
+
const markImpossible = (reason) => { report.impossible.push({ name: cName, reason }); simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category }; };
|
|
52
44
|
|
|
53
45
|
const markRunnable = (isReRun = false, reRunDetails = null) => {
|
|
54
46
|
if (isReRun) report.reRuns.push(reRunDetails);
|
|
@@ -58,33 +50,33 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
58
50
|
|
|
59
51
|
let migrationOldCategory = null;
|
|
60
52
|
if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
|
|
61
|
-
|
|
62
|
-
if (storedHash === STATUS_IMPOSSIBLE) {
|
|
63
|
-
report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
|
|
64
|
-
continue;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// [FIX] Use the shared, strict checking logic from AvailabilityChecker
|
|
68
|
-
// This ensures 'speculator' calculations check 'speculatorPortfolio', etc.
|
|
53
|
+
if (storedHash === STATUS_IMPOSSIBLE) { report.skipped.push({ name: cName, reason: 'Permanently Impossible' }); continue; }
|
|
69
54
|
const rootCheck = checkRootDependencies(calc, rootDataStatus);
|
|
70
55
|
|
|
56
|
+
// Check Root Data Availability
|
|
57
|
+
// LOGIC : Root data is essential for any calculation
|
|
58
|
+
// Therefore if a computation has a dependency on rootdata that does not exist for the dates the computation requires, then the computation is impossible to run.
|
|
59
|
+
// However, to handle edge cases where we might test trigger the computation system early, we do not mark impossible if the computation requires data for today, it might arrive later, we just block and skip.
|
|
60
|
+
|
|
71
61
|
if (!rootCheck.canRun) {
|
|
72
62
|
const missingStr = rootCheck.missing.join(', ');
|
|
73
63
|
if (!isTargetToday) {
|
|
74
|
-
// Historical missing data is usually permanent/impossible
|
|
75
64
|
markImpossible(`Missing Root Data: ${missingStr} (Historical)`);
|
|
76
65
|
} else {
|
|
77
|
-
// Today's missing data might just be late (Blocked)
|
|
78
66
|
report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingStr} (Waiting)` });
|
|
79
67
|
}
|
|
80
68
|
continue;
|
|
81
69
|
}
|
|
82
70
|
|
|
71
|
+
// Check Calculation Dependencies
|
|
72
|
+
// LOGIC : If a calc B depends on calc A, and calc A is impossible, then calc B is always impossible
|
|
73
|
+
// This has a cascading effect, if calc C depends on calc B and calc B depends on calc A and calc A is impossible, then calc B and calc C are also impossible.
|
|
74
|
+
|
|
83
75
|
let dependencyIsImpossible = false;
|
|
84
76
|
const missingDeps = [];
|
|
85
77
|
if (calc.dependencies) {
|
|
86
78
|
for (const dep of calc.dependencies) {
|
|
87
|
-
const normDep
|
|
79
|
+
const normDep = normalizeName(dep);
|
|
88
80
|
const depStored = simulationStatus[normDep];
|
|
89
81
|
if (depStored && depStored.hash === STATUS_IMPOSSIBLE) { dependencyIsImpossible = true; break; }
|
|
90
82
|
if (!isDepSatisfied(dep, simulationStatus, manifestMap)) { missingDeps.push(dep); }
|
|
@@ -94,6 +86,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
94
86
|
if (dependencyIsImpossible) { markImpossible('Dependency is Impossible'); continue; }
|
|
95
87
|
if (missingDeps.length > 0) { report.failedDependency.push({ name: cName, missing: missingDeps }); continue; }
|
|
96
88
|
|
|
89
|
+
// Historical Continuity Check
|
|
90
|
+
// LOGIC : For computations that require historical data, we process them chronologically
|
|
91
|
+
// This is to handle the edge case where calc B runs for Tuesday data, but requires Mondays results from calc B.
|
|
92
|
+
// If we triggered a hash mismatch through updating the code of calc B, it would overwrite the results for Tuesday and Monday but without this,
|
|
93
|
+
// it would never be guaranteed that Monday runs before Tuesday, and so Tuesday would run with the old Monday hash data, or no data.
|
|
94
|
+
// This fixes this edge case by ensuring that historical computations only run if the previous day's computation has run with the latest hash, if not, it blocks and waits.
|
|
95
|
+
|
|
97
96
|
if (calc.isHistorical && prevDailyStatus) {
|
|
98
97
|
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
99
98
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
@@ -105,10 +104,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
105
104
|
}
|
|
106
105
|
}
|
|
107
106
|
}
|
|
108
|
-
|
|
107
|
+
// Final Hash Comparison
|
|
108
|
+
// LOGIC : If the stored hash matches the current hash, we don't need to run the computation again, unless the category stored does not match the current computation category
|
|
109
|
+
// This is to handle the edge case where a developer changes the category of a computation, the stored results need to be moved into the new location so we trigger a re-run to move the data and also delete the old category stored data.
|
|
110
|
+
|
|
109
111
|
if (!storedHash) { markRunnable(); }
|
|
110
112
|
else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); }
|
|
111
|
-
else if (migrationOldCategory) {
|
|
113
|
+
else if (migrationOldCategory) { markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); }
|
|
112
114
|
else { report.skipped.push({ name: cName }); simulationStatus[cName] = { hash: currentHash, category: calc.category }; }
|
|
113
115
|
}
|
|
114
116
|
return report;
|
|
@@ -117,8 +119,9 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
117
119
|
/**
|
|
118
120
|
* DIRECT EXECUTION PIPELINE (For Workers)
|
|
119
121
|
* Skips analysis. Assumes the calculation is valid and runnable.
|
|
122
|
+
* [UPDATED] Accepted previousCategory argument to handle migrations.
|
|
120
123
|
*/
|
|
121
|
-
async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest) {
|
|
124
|
+
async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null) {
|
|
122
125
|
const { logger } = dependencies;
|
|
123
126
|
const pid = generateProcessId(PROCESS_TYPES.EXECUTOR, targetComputation, dateStr);
|
|
124
127
|
|
|
@@ -128,8 +131,13 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
128
131
|
|
|
129
132
|
if (!calcManifest) { throw new Error(`Calculation '${targetComputation}' not found in manifest.`); }
|
|
130
133
|
|
|
134
|
+
// [UPDATED] Attach migration context if present
|
|
135
|
+
if (previousCategory) {
|
|
136
|
+
calcManifest.previousCategory = previousCategory;
|
|
137
|
+
logger.log('INFO', `[Executor] Migration detected for ${calcManifest.name}. Old data will be cleaned from: ${previousCategory}`);
|
|
138
|
+
}
|
|
139
|
+
|
|
131
140
|
// 2. Fetch Root Data Availability
|
|
132
|
-
// Note: this returns { status: {...}, portfolioRefs: null, historyRefs: null, ... }
|
|
133
141
|
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
134
142
|
|
|
135
143
|
if (!rootData) {
|
|
@@ -154,11 +162,8 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
154
162
|
let resultUpdates = {};
|
|
155
163
|
|
|
156
164
|
try {
|
|
157
|
-
if (calcManifest.type === 'standard') {
|
|
158
|
-
|
|
159
|
-
resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
|
|
160
|
-
} else if (calcManifest.type === 'meta') {
|
|
161
|
-
resultUpdates = await MetaExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
|
|
165
|
+
if (calcManifest.type === 'standard') { resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
|
|
166
|
+
} else if (calcManifest.type === 'meta') { resultUpdates = await MetaExecutor.run (new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
|
|
162
167
|
}
|
|
163
168
|
logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
|
|
164
169
|
return { date: dateStr, updates: resultUpdates };
|
|
@@ -168,6 +173,5 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
|
|
|
168
173
|
}
|
|
169
174
|
}
|
|
170
175
|
|
|
171
|
-
async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) { /* Legacy support stub */ }
|
|
172
176
|
|
|
173
|
-
module.exports = {
|
|
177
|
+
module.exports = { executeDispatchTask, groupByPass, analyzeDateExecution };
|
|
@@ -104,6 +104,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
104
104
|
pass: passToRun,
|
|
105
105
|
computation: normalizeName(item.name),
|
|
106
106
|
hash: item.hash || item.newHash, // [NEW] Ensure Hash is passed for Ledger
|
|
107
|
+
previousCategory: item.previousCategory || null, // [UPDATED] Pass migration context
|
|
107
108
|
timestamp: Date.now()
|
|
108
109
|
});
|
|
109
110
|
});
|
|
@@ -54,7 +54,8 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
54
54
|
|
|
55
55
|
if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
// [UPDATED] Destructure previousCategory from payload
|
|
58
|
+
const { date, pass, computation, previousCategory } = data;
|
|
58
59
|
|
|
59
60
|
if (!date || !pass || !computation) {
|
|
60
61
|
logger.log('ERROR', `[Worker] Invalid payload: Missing date, pass, or computation.`, data);
|
|
@@ -80,13 +81,15 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
80
81
|
logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
|
|
81
82
|
|
|
82
83
|
const startTime = Date.now();
|
|
84
|
+
// [UPDATED] Pass previousCategory to executor
|
|
83
85
|
const result = await executeDispatchTask(
|
|
84
86
|
date,
|
|
85
87
|
pass,
|
|
86
88
|
computation,
|
|
87
89
|
config,
|
|
88
90
|
runDependencies,
|
|
89
|
-
computationManifest
|
|
91
|
+
computationManifest,
|
|
92
|
+
previousCategory
|
|
90
93
|
);
|
|
91
94
|
const duration = Date.now() - startTime;
|
|
92
95
|
|
|
@@ -7,6 +7,9 @@ const { updateComputationStatus } = require('./StatusRepository');
|
|
|
7
7
|
const { batchStoreSchemas } = require('../utils/schema_capture');
|
|
8
8
|
const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
|
|
9
9
|
|
|
10
|
+
const { HeuristicValidator } = require('./ResultsValidator'); // Validator
|
|
11
|
+
const validationOverrides = require('../config/validation_overrides'); // Override file
|
|
12
|
+
|
|
10
13
|
async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
|
|
11
14
|
const successUpdates = {};
|
|
12
15
|
const failureReport = []; // [NEW] Track failures per calculation
|
|
@@ -21,21 +24,22 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
|
|
|
21
24
|
const calc = stateObj[name];
|
|
22
25
|
try {
|
|
23
26
|
const result = await calc.getResult();
|
|
27
|
+
|
|
28
|
+
const overrides = validationOverrides[calc.manifest.name] || {};
|
|
29
|
+
const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
|
|
30
|
+
|
|
31
|
+
if (!healthCheck.valid) {
|
|
32
|
+
// We throw a specific error stage so we know to BLOCK it, not retry it.
|
|
33
|
+
throw {
|
|
34
|
+
message: healthCheck.reason,
|
|
35
|
+
stage: 'QUALITY_CIRCUIT_BREAKER'
|
|
36
|
+
};
|
|
37
|
+
}
|
|
24
38
|
|
|
25
39
|
// Validate Result
|
|
26
|
-
const isEmpty = !result ||
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if (isEmpty) {
|
|
31
|
-
if (calc.manifest.hash) {
|
|
32
|
-
successUpdates[name] = {
|
|
33
|
-
hash: false,
|
|
34
|
-
category: calc.manifest.category
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
continue;
|
|
38
|
-
}
|
|
40
|
+
const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0) || (typeof result === 'number' && result === 0);
|
|
41
|
+
|
|
42
|
+
if (isEmpty) { if (calc.manifest.hash) { successUpdates[name] = { hash: false, category: calc.manifest.category }; } continue; }
|
|
39
43
|
|
|
40
44
|
const mainDocRef = db.collection(config.resultsCollection)
|
|
41
45
|
.doc(dStr)
|
|
@@ -89,25 +93,15 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
|
|
|
89
93
|
// Check for Firestore specific limits
|
|
90
94
|
let stage = 'COMMIT_BATCH';
|
|
91
95
|
let msg = commitErr.message;
|
|
92
|
-
if (msg.includes('Transaction too big') || msg.includes('payload is too large')) {
|
|
93
|
-
stage = 'SHARDING_LIMIT_EXCEEDED';
|
|
94
|
-
msg = `Firestore Limit Exceeded: ${msg}`;
|
|
95
|
-
}
|
|
96
|
+
if (msg.includes('Transaction too big') || msg.includes('payload is too large')) { stage = 'SHARDING_LIMIT_EXCEEDED'; msg = `Firestore Limit Exceeded: ${msg}`; }
|
|
96
97
|
throw { message: msg, stack: commitErr.stack, stage };
|
|
97
98
|
}
|
|
98
99
|
|
|
99
100
|
// Log Storage
|
|
100
|
-
if (logger && logger.logStorage) {
|
|
101
|
-
logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
|
|
102
|
-
}
|
|
101
|
+
if (logger && logger.logStorage) { logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded); }
|
|
103
102
|
|
|
104
103
|
// Mark Success
|
|
105
|
-
if (calc.manifest.hash) {
|
|
106
|
-
successUpdates[name] = {
|
|
107
|
-
hash: calc.manifest.hash,
|
|
108
|
-
category: calc.manifest.category
|
|
109
|
-
};
|
|
110
|
-
}
|
|
104
|
+
if (calc.manifest.hash) { successUpdates[name] = { hash: calc.manifest.hash, category: calc.manifest.category }; }
|
|
111
105
|
|
|
112
106
|
// Cleanup Migration
|
|
113
107
|
if (calc.manifest.previousCategory && calc.manifest.previousCategory !== calc.manifest.category) {
|
|
@@ -119,9 +113,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
|
|
|
119
113
|
const stage = e.stage || 'EXECUTION';
|
|
120
114
|
const msg = e.message || 'Unknown error';
|
|
121
115
|
|
|
122
|
-
if (logger && logger.log) {
|
|
123
|
-
logger.log('ERROR', `Commit failed for ${name} [${stage}]`, { processId: pid, error: msg });
|
|
124
|
-
}
|
|
116
|
+
if (logger && logger.log) { logger.log('ERROR', `Commit failed for ${name} [${stage}]`, { processId: pid, error: msg }); }
|
|
125
117
|
|
|
126
118
|
failureReport.push({
|
|
127
119
|
name,
|
|
@@ -132,13 +124,9 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
|
|
|
132
124
|
|
|
133
125
|
if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => {});
|
|
134
126
|
|
|
135
|
-
if (cleanupTasks.length > 0) {
|
|
136
|
-
await Promise.allSettled(cleanupTasks);
|
|
137
|
-
}
|
|
127
|
+
if (cleanupTasks.length > 0) { await Promise.allSettled(cleanupTasks); }
|
|
138
128
|
|
|
139
|
-
if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
|
|
140
|
-
await updateComputationStatus(dStr, successUpdates, config, deps);
|
|
141
|
-
}
|
|
129
|
+
if (!skipStatusWrite && Object.keys(successUpdates).length > 0) { await updateComputationStatus(dStr, successUpdates, config, deps); }
|
|
142
130
|
|
|
143
131
|
// [UPDATE] Return both success and failures so the Worker can log them
|
|
144
132
|
return { successUpdates, failureReport };
|
|
@@ -146,8 +134,12 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
|
|
|
146
134
|
|
|
147
135
|
/**
|
|
148
136
|
* Deletes result documents from a previous category location.
|
|
137
|
+
* This function exists to handle the deleteion of old data,
|
|
138
|
+
* The general use case is that if a developer changes a calculations' category,
|
|
139
|
+
* We want to clean up and delete the old path, the change of a category would trigger a re-run of the calculation to naturally move the data to the new location
|
|
149
140
|
*/
|
|
150
141
|
async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
|
|
142
|
+
|
|
151
143
|
const { db, logger, calculationUtils } = deps;
|
|
152
144
|
const { withRetry } = calculationUtils || { withRetry: (fn) => fn() };
|
|
153
145
|
|
|
@@ -165,10 +157,7 @@ async function deleteOldCalculationData(dateStr, oldCategory, calcName, config,
|
|
|
165
157
|
const batch = db.batch();
|
|
166
158
|
let ops = 0;
|
|
167
159
|
|
|
168
|
-
for (const shardDoc of shardsSnap) {
|
|
169
|
-
batch.delete(shardDoc);
|
|
170
|
-
ops++;
|
|
171
|
-
}
|
|
160
|
+
for (const shardDoc of shardsSnap) { batch.delete(shardDoc); ops++; }
|
|
172
161
|
batch.delete(oldDocRef);
|
|
173
162
|
ops++;
|
|
174
163
|
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview HeuristicValidator.js
|
|
3
|
+
* "Grey Box" validation that infers health using statistical analysis and structural sanity checks.
|
|
4
|
+
* UPDATED: Added NaN detection, Flatline (Variance) checks, and Vector/Array depth checks.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
class HeuristicValidator {
|
|
8
|
+
/**
|
|
9
|
+
* @param {string} calcName - Name for logging
|
|
10
|
+
* @param {Object} data - The result data to inspect
|
|
11
|
+
* @param {Object} [overrides] - Optional central config overrides
|
|
12
|
+
*/
|
|
13
|
+
static analyze(calcName, data, overrides = {}) {
|
|
14
|
+
// 1. Structure Check
|
|
15
|
+
if (!data || typeof data !== 'object') return { valid: true }; // Let scalar types pass
|
|
16
|
+
|
|
17
|
+
const keys = Object.keys(data);
|
|
18
|
+
const totalItems = keys.length;
|
|
19
|
+
|
|
20
|
+
// Skip tiny datasets (statistically insignificant)
|
|
21
|
+
if (totalItems < 5) return { valid: true };
|
|
22
|
+
|
|
23
|
+
// 2. Sampling Configuration
|
|
24
|
+
const sampleSize = Math.min(totalItems, 100);
|
|
25
|
+
const step = Math.floor(totalItems / sampleSize);
|
|
26
|
+
|
|
27
|
+
let zeroCount = 0;
|
|
28
|
+
let nullCount = 0;
|
|
29
|
+
let nanCount = 0; // NEW: Track NaNs
|
|
30
|
+
let emptyVectorCount = 0; // NEW: Track empty arrays in complex objects
|
|
31
|
+
let analyzedCount = 0;
|
|
32
|
+
|
|
33
|
+
// For Variance/Flatline Check
|
|
34
|
+
const numericValues = [];
|
|
35
|
+
|
|
36
|
+
for (let i = 0; i < totalItems; i += step) {
|
|
37
|
+
const key = keys[i];
|
|
38
|
+
const val = data[key];
|
|
39
|
+
if (!val) { // Catch null/undefined immediately
|
|
40
|
+
nullCount++;
|
|
41
|
+
analyzedCount++;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
analyzedCount++;
|
|
45
|
+
|
|
46
|
+
// --- TYPE A: Object / Complex Result ---
|
|
47
|
+
// Example: { "profile": [...], "current_price": 100 } or { "signal": "Buy", "score": 0.5 }
|
|
48
|
+
if (typeof val === 'object') {
|
|
49
|
+
const subValues = Object.values(val);
|
|
50
|
+
|
|
51
|
+
// Dead Object Check: All props are null/0/undefined
|
|
52
|
+
const isDeadObject = subValues.every(v => v === 0 || v === null || v === undefined);
|
|
53
|
+
if (isDeadObject) nullCount++;
|
|
54
|
+
|
|
55
|
+
// NaN Check in Properties
|
|
56
|
+
const hasNan = subValues.some(v => typeof v === 'number' && (isNaN(v) || !isFinite(v)));
|
|
57
|
+
if (hasNan) nanCount++;
|
|
58
|
+
|
|
59
|
+
// Vector/Profile Empty Check (Specific to your System)
|
|
60
|
+
// If result contains 'profile', 'history', 'sparkline', or 'buckets' arrays
|
|
61
|
+
const arrayProps = ['profile', 'history', 'sparkline', 'buckets', 'prices'];
|
|
62
|
+
for (const prop of arrayProps) {
|
|
63
|
+
if (Array.isArray(val[prop]) && val[prop].length === 0) {
|
|
64
|
+
emptyVectorCount++;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Extract primary numeric score for Flatline check (heuristically guessing the 'main' metric)
|
|
69
|
+
const numericProp = subValues.find(v => typeof v === 'number' && v !== 0);
|
|
70
|
+
if (numericProp !== undefined) numericValues.push(numericProp);
|
|
71
|
+
}
|
|
72
|
+
// --- TYPE B: Scalar / Primitive Result ---
|
|
73
|
+
else if (typeof val === 'number') {
|
|
74
|
+
if (val === 0) zeroCount++;
|
|
75
|
+
if (isNaN(val) || !isFinite(val)) nanCount++;
|
|
76
|
+
else numericValues.push(val);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// 3. Thresholds
|
|
81
|
+
const thresholds = {
|
|
82
|
+
maxZeroPct: overrides.maxZeroPct ?? 99,
|
|
83
|
+
maxNullPct: overrides.maxNullPct ?? 90,
|
|
84
|
+
maxNanPct: overrides.maxNanPct ?? 0, // Strict: NaNs are usually bad bugs
|
|
85
|
+
maxFlatlinePct: 95 // If >95% of data is identical, it's suspicious
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
// 4. Calculate Stats
|
|
89
|
+
const zeroPct = (zeroCount / analyzedCount) * 100;
|
|
90
|
+
const nullPct = (nullCount / analyzedCount) * 100;
|
|
91
|
+
const nanPct = (nanCount / analyzedCount) * 100;
|
|
92
|
+
|
|
93
|
+
// 5. Variance / Flatline Analysis
|
|
94
|
+
// If we found numeric values, check if they are all the same
|
|
95
|
+
let isFlatline = false;
|
|
96
|
+
if (numericValues.length > 5) {
|
|
97
|
+
const first = numericValues[0];
|
|
98
|
+
const identicalCount = numericValues.filter(v => Math.abs(v - first) < 0.000001).length;
|
|
99
|
+
const flatlinePct = (identicalCount / numericValues.length) * 100;
|
|
100
|
+
|
|
101
|
+
// Only flag flatline if the value isn't 0 (0 is handled by maxZeroPct)
|
|
102
|
+
if (flatlinePct > thresholds.maxFlatlinePct && Math.abs(first) > 0.0001) {
|
|
103
|
+
isFlatline = true;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// 6. Evaluations
|
|
108
|
+
if (nanPct > thresholds.maxNanPct) {
|
|
109
|
+
return { valid: false, reason: `Mathematical Error: ${nanPct.toFixed(1)}% of sampled results contain NaN or Infinity.` };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (zeroPct > thresholds.maxZeroPct) {
|
|
113
|
+
return { valid: false, reason: `Data Integrity: ${zeroPct.toFixed(1)}% of sampled results are 0. (Suspected Logic Failure)` };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (nullPct > thresholds.maxNullPct) {
|
|
117
|
+
return { valid: false, reason: `Data Integrity: ${nullPct.toFixed(1)}% of sampled results are Empty/Null.` };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (isFlatline) {
|
|
121
|
+
return { valid: false, reason: `Anomaly: Detected Result Flatline. >${thresholds.maxFlatlinePct}% of outputs are identical (non-zero).` };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Special check for Distribution/Profile calculations
|
|
125
|
+
if (calcName.includes('profile') || calcName.includes('distribution')) {
|
|
126
|
+
const vectorEmptyPct = (emptyVectorCount / analyzedCount) * 100;
|
|
127
|
+
if (vectorEmptyPct > 90) {
|
|
128
|
+
return { valid: false, reason: `Data Integrity: ${vectorEmptyPct.toFixed(1)}% of distribution profiles are empty.` };
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return { valid: true };
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
module.exports = { HeuristicValidator };
|
package/index.js
CHANGED
|
@@ -27,7 +27,7 @@ const { handleUpdate } = require('./functions
|
|
|
27
27
|
|
|
28
28
|
// Computation System
|
|
29
29
|
const { build: buildManifest } = require('./functions/computation-system/context/ManifestBuilder');
|
|
30
|
-
const { runDateComputation: runComputationPass } = require('./functions/computation-system/WorkflowOrchestrator');
|
|
30
|
+
// const { runDateComputation: runComputationPass } = require('./functions/computation-system/WorkflowOrchestrator'); Depreciated
|
|
31
31
|
const { dispatchComputationPass } = require('./functions/computation-system/helpers/computation_dispatcher');
|
|
32
32
|
const { handleComputationTask } = require('./functions/computation-system/helpers/computation_worker');
|
|
33
33
|
// [NEW] Import Report Tools
|
|
@@ -88,7 +88,7 @@ const taskEngine = {
|
|
|
88
88
|
};
|
|
89
89
|
|
|
90
90
|
const computationSystem = {
|
|
91
|
-
runComputationPass,
|
|
91
|
+
// runComputationPass, Depreciated
|
|
92
92
|
dispatchComputationPass,
|
|
93
93
|
handleComputationTask,
|
|
94
94
|
dataLoader,
|