bulltrackers-module 1.0.228 → 1.0.230
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Main Orchestrator. Coordinates the topological execution.
|
|
3
|
-
* UPDATED:
|
|
3
|
+
* UPDATED: Implements 'IMPOSSIBLE' state logic for missing root data on historical dates.
|
|
4
4
|
*/
|
|
5
5
|
const { normalizeName } = require('./utils/utils');
|
|
6
6
|
const { checkRootDataAvailability } = require('./data/AvailabilityChecker');
|
|
@@ -10,6 +10,9 @@ const { StandardExecutor } = require('./executor
|
|
|
10
10
|
const { MetaExecutor } = require('./executors/MetaExecutor');
|
|
11
11
|
const { generateProcessId, PROCESS_TYPES } = require('./logger/logger');
|
|
12
12
|
|
|
13
|
+
// New Status Constant
|
|
14
|
+
const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
|
|
15
|
+
|
|
13
16
|
function groupByPass(manifest) {
|
|
14
17
|
return manifest.reduce((acc, calc) => {
|
|
15
18
|
(acc[calc.pass] = acc[calc.pass] || []).push(calc);
|
|
@@ -18,28 +21,32 @@ function groupByPass(manifest) {
|
|
|
18
21
|
}
|
|
19
22
|
|
|
20
23
|
/**
|
|
21
|
-
* Performs strict analysis of what can run
|
|
22
|
-
*
|
|
23
|
-
* 1.
|
|
24
|
-
* 2.
|
|
25
|
-
* 3.
|
|
26
|
-
* 4. No Result -> Run
|
|
27
|
-
* 5. Result Exists & Hash Match -> Skip
|
|
24
|
+
* Performs strict analysis of what can run.
|
|
25
|
+
* IMPOSSIBLE LOGIC:
|
|
26
|
+
* 1. If Root Data is missing AND Date != Today -> IMPOSSIBLE.
|
|
27
|
+
* 2. If Dependency is IMPOSSIBLE -> IMPOSSIBLE.
|
|
28
|
+
* 3. IMPOSSIBLE items are written to DB to prevent future retries.
|
|
28
29
|
*/
|
|
29
30
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
|
|
30
31
|
const report = {
|
|
31
32
|
runnable: [],
|
|
32
|
-
blocked: [], // Missing Root Data
|
|
33
|
-
|
|
33
|
+
blocked: [], // Missing Root Data (Today - Retriable)
|
|
34
|
+
impossible: [], // Missing Root Data (Historical) or Dependency Impossible
|
|
35
|
+
failedDependency: [], // Missing/Stale Dependency (Transient)
|
|
34
36
|
reRuns: [], // Hash Mismatch
|
|
35
37
|
skipped: [] // Already done & valid
|
|
36
38
|
};
|
|
37
39
|
|
|
40
|
+
const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
|
|
41
|
+
|
|
38
42
|
const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
|
|
39
43
|
const norm = normalizeName(depName);
|
|
40
44
|
const storedDepHash = dailyStatus[norm];
|
|
41
45
|
const depManifest = manifestMap.get(norm);
|
|
42
46
|
|
|
47
|
+
// Check 1: Is dependency IMPOSSIBLE? (Logic handled in main loop, but safe to check here)
|
|
48
|
+
if (storedDepHash === STATUS_IMPOSSIBLE) return false;
|
|
49
|
+
|
|
43
50
|
if (!storedDepHash) return false;
|
|
44
51
|
if (!depManifest) return false;
|
|
45
52
|
if (storedDepHash !== depManifest.hash) return false;
|
|
@@ -52,7 +59,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
52
59
|
const storedHash = dailyStatus[cName];
|
|
53
60
|
const currentHash = calc.hash;
|
|
54
61
|
|
|
55
|
-
// 1.
|
|
62
|
+
// 1. Check if ALREADY marked IMPOSSIBLE
|
|
63
|
+
if (storedHash === STATUS_IMPOSSIBLE) {
|
|
64
|
+
report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// 2. Root Data Check
|
|
56
69
|
const missingRoots = [];
|
|
57
70
|
if (calc.rootDataDependencies) {
|
|
58
71
|
for (const dep of calc.rootDataDependencies) {
|
|
@@ -65,27 +78,49 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
65
78
|
}
|
|
66
79
|
|
|
67
80
|
if (missingRoots.length > 0) {
|
|
68
|
-
|
|
81
|
+
// LOGIC: If date is NOT today, missing root data is fatal and permanent.
|
|
82
|
+
if (!isTargetToday) {
|
|
83
|
+
report.impossible.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Historical)` });
|
|
84
|
+
} else {
|
|
85
|
+
report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Waiting)` });
|
|
86
|
+
}
|
|
69
87
|
continue;
|
|
70
88
|
}
|
|
71
89
|
|
|
72
|
-
//
|
|
90
|
+
// 3. Dependency Check
|
|
91
|
+
let dependencyIsImpossible = false;
|
|
73
92
|
const missingDeps = [];
|
|
93
|
+
|
|
74
94
|
if (calc.dependencies) {
|
|
75
95
|
for (const dep of calc.dependencies) {
|
|
96
|
+
const normDep = normalizeName(dep);
|
|
97
|
+
|
|
98
|
+
// Check if the dependency is marked IMPOSSIBLE in the DB
|
|
99
|
+
if (dailyStatus[normDep] === STATUS_IMPOSSIBLE) {
|
|
100
|
+
dependencyIsImpossible = true;
|
|
101
|
+
// We can break early, if one input is impossible, the result is impossible.
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
|
|
76
105
|
if (!isDepSatisfied(dep, dailyStatus, manifestMap)) {
|
|
77
106
|
missingDeps.push(dep);
|
|
78
107
|
}
|
|
79
108
|
}
|
|
80
109
|
}
|
|
81
110
|
|
|
111
|
+
if (dependencyIsImpossible) {
|
|
112
|
+
// Propagate the Impossible Status
|
|
113
|
+
report.impossible.push({ name: cName, reason: 'Dependency is Impossible' });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
|
|
82
117
|
if (missingDeps.length > 0) {
|
|
83
118
|
report.failedDependency.push({ name: cName, missing: missingDeps });
|
|
84
119
|
continue;
|
|
85
120
|
}
|
|
86
121
|
|
|
87
|
-
//
|
|
88
|
-
if (!storedHash) {
|
|
122
|
+
// 4. Hash / State Check
|
|
123
|
+
if (!storedHash || storedHash === false) { // false indicates previous transient failure
|
|
89
124
|
report.runnable.push(calc);
|
|
90
125
|
} else if (storedHash !== currentHash) {
|
|
91
126
|
report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
|
|
@@ -123,23 +158,28 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
|
|
|
123
158
|
const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
|
|
124
159
|
const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap);
|
|
125
160
|
|
|
126
|
-
// 4. LOG ANALYSIS
|
|
161
|
+
// 4. LOG ANALYSIS
|
|
127
162
|
if (logger && typeof logger.logDateAnalysis === 'function') {
|
|
128
163
|
logger.logDateAnalysis(dateStr, analysisReport);
|
|
129
164
|
} else {
|
|
130
|
-
//
|
|
131
|
-
const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} |
|
|
165
|
+
// Safe fallback
|
|
166
|
+
const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | Impossible: ${analysisReport.impossible.length}`;
|
|
132
167
|
if (logger && logger.info) logger.info(logMsg);
|
|
133
168
|
else console.log(logMsg);
|
|
134
169
|
}
|
|
135
170
|
|
|
136
|
-
// 5.
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
171
|
+
// 5. UPDATE STATUS FOR NON-RUNNABLE ITEMS
|
|
172
|
+
const statusUpdates = {};
|
|
173
|
+
|
|
174
|
+
// A. Mark BLOCKED as 'false' (Transient Failure)
|
|
175
|
+
analysisReport.blocked.forEach(item => statusUpdates[item.name] = false);
|
|
176
|
+
analysisReport.failedDependency.forEach(item => statusUpdates[item.name] = false);
|
|
177
|
+
|
|
178
|
+
// B. Mark IMPOSSIBLE as 'IMPOSSIBLE' (Permanent Failure - Overwrites existing status)
|
|
179
|
+
analysisReport.impossible.forEach(item => statusUpdates[item.name] = STATUS_IMPOSSIBLE);
|
|
140
180
|
|
|
141
|
-
if (Object.keys(
|
|
142
|
-
await updateComputationStatus(dateStr,
|
|
181
|
+
if (Object.keys(statusUpdates).length > 0) {
|
|
182
|
+
await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
|
|
143
183
|
}
|
|
144
184
|
|
|
145
185
|
// 6. EXECUTE RUNNABLES
|
|
@@ -151,7 +191,12 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
|
|
|
151
191
|
const finalRunList = calcsInThisPass.filter(c => calcsToRunNames.has(normalizeName(c.name)));
|
|
152
192
|
|
|
153
193
|
if (!finalRunList.length) {
|
|
154
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
date: dateStr,
|
|
196
|
+
updates: {},
|
|
197
|
+
skipped: analysisReport.skipped.length,
|
|
198
|
+
impossible: analysisReport.impossible.length
|
|
199
|
+
};
|
|
155
200
|
}
|
|
156
201
|
|
|
157
202
|
if (logger && logger.log) {
|
|
@@ -1,26 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Structured Logging System for Computation Engine
|
|
3
|
-
*
|
|
4
|
-
* UPDATED: Includes Pre-flight Date Analysis and Storage Observability.
|
|
3
|
+
* UPDATED: Added 'impossible' category handling to Date Analysis.
|
|
5
4
|
*/
|
|
6
5
|
|
|
7
6
|
const crypto = require('crypto');
|
|
8
7
|
|
|
9
|
-
/**
|
|
10
|
-
* Log Levels (Ordered by Severity)
|
|
11
|
-
*/
|
|
12
8
|
const LOG_LEVELS = {
|
|
13
|
-
TRACE: 0,
|
|
14
|
-
DEBUG: 1,
|
|
15
|
-
INFO: 2,
|
|
16
|
-
WARN: 3,
|
|
17
|
-
ERROR: 4,
|
|
18
|
-
FATAL: 5
|
|
9
|
+
TRACE: 0, DEBUG: 1, INFO: 2, WARN: 3, ERROR: 4, FATAL: 5
|
|
19
10
|
};
|
|
20
11
|
|
|
21
|
-
/**
|
|
22
|
-
* Process Types for Tracking
|
|
23
|
-
*/
|
|
24
12
|
const PROCESS_TYPES = {
|
|
25
13
|
MANIFEST: 'manifest',
|
|
26
14
|
ORCHESTRATOR: 'orchestrator',
|
|
@@ -28,30 +16,14 @@ const PROCESS_TYPES = {
|
|
|
28
16
|
STORAGE: 'storage',
|
|
29
17
|
WORKER: 'worker',
|
|
30
18
|
ANALYSIS: 'analysis',
|
|
31
|
-
// Legacy / Specific aliases
|
|
32
|
-
SCHEMA_GENERATION: 'schema_generation',
|
|
33
|
-
COMPUTATION_EXECUTION: 'computation_execution',
|
|
34
|
-
DEPENDENCY_FETCH: 'dependency_fetch',
|
|
35
|
-
DATA_AVAILABILITY: 'data_availability',
|
|
36
19
|
DISPATCH: 'dispatch'
|
|
37
20
|
};
|
|
38
21
|
|
|
39
|
-
/**
|
|
40
|
-
* Generates a deterministic process ID from inputs.
|
|
41
|
-
* Ensures that logs for the same computation/date always have the same Trace ID.
|
|
42
|
-
* @param {string} type - Process type (e.g., 'orchestrator')
|
|
43
|
-
* @param {string} identifier - Unique key (e.g., pass name, calculation name)
|
|
44
|
-
* @param {string} date - Date string YYYY-MM-DD (optional)
|
|
45
|
-
* @returns {string} 16-character hexadecimal process ID
|
|
46
|
-
*/
|
|
47
22
|
function generateProcessId(type, identifier, date = '') {
|
|
48
23
|
const input = `${type}|${identifier}|${date}`;
|
|
49
24
|
return crypto.createHash('sha256').update(input).digest('hex').substring(0, 16);
|
|
50
25
|
}
|
|
51
26
|
|
|
52
|
-
/**
|
|
53
|
-
* Formats a log entry into structured JSON
|
|
54
|
-
*/
|
|
55
27
|
function formatLogEntry(entry) {
|
|
56
28
|
return JSON.stringify({
|
|
57
29
|
timestamp: entry.timestamp,
|
|
@@ -63,16 +35,12 @@ function formatLogEntry(entry) {
|
|
|
63
35
|
message: entry.message,
|
|
64
36
|
context: entry.context,
|
|
65
37
|
metadata: entry.metadata,
|
|
66
|
-
// Specific fields for specialized logs
|
|
67
38
|
stats: entry.stats,
|
|
68
39
|
storage: entry.storage,
|
|
69
40
|
details: entry.details
|
|
70
41
|
});
|
|
71
42
|
}
|
|
72
43
|
|
|
73
|
-
/**
|
|
74
|
-
* Main Structured Logger Class
|
|
75
|
-
*/
|
|
76
44
|
class StructuredLogger {
|
|
77
45
|
constructor(config = {}) {
|
|
78
46
|
this.config = {
|
|
@@ -82,25 +50,15 @@ class StructuredLogger {
|
|
|
82
50
|
includeStackTrace: config.includeStackTrace !== false,
|
|
83
51
|
...config
|
|
84
52
|
};
|
|
85
|
-
|
|
86
53
|
this.activeProcesses = new Map();
|
|
87
54
|
}
|
|
88
55
|
|
|
89
|
-
/**
|
|
90
|
-
* Starts a tracked process and returns a ProcessLogger
|
|
91
|
-
* @param {string} processType - Type from PROCESS_TYPES
|
|
92
|
-
* @param {string} computationName - Optional computation name
|
|
93
|
-
* @param {string} date - Optional date string
|
|
94
|
-
* @returns {ProcessLogger}
|
|
95
|
-
*/
|
|
96
56
|
startProcess(processType, computationName = null, date = null) {
|
|
97
|
-
// Use deterministic ID if components are present, else random/time-based fallback
|
|
98
57
|
const processId = (computationName || date)
|
|
99
58
|
? generateProcessId(processType, computationName || 'general', date)
|
|
100
59
|
: crypto.randomBytes(8).toString('hex');
|
|
101
60
|
|
|
102
61
|
const processLogger = new ProcessLogger(this, processType, processId, computationName, date);
|
|
103
|
-
|
|
104
62
|
this.activeProcesses.set(processId, {
|
|
105
63
|
type: processType,
|
|
106
64
|
computationName,
|
|
@@ -108,13 +66,9 @@ class StructuredLogger {
|
|
|
108
66
|
startTime: Date.now(),
|
|
109
67
|
logger: processLogger
|
|
110
68
|
});
|
|
111
|
-
|
|
112
69
|
return processLogger;
|
|
113
70
|
}
|
|
114
71
|
|
|
115
|
-
/**
|
|
116
|
-
* Ends a tracked process
|
|
117
|
-
*/
|
|
118
72
|
endProcess(processId) {
|
|
119
73
|
const process = this.activeProcesses.get(processId);
|
|
120
74
|
if (process) {
|
|
@@ -126,13 +80,20 @@ class StructuredLogger {
|
|
|
126
80
|
}
|
|
127
81
|
|
|
128
82
|
/**
|
|
129
|
-
* The Main Date Analysis Logger (
|
|
130
|
-
*
|
|
83
|
+
* The Main Date Analysis Logger (UPDATED)
|
|
84
|
+
* Handles 'impossible' category for permanent failures.
|
|
131
85
|
*/
|
|
132
86
|
logDateAnalysis(dateStr, analysisReport) {
|
|
133
|
-
const {
|
|
87
|
+
const {
|
|
88
|
+
runnable = [],
|
|
89
|
+
blocked = [],
|
|
90
|
+
impossible = [], // New Category
|
|
91
|
+
reRuns = [],
|
|
92
|
+
failedDependency = [],
|
|
93
|
+
skipped = []
|
|
94
|
+
} = analysisReport;
|
|
134
95
|
|
|
135
|
-
// 1. Structured Output
|
|
96
|
+
// 1. Structured Output
|
|
136
97
|
if (this.config.enableStructured) {
|
|
137
98
|
console.log(JSON.stringify({
|
|
138
99
|
timestamp: new Date().toISOString(),
|
|
@@ -143,30 +104,43 @@ class StructuredLogger {
|
|
|
143
104
|
stats: {
|
|
144
105
|
runnable: runnable.length,
|
|
145
106
|
blocked: blocked.length,
|
|
107
|
+
impossible: impossible.length,
|
|
146
108
|
reRuns: reRuns.length,
|
|
147
|
-
|
|
109
|
+
failedDependency: failedDependency.length,
|
|
110
|
+
skipped: skipped.length
|
|
148
111
|
},
|
|
149
112
|
details: analysisReport
|
|
150
113
|
}));
|
|
151
114
|
}
|
|
152
115
|
|
|
153
|
-
// 2. Human Readable Output
|
|
116
|
+
// 2. Human Readable Output
|
|
154
117
|
if (this.config.enableConsole) {
|
|
155
|
-
const symbols = {
|
|
118
|
+
const symbols = {
|
|
119
|
+
info: 'ℹ️', warn: '⚠️', check: '✅', block: '⛔',
|
|
120
|
+
cycle: '🔄', skip: '⏭️', dead: '💀'
|
|
121
|
+
};
|
|
156
122
|
|
|
157
123
|
console.log(`\n🔍 === DATE ANALYSIS REPORT: ${dateStr} ===`);
|
|
158
124
|
|
|
159
125
|
if (reRuns.length) {
|
|
160
126
|
console.log(`\n${symbols.cycle} [HASH MISMATCH / RE-RUNS]`);
|
|
161
127
|
reRuns.forEach(item => {
|
|
162
|
-
console.log(` • ${item.name}: Hash changed (Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...)
|
|
128
|
+
console.log(` • ${item.name}: Hash changed. (Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...)`);
|
|
163
129
|
});
|
|
164
130
|
}
|
|
165
131
|
|
|
166
|
-
if (
|
|
167
|
-
console.log(`\n${symbols.
|
|
168
|
-
|
|
169
|
-
console.log(` • ${item.name}:
|
|
132
|
+
if (failedDependency.length) {
|
|
133
|
+
console.log(`\n${symbols.block} [FAILED DEPENDENCIES] (Upstream failure)`);
|
|
134
|
+
failedDependency.forEach(item => {
|
|
135
|
+
console.log(` • ${item.name}: Missing ${item.missing ? item.missing.join(', ') : 'dependencies'}`);
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// NEW: Impossible items
|
|
140
|
+
if (impossible.length) {
|
|
141
|
+
console.log(`\n${symbols.dead} [IMPOSSIBLE] (Permanent Failure - Will not retry)`);
|
|
142
|
+
impossible.forEach(item => {
|
|
143
|
+
console.log(` • ${item.name}: ${item.reason}`);
|
|
170
144
|
});
|
|
171
145
|
}
|
|
172
146
|
|
|
@@ -176,20 +150,26 @@ class StructuredLogger {
|
|
|
176
150
|
}
|
|
177
151
|
|
|
178
152
|
if (blocked.length) {
|
|
179
|
-
console.log(`\n${symbols.
|
|
153
|
+
console.log(`\n${symbols.warn} [BLOCKED] (Waiting for data - Retriable)`);
|
|
180
154
|
blocked.forEach(item => {
|
|
181
155
|
console.log(` • ${item.name}: ${item.reason}`);
|
|
182
156
|
});
|
|
183
157
|
}
|
|
158
|
+
|
|
159
|
+
if (skipped.length) {
|
|
160
|
+
if (skipped.length > 10) {
|
|
161
|
+
console.log(`\n${symbols.skip} [SKIPPED] (${skipped.length} calculations up-to-date)`);
|
|
162
|
+
} else {
|
|
163
|
+
console.log(`\n${symbols.skip} [SKIPPED]`);
|
|
164
|
+
skipped.forEach(item => console.log(` • ${item.name}`));
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
184
168
|
console.log(`\n=============================================\n`);
|
|
185
169
|
}
|
|
186
170
|
}
|
|
187
171
|
|
|
188
|
-
/**
|
|
189
|
-
* Storage Observability Logger (NEW)
|
|
190
|
-
*/
|
|
191
172
|
logStorage(processId, calcName, date, path, sizeBytes, isSharded) {
|
|
192
|
-
// Standard Log Call with extended metadata
|
|
193
173
|
this.log(LOG_LEVELS.INFO, `Results stored for ${calcName}`, {
|
|
194
174
|
storage: {
|
|
195
175
|
path,
|
|
@@ -200,16 +180,10 @@ class StructuredLogger {
|
|
|
200
180
|
}, PROCESS_TYPES.STORAGE, processId, calcName, date);
|
|
201
181
|
}
|
|
202
182
|
|
|
203
|
-
/**
|
|
204
|
-
* Core logging method
|
|
205
|
-
*/
|
|
206
183
|
log(level, message, context = {}, processType = null, processId = null, computationName = null, date = null) {
|
|
207
184
|
const numericLevel = typeof level === 'string' ? LOG_LEVELS[level] : level;
|
|
208
|
-
|
|
209
185
|
if (numericLevel < this.config.minLevel) return;
|
|
210
186
|
|
|
211
|
-
// Support passing "meta" object in place of context for newer calls
|
|
212
|
-
// Logic: If context contains 'processId' or 'storage', it's likely a meta object
|
|
213
187
|
let finalContext = context;
|
|
214
188
|
let finalMetadata = {};
|
|
215
189
|
let finalStats = undefined;
|
|
@@ -222,7 +196,6 @@ class StructuredLogger {
|
|
|
222
196
|
if (context.date) date = context.date;
|
|
223
197
|
if (context.stats) finalStats = context.stats;
|
|
224
198
|
if (context.storage) finalStorage = context.storage;
|
|
225
|
-
// Clean up context to be just the data remaining
|
|
226
199
|
finalContext = { ...context };
|
|
227
200
|
delete finalContext.processId; delete finalContext.processType;
|
|
228
201
|
delete finalContext.computationName; delete finalContext.date;
|
|
@@ -243,25 +216,19 @@ class StructuredLogger {
|
|
|
243
216
|
storage: finalStorage
|
|
244
217
|
};
|
|
245
218
|
|
|
246
|
-
// Add stack trace for errors
|
|
247
219
|
if (numericLevel >= LOG_LEVELS.ERROR && this.config.includeStackTrace && finalContext.stack) {
|
|
248
220
|
entry.metadata.stackTrace = finalContext.stack;
|
|
249
221
|
}
|
|
250
222
|
|
|
251
|
-
// Console output (pretty-printed for development)
|
|
252
223
|
if (this.config.enableConsole) {
|
|
253
224
|
this._consoleLog(entry);
|
|
254
225
|
}
|
|
255
226
|
|
|
256
|
-
// Structured output (for log aggregation systems)
|
|
257
227
|
if (this.config.enableStructured) {
|
|
258
228
|
console.log(formatLogEntry(entry));
|
|
259
229
|
}
|
|
260
230
|
}
|
|
261
231
|
|
|
262
|
-
/**
|
|
263
|
-
* Pretty console output for development
|
|
264
|
-
*/
|
|
265
232
|
_consoleLog(entry) {
|
|
266
233
|
const symbols = { TRACE: '🔍', DEBUG: '🐛', INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', FATAL: '💀' };
|
|
267
234
|
const colors = {
|
|
@@ -273,38 +240,18 @@ class StructuredLogger {
|
|
|
273
240
|
const symbol = symbols[entry.level] || 'ℹ️';
|
|
274
241
|
|
|
275
242
|
let output = `${color}${symbol} [${entry.level}]${reset}`;
|
|
276
|
-
|
|
277
243
|
if (entry.processType) output += ` [${entry.processType}]`;
|
|
278
|
-
if (entry.processId) output += ` [${entry.processId.substring(0, 8)}]`;
|
|
279
244
|
if (entry.computationName) output += ` [${entry.computationName}]`;
|
|
280
|
-
if (entry.date) output += ` [${entry.date}]`;
|
|
281
245
|
|
|
282
246
|
output += ` ${entry.message}`;
|
|
283
247
|
|
|
284
248
|
console.log(output);
|
|
285
|
-
|
|
286
|
-
// Print context if present and not empty
|
|
287
249
|
if (entry.context && Object.keys(entry.context).length > 0) {
|
|
288
|
-
console.log(`
|
|
289
|
-
}
|
|
290
|
-
if (entry.storage) {
|
|
291
|
-
console.log(` ${color}Storage:${reset}`, entry.storage);
|
|
250
|
+
console.log(` Context:`, entry.context);
|
|
292
251
|
}
|
|
293
252
|
}
|
|
294
|
-
|
|
295
|
-
// Convenience methods
|
|
296
|
-
trace(message, context = {}) { this.log(LOG_LEVELS.TRACE, message, context); }
|
|
297
|
-
debug(message, context = {}) { this.log(LOG_LEVELS.DEBUG, message, context); }
|
|
298
|
-
info(message, context = {}) { this.log(LOG_LEVELS.INFO, message, context); }
|
|
299
|
-
warn(message, context = {}) { this.log(LOG_LEVELS.WARN, message, context); }
|
|
300
|
-
error(message, context = {}) { this.log(LOG_LEVELS.ERROR, message, context); }
|
|
301
|
-
fatal(message, context = {}) { this.log(LOG_LEVELS.FATAL, message, context); }
|
|
302
253
|
}
|
|
303
254
|
|
|
304
|
-
/**
|
|
305
|
-
* Process-scoped Logger
|
|
306
|
-
* Automatically includes process context in all log calls
|
|
307
|
-
*/
|
|
308
255
|
class ProcessLogger {
|
|
309
256
|
constructor(parent, processType, processId, computationName, date) {
|
|
310
257
|
this.parent = parent;
|
|
@@ -313,142 +260,20 @@ class ProcessLogger {
|
|
|
313
260
|
this.computationName = computationName;
|
|
314
261
|
this.date = date;
|
|
315
262
|
this.startTime = Date.now();
|
|
316
|
-
this.metrics = {
|
|
317
|
-
operations: 0,
|
|
318
|
-
errors: 0,
|
|
319
|
-
warnings: 0
|
|
320
|
-
};
|
|
263
|
+
this.metrics = { operations: 0, errors: 0, warnings: 0 };
|
|
321
264
|
}
|
|
322
|
-
|
|
323
265
|
log(level, message, context = {}) {
|
|
324
|
-
const numericLevel = typeof level === 'string' ? LOG_LEVELS[level] : level;
|
|
325
|
-
|
|
326
266
|
this.metrics.operations++;
|
|
327
|
-
|
|
328
|
-
this.metrics.errors++;
|
|
329
|
-
}
|
|
330
|
-
if (numericLevel === LOG_LEVELS.WARN) {
|
|
331
|
-
this.metrics.warnings++;
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
this.parent.log(
|
|
335
|
-
level,
|
|
336
|
-
message,
|
|
337
|
-
context,
|
|
338
|
-
this.processType,
|
|
339
|
-
this.processId,
|
|
340
|
-
this.computationName,
|
|
341
|
-
this.date
|
|
342
|
-
);
|
|
267
|
+
this.parent.log(level, message, context, this.processType, this.processId, this.computationName, this.date);
|
|
343
268
|
}
|
|
344
|
-
|
|
345
|
-
/**
|
|
346
|
-
* Complete the process and log summary
|
|
347
|
-
*/
|
|
348
269
|
complete(success = true, finalMessage = null) {
|
|
349
270
|
const duration = Date.now() - this.startTime;
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
? `Process completed successfully`
|
|
354
|
-
: `Process completed with errors`);
|
|
355
|
-
|
|
356
|
-
this.log(level, summaryMessage, {
|
|
357
|
-
stats: {
|
|
358
|
-
duration: `${duration}ms`,
|
|
359
|
-
durationMs: duration,
|
|
360
|
-
operations: this.metrics.operations,
|
|
361
|
-
errors: this.metrics.errors,
|
|
362
|
-
warnings: this.metrics.warnings,
|
|
363
|
-
success
|
|
364
|
-
}
|
|
365
|
-
});
|
|
366
|
-
|
|
271
|
+
this.log(success ? LOG_LEVELS.INFO : LOG_LEVELS.ERROR,
|
|
272
|
+
finalMessage || (success ? 'Process completed' : 'Process failed'),
|
|
273
|
+
{ stats: { durationMs: duration, ...this.metrics, success } });
|
|
367
274
|
this.parent.endProcess(this.processId);
|
|
368
275
|
return duration;
|
|
369
276
|
}
|
|
370
|
-
|
|
371
|
-
// Convenience methods
|
|
372
|
-
trace(message, context = {}) { this.log(LOG_LEVELS.TRACE, message, context); }
|
|
373
|
-
debug(message, context = {}) { this.log(LOG_LEVELS.DEBUG, message, context); }
|
|
374
|
-
info(message, context = {}) { this.log(LOG_LEVELS.INFO, message, context); }
|
|
375
|
-
warn(message, context = {}) { this.log(LOG_LEVELS.WARN, message, context); }
|
|
376
|
-
error(message, context = {}) { this.log(LOG_LEVELS.ERROR, message, context); }
|
|
377
|
-
fatal(message, context = {}) { this.log(LOG_LEVELS.FATAL, message, context); }
|
|
378
277
|
}
|
|
379
278
|
|
|
380
|
-
|
|
381
|
-
* Log Message Templates
|
|
382
|
-
*/
|
|
383
|
-
const LOG_TEMPLATES = {
|
|
384
|
-
// Schema Generation
|
|
385
|
-
SCHEMA_SUCCESS: (computationName) =>
|
|
386
|
-
`Schema generation successful for ${computationName}`,
|
|
387
|
-
SCHEMA_FAILURE: (computationName, reason) =>
|
|
388
|
-
`Schema generation failed for ${computationName}: ${reason}`,
|
|
389
|
-
|
|
390
|
-
// Computation Execution
|
|
391
|
-
COMPUTATION_START: (computationName, date) =>
|
|
392
|
-
`Starting computation ${computationName} for ${date}`,
|
|
393
|
-
COMPUTATION_SUCCESS: (computationName, date) =>
|
|
394
|
-
`Computation successful for ${computationName} on ${date}`,
|
|
395
|
-
COMPUTATION_FAILURE: (computationName, date, reason) =>
|
|
396
|
-
`Computation failed for ${computationName} on ${date}: ${reason}`,
|
|
397
|
-
|
|
398
|
-
// Storage
|
|
399
|
-
STORAGE_SUCCESS: (computationName, date, path, size) =>
|
|
400
|
-
`Results stored for ${computationName} on ${date} at ${path} (${size} bytes)`,
|
|
401
|
-
STORAGE_FAILURE: (computationName, date, path, reason) =>
|
|
402
|
-
`Failed to store results for ${computationName} on ${date} at ${path}: ${reason}`,
|
|
403
|
-
|
|
404
|
-
// Hash Validation
|
|
405
|
-
HASH_MISMATCH: (computationName, storedHash, currentHash) =>
|
|
406
|
-
`Hash mismatch for ${computationName}: stored=${storedHash}, current=${currentHash}`,
|
|
407
|
-
HASH_MATCH: (computationName) =>
|
|
408
|
-
`Hash match for ${computationName}, no code changes detected`,
|
|
409
|
-
HASH_CASCADE: (computationName, affectedComputations) =>
|
|
410
|
-
`Code change in ${computationName} will cascade to: ${affectedComputations.join(', ')}`,
|
|
411
|
-
|
|
412
|
-
// Manifest
|
|
413
|
-
MANIFEST_SUCCESS: (computationCount) =>
|
|
414
|
-
`Manifest built successfully with ${computationCount} computations`,
|
|
415
|
-
MANIFEST_TREE: (tree) =>
|
|
416
|
-
`Dependency tree:\n${tree}`,
|
|
417
|
-
|
|
418
|
-
// Date Analysis
|
|
419
|
-
DATE_ANALYSIS: (date, runnable, notRunnable) =>
|
|
420
|
-
`Date ${date}: ${runnable.length} runnable, ${notRunnable.length} blocked`,
|
|
421
|
-
DATE_MISSING_ROOTDATA: (date, computationName, missingData) =>
|
|
422
|
-
`${computationName} on ${date}: Cannot run due to missing root data: ${missingData.join(', ')}`,
|
|
423
|
-
DATE_MISSING_DEPENDENCY: (date, computationName, missingDep) =>
|
|
424
|
-
`${computationName} on ${date}: Will resolve after ${missingDep} completes`,
|
|
425
|
-
DATE_HASH_RERUN: (date, computationName, affectedDeps) =>
|
|
426
|
-
`${computationName} on ${date}: Hash mismatch, re-running (affects: ${affectedDeps.join(', ')})`,
|
|
427
|
-
|
|
428
|
-
// Availability
|
|
429
|
-
DATA_AVAILABLE: (date, types) =>
|
|
430
|
-
`Data available for ${date}: ${types.join(', ')}`,
|
|
431
|
-
DATA_MISSING: (date, types) =>
|
|
432
|
-
`Data missing for ${date}: ${types.join(', ')}`,
|
|
433
|
-
|
|
434
|
-
// Dispatch/Worker
|
|
435
|
-
DISPATCH_START: (pass, dateCount) =>
|
|
436
|
-
`Dispatching Pass ${pass} for ${dateCount} dates`,
|
|
437
|
-
DISPATCH_COMPLETE: (pass, dispatched) =>
|
|
438
|
-
`Dispatch complete: ${dispatched} tasks for Pass ${pass}`,
|
|
439
|
-
WORKER_TASK_START: (date, pass) =>
|
|
440
|
-
`Worker starting task: Date=${date}, Pass=${pass}`,
|
|
441
|
-
WORKER_TASK_COMPLETE: (date, pass, updateCount) =>
|
|
442
|
-
`Worker completed task: Date=${date}, Pass=${pass}, Updates=${updateCount}`,
|
|
443
|
-
WORKER_TASK_SKIP: (date, pass, reason) =>
|
|
444
|
-
`Worker skipped task: Date=${date}, Pass=${pass}, Reason=${reason}`
|
|
445
|
-
};
|
|
446
|
-
|
|
447
|
-
module.exports = {
|
|
448
|
-
StructuredLogger,
|
|
449
|
-
ProcessLogger,
|
|
450
|
-
LOG_LEVELS,
|
|
451
|
-
PROCESS_TYPES,
|
|
452
|
-
LOG_TEMPLATES,
|
|
453
|
-
generateProcessId
|
|
454
|
-
};
|
|
279
|
+
module.exports = { StructuredLogger, ProcessLogger, LOG_LEVELS, PROCESS_TYPES, generateProcessId };
|