immortal-js 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +577 -0
- package/docs/CHANGELOG.md +21 -0
- package/docs/CONTRIBUTING.md +41 -0
- package/docs/api/chaos.md +179 -0
- package/docs/api/dashboard.md +109 -0
- package/docs/api/isolation.md +191 -0
- package/docs/api/lifecycle.md +187 -0
- package/docs/api/monitoring.md +313 -0
- package/docs/api/plugins.md +217 -0
- package/docs/api/recovery.md +267 -0
- package/docs/api/safe-zone.md +236 -0
- package/docs/api/supervision.md +285 -0
- package/docs/guides/configuration.md +168 -0
- package/docs/guides/express.md +171 -0
- package/docs/guides/fastify.md +188 -0
- package/docs/guides/koa.md +102 -0
- package/docs/guides/nestjs.md +182 -0
- package/docs/guides/testing.md +91 -0
- package/examples/express-basic/index.ts +462 -0
- package/examples/express-basic/package.json +21 -0
- package/examples/express-basic/tsconfig.json +24 -0
- package/examples/fastify-microservice/index.ts +342 -0
- package/examples/fastify-microservice/package.json +22 -0
- package/examples/fastify-microservice/tsconfig.json +24 -0
- package/examples/invoice-service/data/invoices.db +0 -0
- package/examples/invoice-service/data/invoices.db-shm +0 -0
- package/examples/invoice-service/data/invoices.db-wal +0 -0
- package/examples/invoice-service/package.json +25 -0
- package/examples/invoice-service/public/index.html +5025 -0
- package/examples/invoice-service/src/db.ts +608 -0
- package/examples/invoice-service/src/pdf.ts +358 -0
- package/examples/invoice-service/src/server.ts +527 -0
- package/examples/invoice-service/src/store.ts +159 -0
- package/examples/invoice-service/src/types.ts +193 -0
- package/examples/invoice-service/tsconfig.json +23 -0
- package/examples/nestjs-enterprise/app.module.ts +561 -0
- package/examples/nestjs-enterprise/main.ts +67 -0
- package/examples/nestjs-enterprise/package.json +26 -0
- package/examples/nestjs-enterprise/tsconfig.json +27 -0
- package/immortal-js-1.0.0.tgz +0 -0
- package/package.json +33 -0
- package/packages/adapter-express/package.json +34 -0
- package/packages/adapter-express/src/index.ts +349 -0
- package/packages/adapter-express/tsconfig.json +14 -0
- package/packages/adapter-fastify/package.json +56 -0
- package/packages/adapter-fastify/src/plugin.ts +226 -0
- package/packages/adapter-fastify/tsconfig.json +14 -0
- package/packages/adapter-koa/package.json +55 -0
- package/packages/adapter-koa/src/index.ts +207 -0
- package/packages/adapter-koa/tsconfig.json +14 -0
- package/packages/adapter-nestjs/package.json +61 -0
- package/packages/adapter-nestjs/src/immortal.module.ts +313 -0
- package/packages/adapter-nestjs/src/index.ts +14 -0
- package/packages/adapter-nestjs/tsconfig.json +16 -0
- package/packages/core/package.json +56 -0
- package/packages/core/src/chaos/ChaosEngine.ts +249 -0
- package/packages/core/src/config/defaults.ts +200 -0
- package/packages/core/src/config/schema.ts +199 -0
- package/packages/core/src/event-bus.ts +168 -0
- package/packages/core/src/index.ts +164 -0
- package/packages/core/src/isolation/BulkheadPool.ts +279 -0
- package/packages/core/src/isolation/WorkerSandbox.ts +306 -0
- package/packages/core/src/isolation/index.ts +8 -0
- package/packages/core/src/lifecycle/GracefulShutdown.ts +161 -0
- package/packages/core/src/logger.ts +104 -0
- package/packages/core/src/monitoring/DiagnosticsChannel.ts +248 -0
- package/packages/core/src/monitoring/HealthMonitor.ts +191 -0
- package/packages/core/src/monitoring/MemoryLeakGuard.ts +340 -0
- package/packages/core/src/monitoring/MetricsCollector.ts +219 -0
- package/packages/core/src/monitoring/index.ts +10 -0
- package/packages/core/src/plugins/BuiltinPlugins.ts +269 -0
- package/packages/core/src/recovery/CircuitBreaker.ts +334 -0
- package/packages/core/src/recovery/FallbackCache.ts +328 -0
- package/packages/core/src/recovery/RetryEngine.ts +225 -0
- package/packages/core/src/recovery/Timeout.ts +97 -0
- package/packages/core/src/recovery/index.ts +11 -0
- package/packages/core/src/runtime.ts +242 -0
- package/packages/core/src/safe-zone/AsyncBoundary.ts +114 -0
- package/packages/core/src/safe-zone/ErrorTrap.ts +347 -0
- package/packages/core/src/safe-zone/SafeWrapper.ts +317 -0
- package/packages/core/src/safe-zone/index.ts +23 -0
- package/packages/core/src/supervision/ClusterManager.ts +243 -0
- package/packages/core/src/supervision/RestartStrategy.ts +68 -0
- package/packages/core/src/supervision/Supervisor.ts +311 -0
- package/packages/core/src/supervision/index.ts +11 -0
- package/packages/core/src/types.ts +470 -0
- package/packages/core/test/bulkhead.test.ts +310 -0
- package/packages/core/test/circuit-breaker.test.ts +153 -0
- package/packages/core/test/memory-guard.test.ts +213 -0
- package/packages/core/test/retry.test.ts +110 -0
- package/packages/core/test/safe-zone.test.ts +271 -0
- package/packages/core/test/supervisor.test.ts +310 -0
- package/packages/core/tsconfig.json +13 -0
- package/packages/dashboard/package.json +56 -0
- package/packages/dashboard/server/DashboardServer.ts +454 -0
- package/packages/dashboard/tsconfig.json +14 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file runtime.ts
|
|
3
|
+
* @description ImmortalRuntime — the central orchestrator
|
|
4
|
+
* Initializes and wires all five layers together
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { ImmortalConfig, SystemMetrics, ImmortalPlugin } from "./types.js";
|
|
8
|
+
import { mergeWithDefaults } from "./config/defaults.js";
|
|
9
|
+
import { validateConfig } from "./config/schema.js";
|
|
10
|
+
import { createLogger, getLogger } from "./logger.js";
|
|
11
|
+
import { getEventBus } from "./event-bus.js";
|
|
12
|
+
import type { MetricsCollector } from "./monitoring/MetricsCollector.js";
|
|
13
|
+
import type { HealthMonitor } from "./monitoring/HealthMonitor.js";
|
|
14
|
+
import type { MemoryLeakGuard } from "./monitoring/MemoryLeakGuard.js";
|
|
15
|
+
import type { GracefulShutdown } from "./lifecycle/GracefulShutdown.js";
|
|
16
|
+
import type { ErrorTrap } from "./safe-zone/ErrorTrap.js";
|
|
17
|
+
|
|
18
|
+
export interface ImmortalRuntimeAPI {
|
|
19
|
+
config: Required<ImmortalConfig>;
|
|
20
|
+
getMetrics(): SystemMetrics;
|
|
21
|
+
shutdown(signal?: string): Promise<void>;
|
|
22
|
+
addPlugin(plugin: ImmortalPlugin): void;
|
|
23
|
+
isHealthy(): boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export class ImmortalRuntime implements ImmortalRuntimeAPI {
|
|
27
|
+
public readonly config: Required<ImmortalConfig>;
|
|
28
|
+
private _initialized = false;
|
|
29
|
+
private _healthy = true;
|
|
30
|
+
private _metricsCollector: MetricsCollector | undefined;
|
|
31
|
+
private _healthMonitor: HealthMonitor | undefined;
|
|
32
|
+
private _memoryLeakGuard: MemoryLeakGuard | undefined;
|
|
33
|
+
private _gracefulShutdown: GracefulShutdown | undefined;
|
|
34
|
+
private _errorTrap: ErrorTrap | undefined;
|
|
35
|
+
|
|
36
|
+
constructor(userConfig: ImmortalConfig = {}) {
|
|
37
|
+
const validation = validateConfig(userConfig);
|
|
38
|
+
if (!validation.valid) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
`Immortal.js configuration errors:\n${validation.errors.join("\n")}`
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
for (const warning of validation.warnings) {
|
|
44
|
+
console.warn(warning);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
this.config = mergeWithDefaults(userConfig);
|
|
48
|
+
createLogger(this.config.logger as Parameters<typeof createLogger>[0]);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async initialize(): Promise<this> {
|
|
52
|
+
if (this._initialized) return this;
|
|
53
|
+
|
|
54
|
+
const logger = getLogger();
|
|
55
|
+
const bus = getEventBus();
|
|
56
|
+
|
|
57
|
+
logger.info("Immortal.js Runtime initializing...", {
|
|
58
|
+
env: process.env["NODE_ENV"],
|
|
59
|
+
nodeVersion: process.version,
|
|
60
|
+
pid: process.pid,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
try {
|
|
64
|
+
const [
|
|
65
|
+
MetricsCollectorMod,
|
|
66
|
+
HealthMonitorMod,
|
|
67
|
+
MemoryLeakGuardMod,
|
|
68
|
+
GracefulShutdownMod,
|
|
69
|
+
ErrorTrapMod,
|
|
70
|
+
] = await Promise.all([
|
|
71
|
+
import("./monitoring/MetricsCollector.js").catch(() => null),
|
|
72
|
+
import("./monitoring/HealthMonitor.js").catch(() => null),
|
|
73
|
+
import("./monitoring/MemoryLeakGuard.js").catch(() => null),
|
|
74
|
+
import("./lifecycle/GracefulShutdown.js").catch(() => null),
|
|
75
|
+
import("./safe-zone/ErrorTrap.js").catch(() => null),
|
|
76
|
+
]);
|
|
77
|
+
|
|
78
|
+
if (MetricsCollectorMod) {
|
|
79
|
+
this._metricsCollector = new MetricsCollectorMod.MetricsCollector();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (HealthMonitorMod && this._metricsCollector) {
|
|
83
|
+
this._healthMonitor = new HealthMonitorMod.HealthMonitor(
|
|
84
|
+
this.config.healthMonitor,
|
|
85
|
+
bus,
|
|
86
|
+
this._metricsCollector
|
|
87
|
+
);
|
|
88
|
+
this._healthMonitor.start();
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (MemoryLeakGuardMod) {
|
|
92
|
+
this._memoryLeakGuard = new MemoryLeakGuardMod.MemoryLeakGuard(
|
|
93
|
+
this.config.memoryGuard,
|
|
94
|
+
bus
|
|
95
|
+
);
|
|
96
|
+
this._memoryLeakGuard.start();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (GracefulShutdownMod) {
|
|
100
|
+
this._gracefulShutdown = new GracefulShutdownMod.GracefulShutdown(
|
|
101
|
+
this.config.gracefulShutdown,
|
|
102
|
+
bus
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (ErrorTrapMod) {
|
|
107
|
+
this._errorTrap = new ErrorTrapMod.ErrorTrap(bus);
|
|
108
|
+
this._errorTrap.install();
|
|
109
|
+
}
|
|
110
|
+
} catch (err) {
|
|
111
|
+
logger.warn("Some Immortal.js layers could not be loaded", {
|
|
112
|
+
error: (err as Error).message,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Initialize plugins
|
|
117
|
+
const pluginContext = {
|
|
118
|
+
config: this.config,
|
|
119
|
+
emit: (event: Parameters<typeof bus.emit>[1]) => bus.emit("error:captured", event),
|
|
120
|
+
getMetrics: () => this.getMetrics(),
|
|
121
|
+
logger,
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
for (const plugin of this.config.plugins) {
|
|
125
|
+
try {
|
|
126
|
+
await plugin.onInit?.(pluginContext);
|
|
127
|
+
logger.info(`Plugin '${plugin.name}' initialized`);
|
|
128
|
+
} catch (err) {
|
|
129
|
+
logger.error(`Plugin '${plugin.name}' failed to initialize`, {
|
|
130
|
+
error: (err as Error).message,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (this.config.chaos?.enabled && process.env["NODE_ENV"] === "production") {
|
|
136
|
+
logger.error("Chaos mode is BLOCKED in production.");
|
|
137
|
+
if (this.config.chaos) this.config.chaos.enabled = false;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
this._initialized = true;
|
|
141
|
+
logger.info("Immortal.js Runtime ready", {
|
|
142
|
+
layers: ["SafeZone", "Recovery", "Isolation", "Supervision", "Monitoring"],
|
|
143
|
+
plugins: this.config.plugins.map((p) => p.name),
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
return this;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
getMetrics(): SystemMetrics {
|
|
150
|
+
if (this._metricsCollector) {
|
|
151
|
+
return this._metricsCollector.getSnapshot();
|
|
152
|
+
}
|
|
153
|
+
return this._emptyMetrics();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
isHealthy(): boolean {
|
|
157
|
+
return this._healthy;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async shutdown(signal = "manual"): Promise<void> {
|
|
161
|
+
const logger = getLogger();
|
|
162
|
+
logger.info(`Immortal.js Runtime shutting down (signal: ${signal})`);
|
|
163
|
+
|
|
164
|
+
this._healthMonitor?.stop();
|
|
165
|
+
this._memoryLeakGuard?.stop();
|
|
166
|
+
|
|
167
|
+
if (this._gracefulShutdown) {
|
|
168
|
+
await this._gracefulShutdown.execute(signal);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
for (const plugin of this.config.plugins) {
|
|
172
|
+
try {
|
|
173
|
+
await plugin.onShutdown?.(signal);
|
|
174
|
+
} catch (err) {
|
|
175
|
+
logger.error(`Plugin '${plugin.name}' shutdown hook failed`, {
|
|
176
|
+
error: (err as Error).message,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
logger.info("Immortal.js Runtime shutdown complete");
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
addPlugin(plugin: ImmortalPlugin): void {
|
|
185
|
+
this.config.plugins.push(plugin);
|
|
186
|
+
if (this._initialized) {
|
|
187
|
+
const bus = getEventBus();
|
|
188
|
+
const logger = getLogger();
|
|
189
|
+
const ctx = {
|
|
190
|
+
config: this.config,
|
|
191
|
+
emit: (e: Parameters<typeof bus.emit>[1]) => bus.emit("error:captured", e),
|
|
192
|
+
getMetrics: () => this.getMetrics(),
|
|
193
|
+
logger,
|
|
194
|
+
};
|
|
195
|
+
plugin.onInit?.(ctx)?.catch(() => void 0);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
private _emptyMetrics(): SystemMetrics {
|
|
200
|
+
return {
|
|
201
|
+
timestamp: Date.now(),
|
|
202
|
+
eventLoopLag: { p50: 0, p95: 0, p99: 0, max: 0 },
|
|
203
|
+
memory: {
|
|
204
|
+
heapUsedMb: 0, heapTotalMb: 0, externalMb: 0,
|
|
205
|
+
rssMb: 0, heapGrowthRate: 0, gcFrequency: 0,
|
|
206
|
+
},
|
|
207
|
+
cpu: { percentUser: 0, percentSystem: 0 },
|
|
208
|
+
handles: { active: 0 },
|
|
209
|
+
requests: { total: 0, active: 0, perSecond: 0, errorRate: 0 },
|
|
210
|
+
circuits: {},
|
|
211
|
+
bulkheads: {},
|
|
212
|
+
workers: [],
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
218
|
+
// FACTORY FUNCTION
|
|
219
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
220
|
+
|
|
221
|
+
let _instance: ImmortalRuntime | null = null;
|
|
222
|
+
|
|
223
|
+
export async function createImmortal(config: ImmortalConfig = {}): Promise<ImmortalRuntime> {
|
|
224
|
+
if (_instance) {
|
|
225
|
+
getLogger().warn("createImmortal() called more than once — returning existing instance");
|
|
226
|
+
return _instance;
|
|
227
|
+
}
|
|
228
|
+
_instance = new ImmortalRuntime(config);
|
|
229
|
+
await _instance.initialize();
|
|
230
|
+
return _instance;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export function getImmortal(): ImmortalRuntime {
|
|
234
|
+
if (!_instance) {
|
|
235
|
+
throw new Error("Immortal.js runtime not initialized. Call createImmortal() first.");
|
|
236
|
+
}
|
|
237
|
+
return _instance;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export function _resetImmortal(): void {
|
|
241
|
+
_instance = null;
|
|
242
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file AsyncBoundary.ts
|
|
3
|
+
* @description AsyncLocalStorage-based request context propagation
|
|
4
|
+
* Every operation within a request boundary automatically carries its requestId,
|
|
5
|
+
* timing data, and custom metadata — without passing it explicitly through every function call.
|
|
6
|
+
*
|
|
7
|
+
* This is the mechanism that allows Immortal.js to correlate errors, retries, and metrics
|
|
8
|
+
* back to the exact request that triggered them.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
12
|
+
import type { RequestContext } from "../types.js";
|
|
13
|
+
|
|
14
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
15
|
+
// SINGLETON STORAGE
|
|
16
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
|
|
18
|
+
export const asyncLocalStorage = new AsyncLocalStorage<RequestContext>();
|
|
19
|
+
|
|
20
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
21
|
+
// ASYNC BOUNDARY CLASS
|
|
22
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
23
|
+
|
|
24
|
+
export class AsyncBoundary {
|
|
25
|
+
/**
|
|
26
|
+
* Run a function within a request context boundary.
|
|
27
|
+
* All async operations within fn() can access the context via getContext()
|
|
28
|
+
*/
|
|
29
|
+
static run<T>(context: RequestContext, fn: () => Promise<T>): Promise<T> {
|
|
30
|
+
return asyncLocalStorage.run(context, fn);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Get the current request context from anywhere in the call stack.
|
|
35
|
+
* Returns undefined if called outside a boundary (e.g., background tasks).
|
|
36
|
+
*/
|
|
37
|
+
static getContext(): RequestContext | undefined {
|
|
38
|
+
return asyncLocalStorage.getStore();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Get the current requestId safely
|
|
43
|
+
*/
|
|
44
|
+
static getRequestId(): string | undefined {
|
|
45
|
+
return asyncLocalStorage.getStore()?.requestId;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Enrich the current context with additional custom data.
|
|
50
|
+
* Mutates the existing context object in-place (safe because it's per-request).
|
|
51
|
+
*/
|
|
52
|
+
static enrich(data: Partial<RequestContext>): void {
|
|
53
|
+
const ctx = asyncLocalStorage.getStore();
|
|
54
|
+
if (ctx) {
|
|
55
|
+
Object.assign(ctx, data);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Create a new child context (for parallel sub-operations within a request)
|
|
61
|
+
*/
|
|
62
|
+
static fork<T>(
|
|
63
|
+
extraData: Partial<RequestContext>,
|
|
64
|
+
fn: () => Promise<T>
|
|
65
|
+
): Promise<T> {
|
|
66
|
+
const parent = asyncLocalStorage.getStore();
|
|
67
|
+
const child: RequestContext = {
|
|
68
|
+
...(parent ?? { requestId: crypto.randomUUID(), startedAt: Date.now() }),
|
|
69
|
+
...extraData,
|
|
70
|
+
custom: {
|
|
71
|
+
...parent?.custom,
|
|
72
|
+
...extraData.custom,
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
return asyncLocalStorage.run(child, fn);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
80
|
+
// CONVENIENCE FUNCTION API
|
|
81
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Wrap a function to execute within a new request context boundary
|
|
85
|
+
* Useful for background jobs, event handlers, queue processors
|
|
86
|
+
*/
|
|
87
|
+
export function asyncBoundary<Args extends unknown[], R>(
|
|
88
|
+
fn: (...args: Args) => Promise<R>,
|
|
89
|
+
contextFactory?: (...args: Args) => Partial<RequestContext>
|
|
90
|
+
): (...args: Args) => Promise<R> {
|
|
91
|
+
return async (...args: Args): Promise<R> => {
|
|
92
|
+
const baseContext: RequestContext = {
|
|
93
|
+
requestId: crypto.randomUUID(),
|
|
94
|
+
startedAt: Date.now(),
|
|
95
|
+
...(contextFactory?.(...args) ?? {}),
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
return AsyncBoundary.run(baseContext, () => fn(...args));
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Get current request context — shorthand for AsyncBoundary.getContext()
|
|
104
|
+
*/
|
|
105
|
+
export function getRequestContext(): RequestContext | undefined {
|
|
106
|
+
return AsyncBoundary.getContext();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Get current requestId — shorthand
|
|
111
|
+
*/
|
|
112
|
+
export function getRequestId(): string {
|
|
113
|
+
return AsyncBoundary.getRequestId() ?? "unknown";
|
|
114
|
+
}
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file ErrorTrap.ts
|
|
3
|
+
* @description Process-level error interception
|
|
4
|
+
*
|
|
5
|
+
* Handles uncaughtException and unhandledRejection with engineering honesty:
|
|
6
|
+
* - Does NOT silently swallow errors and pretend nothing happened
|
|
7
|
+
* - DOES classify errors as operational vs programming errors
|
|
8
|
+
* - Operational errors → log + continue (if safe)
|
|
9
|
+
* - Programming errors → log + request graceful Worker restart via Supervisor
|
|
10
|
+
*
|
|
11
|
+
* Critical distinction: An uncaughtException may leave Node's internal state
|
|
12
|
+
* corrupted. The ONLY safe response to a true programming error is to restart
|
|
13
|
+
* the process. ErrorTrap ensures this happens in a controlled, supervised way.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { ImmortalEventBus } from "../event-bus.js";
|
|
17
|
+
import type { SerializedError } from "../types.js";
|
|
18
|
+
import { ImmortalError } from "../types.js";
|
|
19
|
+
import { getLogger } from "../logger.js";
|
|
20
|
+
import { AsyncBoundary } from "./AsyncBoundary.js";
|
|
21
|
+
|
|
22
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
23
|
+
// TYPES
|
|
24
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
export type ErrorClassification = "operational" | "programming" | "unknown";
|
|
27
|
+
|
|
28
|
+
export interface CapturedError {
|
|
29
|
+
error: unknown;
|
|
30
|
+
classification: ErrorClassification;
|
|
31
|
+
context?: ReturnType<typeof AsyncBoundary.getContext>;
|
|
32
|
+
timestamp: number;
|
|
33
|
+
processEventType: "uncaughtException" | "unhandledRejection" | "synchronous";
|
|
34
|
+
serialized: SerializedError;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type RestartCallback = () => void;
|
|
38
|
+
|
|
39
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
40
|
+
// ERROR TRAP CLASS
|
|
41
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
export class ErrorTrap {
|
|
44
|
+
private readonly bus: ImmortalEventBus;
|
|
45
|
+
private installed = false;
|
|
46
|
+
private restartCallbacks: RestartCallback[] = [];
|
|
47
|
+
private capturedErrors: CapturedError[] = [];
|
|
48
|
+
private readonly maxErrorHistory = 200;
|
|
49
|
+
|
|
50
|
+
// Track if we're already in a crash-restart cycle to prevent infinite loops
|
|
51
|
+
private restartInProgress = false;
|
|
52
|
+
|
|
53
|
+
constructor(bus: ImmortalEventBus) {
|
|
54
|
+
this.bus = bus;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Install process-level error handlers.
|
|
59
|
+
* Call once during runtime initialization.
|
|
60
|
+
*/
|
|
61
|
+
install(): void {
|
|
62
|
+
if (this.installed) return;
|
|
63
|
+
|
|
64
|
+
process.on("uncaughtException", this.handleUncaughtException.bind(this));
|
|
65
|
+
process.on("unhandledRejection", this.handleUnhandledRejection.bind(this));
|
|
66
|
+
process.on("uncaughtExceptionMonitor", this.handleMonitor.bind(this));
|
|
67
|
+
|
|
68
|
+
this.installed = true;
|
|
69
|
+
getLogger().info("ErrorTrap installed — process-level error handlers active");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Uninstall handlers (for testing only)
|
|
74
|
+
*/
|
|
75
|
+
uninstall(): void {
|
|
76
|
+
process.off("uncaughtException", this.handleUncaughtException.bind(this));
|
|
77
|
+
process.off("unhandledRejection", this.handleUnhandledRejection.bind(this));
|
|
78
|
+
process.off("uncaughtExceptionMonitor", this.handleMonitor.bind(this));
|
|
79
|
+
this.installed = false;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Register a callback to be invoked when a programming error requires restart
|
|
84
|
+
*/
|
|
85
|
+
onRestartRequired(callback: RestartCallback): void {
|
|
86
|
+
this.restartCallbacks.push(callback);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Manually capture an error (from SafeWrapper, middleware, etc.)
|
|
91
|
+
*/
|
|
92
|
+
capture(
|
|
93
|
+
error: unknown,
|
|
94
|
+
context?: { requestId?: string; route?: string; [key: string]: unknown }
|
|
95
|
+
): CapturedError {
|
|
96
|
+
const serialized = serializeError(error);
|
|
97
|
+
const classification = classifyError(error);
|
|
98
|
+
const requestCtx = AsyncBoundary.getContext();
|
|
99
|
+
|
|
100
|
+
const captured: CapturedError = {
|
|
101
|
+
error,
|
|
102
|
+
classification,
|
|
103
|
+
context: requestCtx,
|
|
104
|
+
timestamp: Date.now(),
|
|
105
|
+
processEventType: "synchronous",
|
|
106
|
+
serialized,
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
this.recordError(captured);
|
|
110
|
+
|
|
111
|
+
this.bus.emit("error:captured", {
|
|
112
|
+
requestId: context?.requestId ?? requestCtx?.requestId,
|
|
113
|
+
route: context?.route ?? requestCtx?.route,
|
|
114
|
+
error: serialized,
|
|
115
|
+
data: {
|
|
116
|
+
classification,
|
|
117
|
+
...(context ?? {}),
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
return captured;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Get recent captured error history
|
|
126
|
+
*/
|
|
127
|
+
getHistory(limit = 50): CapturedError[] {
|
|
128
|
+
return this.capturedErrors.slice(-limit);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
132
|
+
// PRIVATE HANDLERS
|
|
133
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
134
|
+
|
|
135
|
+
private handleUncaughtException(error: Error, origin: string): void {
|
|
136
|
+
const logger = getLogger();
|
|
137
|
+
const serialized = serializeError(error);
|
|
138
|
+
const classification = classifyError(error);
|
|
139
|
+
|
|
140
|
+
const captured: CapturedError = {
|
|
141
|
+
error,
|
|
142
|
+
classification,
|
|
143
|
+
context: AsyncBoundary.getContext(),
|
|
144
|
+
timestamp: Date.now(),
|
|
145
|
+
processEventType: "uncaughtException",
|
|
146
|
+
serialized,
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
this.recordError(captured);
|
|
150
|
+
|
|
151
|
+
logger.error("UNCAUGHT EXCEPTION captured", {
|
|
152
|
+
origin,
|
|
153
|
+
classification,
|
|
154
|
+
...serialized,
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
this.bus.emit("error:captured", {
|
|
158
|
+
error: serialized,
|
|
159
|
+
data: { classification, origin, processEvent: "uncaughtException" },
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
if (classification === "programming") {
|
|
163
|
+
logger.error(
|
|
164
|
+
"Programming error detected — requesting graceful restart to prevent corrupted state"
|
|
165
|
+
);
|
|
166
|
+
this.requestRestart();
|
|
167
|
+
}
|
|
168
|
+
// Operational errors: log and continue — state is not expected to be corrupted
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private handleUnhandledRejection(reason: unknown, promise: Promise<unknown>): void {
|
|
172
|
+
const logger = getLogger();
|
|
173
|
+
const serialized = serializeError(reason);
|
|
174
|
+
const classification = classifyError(reason);
|
|
175
|
+
|
|
176
|
+
const captured: CapturedError = {
|
|
177
|
+
error: reason,
|
|
178
|
+
classification,
|
|
179
|
+
context: AsyncBoundary.getContext(),
|
|
180
|
+
timestamp: Date.now(),
|
|
181
|
+
processEventType: "unhandledRejection",
|
|
182
|
+
serialized,
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
this.recordError(captured);
|
|
186
|
+
|
|
187
|
+
logger.warn("UNHANDLED REJECTION captured", {
|
|
188
|
+
classification,
|
|
189
|
+
promise: String(promise),
|
|
190
|
+
...serialized,
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
this.bus.emit("error:captured", {
|
|
194
|
+
error: serialized,
|
|
195
|
+
data: {
|
|
196
|
+
classification,
|
|
197
|
+
processEvent: "unhandledRejection",
|
|
198
|
+
promiseString: String(promise),
|
|
199
|
+
},
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
// Unhandled rejections are less likely to corrupt state than uncaughtException
|
|
203
|
+
// but programming errors still warrant a restart
|
|
204
|
+
if (classification === "programming") {
|
|
205
|
+
logger.error("Programming error in unhandled rejection — requesting graceful restart");
|
|
206
|
+
this.requestRestart();
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
private handleMonitor(error: Error, origin: string): void {
|
|
211
|
+
// uncaughtExceptionMonitor fires BEFORE uncaughtException handlers
|
|
212
|
+
// Use it for monitoring/telemetry only — don't interfere with error flow
|
|
213
|
+
getLogger().debug("uncaughtExceptionMonitor fired", {
|
|
214
|
+
errorName: error.name,
|
|
215
|
+
origin,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private requestRestart(): void {
|
|
220
|
+
if (this.restartInProgress) return;
|
|
221
|
+
this.restartInProgress = true;
|
|
222
|
+
|
|
223
|
+
this.bus.emit("error:escalated", {
|
|
224
|
+
data: { reason: "programming-error", pid: process.pid },
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
for (const callback of this.restartCallbacks) {
|
|
228
|
+
try {
|
|
229
|
+
callback();
|
|
230
|
+
} catch {
|
|
231
|
+
// Ignore errors in restart callbacks
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// If no restart callbacks registered (standalone mode), force exit
|
|
236
|
+
// so the process manager (PM2, Kubernetes, systemd) restarts us
|
|
237
|
+
if (this.restartCallbacks.length === 0) {
|
|
238
|
+
getLogger().error("No restart callbacks — forcing process exit for clean restart");
|
|
239
|
+
setTimeout(() => process.exit(1), 100).unref();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
private recordError(captured: CapturedError): void {
|
|
244
|
+
this.capturedErrors.push(captured);
|
|
245
|
+
if (this.capturedErrors.length > this.maxErrorHistory) {
|
|
246
|
+
this.capturedErrors.shift();
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
252
|
+
// UTILITIES
|
|
253
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Classify an error as operational (expected, safe to continue) or
|
|
257
|
+
* programming (unexpected, may have corrupted state)
|
|
258
|
+
*
|
|
259
|
+
* Engineering principle: When in doubt, classify as programming error
|
|
260
|
+
* (safer to restart than to continue in potentially corrupted state)
|
|
261
|
+
*/
|
|
262
|
+
export function classifyError(error: unknown): ErrorClassification {
|
|
263
|
+
if (error instanceof ImmortalError) {
|
|
264
|
+
return error.isOperational ? "operational" : "programming";
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (!(error instanceof Error)) return "unknown";
|
|
268
|
+
|
|
269
|
+
// Node.js system errors
|
|
270
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
271
|
+
const systemErrorCodes = [
|
|
272
|
+
"ENOENT", "EACCES", "EADDRINUSE", "ECONNREFUSED", "ECONNRESET",
|
|
273
|
+
"ETIMEDOUT", "EPIPE", "EHOSTUNREACH", "ENETUNREACH", "ENOMEM",
|
|
274
|
+
];
|
|
275
|
+
if (code && systemErrorCodes.includes(code)) return "operational";
|
|
276
|
+
|
|
277
|
+
// HTTP/network operational errors (by status code)
|
|
278
|
+
const statusCode = (error as { statusCode?: number; status?: number }).statusCode
|
|
279
|
+
?? (error as { statusCode?: number; status?: number }).status;
|
|
280
|
+
if (statusCode && statusCode < 500) return "operational";
|
|
281
|
+
|
|
282
|
+
// Explicit operational flag
|
|
283
|
+
if ((error as { isOperational?: boolean }).isOperational === true) return "operational";
|
|
284
|
+
if ((error as { isOperational?: boolean }).isOperational === false) return "programming";
|
|
285
|
+
|
|
286
|
+
// Well-known programming error types
|
|
287
|
+
if (
|
|
288
|
+
error instanceof TypeError ||
|
|
289
|
+
error instanceof RangeError ||
|
|
290
|
+
error instanceof SyntaxError ||
|
|
291
|
+
error instanceof ReferenceError
|
|
292
|
+
) {
|
|
293
|
+
return "programming";
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// URIError, EvalError, AssertionError — programming errors
|
|
297
|
+
if (
|
|
298
|
+
error.name === "AssertionError" ||
|
|
299
|
+
error.name === "URIError" ||
|
|
300
|
+
error.name === "EvalError"
|
|
301
|
+
) {
|
|
302
|
+
return "programming";
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// Default: treat as operational (conservative — avoid unnecessary restarts)
|
|
306
|
+
return "operational";
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Serialize any error into a safe, loggable, JSON-compatible structure
|
|
311
|
+
*/
|
|
312
|
+
export function serializeError(error: unknown): SerializedError {
|
|
313
|
+
if (error instanceof Error) {
|
|
314
|
+
const ext = error as Error & {
|
|
315
|
+
code?: string;
|
|
316
|
+
statusCode?: number;
|
|
317
|
+
status?: number;
|
|
318
|
+
isOperational?: boolean;
|
|
319
|
+
context?: Record<string, unknown>;
|
|
320
|
+
};
|
|
321
|
+
const result: SerializedError = {
|
|
322
|
+
name: error.name,
|
|
323
|
+
message: error.message,
|
|
324
|
+
};
|
|
325
|
+
const stack = process.env["NODE_ENV"] !== "production" ? error.stack : undefined;
|
|
326
|
+
if (stack !== undefined) result.stack = stack;
|
|
327
|
+
if (ext.code !== undefined) result.code = ext.code;
|
|
328
|
+
const statusCode = ext.statusCode ?? ext.status;
|
|
329
|
+
if (statusCode !== undefined) result.statusCode = statusCode;
|
|
330
|
+
if (ext.isOperational !== undefined) result.isOperational = ext.isOperational;
|
|
331
|
+
if (ext.context !== undefined) result.context = ext.context;
|
|
332
|
+
return result;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (typeof error === "string") {
|
|
336
|
+
return { name: "Error", message: error };
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (typeof error === "object" && error !== null) {
|
|
340
|
+
return {
|
|
341
|
+
name: "UnknownError",
|
|
342
|
+
message: JSON.stringify(error),
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return { name: "UnknownError", message: String(error) };
|
|
347
|
+
}
|