immortal-js 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +577 -0
- package/docs/CHANGELOG.md +21 -0
- package/docs/CONTRIBUTING.md +41 -0
- package/docs/api/chaos.md +179 -0
- package/docs/api/dashboard.md +109 -0
- package/docs/api/isolation.md +191 -0
- package/docs/api/lifecycle.md +187 -0
- package/docs/api/monitoring.md +313 -0
- package/docs/api/plugins.md +217 -0
- package/docs/api/recovery.md +267 -0
- package/docs/api/safe-zone.md +236 -0
- package/docs/api/supervision.md +285 -0
- package/docs/guides/configuration.md +168 -0
- package/docs/guides/express.md +171 -0
- package/docs/guides/fastify.md +188 -0
- package/docs/guides/koa.md +102 -0
- package/docs/guides/nestjs.md +182 -0
- package/docs/guides/testing.md +91 -0
- package/examples/express-basic/index.ts +462 -0
- package/examples/express-basic/package.json +21 -0
- package/examples/express-basic/tsconfig.json +24 -0
- package/examples/fastify-microservice/index.ts +342 -0
- package/examples/fastify-microservice/package.json +22 -0
- package/examples/fastify-microservice/tsconfig.json +24 -0
- package/examples/invoice-service/data/invoices.db +0 -0
- package/examples/invoice-service/data/invoices.db-shm +0 -0
- package/examples/invoice-service/data/invoices.db-wal +0 -0
- package/examples/invoice-service/package.json +25 -0
- package/examples/invoice-service/public/index.html +5025 -0
- package/examples/invoice-service/src/db.ts +608 -0
- package/examples/invoice-service/src/pdf.ts +358 -0
- package/examples/invoice-service/src/server.ts +527 -0
- package/examples/invoice-service/src/store.ts +159 -0
- package/examples/invoice-service/src/types.ts +193 -0
- package/examples/invoice-service/tsconfig.json +23 -0
- package/examples/nestjs-enterprise/app.module.ts +561 -0
- package/examples/nestjs-enterprise/main.ts +67 -0
- package/examples/nestjs-enterprise/package.json +26 -0
- package/examples/nestjs-enterprise/tsconfig.json +27 -0
- package/immortal-js-1.0.0.tgz +0 -0
- package/package.json +33 -0
- package/packages/adapter-express/package.json +34 -0
- package/packages/adapter-express/src/index.ts +349 -0
- package/packages/adapter-express/tsconfig.json +14 -0
- package/packages/adapter-fastify/package.json +56 -0
- package/packages/adapter-fastify/src/plugin.ts +226 -0
- package/packages/adapter-fastify/tsconfig.json +14 -0
- package/packages/adapter-koa/package.json +55 -0
- package/packages/adapter-koa/src/index.ts +207 -0
- package/packages/adapter-koa/tsconfig.json +14 -0
- package/packages/adapter-nestjs/package.json +61 -0
- package/packages/adapter-nestjs/src/immortal.module.ts +313 -0
- package/packages/adapter-nestjs/src/index.ts +14 -0
- package/packages/adapter-nestjs/tsconfig.json +16 -0
- package/packages/core/package.json +56 -0
- package/packages/core/src/chaos/ChaosEngine.ts +249 -0
- package/packages/core/src/config/defaults.ts +200 -0
- package/packages/core/src/config/schema.ts +199 -0
- package/packages/core/src/event-bus.ts +168 -0
- package/packages/core/src/index.ts +164 -0
- package/packages/core/src/isolation/BulkheadPool.ts +279 -0
- package/packages/core/src/isolation/WorkerSandbox.ts +306 -0
- package/packages/core/src/isolation/index.ts +8 -0
- package/packages/core/src/lifecycle/GracefulShutdown.ts +161 -0
- package/packages/core/src/logger.ts +104 -0
- package/packages/core/src/monitoring/DiagnosticsChannel.ts +248 -0
- package/packages/core/src/monitoring/HealthMonitor.ts +191 -0
- package/packages/core/src/monitoring/MemoryLeakGuard.ts +340 -0
- package/packages/core/src/monitoring/MetricsCollector.ts +219 -0
- package/packages/core/src/monitoring/index.ts +10 -0
- package/packages/core/src/plugins/BuiltinPlugins.ts +269 -0
- package/packages/core/src/recovery/CircuitBreaker.ts +334 -0
- package/packages/core/src/recovery/FallbackCache.ts +328 -0
- package/packages/core/src/recovery/RetryEngine.ts +225 -0
- package/packages/core/src/recovery/Timeout.ts +97 -0
- package/packages/core/src/recovery/index.ts +11 -0
- package/packages/core/src/runtime.ts +242 -0
- package/packages/core/src/safe-zone/AsyncBoundary.ts +114 -0
- package/packages/core/src/safe-zone/ErrorTrap.ts +347 -0
- package/packages/core/src/safe-zone/SafeWrapper.ts +317 -0
- package/packages/core/src/safe-zone/index.ts +23 -0
- package/packages/core/src/supervision/ClusterManager.ts +243 -0
- package/packages/core/src/supervision/RestartStrategy.ts +68 -0
- package/packages/core/src/supervision/Supervisor.ts +311 -0
- package/packages/core/src/supervision/index.ts +11 -0
- package/packages/core/src/types.ts +470 -0
- package/packages/core/test/bulkhead.test.ts +310 -0
- package/packages/core/test/circuit-breaker.test.ts +153 -0
- package/packages/core/test/memory-guard.test.ts +213 -0
- package/packages/core/test/retry.test.ts +110 -0
- package/packages/core/test/safe-zone.test.ts +271 -0
- package/packages/core/test/supervisor.test.ts +310 -0
- package/packages/core/tsconfig.json +13 -0
- package/packages/dashboard/package.json +56 -0
- package/packages/dashboard/server/DashboardServer.ts +454 -0
- package/packages/dashboard/tsconfig.json +14 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file retry.test.ts
|
|
3
|
+
* @description Comprehensive tests for RetryEngine
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
7
|
+
|
|
8
|
+
// ── Mocked dependencies ────────────────────────────────────────────────────
|
|
9
|
+
vi.mock("../src/event-bus.js", () => ({
|
|
10
|
+
getEventBus: () => ({ emit: vi.fn() }),
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
vi.mock("../src/logger.js", () => ({
|
|
14
|
+
getLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
|
|
15
|
+
}));
|
|
16
|
+
|
|
17
|
+
vi.mock("../src/safe-zone/AsyncBoundary.js", () => ({
|
|
18
|
+
AsyncBoundary: { getRequestId: () => "test-request-123" },
|
|
19
|
+
}));
|
|
20
|
+
|
|
21
|
+
// Import after mocks
|
|
22
|
+
const { RetryEngine, withRetry, calculateBackoff } = await import("../src/recovery/RetryEngine.js");
|
|
23
|
+
const { RetryExhaustedError } = await import("../src/types.js");
|
|
24
|
+
|
|
25
|
+
describe("RetryEngine", () => {
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
RetryEngine.resetBudget();
|
|
28
|
+
vi.useFakeTimers();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it("should succeed on first attempt without retrying", async () => {
|
|
32
|
+
vi.useRealTimers();
|
|
33
|
+
const engine = new RetryEngine({ maxAttempts: 3 });
|
|
34
|
+
const fn = vi.fn().mockResolvedValue("success");
|
|
35
|
+
|
|
36
|
+
const result = await engine.execute(fn, "test-op");
|
|
37
|
+
|
|
38
|
+
expect(result).toBe("success");
|
|
39
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("should retry transient failures and succeed", async () => {
|
|
43
|
+
vi.useRealTimers();
|
|
44
|
+
const engine = new RetryEngine({
|
|
45
|
+
maxAttempts: 3,
|
|
46
|
+
baseDelayMs: 1,
|
|
47
|
+
maxDelayMs: 10,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
let callCount = 0;
|
|
51
|
+
const fn = vi.fn().mockImplementation(async () => {
|
|
52
|
+
callCount++;
|
|
53
|
+
if (callCount < 3) {
|
|
54
|
+
const err = new Error("ECONNRESET");
|
|
55
|
+
(err as NodeJS.ErrnoException).code = "ECONNRESET";
|
|
56
|
+
throw err;
|
|
57
|
+
}
|
|
58
|
+
return "recovered";
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
const result = await engine.execute(fn, "retry-test");
|
|
62
|
+
|
|
63
|
+
expect(result).toBe("recovered");
|
|
64
|
+
expect(fn).toHaveBeenCalledTimes(3);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("should throw RetryExhaustedError after maxAttempts", async () => {
|
|
68
|
+
vi.useRealTimers();
|
|
69
|
+
const engine = new RetryEngine({
|
|
70
|
+
maxAttempts: 2,
|
|
71
|
+
baseDelayMs: 1,
|
|
72
|
+
maxDelayMs: 5,
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const err = new Error("ECONNRESET");
|
|
76
|
+
(err as NodeJS.ErrnoException).code = "ECONNRESET";
|
|
77
|
+
const fn = vi.fn().mockRejectedValue(err);
|
|
78
|
+
|
|
79
|
+
await expect(engine.execute(fn, "exhaust-test")).rejects.toBeInstanceOf(RetryExhaustedError);
|
|
80
|
+
expect(fn).toHaveBeenCalledTimes(2);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("should NOT retry non-retryable errors (e.g. 400 Bad Request)", async () => {
|
|
84
|
+
vi.useRealTimers();
|
|
85
|
+
const engine = new RetryEngine({ maxAttempts: 5, baseDelayMs: 1 });
|
|
86
|
+
|
|
87
|
+
const err = new Error("Bad Request");
|
|
88
|
+
(err as Error & { statusCode: number }).statusCode = 400;
|
|
89
|
+
const fn = vi.fn().mockRejectedValue(err);
|
|
90
|
+
|
|
91
|
+
await expect(engine.execute(fn)).rejects.toThrow("Bad Request");
|
|
92
|
+
// Should only have been called once (no retries for 400)
|
|
93
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("calculateBackoff should return value within expected range", () => {
|
|
97
|
+
const delay = calculateBackoff(1, 200, 10000, 0.3);
|
|
98
|
+
expect(delay).toBeGreaterThan(0);
|
|
99
|
+
expect(delay).toBeLessThanOrEqual(10000);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it("withRetry functional API should work", async () => {
|
|
103
|
+
vi.useRealTimers();
|
|
104
|
+
const result = await withRetry(
|
|
105
|
+
async () => "functional-result",
|
|
106
|
+
{ maxAttempts: 3, baseDelayMs: 1 }
|
|
107
|
+
);
|
|
108
|
+
expect(result).toBe("functional-result");
|
|
109
|
+
});
|
|
110
|
+
});
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file safe-zone.test.ts
|
|
3
|
+
* @description Tests for the Safe Zone layer.
|
|
4
|
+
*
|
|
5
|
+
* Actual API shapes (from source):
|
|
6
|
+
* AsyncBoundary — all methods are static:
|
|
7
|
+
* AsyncBoundary.run(ctx, fn)
|
|
8
|
+
* AsyncBoundary.getContext()
|
|
9
|
+
* AsyncBoundary.getRequestId()
|
|
10
|
+
* AsyncBoundary.enrich(partial)
|
|
11
|
+
*
|
|
12
|
+
* classifyError(err) → "operational" | "programming" | "unknown"
|
|
13
|
+
* serializeError(err) → SerializedError
|
|
14
|
+
*
|
|
15
|
+
* NOTE: classifyError classifies generic Error as "operational" (conservative).
|
|
16
|
+
* Only TypeError/RangeError/SyntaxError/ReferenceError etc. are "programming".
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { describe, it, expect } from "vitest";
|
|
20
|
+
import {
|
|
21
|
+
AsyncBoundary,
|
|
22
|
+
asyncBoundary,
|
|
23
|
+
getRequestContext,
|
|
24
|
+
getRequestId,
|
|
25
|
+
} from "../src/safe-zone/AsyncBoundary.js";
|
|
26
|
+
import {
|
|
27
|
+
classifyError,
|
|
28
|
+
serializeError,
|
|
29
|
+
} from "../src/safe-zone/ErrorTrap.js";
|
|
30
|
+
import type { RequestContext } from "../src/types.js";
|
|
31
|
+
|
|
32
|
+
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
function sleep(ms: number): Promise<void> {
|
|
35
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function makeCtx(id = "test-req"): RequestContext {
|
|
39
|
+
return { requestId: id, startedAt: Date.now() };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// ─── AsyncBoundary — static API ───────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
describe("AsyncBoundary.run — context propagation", () => {
|
|
45
|
+
it("returns undefined context outside run()", () => {
|
|
46
|
+
// After all prior tests complete, we should be outside any boundary
|
|
47
|
+
const ctx = AsyncBoundary.getContext();
|
|
48
|
+
// May or may not be defined depending on test isolation — just don't throw
|
|
49
|
+
expect(ctx === undefined || typeof ctx === "object").toBe(true);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("provides context inside run()", async () => {
|
|
53
|
+
await AsyncBoundary.run(makeCtx("ctx-001"), async () => {
|
|
54
|
+
const ctx = AsyncBoundary.getContext();
|
|
55
|
+
expect(ctx).toBeDefined();
|
|
56
|
+
expect(ctx?.requestId).toBe("ctx-001");
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("propagates context across awaited async operations", async () => {
|
|
61
|
+
await AsyncBoundary.run(makeCtx("propagation-test"), async () => {
|
|
62
|
+
await Promise.resolve();
|
|
63
|
+
await new Promise<void>((r) => setTimeout(r, 10));
|
|
64
|
+
const ctx = AsyncBoundary.getContext();
|
|
65
|
+
expect(ctx?.requestId).toBe("propagation-test");
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("isolates contexts between concurrent run() calls", async () => {
|
|
70
|
+
const results: string[] = [];
|
|
71
|
+
|
|
72
|
+
await Promise.all([
|
|
73
|
+
AsyncBoundary.run(makeCtx("ctx-A"), async () => {
|
|
74
|
+
await sleep(20);
|
|
75
|
+
results.push(AsyncBoundary.getContext()?.requestId ?? "missing");
|
|
76
|
+
}),
|
|
77
|
+
AsyncBoundary.run(makeCtx("ctx-B"), async () => {
|
|
78
|
+
await sleep(5);
|
|
79
|
+
results.push(AsyncBoundary.getContext()?.requestId ?? "missing");
|
|
80
|
+
}),
|
|
81
|
+
]);
|
|
82
|
+
|
|
83
|
+
expect(results).toContain("ctx-A");
|
|
84
|
+
expect(results).toContain("ctx-B");
|
|
85
|
+
expect(results).toHaveLength(2);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("enrich() adds fields to the active context", async () => {
|
|
89
|
+
await AsyncBoundary.run(makeCtx("enrich-test"), async () => {
|
|
90
|
+
AsyncBoundary.enrich({ custom: { userId: "user-42" } });
|
|
91
|
+
const ctx = AsyncBoundary.getContext();
|
|
92
|
+
expect(ctx?.custom?.["userId"]).toBe("user-42");
|
|
93
|
+
expect(ctx?.requestId).toBe("enrich-test");
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("getRequestId() returns requestId when inside run()", async () => {
|
|
98
|
+
await AsyncBoundary.run(makeCtx("known-id"), async () => {
|
|
99
|
+
const id = AsyncBoundary.getRequestId();
|
|
100
|
+
expect(id).toBe("known-id");
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("getRequestId() returns undefined when outside run()", () => {
|
|
105
|
+
// Relies on the fact that we're between async boundaries here
|
|
106
|
+
// This just verifies it doesn't throw
|
|
107
|
+
expect(() => AsyncBoundary.getRequestId()).not.toThrow();
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("passes return value from run()", async () => {
|
|
111
|
+
const result = await AsyncBoundary.run(makeCtx("ret"), async () => 42);
|
|
112
|
+
expect(result).toBe(42);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
// ─── Convenience exports ──────────────────────────────────────────────────────
|
|
117
|
+
|
|
118
|
+
describe("getRequestContext / getRequestId convenience functions", () => {
|
|
119
|
+
it("getRequestContext returns same as AsyncBoundary.getContext() inside run()", async () => {
|
|
120
|
+
await AsyncBoundary.run(makeCtx("convenience-test"), async () => {
|
|
121
|
+
const ctx = getRequestContext();
|
|
122
|
+
expect(ctx?.requestId).toBe("convenience-test");
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("getRequestId returns 'unknown' when outside any context", () => {
|
|
127
|
+
const id = getRequestId();
|
|
128
|
+
expect(typeof id).toBe("string");
|
|
129
|
+
// May be "unknown" or the actual requestId if a parent run() is active
|
|
130
|
+
expect(id.length).toBeGreaterThan(0);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
// ─── asyncBoundary HOF ────────────────────────────────────────────────────────
|
|
135
|
+
|
|
136
|
+
describe("asyncBoundary HOF wrapper", () => {
|
|
137
|
+
it("executes the wrapped function inside a fresh context", async () => {
|
|
138
|
+
let capturedId: string | undefined;
|
|
139
|
+
|
|
140
|
+
const wrapped = asyncBoundary(async (..._args: unknown[]) => {
|
|
141
|
+
capturedId = AsyncBoundary.getContext()?.requestId;
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
await wrapped();
|
|
145
|
+
expect(typeof capturedId).toBe("string");
|
|
146
|
+
expect(capturedId).not.toBe(""); // auto-generated UUID
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("passes through the return value", async () => {
|
|
150
|
+
const wrapped = asyncBoundary(async () => 99 as unknown);
|
|
151
|
+
const result = await wrapped();
|
|
152
|
+
expect(result).toBe(99);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("passes arguments through to the wrapped function", async () => {
|
|
156
|
+
let received: number[] = [];
|
|
157
|
+
const wrapped = asyncBoundary(async (a: unknown, b: unknown) => {
|
|
158
|
+
received = [a as number, b as number];
|
|
159
|
+
});
|
|
160
|
+
await wrapped(3, 7);
|
|
161
|
+
expect(received).toEqual([3, 7]);
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
// ─── classifyError ────────────────────────────────────────────────────────────
|
|
166
|
+
|
|
167
|
+
describe("classifyError", () => {
|
|
168
|
+
it("classifies TypeError as programming", () => {
|
|
169
|
+
expect(classifyError(new TypeError("bad type"))).toBe("programming");
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("classifies ReferenceError as programming", () => {
|
|
173
|
+
expect(classifyError(new ReferenceError("not defined"))).toBe("programming");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("classifies SyntaxError as programming", () => {
|
|
177
|
+
expect(classifyError(new SyntaxError("unexpected token"))).toBe("programming");
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it("classifies RangeError as programming", () => {
|
|
181
|
+
expect(classifyError(new RangeError("out of range"))).toBe("programming");
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it("classifies ECONNREFUSED as operational", () => {
|
|
185
|
+
const err = new Error("connection refused");
|
|
186
|
+
(err as NodeJS.ErrnoException).code = "ECONNREFUSED";
|
|
187
|
+
expect(classifyError(err)).toBe("operational");
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it("classifies ETIMEDOUT as operational", () => {
|
|
191
|
+
const err = new Error("timeout");
|
|
192
|
+
(err as NodeJS.ErrnoException).code = "ETIMEDOUT";
|
|
193
|
+
expect(classifyError(err)).toBe("operational");
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("classifies ENOTFOUND as operational", () => {
|
|
197
|
+
const err = new Error("ENOTFOUND");
|
|
198
|
+
(err as NodeJS.ErrnoException).code = "ENOTFOUND";
|
|
199
|
+
expect(classifyError(err)).toBe("operational");
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("classifies errors with isOperational=true as operational", () => {
|
|
203
|
+
const err = Object.assign(new Error("custom"), { isOperational: true });
|
|
204
|
+
expect(classifyError(err)).toBe("operational");
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("classifies errors with isOperational=false as programming", () => {
|
|
208
|
+
const err = Object.assign(new Error("custom"), { isOperational: false });
|
|
209
|
+
expect(classifyError(err)).toBe("programming");
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it("classifies 4xx HTTP errors as operational", () => {
|
|
213
|
+
const err = Object.assign(new Error("not found"), { statusCode: 404 });
|
|
214
|
+
expect(classifyError(err)).toBe("operational");
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("classifies generic Error() conservatively as operational", () => {
|
|
218
|
+
// Immortal classifies unknowns as operational to avoid unnecessary restarts
|
|
219
|
+
expect(classifyError(new Error("some message"))).toBe("operational");
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
it("classifies non-Error throws as unknown", () => {
|
|
223
|
+
expect(classifyError("just a string")).toBe("unknown");
|
|
224
|
+
expect(classifyError(42)).toBe("unknown");
|
|
225
|
+
expect(classifyError(null)).toBe("unknown");
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
// ─── serializeError ──────────────────────────────────────────────────────────
|
|
230
|
+
|
|
231
|
+
describe("serializeError", () => {
|
|
232
|
+
it("serializes a standard Error correctly", () => {
|
|
233
|
+
const err = new Error("test message");
|
|
234
|
+
const s = serializeError(err);
|
|
235
|
+
expect(s.name).toBe("Error");
|
|
236
|
+
expect(s.message).toBe("test message");
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it("includes name from Error subclasses", () => {
|
|
240
|
+
class PaymentError extends Error {
|
|
241
|
+
constructor(msg: string) { super(msg); this.name = "PaymentError"; }
|
|
242
|
+
}
|
|
243
|
+
const s = serializeError(new PaymentError("declined"));
|
|
244
|
+
expect(s.name).toBe("PaymentError");
|
|
245
|
+
expect(s.message).toBe("declined");
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("serializes raw string throws", () => {
|
|
249
|
+
const s = serializeError("raw string");
|
|
250
|
+
expect(s.name).toBe("Error");
|
|
251
|
+
expect(s.message).toBe("raw string");
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it("serializes null without throwing", () => {
|
|
255
|
+
const s = serializeError(null);
|
|
256
|
+
expect(s.name).toBe("UnknownError");
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it("serializes plain object without throwing", () => {
|
|
260
|
+
const s = serializeError({ code: 500, msg: "oops" });
|
|
261
|
+
expect(s.name).toBe("UnknownError");
|
|
262
|
+
expect(typeof s.message).toBe("string");
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
it("includes code and statusCode if present", () => {
|
|
266
|
+
const err = Object.assign(new Error("bad"), { code: "ERR_X", statusCode: 503 });
|
|
267
|
+
const s = serializeError(err);
|
|
268
|
+
expect(s.code).toBe("ERR_X");
|
|
269
|
+
expect(s.statusCode).toBe(503);
|
|
270
|
+
});
|
|
271
|
+
});
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file supervisor.test.ts
|
|
3
|
+
* @description Tests for the Supervisor and RestartStrategy layers.
|
|
4
|
+
*
|
|
5
|
+
* Actual API shapes (from source):
|
|
6
|
+
* - computeRestartDecision(failedId, allWorkers: WorkerEntry[], strategy)
|
|
7
|
+
* → { workersToRestart: string[], strategy }
|
|
8
|
+
* - new Supervisor(options?: { config?, onEscalation? })
|
|
9
|
+
* - supervisor.register(id, spawnFn) ← spawnFn returns { terminate? }
|
|
10
|
+
* - supervisor.onWorkerExit(id, exitCode?)
|
|
11
|
+
* - supervisor.stop(id) — async
|
|
12
|
+
* - supervisor.stopAll() — async
|
|
13
|
+
* - supervisor.getWorkerStatuses() → WorkerStatus[]
|
|
14
|
+
* - supervisor.updateWorkerMetrics(id, { memoryMb?, cpuPercent?, pid? })
|
|
15
|
+
* - supervisor.requestGracefulRestart(id, reason?)
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { describe, it, expect, vi, afterEach } from "vitest";
|
|
19
|
+
import { Supervisor } from "../src/supervision/Supervisor.js";
|
|
20
|
+
import { computeRestartDecision } from "../src/supervision/RestartStrategy.js";
|
|
21
|
+
import type { WorkerEntry } from "../src/supervision/RestartStrategy.js";
|
|
22
|
+
|
|
23
|
+
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
24
|
+
|
|
25
|
+
function sleep(ms: number): Promise<void> {
|
|
26
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** A spawnFn that always returns a simple terminatable object */
|
|
30
|
+
function makeSpawn() {
|
|
31
|
+
const terminated: boolean[] = [];
|
|
32
|
+
const fn = vi.fn(() => ({
|
|
33
|
+
terminate: () => {
|
|
34
|
+
terminated.push(true);
|
|
35
|
+
},
|
|
36
|
+
}));
|
|
37
|
+
return { fn, terminated };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// ─── computeRestartDecision — pure function ───────────────────────────────────
|
|
41
|
+
|
|
42
|
+
describe("computeRestartDecision — one_for_one", () => {
|
|
43
|
+
const workers: WorkerEntry[] = [
|
|
44
|
+
{ id: "w1", index: 0, restart: vi.fn() },
|
|
45
|
+
{ id: "w2", index: 1, restart: vi.fn() },
|
|
46
|
+
{ id: "w3", index: 2, restart: vi.fn() },
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
it("should restart only the failed worker", () => {
|
|
50
|
+
const d = computeRestartDecision("w2", workers, "one_for_one");
|
|
51
|
+
expect(d.workersToRestart).toEqual(["w2"]);
|
|
52
|
+
expect(d.strategy).toBe("one_for_one");
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("should return empty list for unknown worker", () => {
|
|
56
|
+
const d = computeRestartDecision("w-unknown", workers, "one_for_one");
|
|
57
|
+
expect(d.workersToRestart).toHaveLength(0);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
describe("computeRestartDecision — one_for_all", () => {
|
|
62
|
+
const workers: WorkerEntry[] = [
|
|
63
|
+
{ id: "w1", index: 0, restart: vi.fn() },
|
|
64
|
+
{ id: "w2", index: 1, restart: vi.fn() },
|
|
65
|
+
{ id: "w3", index: 2, restart: vi.fn() },
|
|
66
|
+
];
|
|
67
|
+
|
|
68
|
+
it("should restart ALL workers when any one fails", () => {
|
|
69
|
+
const d = computeRestartDecision("w1", workers, "one_for_all");
|
|
70
|
+
expect(d.workersToRestart.sort()).toEqual(["w1", "w2", "w3"]);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("computeRestartDecision — rest_for_one", () => {
|
|
75
|
+
const workers: WorkerEntry[] = [
|
|
76
|
+
{ id: "w1", index: 0, restart: vi.fn() },
|
|
77
|
+
{ id: "w2", index: 1, restart: vi.fn() },
|
|
78
|
+
{ id: "w3", index: 2, restart: vi.fn() },
|
|
79
|
+
{ id: "w4", index: 3, restart: vi.fn() },
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
it("should restart failed + all subsequent workers", () => {
|
|
83
|
+
const d = computeRestartDecision("w2", workers, "rest_for_one");
|
|
84
|
+
expect(d.workersToRestart).toContain("w2");
|
|
85
|
+
expect(d.workersToRestart).toContain("w3");
|
|
86
|
+
expect(d.workersToRestart).toContain("w4");
|
|
87
|
+
expect(d.workersToRestart).not.toContain("w1");
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("should restart only the tail when last worker fails", () => {
|
|
91
|
+
const d = computeRestartDecision("w4", workers, "rest_for_one");
|
|
92
|
+
expect(d.workersToRestart).toEqual(["w4"]);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("should restart all when the first worker fails", () => {
|
|
96
|
+
const d = computeRestartDecision("w1", workers, "rest_for_one");
|
|
97
|
+
expect(d.workersToRestart.sort()).toEqual(["w1", "w2", "w3", "w4"]);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// ─── Supervisor — registration ────────────────────────────────────────────────
|
|
102
|
+
|
|
103
|
+
describe("Supervisor — register", () => {
|
|
104
|
+
it("should register a worker and call spawnFn once", async () => {
|
|
105
|
+
const { fn } = makeSpawn();
|
|
106
|
+
const sup = new Supervisor();
|
|
107
|
+
sup.register("api", fn);
|
|
108
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
109
|
+
await sup.stopAll();
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("should throw if the same id is registered twice", async () => {
|
|
113
|
+
const { fn } = makeSpawn();
|
|
114
|
+
const sup = new Supervisor();
|
|
115
|
+
sup.register("dup", fn);
|
|
116
|
+
expect(() => sup.register("dup", fn)).toThrow(/already registered/i);
|
|
117
|
+
await sup.stopAll();
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("should reflect registered worker in getWorkerStatuses()", async () => {
|
|
121
|
+
const { fn } = makeSpawn();
|
|
122
|
+
const sup = new Supervisor();
|
|
123
|
+
sup.register("worker-1", fn);
|
|
124
|
+
const statuses = sup.getWorkerStatuses();
|
|
125
|
+
expect(statuses).toHaveLength(1);
|
|
126
|
+
expect(statuses[0]!.id).toBe("worker-1");
|
|
127
|
+
expect(statuses[0]!.state).toBe("running");
|
|
128
|
+
await sup.stopAll();
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
// ─── Supervisor — restart on exit ────────────────────────────────────────────
|
|
133
|
+
|
|
134
|
+
describe("Supervisor — onWorkerExit triggers restart", () => {
|
|
135
|
+
it("should call spawnFn again after a crash", async () => {
|
|
136
|
+
const { fn } = makeSpawn();
|
|
137
|
+
const sup = new Supervisor({
|
|
138
|
+
config: { strategy: "one_for_one", baseRestartDelayMs: 10 },
|
|
139
|
+
});
|
|
140
|
+
sup.register("crash-worker", fn);
|
|
141
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
142
|
+
|
|
143
|
+
sup.onWorkerExit("crash-worker", 1);
|
|
144
|
+
await sleep(60);
|
|
145
|
+
|
|
146
|
+
expect(fn).toHaveBeenCalledTimes(2); // restarted once
|
|
147
|
+
await sup.stopAll();
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it("should respect baseRestartDelayMs before restarting", async () => {
|
|
151
|
+
const { fn } = makeSpawn();
|
|
152
|
+
const sup = new Supervisor({
|
|
153
|
+
config: { strategy: "one_for_one", baseRestartDelayMs: 150 },
|
|
154
|
+
});
|
|
155
|
+
sup.register("slow-restart", fn);
|
|
156
|
+
sup.onWorkerExit("slow-restart", 1);
|
|
157
|
+
|
|
158
|
+
await sleep(30); // not yet
|
|
159
|
+
expect(fn).toHaveBeenCalledTimes(1); // only initial spawn
|
|
160
|
+
|
|
161
|
+
await sleep(200); // now past delay
|
|
162
|
+
expect(fn).toHaveBeenCalledTimes(2);
|
|
163
|
+
await sup.stopAll();
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it("should ignore exits for unknown worker ids", async () => {
|
|
167
|
+
const sup = new Supervisor();
|
|
168
|
+
expect(() => sup.onWorkerExit("ghost", 1)).not.toThrow();
|
|
169
|
+
await sup.stopAll();
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// ─── Supervisor — escalation ceiling ─────────────────────────────────────────
|
|
174
|
+
|
|
175
|
+
describe("Supervisor — escalation ceiling", () => {
|
|
176
|
+
it("should call onEscalation after maxRestartsInWindow crashes", async () => {
|
|
177
|
+
const onEscalation = vi.fn();
|
|
178
|
+
const { fn } = makeSpawn();
|
|
179
|
+
|
|
180
|
+
const sup = new Supervisor({
|
|
181
|
+
config: {
|
|
182
|
+
strategy: "one_for_one",
|
|
183
|
+
maxRestartsInWindow: 3,
|
|
184
|
+
windowMs: 10_000,
|
|
185
|
+
baseRestartDelayMs: 5,
|
|
186
|
+
},
|
|
187
|
+
onEscalation,
|
|
188
|
+
});
|
|
189
|
+
sup.register("crasher", fn);
|
|
190
|
+
|
|
191
|
+
// Trigger 4 exits — 3 allowed, 4th escalates
|
|
192
|
+
sup.onWorkerExit("crasher", 1); await sleep(20);
|
|
193
|
+
sup.onWorkerExit("crasher", 1); await sleep(20);
|
|
194
|
+
sup.onWorkerExit("crasher", 1); await sleep(20);
|
|
195
|
+
sup.onWorkerExit("crasher", 1); await sleep(20);
|
|
196
|
+
|
|
197
|
+
expect(onEscalation).toHaveBeenCalledTimes(1);
|
|
198
|
+
const [calledId] = onEscalation.mock.calls[0] as [string];
|
|
199
|
+
expect(calledId).toBe("crasher");
|
|
200
|
+
await sup.stopAll();
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("should set worker status to escalated and stop restarting", async () => {
|
|
204
|
+
const onEscalation = vi.fn();
|
|
205
|
+
const { fn } = makeSpawn();
|
|
206
|
+
|
|
207
|
+
const sup = new Supervisor({
|
|
208
|
+
config: {
|
|
209
|
+
strategy: "one_for_one",
|
|
210
|
+
maxRestartsInWindow: 2,
|
|
211
|
+
windowMs: 10_000,
|
|
212
|
+
baseRestartDelayMs: 5,
|
|
213
|
+
},
|
|
214
|
+
onEscalation,
|
|
215
|
+
});
|
|
216
|
+
sup.register("looper", fn);
|
|
217
|
+
|
|
218
|
+
// Crash 3 times (limit = 2, so 3rd is escalation)
|
|
219
|
+
sup.onWorkerExit("looper", 1); await sleep(20);
|
|
220
|
+
sup.onWorkerExit("looper", 1); await sleep(20);
|
|
221
|
+
sup.onWorkerExit("looper", 1); await sleep(20);
|
|
222
|
+
|
|
223
|
+
const statuses = sup.getWorkerStatuses();
|
|
224
|
+
const w = statuses.find((s) => s.id === "looper");
|
|
225
|
+
expect(w?.state).toBe("escalated");
|
|
226
|
+
await sup.stopAll();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
// ─── Supervisor — metrics update ─────────────────────────────────────────────
|
|
231
|
+
|
|
232
|
+
describe("Supervisor — updateWorkerMetrics", () => {
|
|
233
|
+
it("should store metrics for a registered worker", async () => {
|
|
234
|
+
const { fn } = makeSpawn();
|
|
235
|
+
const sup = new Supervisor();
|
|
236
|
+
sup.register("metrics-w", fn);
|
|
237
|
+
|
|
238
|
+
sup.updateWorkerMetrics("metrics-w", { memoryMb: 120, cpuPercent: 45 });
|
|
239
|
+
|
|
240
|
+
const statuses = sup.getWorkerStatuses();
|
|
241
|
+
const w = statuses.find((s) => s.id === "metrics-w")!;
|
|
242
|
+
expect(w.memoryMb).toBe(120);
|
|
243
|
+
expect(w.cpuPercent).toBe(45);
|
|
244
|
+
await sup.stopAll();
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
it("should silently ignore unknown worker ids", async () => {
|
|
248
|
+
const sup = new Supervisor();
|
|
249
|
+
expect(() =>
|
|
250
|
+
sup.updateWorkerMetrics("ghost", { memoryMb: 50, cpuPercent: 10 })
|
|
251
|
+
).not.toThrow();
|
|
252
|
+
await sup.stopAll();
|
|
253
|
+
});
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
// ─── Supervisor — requestGracefulRestart ──────────────────────────────────────
|
|
257
|
+
|
|
258
|
+
describe("Supervisor — requestGracefulRestart", () => {
|
|
259
|
+
it("should trigger terminate + re-register within delay", async () => {
|
|
260
|
+
const { fn, terminated } = makeSpawn();
|
|
261
|
+
const sup = new Supervisor({
|
|
262
|
+
config: { strategy: "one_for_one", baseRestartDelayMs: 10 },
|
|
263
|
+
});
|
|
264
|
+
sup.register("api-server", fn);
|
|
265
|
+
|
|
266
|
+
sup.requestGracefulRestart("api-server");
|
|
267
|
+
await sleep(80);
|
|
268
|
+
|
|
269
|
+
expect(terminated.length).toBeGreaterThan(0); // terminate was called
|
|
270
|
+
expect(fn.mock.calls.length).toBeGreaterThan(1); // re-spawned
|
|
271
|
+
await sup.stopAll();
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it("should not restart if supervisor is stopped", async () => {
|
|
275
|
+
const { fn } = makeSpawn();
|
|
276
|
+
const sup = new Supervisor({
|
|
277
|
+
config: { strategy: "one_for_one", baseRestartDelayMs: 10 },
|
|
278
|
+
});
|
|
279
|
+
sup.register("w", fn);
|
|
280
|
+
await sup.stopAll();
|
|
281
|
+
|
|
282
|
+
const callsBeforeExit = fn.mock.calls.length;
|
|
283
|
+
sup.onWorkerExit("w", 1);
|
|
284
|
+
await sleep(60);
|
|
285
|
+
// No additional spawns after stopAll
|
|
286
|
+
expect(fn.mock.calls.length).toBe(callsBeforeExit);
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
// ─── Supervisor — stop individual worker ─────────────────────────────────────
|
|
291
|
+
|
|
292
|
+
describe("Supervisor — stop single worker", () => {
|
|
293
|
+
it("should remove worker from statuses after stop()", async () => {
|
|
294
|
+
const { fn } = makeSpawn();
|
|
295
|
+
const sup = new Supervisor();
|
|
296
|
+
sup.register("to-stop", fn);
|
|
297
|
+
await sup.stop("to-stop");
|
|
298
|
+
|
|
299
|
+
const statuses = sup.getWorkerStatuses();
|
|
300
|
+
expect(statuses.find((s) => s.id === "to-stop")).toBeUndefined();
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
it("should call terminate on the worker instance", async () => {
|
|
304
|
+
const { fn, terminated } = makeSpawn();
|
|
305
|
+
const sup = new Supervisor();
|
|
306
|
+
sup.register("terminate-me", fn);
|
|
307
|
+
await sup.stop("terminate-me");
|
|
308
|
+
expect(terminated).toHaveLength(1);
|
|
309
|
+
});
|
|
310
|
+
});
|