@omnitron-dev/titan-pm 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +27 -0
- package/dist/common-types.d.ts +59 -0
- package/dist/common-types.d.ts.map +1 -0
- package/dist/common-types.js +23 -0
- package/dist/common-types.js.map +1 -0
- package/dist/decorators.d.ts +41 -0
- package/dist/decorators.d.ts.map +1 -0
- package/dist/decorators.js +372 -0
- package/dist/decorators.js.map +1 -0
- package/dist/fork-worker.d.ts +2 -0
- package/dist/fork-worker.d.ts.map +1 -0
- package/dist/fork-worker.js +66 -0
- package/dist/fork-worker.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/netron-client.d.ts +46 -0
- package/dist/netron-client.d.ts.map +1 -0
- package/dist/netron-client.js +222 -0
- package/dist/netron-client.js.map +1 -0
- package/dist/pm.module.d.ts +23 -0
- package/dist/pm.module.d.ts.map +1 -0
- package/dist/pm.module.js +176 -0
- package/dist/pm.module.js.map +1 -0
- package/dist/process-health.d.ts +37 -0
- package/dist/process-health.d.ts.map +1 -0
- package/dist/process-health.js +245 -0
- package/dist/process-health.js.map +1 -0
- package/dist/process-manager.d.ts +42 -0
- package/dist/process-manager.d.ts.map +1 -0
- package/dist/process-manager.js +435 -0
- package/dist/process-manager.js.map +1 -0
- package/dist/process-metrics.d.ts +22 -0
- package/dist/process-metrics.d.ts.map +1 -0
- package/dist/process-metrics.js +169 -0
- package/dist/process-metrics.js.map +1 -0
- package/dist/process-pool.d.ts +107 -0
- package/dist/process-pool.d.ts.map +1 -0
- package/dist/process-pool.js +1035 -0
- package/dist/process-pool.js.map +1 -0
- package/dist/process-registry.d.ts +14 -0
- package/dist/process-registry.d.ts.map +1 -0
- package/dist/process-registry.js +59 -0
- package/dist/process-registry.js.map +1 -0
- package/dist/process-spawner.d.ts +64 -0
- package/dist/process-spawner.d.ts.map +1 -0
- package/dist/process-spawner.js +596 -0
- package/dist/process-spawner.js.map +1 -0
- package/dist/process-supervisor.d.ts +43 -0
- package/dist/process-supervisor.d.ts.map +1 -0
- package/dist/process-supervisor.js +319 -0
- package/dist/process-supervisor.js.map +1 -0
- package/dist/process-workflow.d.ts +31 -0
- package/dist/process-workflow.d.ts.map +1 -0
- package/dist/process-workflow.js +265 -0
- package/dist/process-workflow.js.map +1 -0
- package/dist/service-proxy.d.ts +29 -0
- package/dist/service-proxy.d.ts.map +1 -0
- package/dist/service-proxy.js +245 -0
- package/dist/service-proxy.js.map +1 -0
- package/dist/types.d.ts +523 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +40 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/deque.d.ts +17 -0
- package/dist/utils/deque.d.ts.map +1 -0
- package/dist/utils/deque.js +86 -0
- package/dist/utils/deque.js.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/worker-runtime.d.ts +2 -0
- package/dist/worker-runtime.d.ts.map +1 -0
- package/dist/worker-runtime.js +503 -0
- package/dist/worker-runtime.js.map +1 -0
- package/package.json +39 -0
|
@@ -0,0 +1,1035 @@
|
|
|
1
|
+
import { cpus } from 'node:os';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
4
|
+
import { Errors } from '@omnitron-dev/titan/errors';
|
|
5
|
+
import { PoolStrategy } from './types.js';
|
|
6
|
+
import { Deque } from './utils/index.js';
|
|
7
|
+
export class PoolBackpressureError extends Error {
|
|
8
|
+
constructor(queueSize, retryAfterMs = 2000) {
|
|
9
|
+
super(`Pool queue is full (${queueSize} pending). Retry after ${retryAfterMs}ms.`);
|
|
10
|
+
this.name = 'PoolBackpressureError';
|
|
11
|
+
this.retryAfterMs = retryAfterMs;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export class ProcessPool {
|
|
15
|
+
constructor(manager, processPathOrClass, options, logger) {
|
|
16
|
+
this.manager = manager;
|
|
17
|
+
this.processPathOrClass = processPathOrClass;
|
|
18
|
+
this.logger = logger;
|
|
19
|
+
this.workers = new Map();
|
|
20
|
+
this.queue = new Deque();
|
|
21
|
+
this.isInitialized = false;
|
|
22
|
+
this.isShuttingDown = false;
|
|
23
|
+
this.isDraining = false;
|
|
24
|
+
this.currentRoundRobinIndex = 0;
|
|
25
|
+
this.requestIdCounter = 0;
|
|
26
|
+
this.workerIndexCounter = 0;
|
|
27
|
+
this.totalRequests = 0;
|
|
28
|
+
this.totalErrors = 0;
|
|
29
|
+
this.totalResponseTime = 0;
|
|
30
|
+
this.successfulRequests = 0;
|
|
31
|
+
this.lastScaleCheck = Date.now();
|
|
32
|
+
this.scaleHistory = [];
|
|
33
|
+
this.unhealthyWorkers = new Set();
|
|
34
|
+
this.queueBatchSize = 10;
|
|
35
|
+
this.queueProcessingScheduled = false;
|
|
36
|
+
this.circuitBreaker = {
|
|
37
|
+
isOpen: false,
|
|
38
|
+
failures: 0,
|
|
39
|
+
lastFailTime: 0,
|
|
40
|
+
successAfterOpen: 0,
|
|
41
|
+
};
|
|
42
|
+
this.emitter = new EventEmitter();
|
|
43
|
+
this.healthyWorkersCache = [];
|
|
44
|
+
this.healthyWorkersCacheValid = false;
|
|
45
|
+
this.poolOptions = this.normalizeOptions(options);
|
|
46
|
+
this.memoryLimitBytes = this.parseMemoryLimit(options.memoryLimit || '512MB');
|
|
47
|
+
this.memoryWarningThreshold = options.memoryWarningThreshold || 0.8;
|
|
48
|
+
}
|
|
49
|
+
parseMemoryLimit(limit) {
|
|
50
|
+
if (typeof limit === 'number')
|
|
51
|
+
return limit;
|
|
52
|
+
const match = limit.match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB)?$/i);
|
|
53
|
+
if (!match)
|
|
54
|
+
return 512 * 1024 * 1024;
|
|
55
|
+
const value = parseFloat(match[1]);
|
|
56
|
+
const unit = (match[2] || 'B').toUpperCase();
|
|
57
|
+
const multipliers = {
|
|
58
|
+
B: 1,
|
|
59
|
+
KB: 1024,
|
|
60
|
+
MB: 1024 * 1024,
|
|
61
|
+
GB: 1024 * 1024 * 1024,
|
|
62
|
+
};
|
|
63
|
+
return value * (multipliers[unit] || 1);
|
|
64
|
+
}
|
|
65
|
+
get processName() {
|
|
66
|
+
return typeof this.processPathOrClass === 'string'
|
|
67
|
+
? path.basename(this.processPathOrClass, '.js')
|
|
68
|
+
: this.processPathOrClass.name;
|
|
69
|
+
}
|
|
70
|
+
normalizeOptions(options) {
|
|
71
|
+
const defaults = {
|
|
72
|
+
size: options.size || 'auto',
|
|
73
|
+
strategy: options.strategy || PoolStrategy.ROUND_ROBIN,
|
|
74
|
+
maxQueueSize: options.maxQueueSize ?? 100,
|
|
75
|
+
requestTimeout: options.requestTimeout ?? 30000,
|
|
76
|
+
recycleAfter: options.recycleAfter ?? 10000,
|
|
77
|
+
maxLifetime: options.maxLifetime ?? 3600000,
|
|
78
|
+
warmup: options.warmup ?? false,
|
|
79
|
+
metrics: options.metrics ?? true,
|
|
80
|
+
healthCheck: options.healthCheck ?? {
|
|
81
|
+
enabled: true,
|
|
82
|
+
interval: 30000,
|
|
83
|
+
unhealthyThreshold: 3,
|
|
84
|
+
},
|
|
85
|
+
autoScale: options.autoScale ?? {
|
|
86
|
+
enabled: false,
|
|
87
|
+
min: 1,
|
|
88
|
+
max: 10,
|
|
89
|
+
targetCPU: 70,
|
|
90
|
+
targetMemory: 80,
|
|
91
|
+
scaleUpThreshold: 0.8,
|
|
92
|
+
scaleDownThreshold: 0.3,
|
|
93
|
+
cooldownPeriod: 60000,
|
|
94
|
+
},
|
|
95
|
+
circuitBreaker: options.circuitBreaker ?? {
|
|
96
|
+
enabled: false,
|
|
97
|
+
threshold: 5,
|
|
98
|
+
timeout: 60000,
|
|
99
|
+
halfOpenRequests: 3,
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
return { ...defaults, ...options };
|
|
103
|
+
}
|
|
104
|
+
async initialize() {
|
|
105
|
+
if (this.isInitialized)
|
|
106
|
+
return;
|
|
107
|
+
const size = this.getPoolSize();
|
|
108
|
+
this.logger.info({
|
|
109
|
+
size,
|
|
110
|
+
class: this.processName,
|
|
111
|
+
strategy: this.poolOptions.strategy,
|
|
112
|
+
}, 'Initializing process pool');
|
|
113
|
+
const spawnPromises = [];
|
|
114
|
+
for (let i = 0; i < size; i++) {
|
|
115
|
+
spawnPromises.push(this.spawnWorker());
|
|
116
|
+
}
|
|
117
|
+
await Promise.all(spawnPromises);
|
|
118
|
+
this.isInitialized = true;
|
|
119
|
+
if (this.poolOptions.healthCheck?.enabled) {
|
|
120
|
+
this.setupHealthMonitoring();
|
|
121
|
+
}
|
|
122
|
+
if (this.poolOptions.autoScale?.enabled) {
|
|
123
|
+
this.setupAutoScaling();
|
|
124
|
+
}
|
|
125
|
+
if (this.poolOptions.warmup) {
|
|
126
|
+
await this.warmupPool();
|
|
127
|
+
}
|
|
128
|
+
if (this.poolOptions.recycleAfter || this.poolOptions.maxLifetime) {
|
|
129
|
+
this.setupRecycling();
|
|
130
|
+
}
|
|
131
|
+
this.setupMemoryMonitoring();
|
|
132
|
+
this.setupHeartbeatMonitoring();
|
|
133
|
+
this.emit('pool:initialized', { size, class: this.processName });
|
|
134
|
+
}
|
|
135
|
+
get size() {
|
|
136
|
+
return this.workers.size;
|
|
137
|
+
}
|
|
138
|
+
get active() {
|
|
139
|
+
let activeCount = 0;
|
|
140
|
+
this.workers.forEach((worker) => {
|
|
141
|
+
if (worker.processing.size > 0)
|
|
142
|
+
activeCount++;
|
|
143
|
+
});
|
|
144
|
+
return activeCount;
|
|
145
|
+
}
|
|
146
|
+
get pending() {
|
|
147
|
+
return this.queue.length;
|
|
148
|
+
}
|
|
149
|
+
getWorkerIds() {
|
|
150
|
+
return Array.from(this.workers.keys());
|
|
151
|
+
}
|
|
152
|
+
getWorkerHandle(workerId) {
|
|
153
|
+
const worker = this.workers.get(workerId);
|
|
154
|
+
if (!worker)
|
|
155
|
+
return null;
|
|
156
|
+
const proxy = worker.proxy;
|
|
157
|
+
const pmProcessId = proxy?.__processId;
|
|
158
|
+
if (!pmProcessId)
|
|
159
|
+
return null;
|
|
160
|
+
const handle = this.manager.workers?.get(pmProcessId);
|
|
161
|
+
return handle ?? null;
|
|
162
|
+
}
|
|
163
|
+
get metrics() {
|
|
164
|
+
const avgResponseTime = this.totalRequests > 0 ? this.totalResponseTime / this.totalRequests : 0;
|
|
165
|
+
const errorRate = this.totalRequests > 0 ? this.totalErrors / this.totalRequests : 0;
|
|
166
|
+
let totalCpu = 0;
|
|
167
|
+
let totalMemory = 0;
|
|
168
|
+
let healthyWorkers = 0;
|
|
169
|
+
this.workers.forEach((worker) => {
|
|
170
|
+
if (worker.metrics) {
|
|
171
|
+
totalCpu += worker.metrics.cpu;
|
|
172
|
+
totalMemory += worker.metrics.memory;
|
|
173
|
+
}
|
|
174
|
+
if (worker.health === 'healthy')
|
|
175
|
+
healthyWorkers++;
|
|
176
|
+
});
|
|
177
|
+
const avgCpu = this.workers.size > 0 ? totalCpu / this.workers.size : 0;
|
|
178
|
+
const avgMemory = this.workers.size > 0 ? totalMemory / this.workers.size : 0;
|
|
179
|
+
return {
|
|
180
|
+
cpu: avgCpu,
|
|
181
|
+
memory: avgMemory,
|
|
182
|
+
queueSize: this.queue.length,
|
|
183
|
+
activeWorkers: this.active,
|
|
184
|
+
totalWorkers: this.workers.size,
|
|
185
|
+
idleWorkers: this.workers.size - this.active,
|
|
186
|
+
healthyWorkers,
|
|
187
|
+
unhealthyWorkers: this.unhealthyWorkers.size,
|
|
188
|
+
totalRequests: this.totalRequests,
|
|
189
|
+
successfulRequests: this.successfulRequests,
|
|
190
|
+
failedRequests: this.totalErrors,
|
|
191
|
+
avgResponseTime,
|
|
192
|
+
errorRate,
|
|
193
|
+
throughput: this.calculateThroughput(),
|
|
194
|
+
saturation: this.calculateSaturation(),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
get circuitBreakerState() {
|
|
198
|
+
const isEnabled = this.poolOptions.circuitBreaker?.enabled ?? false;
|
|
199
|
+
let state = 'closed';
|
|
200
|
+
if (this.circuitBreaker.isOpen) {
|
|
201
|
+
state = this.shouldTryHalfOpen() ? 'half-open' : 'open';
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
state,
|
|
205
|
+
failures: this.circuitBreaker.failures,
|
|
206
|
+
lastFailTime: this.circuitBreaker.lastFailTime,
|
|
207
|
+
isEnabled,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
async scale(newSize) {
|
|
211
|
+
if (this.isShuttingDown) {
|
|
212
|
+
throw Errors.conflict('Cannot scale pool during shutdown');
|
|
213
|
+
}
|
|
214
|
+
const currentSize = this.workers.size;
|
|
215
|
+
if (newSize === currentSize)
|
|
216
|
+
return;
|
|
217
|
+
this.logger.info({
|
|
218
|
+
currentSize,
|
|
219
|
+
newSize,
|
|
220
|
+
class: this.processName,
|
|
221
|
+
}, 'Scaling process pool');
|
|
222
|
+
this.scaleHistory.push({
|
|
223
|
+
timestamp: Date.now(),
|
|
224
|
+
action: newSize > currentSize ? 'up' : 'down',
|
|
225
|
+
from: currentSize,
|
|
226
|
+
to: newSize,
|
|
227
|
+
});
|
|
228
|
+
if (this.scaleHistory.length > 100) {
|
|
229
|
+
this.scaleHistory = this.scaleHistory.slice(-100);
|
|
230
|
+
}
|
|
231
|
+
if (newSize > currentSize) {
|
|
232
|
+
await this.scaleUp(newSize - currentSize);
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
await this.scaleDown(currentSize - newSize);
|
|
236
|
+
}
|
|
237
|
+
this.emit('pool:scaled', {
|
|
238
|
+
from: currentSize,
|
|
239
|
+
to: newSize,
|
|
240
|
+
class: this.processName,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
async drain() {
|
|
244
|
+
this.logger.info({ class: this.processName }, 'Draining process pool');
|
|
245
|
+
this.isDraining = true;
|
|
246
|
+
while (this.queue.length > 0) {
|
|
247
|
+
const request = this.queue.shift();
|
|
248
|
+
if (request) {
|
|
249
|
+
if (request.timeout) {
|
|
250
|
+
clearTimeout(request.timeout);
|
|
251
|
+
}
|
|
252
|
+
request.reject(Errors.conflict('Pool is draining'));
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
await this.waitForActiveRequests();
|
|
256
|
+
this.emit('pool:drained', { class: this.processName });
|
|
257
|
+
}
|
|
258
|
+
async destroy() {
|
|
259
|
+
this.logger.info({ class: this.processName }, 'Destroying process pool');
|
|
260
|
+
this.isShuttingDown = true;
|
|
261
|
+
if (this.autoScaleTimer) {
|
|
262
|
+
clearInterval(this.autoScaleTimer);
|
|
263
|
+
this.autoScaleTimer = undefined;
|
|
264
|
+
}
|
|
265
|
+
if (this.healthCheckTimer) {
|
|
266
|
+
clearInterval(this.healthCheckTimer);
|
|
267
|
+
this.healthCheckTimer = undefined;
|
|
268
|
+
}
|
|
269
|
+
if (this.recycleTimer) {
|
|
270
|
+
clearInterval(this.recycleTimer);
|
|
271
|
+
this.recycleTimer = undefined;
|
|
272
|
+
}
|
|
273
|
+
if (this.heartbeatTimer) {
|
|
274
|
+
clearInterval(this.heartbeatTimer);
|
|
275
|
+
this.heartbeatTimer = undefined;
|
|
276
|
+
}
|
|
277
|
+
if (this.memoryMonitorTimer) {
|
|
278
|
+
clearInterval(this.memoryMonitorTimer);
|
|
279
|
+
this.memoryMonitorTimer = undefined;
|
|
280
|
+
}
|
|
281
|
+
await this.drain();
|
|
282
|
+
const shutdownPromises = Array.from(this.workers.values()).map((worker) => this.shutdownWorker(worker.id));
|
|
283
|
+
await Promise.all(shutdownPromises);
|
|
284
|
+
this.workers.clear();
|
|
285
|
+
this.isInitialized = false;
|
|
286
|
+
this.isShuttingDown = false;
|
|
287
|
+
this.isDraining = false;
|
|
288
|
+
this.emit('pool:destroyed', { class: this.processName });
|
|
289
|
+
this.emitter.removeAllListeners();
|
|
290
|
+
}
|
|
291
|
+
async execute(method, ...args) {
|
|
292
|
+
if (this.poolOptions.circuitBreaker?.enabled && this.circuitBreaker.isOpen) {
|
|
293
|
+
if (!this.shouldTryHalfOpen()) {
|
|
294
|
+
throw Errors.conflict('Circuit breaker is open');
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
if (this.isDraining) {
|
|
298
|
+
throw Errors.conflict('Pool is draining, not accepting new requests');
|
|
299
|
+
}
|
|
300
|
+
if (this.workers.size === 0 ||
|
|
301
|
+
(this.active >= this.workers.size && this.queue.length < (this.poolOptions.maxQueueSize || 100))) {
|
|
302
|
+
return this.queueRequest(method, args);
|
|
303
|
+
}
|
|
304
|
+
const worker = await this.selectWorker();
|
|
305
|
+
return this.executeOnWorker(worker, method, args);
|
|
306
|
+
}
|
|
307
|
+
getPoolSize() {
|
|
308
|
+
if (this.poolOptions.size === 'auto') {
|
|
309
|
+
return cpus().length;
|
|
310
|
+
}
|
|
311
|
+
return typeof this.poolOptions.size === 'number' ? this.poolOptions.size : 2;
|
|
312
|
+
}
|
|
313
|
+
async spawnWorker() {
|
|
314
|
+
try {
|
|
315
|
+
const workerIndex = this.workerIndexCounter++;
|
|
316
|
+
const baseOptions = this.poolOptions.spawnOptions ?? {};
|
|
317
|
+
const factoryOptions = this.poolOptions.spawnOptionsFactory
|
|
318
|
+
? this.poolOptions.spawnOptionsFactory(workerIndex)
|
|
319
|
+
: {};
|
|
320
|
+
const proxy = await this.manager.spawn(this.processPathOrClass, {
|
|
321
|
+
name: `${this.processName}-pool-${workerIndex}`,
|
|
322
|
+
...baseOptions,
|
|
323
|
+
...factoryOptions,
|
|
324
|
+
dependencies: {
|
|
325
|
+
...baseOptions.dependencies,
|
|
326
|
+
...factoryOptions.dependencies,
|
|
327
|
+
},
|
|
328
|
+
});
|
|
329
|
+
const workerId = proxy.__processId;
|
|
330
|
+
const now = Date.now();
|
|
331
|
+
const worker = {
|
|
332
|
+
id: workerId,
|
|
333
|
+
proxy,
|
|
334
|
+
requests: 0,
|
|
335
|
+
totalRequestTime: 0,
|
|
336
|
+
lastUsed: now,
|
|
337
|
+
created: now,
|
|
338
|
+
health: 'healthy',
|
|
339
|
+
currentLoad: 0,
|
|
340
|
+
processing: new Set(),
|
|
341
|
+
errors: 0,
|
|
342
|
+
restarts: 0,
|
|
343
|
+
memoryUsage: 0,
|
|
344
|
+
peakMemoryUsage: 0,
|
|
345
|
+
consecutiveFailures: 0,
|
|
346
|
+
lastHeartbeat: now,
|
|
347
|
+
weight: 100,
|
|
348
|
+
};
|
|
349
|
+
this.workers.set(workerId, worker);
|
|
350
|
+
this.invalidateHealthyWorkersCache();
|
|
351
|
+
this.logger.debug({
|
|
352
|
+
workerId,
|
|
353
|
+
totalWorkers: this.workers.size,
|
|
354
|
+
}, 'Spawned pool worker');
|
|
355
|
+
this.emit('worker:spawned', { workerId, class: this.processName });
|
|
356
|
+
}
|
|
357
|
+
catch (error) {
|
|
358
|
+
this.logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Failed to spawn worker');
|
|
359
|
+
throw error;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
async shutdownWorker(workerId) {
|
|
363
|
+
const worker = this.workers.get(workerId);
|
|
364
|
+
if (!worker)
|
|
365
|
+
return;
|
|
366
|
+
try {
|
|
367
|
+
if (worker.processing.size > 0) {
|
|
368
|
+
this.logger.debug({
|
|
369
|
+
workerId,
|
|
370
|
+
activeRequests: worker.processing.size,
|
|
371
|
+
}, 'Waiting for worker to finish active requests');
|
|
372
|
+
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
373
|
+
}
|
|
374
|
+
await this.manager.kill(workerId);
|
|
375
|
+
this.workers.delete(workerId);
|
|
376
|
+
this.unhealthyWorkers.delete(workerId);
|
|
377
|
+
this.invalidateHealthyWorkersCache();
|
|
378
|
+
this.logger.debug({ workerId }, 'Shutdown pool worker');
|
|
379
|
+
this.emit('worker:shutdown', { workerId, class: this.processName });
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
this.logger.error({ error, workerId }, 'Error shutting down worker');
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
invalidateHealthyWorkersCache() {
|
|
386
|
+
this.healthyWorkersCacheValid = false;
|
|
387
|
+
}
|
|
388
|
+
getHealthyWorkers() {
|
|
389
|
+
if (!this.healthyWorkersCacheValid) {
|
|
390
|
+
this.healthyWorkersCache = Array.from(this.workers.values()).filter((w) => w.health === 'healthy');
|
|
391
|
+
this.healthyWorkersCacheValid = true;
|
|
392
|
+
}
|
|
393
|
+
return this.healthyWorkersCache;
|
|
394
|
+
}
|
|
395
|
+
async selectWorker() {
|
|
396
|
+
const healthyWorkers = this.getHealthyWorkers();
|
|
397
|
+
if (healthyWorkers.length === 0) {
|
|
398
|
+
if (this.workers.size === 0) {
|
|
399
|
+
throw Errors.conflict('No workers available in pool');
|
|
400
|
+
}
|
|
401
|
+
const firstWorker = this.workers.values().next().value;
|
|
402
|
+
if (!firstWorker) {
|
|
403
|
+
throw Errors.conflict('No workers available in pool');
|
|
404
|
+
}
|
|
405
|
+
return firstWorker;
|
|
406
|
+
}
|
|
407
|
+
const strategy = this.poolOptions.strategy || PoolStrategy.ROUND_ROBIN;
|
|
408
|
+
switch (strategy) {
|
|
409
|
+
case PoolStrategy.ROUND_ROBIN:
|
|
410
|
+
return this.selectRoundRobin(healthyWorkers);
|
|
411
|
+
case PoolStrategy.LEAST_LOADED:
|
|
412
|
+
return this.selectLeastLoaded(healthyWorkers);
|
|
413
|
+
case PoolStrategy.RANDOM:
|
|
414
|
+
return this.selectRandom(healthyWorkers);
|
|
415
|
+
case PoolStrategy.WEIGHTED:
|
|
416
|
+
return this.selectWeighted(healthyWorkers);
|
|
417
|
+
case PoolStrategy.ADAPTIVE:
|
|
418
|
+
return this.selectAdaptive(healthyWorkers);
|
|
419
|
+
case PoolStrategy.CONSISTENT_HASH:
|
|
420
|
+
return this.selectConsistentHash(healthyWorkers);
|
|
421
|
+
case PoolStrategy.LATENCY:
|
|
422
|
+
case PoolStrategy.LEAST_RESPONSE_TIME:
|
|
423
|
+
return this.selectByLatency(healthyWorkers);
|
|
424
|
+
case PoolStrategy.LEAST_CONNECTIONS:
|
|
425
|
+
return this.selectLeastConnections(healthyWorkers);
|
|
426
|
+
case PoolStrategy.WEIGHTED_ROUND_ROBIN:
|
|
427
|
+
return this.selectWeightedRoundRobin(healthyWorkers);
|
|
428
|
+
case PoolStrategy.IP_HASH:
|
|
429
|
+
return this.selectConsistentHash(healthyWorkers);
|
|
430
|
+
case PoolStrategy.POWER_OF_TWO:
|
|
431
|
+
return this.selectPowerOfTwoChoices(healthyWorkers);
|
|
432
|
+
default:
|
|
433
|
+
return this.selectRoundRobin(healthyWorkers);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
selectRoundRobin(workers) {
|
|
437
|
+
if (workers.length === 0) {
|
|
438
|
+
throw Errors.conflict('No workers available');
|
|
439
|
+
}
|
|
440
|
+
const worker = workers[this.currentRoundRobinIndex % workers.length];
|
|
441
|
+
this.currentRoundRobinIndex++;
|
|
442
|
+
return worker;
|
|
443
|
+
}
|
|
444
|
+
selectLeastLoaded(workers) {
|
|
445
|
+
return workers.reduce((min, worker) => (worker.currentLoad < min.currentLoad ? worker : min));
|
|
446
|
+
}
|
|
447
|
+
selectRandom(workers) {
|
|
448
|
+
if (workers.length === 0) {
|
|
449
|
+
throw Errors.conflict('No workers available');
|
|
450
|
+
}
|
|
451
|
+
const index = Math.floor(Math.random() * workers.length);
|
|
452
|
+
return workers[index];
|
|
453
|
+
}
|
|
454
|
+
selectWeighted(workers) {
|
|
455
|
+
if (workers.length === 0) {
|
|
456
|
+
throw Errors.conflict('No workers available');
|
|
457
|
+
}
|
|
458
|
+
const weights = workers.map((w) => 1 / (w.currentLoad + 1));
|
|
459
|
+
const totalWeight = weights.reduce((sum, w) => sum + w, 0);
|
|
460
|
+
let random = Math.random() * totalWeight;
|
|
461
|
+
for (let i = 0; i < workers.length; i++) {
|
|
462
|
+
random -= weights[i];
|
|
463
|
+
if (random <= 0) {
|
|
464
|
+
return workers[i];
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
return workers[workers.length - 1];
|
|
468
|
+
}
|
|
469
|
+
selectAdaptive(workers) {
|
|
470
|
+
if (workers.length === 0) {
|
|
471
|
+
throw Errors.conflict('No workers available');
|
|
472
|
+
}
|
|
473
|
+
let bestWorker = workers[0];
|
|
474
|
+
let bestScore = this.computeWorkerScore(bestWorker);
|
|
475
|
+
for (let i = 1; i < workers.length; i++) {
|
|
476
|
+
const worker = workers[i];
|
|
477
|
+
const score = this.computeWorkerScore(worker);
|
|
478
|
+
if (score < bestScore) {
|
|
479
|
+
bestScore = score;
|
|
480
|
+
bestWorker = worker;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return bestWorker;
|
|
484
|
+
}
|
|
485
|
+
computeWorkerScore(worker) {
|
|
486
|
+
const avgResponseTime = worker.requests > 0 ? worker.totalRequestTime / worker.requests : 0;
|
|
487
|
+
const errorRate = worker.requests > 0 ? worker.errors / worker.requests : 0;
|
|
488
|
+
return worker.currentLoad * 0.3 + avgResponseTime * 0.3 + errorRate * 0.2 + worker.processing.size * 0.2;
|
|
489
|
+
}
|
|
490
|
+
selectConsistentHash(workers) {
|
|
491
|
+
if (workers.length === 0) {
|
|
492
|
+
throw Errors.conflict('No workers available');
|
|
493
|
+
}
|
|
494
|
+
const hash = this.requestIdCounter % workers.length;
|
|
495
|
+
return workers[hash];
|
|
496
|
+
}
|
|
497
|
+
selectByLatency(workers) {
|
|
498
|
+
return workers.reduce((best, worker) => {
|
|
499
|
+
const workerAvg = worker.requests > 0 ? worker.totalRequestTime / worker.requests : 0;
|
|
500
|
+
const bestAvg = best.requests > 0 ? best.totalRequestTime / best.requests : 0;
|
|
501
|
+
return workerAvg < bestAvg ? worker : best;
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
selectLeastConnections(workers) {
|
|
505
|
+
if (workers.length === 0) {
|
|
506
|
+
throw Errors.conflict('No workers available');
|
|
507
|
+
}
|
|
508
|
+
const minConnections = Math.min(...workers.map((w) => w.processing.size));
|
|
509
|
+
const candidates = workers.filter((w) => w.processing.size === minConnections);
|
|
510
|
+
const selectedIndex = this.currentRoundRobinIndex % candidates.length;
|
|
511
|
+
this.currentRoundRobinIndex++;
|
|
512
|
+
return candidates[selectedIndex];
|
|
513
|
+
}
|
|
514
|
+
selectWeightedRoundRobin(workers) {
|
|
515
|
+
if (workers.length === 0) {
|
|
516
|
+
throw Errors.conflict('No workers available');
|
|
517
|
+
}
|
|
518
|
+
const weights = workers.map((w) => Math.max(1, 10 - w.currentLoad));
|
|
519
|
+
const totalWeight = weights.reduce((sum, w) => sum + w, 0);
|
|
520
|
+
let targetWeight = (this.currentRoundRobinIndex % totalWeight) + 1;
|
|
521
|
+
this.currentRoundRobinIndex++;
|
|
522
|
+
for (let i = 0; i < workers.length; i++) {
|
|
523
|
+
targetWeight -= weights[i];
|
|
524
|
+
if (targetWeight <= 0) {
|
|
525
|
+
return workers[i];
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
return workers[workers.length - 1];
|
|
529
|
+
}
|
|
530
|
+
selectPowerOfTwoChoices(workers) {
|
|
531
|
+
if (workers.length === 0) {
|
|
532
|
+
throw Errors.conflict('No workers available');
|
|
533
|
+
}
|
|
534
|
+
if (workers.length === 1) {
|
|
535
|
+
return workers[0];
|
|
536
|
+
}
|
|
537
|
+
const idx1 = Math.floor(Math.random() * workers.length);
|
|
538
|
+
let idx2 = Math.floor(Math.random() * (workers.length - 1));
|
|
539
|
+
if (idx2 >= idx1)
|
|
540
|
+
idx2++;
|
|
541
|
+
const worker1 = workers[idx1];
|
|
542
|
+
const worker2 = workers[idx2];
|
|
543
|
+
const score1 = this.computeP2CScore(worker1);
|
|
544
|
+
const score2 = this.computeP2CScore(worker2);
|
|
545
|
+
return score1 <= score2 ? worker1 : worker2;
|
|
546
|
+
}
|
|
547
|
+
computeP2CScore(worker) {
|
|
548
|
+
let score = worker.processing.size * 10;
|
|
549
|
+
const memoryRatio = worker.memoryUsage / this.memoryLimitBytes;
|
|
550
|
+
if (memoryRatio > 0.8) {
|
|
551
|
+
score += 50;
|
|
552
|
+
}
|
|
553
|
+
else if (memoryRatio > 0.5) {
|
|
554
|
+
score += 20;
|
|
555
|
+
}
|
|
556
|
+
if (worker.requests > 0) {
|
|
557
|
+
const errorRate = worker.errors / worker.requests;
|
|
558
|
+
score += errorRate * 100;
|
|
559
|
+
}
|
|
560
|
+
score *= 100 / Math.max(worker.weight, 1);
|
|
561
|
+
return score;
|
|
562
|
+
}
|
|
563
|
+
async executeOnWorker(worker, method, args) {
|
|
564
|
+
const requestId = `req-${++this.requestIdCounter}`;
|
|
565
|
+
const startTime = Date.now();
|
|
566
|
+
worker.processing.add(requestId);
|
|
567
|
+
worker.currentLoad++;
|
|
568
|
+
this.totalRequests++;
|
|
569
|
+
try {
|
|
570
|
+
const timeout = this.poolOptions.requestTimeout;
|
|
571
|
+
let result;
|
|
572
|
+
if (timeout) {
|
|
573
|
+
result = await Promise.race([
|
|
574
|
+
worker.proxy[method](...args),
|
|
575
|
+
new Promise((_, reject) => setTimeout(() => reject(Errors.timeout('Request', timeout)), timeout)),
|
|
576
|
+
]);
|
|
577
|
+
}
|
|
578
|
+
else {
|
|
579
|
+
result = await worker.proxy[method](...args);
|
|
580
|
+
}
|
|
581
|
+
const responseTime = Date.now() - startTime;
|
|
582
|
+
worker.totalRequestTime += responseTime;
|
|
583
|
+
worker.requests++;
|
|
584
|
+
this.totalResponseTime += responseTime;
|
|
585
|
+
this.successfulRequests++;
|
|
586
|
+
if (this.poolOptions.circuitBreaker?.enabled && this.circuitBreaker.isOpen) {
|
|
587
|
+
this.circuitBreaker.successAfterOpen++;
|
|
588
|
+
if (this.circuitBreaker.successAfterOpen >= (this.poolOptions.circuitBreaker.halfOpenRequests || 3)) {
|
|
589
|
+
this.closeCircuitBreaker();
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
return result;
|
|
593
|
+
}
|
|
594
|
+
catch (error) {
|
|
595
|
+
worker.errors++;
|
|
596
|
+
this.totalErrors++;
|
|
597
|
+
if (this.poolOptions.circuitBreaker?.enabled) {
|
|
598
|
+
this.handleCircuitBreakerFailure();
|
|
599
|
+
}
|
|
600
|
+
if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold || 3)) {
|
|
601
|
+
worker.health = 'unhealthy';
|
|
602
|
+
this.unhealthyWorkers.add(worker.id);
|
|
603
|
+
this.invalidateHealthyWorkersCache();
|
|
604
|
+
this.emit('worker:unhealthy', { workerId: worker.id });
|
|
605
|
+
}
|
|
606
|
+
throw error;
|
|
607
|
+
}
|
|
608
|
+
finally {
|
|
609
|
+
worker.processing.delete(requestId);
|
|
610
|
+
worker.currentLoad--;
|
|
611
|
+
worker.lastUsed = Date.now();
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
async queueRequest(method, args) {
|
|
615
|
+
if (this.queue.length >= (this.poolOptions.maxQueueSize || 100)) {
|
|
616
|
+
throw new PoolBackpressureError(this.queue.length);
|
|
617
|
+
}
|
|
618
|
+
return new Promise((resolve, reject) => {
|
|
619
|
+
const request = {
|
|
620
|
+
id: `req-${++this.requestIdCounter}`,
|
|
621
|
+
method,
|
|
622
|
+
args,
|
|
623
|
+
resolve,
|
|
624
|
+
reject,
|
|
625
|
+
timestamp: Date.now(),
|
|
626
|
+
retries: 0,
|
|
627
|
+
};
|
|
628
|
+
if (this.poolOptions.requestTimeout) {
|
|
629
|
+
request.timeout = setTimeout(() => {
|
|
630
|
+
const index = this.queue.indexOf(request);
|
|
631
|
+
if (index !== -1) {
|
|
632
|
+
this.queue.splice(index, 1);
|
|
633
|
+
reject(Errors.timeout('Request in queue', this.poolOptions.requestTimeout));
|
|
634
|
+
}
|
|
635
|
+
}, this.poolOptions.requestTimeout);
|
|
636
|
+
}
|
|
637
|
+
this.queue.push(request);
|
|
638
|
+
this.emit('request:queued', {
|
|
639
|
+
queueSize: this.queue.length,
|
|
640
|
+
method,
|
|
641
|
+
});
|
|
642
|
+
this.processQueue();
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
async processQueue() {
|
|
646
|
+
if (this.queue.length === 0 || this.isDraining)
|
|
647
|
+
return;
|
|
648
|
+
if (this.queueProcessingScheduled)
|
|
649
|
+
return;
|
|
650
|
+
this.queueProcessingScheduled = true;
|
|
651
|
+
queueMicrotask(() => {
|
|
652
|
+
this.queueProcessingScheduled = false;
|
|
653
|
+
this.processQueueBatch();
|
|
654
|
+
});
|
|
655
|
+
}
|
|
656
|
+
processQueueBatch() {
|
|
657
|
+
if (this.queue.length === 0 || this.isDraining)
|
|
658
|
+
return;
|
|
659
|
+
const availableWorkers = Array.from(this.workers.values()).filter((w) => w.health === 'healthy' && w.currentLoad < 2);
|
|
660
|
+
if (availableWorkers.length === 0)
|
|
661
|
+
return;
|
|
662
|
+
let processed = 0;
|
|
663
|
+
const maxBatch = Math.min(this.queueBatchSize, this.queue.length, availableWorkers.length);
|
|
664
|
+
while (processed < maxBatch && this.queue.length > 0 && availableWorkers.length > 0) {
|
|
665
|
+
const request = this.queue.shift();
|
|
666
|
+
if (!request)
|
|
667
|
+
break;
|
|
668
|
+
if (request.timeout) {
|
|
669
|
+
clearTimeout(request.timeout);
|
|
670
|
+
}
|
|
671
|
+
const workerIdx = availableWorkers.length === 1
|
|
672
|
+
? 0
|
|
673
|
+
: this.selectBestFromTwo(availableWorkers, 0, Math.min(1, availableWorkers.length - 1));
|
|
674
|
+
const worker = availableWorkers[workerIdx];
|
|
675
|
+
if (worker.currentLoad >= 1) {
|
|
676
|
+
availableWorkers.splice(workerIdx, 1);
|
|
677
|
+
}
|
|
678
|
+
this.executeOnWorker(worker, request.method, request.args).then(request.resolve).catch(request.reject);
|
|
679
|
+
processed++;
|
|
680
|
+
}
|
|
681
|
+
if (this.queue.length > 0) {
|
|
682
|
+
this.processQueue();
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
selectBestFromTwo(workers, idx1, idx2) {
|
|
686
|
+
const worker1 = workers[idx1];
|
|
687
|
+
const worker2 = workers[idx2];
|
|
688
|
+
const score1 = worker1.currentLoad * 10 + (worker1.memoryUsage / this.memoryLimitBytes) * 5;
|
|
689
|
+
const score2 = worker2.currentLoad * 10 + (worker2.memoryUsage / this.memoryLimitBytes) * 5;
|
|
690
|
+
return score1 <= score2 ? idx1 : idx2;
|
|
691
|
+
}
|
|
692
|
+
setupHealthMonitoring() {
|
|
693
|
+
const baseInterval = this.poolOptions.healthCheck?.interval || 30000;
|
|
694
|
+
const HEALTH_CHECK_CONCURRENCY = 10;
|
|
695
|
+
const MAX_BACKOFF_MULTIPLIER = 8;
|
|
696
|
+
this.healthCheckTimer = setInterval(async () => {
|
|
697
|
+
let healthChanged = false;
|
|
698
|
+
const workers = Array.from(this.workers.values());
|
|
699
|
+
const now = Date.now();
|
|
700
|
+
const checkWorkerHealth = async (worker) => {
|
|
701
|
+
const previousHealth = worker.health;
|
|
702
|
+
if (worker.consecutiveFailures > 0) {
|
|
703
|
+
const backoffMultiplier = Math.min(Math.pow(2, worker.consecutiveFailures - 1), MAX_BACKOFF_MULTIPLIER);
|
|
704
|
+
const nextCheckTime = worker.lastHeartbeat + baseInterval * backoffMultiplier;
|
|
705
|
+
if (now < nextCheckTime) {
|
|
706
|
+
return false;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
try {
|
|
710
|
+
if ('__getHealth' in worker.proxy) {
|
|
711
|
+
const health = await worker.proxy.__getHealth();
|
|
712
|
+
worker.health =
|
|
713
|
+
health.status === 'healthy' ? 'healthy' : health.status === 'degraded' ? 'degraded' : 'unhealthy';
|
|
714
|
+
if (worker.health === 'healthy') {
|
|
715
|
+
worker.consecutiveFailures = 0;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
if ('__getMetrics' in worker.proxy) {
|
|
719
|
+
worker.metrics = await worker.proxy.__getMetrics();
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
catch (error) {
|
|
723
|
+
this.logger.warn({
|
|
724
|
+
error,
|
|
725
|
+
workerId: worker.id,
|
|
726
|
+
consecutiveFailures: worker.consecutiveFailures + 1,
|
|
727
|
+
}, 'Health check failed');
|
|
728
|
+
worker.health = 'unhealthy';
|
|
729
|
+
worker.consecutiveFailures++;
|
|
730
|
+
}
|
|
731
|
+
return previousHealth !== worker.health;
|
|
732
|
+
};
|
|
733
|
+
for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
|
|
734
|
+
const batch = workers.slice(i, i + HEALTH_CHECK_CONCURRENCY);
|
|
735
|
+
const results = await Promise.all(batch.map(checkWorkerHealth));
|
|
736
|
+
if (results.some((changed) => changed)) {
|
|
737
|
+
healthChanged = true;
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
if (healthChanged) {
|
|
741
|
+
this.invalidateHealthyWorkersCache();
|
|
742
|
+
}
|
|
743
|
+
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold || 3;
|
|
744
|
+
for (const worker of workers) {
|
|
745
|
+
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
746
|
+
this.logger.info({
|
|
747
|
+
workerId: worker.id,
|
|
748
|
+
consecutiveFailures: worker.consecutiveFailures,
|
|
749
|
+
restarts: worker.restarts,
|
|
750
|
+
}, 'Replacing unhealthy worker');
|
|
751
|
+
worker.restarts++;
|
|
752
|
+
await this.replaceWorker(worker.id);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}, baseInterval);
|
|
756
|
+
}
|
|
757
|
+
setupAutoScaling() {
|
|
758
|
+
if (!this.poolOptions.autoScale?.enabled)
|
|
759
|
+
return;
|
|
760
|
+
const checkInterval = this.poolOptions.autoScale?.checkInterval ?? 10_000;
|
|
761
|
+
this.autoScaleTimer = setInterval(async () => {
|
|
762
|
+
if (this.isShuttingDown || this.isDraining)
|
|
763
|
+
return;
|
|
764
|
+
try {
|
|
765
|
+
const now = Date.now();
|
|
766
|
+
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod || 60000;
|
|
767
|
+
if (now - this.lastScaleCheck < cooldownPeriod)
|
|
768
|
+
return;
|
|
769
|
+
const metrics = this.metrics;
|
|
770
|
+
const config = this.poolOptions.autoScale;
|
|
771
|
+
const queueThreshold = config.queueThreshold ?? 50;
|
|
772
|
+
const shouldScaleUp = metrics.cpu > (config.targetCPU || 70) ||
|
|
773
|
+
metrics.memory > (config.targetMemory || 80) ||
|
|
774
|
+
(metrics.saturation || 0) > (config.scaleUpThreshold || 0.8) ||
|
|
775
|
+
this.queue.length > queueThreshold;
|
|
776
|
+
const scaleDownCpu = (config.targetCPU ?? 70) * 0.4;
|
|
777
|
+
const scaleDownMemory = (config.targetMemory ?? 80) * 0.5;
|
|
778
|
+
const shouldScaleDown = metrics.cpu < scaleDownCpu &&
|
|
779
|
+
metrics.memory < scaleDownMemory &&
|
|
780
|
+
(metrics.saturation || 0) < (config.scaleDownThreshold || 0.3);
|
|
781
|
+
if (shouldScaleUp && this.workers.size < (config.max || 10)) {
|
|
782
|
+
const queuePressure = this.queue.length / queueThreshold;
|
|
783
|
+
const scaleStep = Math.max(1, Math.ceil(queuePressure));
|
|
784
|
+
const maxWorkers = config.max ?? 10;
|
|
785
|
+
const newSize = Math.min(this.workers.size + scaleStep, maxWorkers);
|
|
786
|
+
await this.scale(newSize);
|
|
787
|
+
this.lastScaleCheck = now;
|
|
788
|
+
}
|
|
789
|
+
else if (shouldScaleDown && this.workers.size > (config.min || 1)) {
|
|
790
|
+
const newSize = Math.max(this.workers.size - 1, config.min || 1);
|
|
791
|
+
await this.scale(newSize);
|
|
792
|
+
this.lastScaleCheck = now;
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
catch (error) {
|
|
796
|
+
this.logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error during auto-scaling check');
|
|
797
|
+
}
|
|
798
|
+
}, checkInterval);
|
|
799
|
+
}
|
|
800
|
+
setupRecycling() {
|
|
801
|
+
this.recycleTimer = setInterval(async () => {
|
|
802
|
+
if (this.isShuttingDown)
|
|
803
|
+
return;
|
|
804
|
+
try {
|
|
805
|
+
const now = Date.now();
|
|
806
|
+
const maxLifetime = this.poolOptions.maxLifetime || 3600000;
|
|
807
|
+
const recycleAfter = this.poolOptions.recycleAfter || 10000;
|
|
808
|
+
for (const worker of this.workers.values()) {
|
|
809
|
+
const ageBasedRecycle = now - worker.created > maxLifetime;
|
|
810
|
+
const requestBasedRecycle = worker.requests > recycleAfter;
|
|
811
|
+
const memoryBasedRecycle = worker.memoryUsage > this.memoryLimitBytes;
|
|
812
|
+
const memoryWarningRecycle = worker.memoryUsage > this.memoryLimitBytes * this.memoryWarningThreshold &&
|
|
813
|
+
worker.peakMemoryUsage > this.memoryLimitBytes * 0.9;
|
|
814
|
+
const shouldRecycle = ageBasedRecycle || requestBasedRecycle || memoryBasedRecycle || memoryWarningRecycle;
|
|
815
|
+
if (shouldRecycle) {
|
|
816
|
+
const reason = memoryBasedRecycle
|
|
817
|
+
? 'memory_limit'
|
|
818
|
+
: memoryWarningRecycle
|
|
819
|
+
? 'memory_warning'
|
|
820
|
+
: requestBasedRecycle
|
|
821
|
+
? 'request_count'
|
|
822
|
+
: 'max_lifetime';
|
|
823
|
+
this.logger.info({
|
|
824
|
+
workerId: worker.id,
|
|
825
|
+
reason,
|
|
826
|
+
memoryUsage: Math.round(worker.memoryUsage / 1024 / 1024),
|
|
827
|
+
requests: worker.requests,
|
|
828
|
+
age: Math.round((now - worker.created) / 1000),
|
|
829
|
+
}, 'Recycling worker');
|
|
830
|
+
await this.replaceWorker(worker.id);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
catch (error) {
|
|
835
|
+
this.logger.error({ error }, 'Error during worker recycling');
|
|
836
|
+
}
|
|
837
|
+
}, 60000);
|
|
838
|
+
}
|
|
839
|
+
setupMemoryMonitoring() {
|
|
840
|
+
const MEMORY_CHECK_INTERVAL = 15000;
|
|
841
|
+
this.memoryMonitorTimer = setInterval(async () => {
|
|
842
|
+
if (this.isShuttingDown)
|
|
843
|
+
return;
|
|
844
|
+
const workers = Array.from(this.workers.values());
|
|
845
|
+
await Promise.all(workers.map(async (worker) => {
|
|
846
|
+
try {
|
|
847
|
+
if ('__getMetrics' in worker.proxy) {
|
|
848
|
+
const metrics = await worker.proxy.__getMetrics();
|
|
849
|
+
if (metrics && typeof metrics.memory === 'number') {
|
|
850
|
+
worker.memoryUsage = metrics.memory;
|
|
851
|
+
worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, metrics.memory);
|
|
852
|
+
const memoryRatio = metrics.memory / this.memoryLimitBytes;
|
|
853
|
+
if (memoryRatio > 0.9) {
|
|
854
|
+
worker.weight = 10;
|
|
855
|
+
}
|
|
856
|
+
else if (memoryRatio > 0.7) {
|
|
857
|
+
worker.weight = 50;
|
|
858
|
+
}
|
|
859
|
+
else {
|
|
860
|
+
worker.weight = 100;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
catch (error) {
|
|
866
|
+
this.logger.debug({ error, workerId: worker.id }, 'Failed to collect memory metrics');
|
|
867
|
+
}
|
|
868
|
+
}));
|
|
869
|
+
const totalMemory = workers.reduce((sum, w) => sum + w.memoryUsage, 0);
|
|
870
|
+
const avgMemory = workers.length > 0 ? totalMemory / workers.length : 0;
|
|
871
|
+
this.emit('pool:memory', {
|
|
872
|
+
totalMemory,
|
|
873
|
+
avgMemory,
|
|
874
|
+
workers: workers.map((w) => ({
|
|
875
|
+
id: w.id,
|
|
876
|
+
memory: w.memoryUsage,
|
|
877
|
+
peak: w.peakMemoryUsage,
|
|
878
|
+
})),
|
|
879
|
+
});
|
|
880
|
+
}, MEMORY_CHECK_INTERVAL);
|
|
881
|
+
}
|
|
882
|
+
setupHeartbeatMonitoring() {
|
|
883
|
+
const HEARTBEAT_INTERVAL = 10000;
|
|
884
|
+
const HEARTBEAT_TIMEOUT = 5000;
|
|
885
|
+
const MAX_MISSED_HEARTBEATS = 3;
|
|
886
|
+
this.heartbeatTimer = setInterval(async () => {
|
|
887
|
+
if (this.isShuttingDown)
|
|
888
|
+
return;
|
|
889
|
+
const now = Date.now();
|
|
890
|
+
const workers = Array.from(this.workers.values());
|
|
891
|
+
await Promise.all(workers.map(async (worker) => {
|
|
892
|
+
try {
|
|
893
|
+
const heartbeatPromise = this.sendHeartbeat(worker);
|
|
894
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(false), HEARTBEAT_TIMEOUT));
|
|
895
|
+
const responded = await Promise.race([heartbeatPromise, timeoutPromise]);
|
|
896
|
+
if (responded) {
|
|
897
|
+
worker.lastHeartbeat = now;
|
|
898
|
+
worker.consecutiveFailures = 0;
|
|
899
|
+
}
|
|
900
|
+
else {
|
|
901
|
+
worker.consecutiveFailures++;
|
|
902
|
+
if (worker.consecutiveFailures >= MAX_MISSED_HEARTBEATS) {
|
|
903
|
+
this.logger.warn({
|
|
904
|
+
workerId: worker.id,
|
|
905
|
+
missedHeartbeats: worker.consecutiveFailures,
|
|
906
|
+
}, 'Worker unresponsive - marking unhealthy');
|
|
907
|
+
worker.health = 'unhealthy';
|
|
908
|
+
this.unhealthyWorkers.add(worker.id);
|
|
909
|
+
this.invalidateHealthyWorkersCache();
|
|
910
|
+
this.emit('worker:unresponsive', { workerId: worker.id });
|
|
911
|
+
if (!this.isShuttingDown) {
|
|
912
|
+
await this.replaceWorker(worker.id);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
catch (error) {
|
|
918
|
+
worker.consecutiveFailures++;
|
|
919
|
+
this.logger.debug({ error, workerId: worker.id }, 'Heartbeat failed');
|
|
920
|
+
}
|
|
921
|
+
}));
|
|
922
|
+
}, HEARTBEAT_INTERVAL);
|
|
923
|
+
}
|
|
924
|
+
async sendHeartbeat(worker) {
|
|
925
|
+
try {
|
|
926
|
+
if ('__getHealth' in worker.proxy) {
|
|
927
|
+
await worker.proxy.__getHealth();
|
|
928
|
+
return true;
|
|
929
|
+
}
|
|
930
|
+
return true;
|
|
931
|
+
}
|
|
932
|
+
catch {
|
|
933
|
+
return false;
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
async replaceWorker(workerId) {
|
|
937
|
+
this.logger.info({ workerId }, 'Replacing pool worker');
|
|
938
|
+
await this.spawnWorker();
|
|
939
|
+
await this.shutdownWorker(workerId);
|
|
940
|
+
this.emit('worker:replaced', { workerId });
|
|
941
|
+
}
|
|
942
|
+
async warmupPool() {
|
|
943
|
+
this.logger.info('Warming up process pool');
|
|
944
|
+
const warmupPromises = Array.from(this.workers.values()).map(async (worker) => {
|
|
945
|
+
try {
|
|
946
|
+
if ('__getHealth' in worker.proxy) {
|
|
947
|
+
await worker.proxy.__getHealth();
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
catch (error) {
|
|
951
|
+
this.logger.debug({ error, workerId: worker.id }, 'Warmup failed');
|
|
952
|
+
}
|
|
953
|
+
});
|
|
954
|
+
await Promise.all(warmupPromises);
|
|
955
|
+
}
|
|
956
|
+
async scaleUp(count) {
|
|
957
|
+
const spawnPromises = [];
|
|
958
|
+
for (let i = 0; i < count; i++) {
|
|
959
|
+
spawnPromises.push(this.spawnWorker());
|
|
960
|
+
}
|
|
961
|
+
await Promise.all(spawnPromises);
|
|
962
|
+
}
|
|
963
|
+
async scaleDown(count) {
|
|
964
|
+
const sortedWorkers = Array.from(this.workers.values()).sort((a, b) => a.currentLoad - b.currentLoad);
|
|
965
|
+
const toRemove = sortedWorkers.slice(0, count);
|
|
966
|
+
await Promise.all(toRemove.map((worker) => this.shutdownWorker(worker.id)));
|
|
967
|
+
}
|
|
968
|
+
async waitForActiveRequests() {
|
|
969
|
+
const maxWait = 30000;
|
|
970
|
+
const startTime = Date.now();
|
|
971
|
+
while (this.active > 0 && Date.now() - startTime < maxWait) {
|
|
972
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
973
|
+
}
|
|
974
|
+
if (this.active > 0) {
|
|
975
|
+
this.logger.warn({ active: this.active }, 'Timed out waiting for active requests');
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
calculateThroughput() {
|
|
979
|
+
const recentHistory = this.scaleHistory.filter((h) => Date.now() - h.timestamp < 60000);
|
|
980
|
+
if (recentHistory.length === 0)
|
|
981
|
+
return 0;
|
|
982
|
+
const firstEntry = recentHistory[0];
|
|
983
|
+
if (!firstEntry)
|
|
984
|
+
return 0;
|
|
985
|
+
const duration = (Date.now() - firstEntry.timestamp) / 1000;
|
|
986
|
+
return duration > 0 ? this.successfulRequests / duration : 0;
|
|
987
|
+
}
|
|
988
|
+
calculateSaturation() {
|
|
989
|
+
const totalCapacity = this.workers.size * 2;
|
|
990
|
+
const currentLoad = this.active + this.queue.length;
|
|
991
|
+
return totalCapacity > 0 ? currentLoad / totalCapacity : 0;
|
|
992
|
+
}
|
|
993
|
+
handleCircuitBreakerFailure() {
|
|
994
|
+
this.circuitBreaker.failures++;
|
|
995
|
+
this.circuitBreaker.lastFailTime = Date.now();
|
|
996
|
+
const threshold = this.poolOptions.circuitBreaker?.threshold || 5;
|
|
997
|
+
if (this.circuitBreaker.failures >= threshold && !this.circuitBreaker.isOpen) {
|
|
998
|
+
this.openCircuitBreaker();
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
openCircuitBreaker() {
|
|
1002
|
+
this.circuitBreaker.isOpen = true;
|
|
1003
|
+
this.circuitBreaker.successAfterOpen = 0;
|
|
1004
|
+
this.logger.warn('Circuit breaker opened');
|
|
1005
|
+
this.emit('circuitbreaker:open');
|
|
1006
|
+
const timeout = this.poolOptions.circuitBreaker?.timeout || 60000;
|
|
1007
|
+
setTimeout(() => {
|
|
1008
|
+
if (this.circuitBreaker.isOpen) {
|
|
1009
|
+
this.logger.info('Circuit breaker entering half-open state');
|
|
1010
|
+
this.emit('circuitbreaker:halfopen');
|
|
1011
|
+
}
|
|
1012
|
+
}, timeout);
|
|
1013
|
+
}
|
|
1014
|
+
closeCircuitBreaker() {
|
|
1015
|
+
this.circuitBreaker.isOpen = false;
|
|
1016
|
+
this.circuitBreaker.failures = 0;
|
|
1017
|
+
this.circuitBreaker.successAfterOpen = 0;
|
|
1018
|
+
this.logger.info('Circuit breaker closed');
|
|
1019
|
+
this.emit('circuitbreaker:close');
|
|
1020
|
+
}
|
|
1021
|
+
shouldTryHalfOpen() {
|
|
1022
|
+
const timeout = this.poolOptions.circuitBreaker?.timeout || 60000;
|
|
1023
|
+
return Date.now() - this.circuitBreaker.lastFailTime > timeout;
|
|
1024
|
+
}
|
|
1025
|
+
emit(event, data) {
|
|
1026
|
+
this.emitter.emit(event, data);
|
|
1027
|
+
}
|
|
1028
|
+
on(event, listener) {
|
|
1029
|
+
this.emitter.on(event, listener);
|
|
1030
|
+
}
|
|
1031
|
+
off(event, listener) {
|
|
1032
|
+
this.emitter.off(event, listener);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
//# sourceMappingURL=process-pool.js.map
|