comfyui-node 1.6.1 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/call-wrapper.js +856 -856
- package/dist/index.d.ts +13 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/multipool/client-registry.d.ts +32 -11
- package/dist/multipool/client-registry.d.ts.map +1 -1
- package/dist/multipool/client-registry.js +152 -25
- package/dist/multipool/client-registry.js.map +1 -1
- package/dist/multipool/helpers.d.ts +5 -0
- package/dist/multipool/helpers.d.ts.map +1 -0
- package/dist/multipool/helpers.js +53 -0
- package/dist/multipool/helpers.js.map +1 -0
- package/dist/multipool/index.d.ts +2 -1
- package/dist/multipool/index.d.ts.map +1 -1
- package/dist/multipool/index.js +2 -1
- package/dist/multipool/index.js.map +1 -1
- package/dist/multipool/interfaces.d.ts +12 -4
- package/dist/multipool/interfaces.d.ts.map +1 -1
- package/dist/multipool/interfaces.js +1 -1
- package/dist/multipool/job-profiler.d.ts +128 -0
- package/dist/multipool/job-profiler.d.ts.map +1 -0
- package/dist/multipool/job-profiler.js +222 -0
- package/dist/multipool/job-profiler.js.map +1 -0
- package/dist/multipool/job-queue-processor.d.ts +27 -11
- package/dist/multipool/job-queue-processor.d.ts.map +1 -1
- package/dist/multipool/job-queue-processor.js +196 -9
- package/dist/multipool/job-queue-processor.js.map +1 -1
- package/dist/multipool/job-state-registry.d.ts +67 -0
- package/dist/multipool/job-state-registry.d.ts.map +1 -0
- package/dist/multipool/job-state-registry.js +283 -0
- package/dist/multipool/job-state-registry.js.map +1 -0
- package/dist/multipool/logger.d.ts +30 -0
- package/dist/multipool/logger.d.ts.map +1 -0
- package/dist/multipool/logger.js +75 -0
- package/dist/multipool/logger.js.map +1 -0
- package/dist/multipool/multi-workflow-pool.d.ts +42 -23
- package/dist/multipool/multi-workflow-pool.d.ts.map +1 -1
- package/dist/multipool/multi-workflow-pool.js +313 -72
- package/dist/multipool/multi-workflow-pool.js.map +1 -1
- package/dist/multipool/pool-event-manager.d.ts +10 -10
- package/dist/multipool/pool-event-manager.js +27 -27
- package/dist/multipool/tests/error-classification-tests.d.ts +2 -0
- package/dist/multipool/tests/error-classification-tests.d.ts.map +1 -0
- package/dist/multipool/tests/error-classification-tests.js +374 -0
- package/dist/multipool/tests/error-classification-tests.js.map +1 -0
- package/dist/multipool/tests/job-state-registry.d.ts +16 -16
- package/dist/multipool/tests/job-state-registry.js +23 -23
- package/dist/multipool/tests/multipool-basic.d.ts +11 -1
- package/dist/multipool/tests/multipool-basic.d.ts.map +1 -1
- package/dist/multipool/tests/multipool-basic.js +141 -3
- package/dist/multipool/tests/multipool-basic.js.map +1 -1
- package/dist/multipool/tests/profiling-demo.d.ts +7 -0
- package/dist/multipool/tests/profiling-demo.d.ts.map +1 -0
- package/dist/multipool/tests/profiling-demo.js +88 -0
- package/dist/multipool/tests/profiling-demo.js.map +1 -0
- package/dist/multipool/tests/prompt-generator.d.ts +10 -0
- package/dist/multipool/tests/prompt-generator.d.ts.map +1 -0
- package/dist/multipool/tests/prompt-generator.js +26 -0
- package/dist/multipool/tests/prompt-generator.js.map +1 -0
- package/dist/multipool/tests/test-helpers.d.ts +4 -0
- package/dist/multipool/tests/test-helpers.d.ts.map +1 -0
- package/dist/multipool/tests/test-helpers.js +10 -0
- package/dist/multipool/tests/test-helpers.js.map +1 -0
- package/dist/multipool/tests/two-stage-edit-simulation.d.ts +32 -0
- package/dist/multipool/tests/two-stage-edit-simulation.d.ts.map +1 -0
- package/dist/multipool/tests/two-stage-edit-simulation.js +299 -0
- package/dist/multipool/tests/two-stage-edit-simulation.js.map +1 -0
- package/dist/multipool/workflow.d.ts +178 -173
- package/dist/multipool/workflow.d.ts.map +1 -1
- package/dist/multipool/workflow.js +333 -271
- package/dist/multipool/workflow.js.map +1 -1
- package/dist/pool/SmartPool.d.ts +143 -143
- package/dist/pool/SmartPool.js +676 -676
- package/dist/pool/SmartPoolV2.d.ts +119 -119
- package/dist/pool/SmartPoolV2.js +586 -586
- package/dist/pool/WorkflowPool.d.ts +202 -202
- package/dist/pool/WorkflowPool.d.ts.map +1 -1
- package/dist/pool/WorkflowPool.js +845 -840
- package/dist/pool/WorkflowPool.js.map +1 -1
- package/dist/pool/client/ClientManager.d.ts +86 -86
- package/dist/pool/client/ClientManager.js +215 -215
- package/dist/pool/index.d.ts +9 -11
- package/dist/pool/index.d.ts.map +1 -1
- package/dist/pool/index.js +3 -5
- package/dist/pool/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,216 +1,216 @@
|
|
|
1
|
-
import { TypedEventTarget } from "../../typed-event-target.js";
|
|
2
|
-
export class ClientManager extends TypedEventTarget {
|
|
3
|
-
clients = [];
|
|
4
|
-
strategy;
|
|
5
|
-
healthCheckInterval = null;
|
|
6
|
-
healthCheckIntervalMs;
|
|
7
|
-
debugLogs = process.env.WORKFLOW_POOL_DEBUG === "1";
|
|
8
|
-
/**
|
|
9
|
-
* Grace period after reconnection before client is considered stable (default: 10 seconds).
|
|
10
|
-
* ComfyUI sometimes quickly disconnects/reconnects after job execution.
|
|
11
|
-
* During this grace period, the client won't be used for new jobs.
|
|
12
|
-
*/
|
|
13
|
-
reconnectionGracePeriodMs = 10000;
|
|
14
|
-
/**
|
|
15
|
-
* Create a new ClientManager for managing ComfyUI client connections.
|
|
16
|
-
*
|
|
17
|
-
* @param strategy - Failover strategy for handling client failures
|
|
18
|
-
* @param opts - Configuration options
|
|
19
|
-
* @param opts.healthCheckIntervalMs - Interval (ms) for health check pings to keep connections alive.
|
|
20
|
-
* Set to 0 to disable. Default: 30000 (30 seconds).
|
|
21
|
-
*/
|
|
22
|
-
constructor(strategy, opts) {
|
|
23
|
-
super();
|
|
24
|
-
this.strategy = strategy;
|
|
25
|
-
this.healthCheckIntervalMs = opts?.healthCheckIntervalMs ?? 30000; // Default: 30 seconds
|
|
26
|
-
}
|
|
27
|
-
emitBlocked(clientId, workflowHash) {
|
|
28
|
-
this.dispatchEvent(new CustomEvent("client:blocked_workflow", { detail: { clientId, workflowHash } }));
|
|
29
|
-
}
|
|
30
|
-
emitUnblocked(clientId, workflowHash) {
|
|
31
|
-
this.dispatchEvent(new CustomEvent("client:unblocked_workflow", { detail: { clientId, workflowHash } }));
|
|
32
|
-
}
|
|
33
|
-
async initialize(clients) {
|
|
34
|
-
for (const client of clients) {
|
|
35
|
-
await this.addClient(client);
|
|
36
|
-
}
|
|
37
|
-
this.startHealthCheck();
|
|
38
|
-
}
|
|
39
|
-
async addClient(client) {
|
|
40
|
-
await client.init();
|
|
41
|
-
const managed = {
|
|
42
|
-
client,
|
|
43
|
-
id: client.id,
|
|
44
|
-
online: true,
|
|
45
|
-
busy: false,
|
|
46
|
-
lastSeenAt: Date.now(),
|
|
47
|
-
supportedWorkflows: new Set()
|
|
48
|
-
};
|
|
49
|
-
this.clients.push(managed);
|
|
50
|
-
client.on("disconnected", () => {
|
|
51
|
-
managed.online = false;
|
|
52
|
-
managed.busy = false;
|
|
53
|
-
managed.lastSeenAt = Date.now();
|
|
54
|
-
managed.lastDisconnectedAt = Date.now();
|
|
55
|
-
this.dispatchEvent(new CustomEvent("client:state", {
|
|
56
|
-
detail: { clientId: managed.id, online: false, busy: false, lastError: managed.lastError }
|
|
57
|
-
}));
|
|
58
|
-
});
|
|
59
|
-
client.on("reconnected", () => {
|
|
60
|
-
const now = Date.now();
|
|
61
|
-
managed.online = true;
|
|
62
|
-
managed.lastSeenAt = now;
|
|
63
|
-
managed.reconnectionStableAt = now + this.reconnectionGracePeriodMs;
|
|
64
|
-
// Log if this is a quick reconnect (within 30 seconds of disconnect)
|
|
65
|
-
if (managed.lastDisconnectedAt && (now - managed.lastDisconnectedAt) < 30000) {
|
|
66
|
-
console.warn(`[ClientManager] Client ${managed.id} reconnected ${((now - managed.lastDisconnectedAt) / 1000).toFixed(1)}s after disconnect. ` +
|
|
67
|
-
`Grace period active until ${new Date(managed.reconnectionStableAt).toISOString()}`);
|
|
68
|
-
}
|
|
69
|
-
this.dispatchEvent(new CustomEvent("client:state", {
|
|
70
|
-
detail: { clientId: managed.id, online: true, busy: managed.busy }
|
|
71
|
-
}));
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
list() {
|
|
75
|
-
return [...this.clients];
|
|
76
|
-
}
|
|
77
|
-
getClient(clientId) {
|
|
78
|
-
return this.clients.find((c) => c.id === clientId);
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Checks if a client is truly available for work.
|
|
82
|
-
* A client must be online, not busy, AND past the reconnection grace period.
|
|
83
|
-
*/
|
|
84
|
-
isClientStable(client) {
|
|
85
|
-
if (!client.online || client.busy) {
|
|
86
|
-
return false;
|
|
87
|
-
}
|
|
88
|
-
// If client recently reconnected, wait for grace period
|
|
89
|
-
if (client.reconnectionStableAt && Date.now() < client.reconnectionStableAt) {
|
|
90
|
-
return false;
|
|
91
|
-
}
|
|
92
|
-
return true;
|
|
93
|
-
}
|
|
94
|
-
canClientRunJob(client, job) {
|
|
95
|
-
if (job.options.preferredClientIds?.length && !job.options.preferredClientIds.includes(client.id)) {
|
|
96
|
-
return false; // Job has preferred clients and this isn't one of them
|
|
97
|
-
}
|
|
98
|
-
if (job.options.excludeClientIds?.includes(client.id)) {
|
|
99
|
-
return false; // Job has excluded clients and this is one of them
|
|
100
|
-
}
|
|
101
|
-
if (this.strategy.shouldSkipClient(client, job)) {
|
|
102
|
-
return false; // Strategy says to skip
|
|
103
|
-
}
|
|
104
|
-
return true;
|
|
105
|
-
}
|
|
106
|
-
claim(job, specificClientId) {
|
|
107
|
-
let chosen;
|
|
108
|
-
if (specificClientId) {
|
|
109
|
-
const candidate = this.clients.find(c => c.id === specificClientId);
|
|
110
|
-
if (candidate && this.isClientStable(candidate) && this.canClientRunJob(candidate, job)) {
|
|
111
|
-
chosen = candidate;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
const candidates = this.clients.filter((c) => this.isClientStable(c));
|
|
116
|
-
chosen = candidates.find(c => this.canClientRunJob(c, job));
|
|
117
|
-
}
|
|
118
|
-
if (!chosen) {
|
|
119
|
-
return null;
|
|
120
|
-
}
|
|
121
|
-
chosen.busy = true;
|
|
122
|
-
chosen.busy = true;
|
|
123
|
-
chosen.lastSeenAt = Date.now();
|
|
124
|
-
return {
|
|
125
|
-
client: chosen.client,
|
|
126
|
-
clientId: chosen.id,
|
|
127
|
-
release: (opts) => {
|
|
128
|
-
if (this.debugLogs) {
|
|
129
|
-
console.log(`[ClientManager.release] Releasing client ${chosen.id} for job ${job.jobId}. Setting busy: false.`);
|
|
130
|
-
}
|
|
131
|
-
chosen.busy = false;
|
|
132
|
-
if (opts?.success) {
|
|
133
|
-
const wasBlocked = this.strategy.isWorkflowBlocked?.(chosen, job.workflowHash) ?? false;
|
|
134
|
-
this.strategy.recordSuccess(chosen, job);
|
|
135
|
-
const stillBlocked = this.strategy.isWorkflowBlocked?.(chosen, job.workflowHash) ?? false;
|
|
136
|
-
if (wasBlocked && !stillBlocked) {
|
|
137
|
-
this.emitUnblocked(chosen.id, job.workflowHash);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
this.dispatchEvent(new CustomEvent("client:state", {
|
|
141
|
-
detail: { clientId: chosen.id, online: chosen.online, busy: chosen.busy }
|
|
142
|
-
}));
|
|
143
|
-
}
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
recordFailure(clientId, job, error) {
|
|
147
|
-
const client = this.clients.find((c) => c.id === clientId);
|
|
148
|
-
if (!client) {
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
client.lastError = error;
|
|
152
|
-
client.busy = false;
|
|
153
|
-
const wasBlocked = this.strategy.isWorkflowBlocked?.(client, job.workflowHash) ?? false;
|
|
154
|
-
this.strategy.recordFailure(client, job, error);
|
|
155
|
-
const isBlocked = this.strategy.isWorkflowBlocked?.(client, job.workflowHash) ?? false;
|
|
156
|
-
if (!wasBlocked && isBlocked) {
|
|
157
|
-
this.emitBlocked(client.id, job.workflowHash);
|
|
158
|
-
}
|
|
159
|
-
this.dispatchEvent(new CustomEvent("client:state", {
|
|
160
|
-
detail: { clientId: client.id, online: client.online, busy: client.busy, lastError: error }
|
|
161
|
-
}));
|
|
162
|
-
}
|
|
163
|
-
/**
|
|
164
|
-
* Start periodic health check to keep connections alive and detect issues early.
|
|
165
|
-
* Pings idle clients by polling their queue status.
|
|
166
|
-
*/
|
|
167
|
-
startHealthCheck() {
|
|
168
|
-
if (this.healthCheckInterval) {
|
|
169
|
-
return; // Already running
|
|
170
|
-
}
|
|
171
|
-
this.healthCheckInterval = setInterval(() => {
|
|
172
|
-
this.performHealthCheck().catch((error) => {
|
|
173
|
-
console.error("[ClientManager] Health check error:", error);
|
|
174
|
-
});
|
|
175
|
-
}, this.healthCheckIntervalMs);
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Perform health check on all clients.
|
|
179
|
-
* Polls queue status to keep WebSocket alive and detect connection issues.
|
|
180
|
-
* IMPORTANT: Pings ALL online clients (including busy ones) to prevent WebSocket timeout during heavy load.
|
|
181
|
-
*/
|
|
182
|
-
async performHealthCheck() {
|
|
183
|
-
for (const managed of this.clients) {
|
|
184
|
-
// Ping ALL online clients (not just idle ones) to keep WebSocket alive during heavy load
|
|
185
|
-
if (managed.online) {
|
|
186
|
-
try {
|
|
187
|
-
// Lightweight ping: poll queue status (triggers WebSocket activity via fetchApi)
|
|
188
|
-
await managed.client.getQueue();
|
|
189
|
-
managed.lastSeenAt = Date.now();
|
|
190
|
-
}
|
|
191
|
-
catch (error) {
|
|
192
|
-
// Health check failed - client may have connection issues
|
|
193
|
-
console.warn(`[ClientManager] Health check failed for client ${managed.id}:`, error);
|
|
194
|
-
// Don't mark as offline here - let the WebSocket disconnect event handle that
|
|
195
|
-
// This prevents false positives from temporary network hiccups
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
/**
|
|
201
|
-
* Stop health check interval (called during shutdown).
|
|
202
|
-
*/
|
|
203
|
-
stopHealthCheck() {
|
|
204
|
-
if (this.healthCheckInterval) {
|
|
205
|
-
clearInterval(this.healthCheckInterval);
|
|
206
|
-
this.healthCheckInterval = null;
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* Cleanup resources when destroying the manager.
|
|
211
|
-
*/
|
|
212
|
-
destroy() {
|
|
213
|
-
this.stopHealthCheck();
|
|
214
|
-
}
|
|
215
|
-
}
|
|
1
|
+
import { TypedEventTarget } from "../../typed-event-target.js";
|
|
2
|
+
export class ClientManager extends TypedEventTarget {
|
|
3
|
+
clients = [];
|
|
4
|
+
strategy;
|
|
5
|
+
healthCheckInterval = null;
|
|
6
|
+
healthCheckIntervalMs;
|
|
7
|
+
debugLogs = process.env.WORKFLOW_POOL_DEBUG === "1";
|
|
8
|
+
/**
|
|
9
|
+
* Grace period after reconnection before client is considered stable (default: 10 seconds).
|
|
10
|
+
* ComfyUI sometimes quickly disconnects/reconnects after job execution.
|
|
11
|
+
* During this grace period, the client won't be used for new jobs.
|
|
12
|
+
*/
|
|
13
|
+
reconnectionGracePeriodMs = 10000;
|
|
14
|
+
/**
|
|
15
|
+
* Create a new ClientManager for managing ComfyUI client connections.
|
|
16
|
+
*
|
|
17
|
+
* @param strategy - Failover strategy for handling client failures
|
|
18
|
+
* @param opts - Configuration options
|
|
19
|
+
* @param opts.healthCheckIntervalMs - Interval (ms) for health check pings to keep connections alive.
|
|
20
|
+
* Set to 0 to disable. Default: 30000 (30 seconds).
|
|
21
|
+
*/
|
|
22
|
+
constructor(strategy, opts) {
|
|
23
|
+
super();
|
|
24
|
+
this.strategy = strategy;
|
|
25
|
+
this.healthCheckIntervalMs = opts?.healthCheckIntervalMs ?? 30000; // Default: 30 seconds
|
|
26
|
+
}
|
|
27
|
+
emitBlocked(clientId, workflowHash) {
|
|
28
|
+
this.dispatchEvent(new CustomEvent("client:blocked_workflow", { detail: { clientId, workflowHash } }));
|
|
29
|
+
}
|
|
30
|
+
emitUnblocked(clientId, workflowHash) {
|
|
31
|
+
this.dispatchEvent(new CustomEvent("client:unblocked_workflow", { detail: { clientId, workflowHash } }));
|
|
32
|
+
}
|
|
33
|
+
async initialize(clients) {
|
|
34
|
+
for (const client of clients) {
|
|
35
|
+
await this.addClient(client);
|
|
36
|
+
}
|
|
37
|
+
this.startHealthCheck();
|
|
38
|
+
}
|
|
39
|
+
async addClient(client) {
|
|
40
|
+
await client.init();
|
|
41
|
+
const managed = {
|
|
42
|
+
client,
|
|
43
|
+
id: client.id,
|
|
44
|
+
online: true,
|
|
45
|
+
busy: false,
|
|
46
|
+
lastSeenAt: Date.now(),
|
|
47
|
+
supportedWorkflows: new Set()
|
|
48
|
+
};
|
|
49
|
+
this.clients.push(managed);
|
|
50
|
+
client.on("disconnected", () => {
|
|
51
|
+
managed.online = false;
|
|
52
|
+
managed.busy = false;
|
|
53
|
+
managed.lastSeenAt = Date.now();
|
|
54
|
+
managed.lastDisconnectedAt = Date.now();
|
|
55
|
+
this.dispatchEvent(new CustomEvent("client:state", {
|
|
56
|
+
detail: { clientId: managed.id, online: false, busy: false, lastError: managed.lastError }
|
|
57
|
+
}));
|
|
58
|
+
});
|
|
59
|
+
client.on("reconnected", () => {
|
|
60
|
+
const now = Date.now();
|
|
61
|
+
managed.online = true;
|
|
62
|
+
managed.lastSeenAt = now;
|
|
63
|
+
managed.reconnectionStableAt = now + this.reconnectionGracePeriodMs;
|
|
64
|
+
// Log if this is a quick reconnect (within 30 seconds of disconnect)
|
|
65
|
+
if (managed.lastDisconnectedAt && (now - managed.lastDisconnectedAt) < 30000) {
|
|
66
|
+
console.warn(`[ClientManager] Client ${managed.id} reconnected ${((now - managed.lastDisconnectedAt) / 1000).toFixed(1)}s after disconnect. ` +
|
|
67
|
+
`Grace period active until ${new Date(managed.reconnectionStableAt).toISOString()}`);
|
|
68
|
+
}
|
|
69
|
+
this.dispatchEvent(new CustomEvent("client:state", {
|
|
70
|
+
detail: { clientId: managed.id, online: true, busy: managed.busy }
|
|
71
|
+
}));
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
list() {
|
|
75
|
+
return [...this.clients];
|
|
76
|
+
}
|
|
77
|
+
getClient(clientId) {
|
|
78
|
+
return this.clients.find((c) => c.id === clientId);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Checks if a client is truly available for work.
|
|
82
|
+
* A client must be online, not busy, AND past the reconnection grace period.
|
|
83
|
+
*/
|
|
84
|
+
isClientStable(client) {
|
|
85
|
+
if (!client.online || client.busy) {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
// If client recently reconnected, wait for grace period
|
|
89
|
+
if (client.reconnectionStableAt && Date.now() < client.reconnectionStableAt) {
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
return true;
|
|
93
|
+
}
|
|
94
|
+
canClientRunJob(client, job) {
|
|
95
|
+
if (job.options.preferredClientIds?.length && !job.options.preferredClientIds.includes(client.id)) {
|
|
96
|
+
return false; // Job has preferred clients and this isn't one of them
|
|
97
|
+
}
|
|
98
|
+
if (job.options.excludeClientIds?.includes(client.id)) {
|
|
99
|
+
return false; // Job has excluded clients and this is one of them
|
|
100
|
+
}
|
|
101
|
+
if (this.strategy.shouldSkipClient(client, job)) {
|
|
102
|
+
return false; // Strategy says to skip
|
|
103
|
+
}
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
claim(job, specificClientId) {
|
|
107
|
+
let chosen;
|
|
108
|
+
if (specificClientId) {
|
|
109
|
+
const candidate = this.clients.find(c => c.id === specificClientId);
|
|
110
|
+
if (candidate && this.isClientStable(candidate) && this.canClientRunJob(candidate, job)) {
|
|
111
|
+
chosen = candidate;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
const candidates = this.clients.filter((c) => this.isClientStable(c));
|
|
116
|
+
chosen = candidates.find(c => this.canClientRunJob(c, job));
|
|
117
|
+
}
|
|
118
|
+
if (!chosen) {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
chosen.busy = true;
|
|
122
|
+
chosen.busy = true;
|
|
123
|
+
chosen.lastSeenAt = Date.now();
|
|
124
|
+
return {
|
|
125
|
+
client: chosen.client,
|
|
126
|
+
clientId: chosen.id,
|
|
127
|
+
release: (opts) => {
|
|
128
|
+
if (this.debugLogs) {
|
|
129
|
+
console.log(`[ClientManager.release] Releasing client ${chosen.id} for job ${job.jobId}. Setting busy: false.`);
|
|
130
|
+
}
|
|
131
|
+
chosen.busy = false;
|
|
132
|
+
if (opts?.success) {
|
|
133
|
+
const wasBlocked = this.strategy.isWorkflowBlocked?.(chosen, job.workflowHash) ?? false;
|
|
134
|
+
this.strategy.recordSuccess(chosen, job);
|
|
135
|
+
const stillBlocked = this.strategy.isWorkflowBlocked?.(chosen, job.workflowHash) ?? false;
|
|
136
|
+
if (wasBlocked && !stillBlocked) {
|
|
137
|
+
this.emitUnblocked(chosen.id, job.workflowHash);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
this.dispatchEvent(new CustomEvent("client:state", {
|
|
141
|
+
detail: { clientId: chosen.id, online: chosen.online, busy: chosen.busy }
|
|
142
|
+
}));
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
recordFailure(clientId, job, error) {
|
|
147
|
+
const client = this.clients.find((c) => c.id === clientId);
|
|
148
|
+
if (!client) {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
client.lastError = error;
|
|
152
|
+
client.busy = false;
|
|
153
|
+
const wasBlocked = this.strategy.isWorkflowBlocked?.(client, job.workflowHash) ?? false;
|
|
154
|
+
this.strategy.recordFailure(client, job, error);
|
|
155
|
+
const isBlocked = this.strategy.isWorkflowBlocked?.(client, job.workflowHash) ?? false;
|
|
156
|
+
if (!wasBlocked && isBlocked) {
|
|
157
|
+
this.emitBlocked(client.id, job.workflowHash);
|
|
158
|
+
}
|
|
159
|
+
this.dispatchEvent(new CustomEvent("client:state", {
|
|
160
|
+
detail: { clientId: client.id, online: client.online, busy: client.busy, lastError: error }
|
|
161
|
+
}));
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Start periodic health check to keep connections alive and detect issues early.
|
|
165
|
+
* Pings idle clients by polling their queue status.
|
|
166
|
+
*/
|
|
167
|
+
startHealthCheck() {
|
|
168
|
+
if (this.healthCheckInterval) {
|
|
169
|
+
return; // Already running
|
|
170
|
+
}
|
|
171
|
+
this.healthCheckInterval = setInterval(() => {
|
|
172
|
+
this.performHealthCheck().catch((error) => {
|
|
173
|
+
console.error("[ClientManager] Health check error:", error);
|
|
174
|
+
});
|
|
175
|
+
}, this.healthCheckIntervalMs);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Perform health check on all clients.
|
|
179
|
+
* Polls queue status to keep WebSocket alive and detect connection issues.
|
|
180
|
+
* IMPORTANT: Pings ALL online clients (including busy ones) to prevent WebSocket timeout during heavy load.
|
|
181
|
+
*/
|
|
182
|
+
async performHealthCheck() {
|
|
183
|
+
for (const managed of this.clients) {
|
|
184
|
+
// Ping ALL online clients (not just idle ones) to keep WebSocket alive during heavy load
|
|
185
|
+
if (managed.online) {
|
|
186
|
+
try {
|
|
187
|
+
// Lightweight ping: poll queue status (triggers WebSocket activity via fetchApi)
|
|
188
|
+
await managed.client.getQueue();
|
|
189
|
+
managed.lastSeenAt = Date.now();
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
// Health check failed - client may have connection issues
|
|
193
|
+
console.warn(`[ClientManager] Health check failed for client ${managed.id}:`, error);
|
|
194
|
+
// Don't mark as offline here - let the WebSocket disconnect event handle that
|
|
195
|
+
// This prevents false positives from temporary network hiccups
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Stop health check interval (called during shutdown).
|
|
202
|
+
*/
|
|
203
|
+
stopHealthCheck() {
|
|
204
|
+
if (this.healthCheckInterval) {
|
|
205
|
+
clearInterval(this.healthCheckInterval);
|
|
206
|
+
this.healthCheckInterval = null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Cleanup resources when destroying the manager.
|
|
211
|
+
*/
|
|
212
|
+
destroy() {
|
|
213
|
+
this.stopHealthCheck();
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
216
|
//# sourceMappingURL=ClientManager.js.map
|
package/dist/pool/index.d.ts
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
export { WorkflowPool } from "./WorkflowPool.js";
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
4
|
-
export type {
|
|
5
|
-
export type {
|
|
6
|
-
export
|
|
7
|
-
export type {
|
|
8
|
-
export {
|
|
9
|
-
export type {
|
|
10
|
-
export { SmartFailoverStrategy } from "./failover/SmartFailoverStrategy.js";
|
|
11
|
-
export type { JobProfileStats, NodeExecutionProfile } from "./profiling/JobProfiler.js";
|
|
1
|
+
export { WorkflowPool } from "./WorkflowPool.js";
|
|
2
|
+
export type { WorkflowPoolOpts } from "./WorkflowPool.js";
|
|
3
|
+
export type { WorkflowPoolEventMap } from "./types/events.js";
|
|
4
|
+
export type { JobRecord, JobStatus, WorkflowJobOptions } from "./types/job.js";
|
|
5
|
+
export type { QueueAdapter, QueueReservation, QueueStats } from "./queue/QueueAdapter.js";
|
|
6
|
+
export { MemoryQueueAdapter } from "./queue/adapters/memory.js";
|
|
7
|
+
export type { FailoverStrategy } from "./failover/Strategy.js";
|
|
8
|
+
export { SmartFailoverStrategy } from "./failover/SmartFailoverStrategy.js";
|
|
9
|
+
export type { JobProfileStats, NodeExecutionProfile } from "./profiling/JobProfiler.js";
|
|
12
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/pool/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pool/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pool/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC9D,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC"}
|
package/dist/pool/index.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
export { WorkflowPool } from "./WorkflowPool.js";
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
4
|
-
export { MemoryQueueAdapter } from "./queue/adapters/memory.js";
|
|
5
|
-
export { SmartFailoverStrategy } from "./failover/SmartFailoverStrategy.js";
|
|
1
|
+
export { WorkflowPool } from "./WorkflowPool.js";
|
|
2
|
+
export { MemoryQueueAdapter } from "./queue/adapters/memory.js";
|
|
3
|
+
export { SmartFailoverStrategy } from "./failover/SmartFailoverStrategy.js";
|
|
6
4
|
//# sourceMappingURL=index.js.map
|
package/dist/pool/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pool/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pool/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC"}
|