comfyui-node 1.6.2 ā 1.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +18 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -7
- package/dist/index.js.map +1 -1
- package/dist/multipool/client-registry.d.ts +23 -32
- package/dist/multipool/client-registry.d.ts.map +1 -1
- package/dist/multipool/client-registry.js +152 -152
- package/dist/multipool/client-registry.js.map +1 -1
- package/dist/multipool/helpers.js +52 -52
- package/dist/multipool/helpers.js.map +1 -1
- package/dist/multipool/index.js +2 -2
- package/dist/multipool/interfaces.d.ts +135 -12
- package/dist/multipool/interfaces.d.ts.map +1 -1
- package/dist/multipool/interfaces.js +1 -1
- package/dist/multipool/job-profiler.d.ts +64 -127
- package/dist/multipool/job-profiler.d.ts.map +1 -1
- package/dist/multipool/job-profiler.js +221 -221
- package/dist/multipool/job-profiler.js.map +1 -1
- package/dist/multipool/job-queue-processor.d.ts +23 -27
- package/dist/multipool/job-queue-processor.d.ts.map +1 -1
- package/dist/multipool/job-queue-processor.js +196 -196
- package/dist/multipool/job-queue-processor.js.map +1 -1
- package/dist/multipool/job-state-registry.d.ts +42 -66
- package/dist/multipool/job-state-registry.d.ts.map +1 -1
- package/dist/multipool/job-state-registry.js +282 -282
- package/dist/multipool/job-state-registry.js.map +1 -1
- package/dist/multipool/multi-workflow-pool.d.ts +101 -42
- package/dist/multipool/multi-workflow-pool.d.ts.map +1 -1
- package/dist/multipool/multi-workflow-pool.js +424 -313
- package/dist/multipool/multi-workflow-pool.js.map +1 -1
- package/dist/multipool/pool-event-manager.d.ts +10 -10
- package/dist/multipool/pool-event-manager.d.ts.map +1 -1
- package/dist/multipool/pool-event-manager.js +27 -27
- package/dist/multipool/tests/client-registry-api-demo.d.ts +7 -0
- package/dist/multipool/tests/client-registry-api-demo.d.ts.map +1 -0
- package/dist/multipool/tests/client-registry-api-demo.js +136 -0
- package/dist/multipool/tests/client-registry-api-demo.js.map +1 -0
- package/dist/multipool/tests/client-registry.spec.d.ts +2 -0
- package/dist/multipool/tests/client-registry.spec.d.ts.map +1 -0
- package/dist/multipool/tests/client-registry.spec.js +191 -0
- package/dist/multipool/tests/client-registry.spec.js.map +1 -0
- package/dist/multipool/tests/error-classification-tests.js +373 -373
- package/dist/multipool/tests/event-forwarding-demo.d.ts +7 -0
- package/dist/multipool/tests/event-forwarding-demo.d.ts.map +1 -0
- package/dist/multipool/tests/event-forwarding-demo.js +88 -0
- package/dist/multipool/tests/event-forwarding-demo.js.map +1 -0
- package/dist/multipool/tests/helpers.spec.d.ts +2 -0
- package/dist/multipool/tests/helpers.spec.d.ts.map +1 -0
- package/dist/multipool/tests/helpers.spec.js +100 -0
- package/dist/multipool/tests/helpers.spec.js.map +1 -0
- package/dist/multipool/tests/job-queue-processor.spec.d.ts +2 -0
- package/dist/multipool/tests/job-queue-processor.spec.d.ts.map +1 -0
- package/dist/multipool/tests/job-queue-processor.spec.js +89 -0
- package/dist/multipool/tests/job-queue-processor.spec.js.map +1 -0
- package/dist/multipool/tests/job-state-registry.spec.d.ts +2 -0
- package/dist/multipool/tests/job-state-registry.spec.d.ts.map +1 -0
- package/dist/multipool/tests/job-state-registry.spec.js +143 -0
- package/dist/multipool/tests/job-state-registry.spec.js.map +1 -0
- package/dist/multipool/tests/multipool-basic.js +141 -141
- package/dist/multipool/tests/profiling-demo.js +87 -87
- package/dist/multipool/tests/profiling-demo.js.map +1 -1
- package/dist/multipool/tests/two-stage-edit-simulation.js +298 -298
- package/dist/multipool/tests/two-stage-edit-simulation.js.map +1 -1
- package/dist/multipool/workflow.d.ts +178 -178
- package/dist/multipool/workflow.d.ts.map +1 -1
- package/dist/multipool/workflow.js +333 -333
- package/package.json +1 -1
|
@@ -1,314 +1,425 @@
|
|
|
1
|
-
import { ClientRegistry } from "./client-registry.js";
|
|
2
|
-
import { PoolEventManager } from "./pool-event-manager.js";
|
|
3
|
-
import { JobStateRegistry } from "./job-state-registry.js";
|
|
4
|
-
import { JobQueueProcessor } from "./job-queue-processor.js";
|
|
5
|
-
import { createLogger } from "./logger.js";
|
|
6
|
-
/**
|
|
7
|
-
* MultiWorkflowPool class to manage heterogeneous clusters of ComfyUI workers with different workflow capabilities.
|
|
8
|
-
* Using a fully event driven architecture to handle client connections, job submissions, and failover strategies.
|
|
9
|
-
* Zero polling is used; all operations are event driven. Maximizes responsiveness and scalability.
|
|
10
|
-
*/
|
|
11
|
-
export class MultiWorkflowPool {
|
|
12
|
-
// Event manager for handling pool events
|
|
13
|
-
events;
|
|
14
|
-
// Registry for managing clients in the pool
|
|
15
|
-
clientRegistry;
|
|
16
|
-
// Registry for managing job state
|
|
17
|
-
jobRegistry;
|
|
18
|
-
// Multi queue map, one per workflow based on the workflow hash
|
|
19
|
-
queues = new Map();
|
|
20
|
-
// Pool configuration
|
|
21
|
-
options;
|
|
22
|
-
// Logger instance
|
|
23
|
-
logger;
|
|
24
|
-
monitoringInterval;
|
|
25
|
-
constructor(options) {
|
|
26
|
-
this.options = {
|
|
27
|
-
connectionTimeoutMs: options?.connectionTimeoutMs ?? 10000,
|
|
28
|
-
enableMonitoring: options?.enableMonitoring ?? false,
|
|
29
|
-
monitoringIntervalMs: options?.monitoringIntervalMs ?? 60000,
|
|
30
|
-
logLevel: options?.logLevel ?? "warn",
|
|
31
|
-
enableProfiling: options?.enableProfiling ?? false
|
|
32
|
-
};
|
|
33
|
-
this.logger = createLogger("MultiWorkflowPool", this.options.logLevel);
|
|
34
|
-
this.events = new PoolEventManager(this);
|
|
35
|
-
this.clientRegistry = new ClientRegistry(this, this.logger);
|
|
36
|
-
this.jobRegistry = new JobStateRegistry(this, this.clientRegistry);
|
|
37
|
-
// Create general queue for workflows without specific hashes
|
|
38
|
-
this.queues.set("general", new JobQueueProcessor(this.jobRegistry, this.clientRegistry, "general", this.logger));
|
|
39
|
-
// Monitoring
|
|
40
|
-
if (this.options.enableMonitoring) {
|
|
41
|
-
this.monitoringInterval = setInterval(() => {
|
|
42
|
-
this.printStatusSummary();
|
|
43
|
-
}, this.options.monitoringIntervalMs);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
// PUBLIC API
|
|
47
|
-
async init() {
|
|
48
|
-
if (this.clientRegistry.clients.size === 0) {
|
|
49
|
-
throw new Error("No clients registered in the pool. Please add clients before initializing the pool.");
|
|
50
|
-
}
|
|
51
|
-
const connectionPromises = [];
|
|
52
|
-
for (const client of this.clientRegistry.clients.values()) {
|
|
53
|
-
connectionPromises.push(new Promise(async (resolve, reject) => {
|
|
54
|
-
let timeout = setTimeout(() => {
|
|
55
|
-
client.api.abortReconnect();
|
|
56
|
-
reject(new Error(`Connection to client ${client.url} timed out`));
|
|
57
|
-
}, this.options.connectionTimeoutMs);
|
|
58
|
-
try {
|
|
59
|
-
const readyApi = await client.api.init(1);
|
|
60
|
-
clearTimeout(timeout);
|
|
61
|
-
timeout = null;
|
|
62
|
-
this.logger.info(`Connected to ${client.url}`);
|
|
63
|
-
client.api = readyApi;
|
|
64
|
-
this.attachHandlersToClient(client);
|
|
65
|
-
const queueStatus = await client.api.getQueue();
|
|
66
|
-
if (queueStatus.queue_running.length === 0 && queueStatus.queue_pending.length === 0) {
|
|
67
|
-
this.logger.debug(`Client ${client.url} is idle.`);
|
|
68
|
-
client.state = "idle";
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
client.state = "busy";
|
|
72
|
-
}
|
|
73
|
-
resolve();
|
|
74
|
-
}
|
|
75
|
-
catch (e) {
|
|
76
|
-
client.state = "offline";
|
|
77
|
-
reject(e);
|
|
78
|
-
}
|
|
79
|
-
finally {
|
|
80
|
-
if (timeout) {
|
|
81
|
-
clearTimeout(timeout);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}));
|
|
85
|
-
}
|
|
86
|
-
const promiseResults = await Promise.allSettled(connectionPromises);
|
|
87
|
-
const failedConnections = promiseResults.filter(result => result.status === "rejected");
|
|
88
|
-
if (failedConnections.length > 0) {
|
|
89
|
-
this.logger.warn(`Warning: ${failedConnections.length} client(s) failed to connect.`);
|
|
90
|
-
failedConnections.forEach((result) => {
|
|
91
|
-
if (result.status === "rejected") {
|
|
92
|
-
this.logger.error("Connection failed:", result.reason);
|
|
93
|
-
}
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
// Throw error if all connections failed
|
|
97
|
-
if (failedConnections.length === this.clientRegistry.clients.size) {
|
|
98
|
-
throw new Error("All clients failed to connect. Pool initialization failed.");
|
|
99
|
-
}
|
|
100
|
-
this.logger.info(`Initialization complete. ${this.clientRegistry.clients.size - failedConnections.length} client(s) connected successfully.`);
|
|
101
|
-
}
|
|
102
|
-
async shutdown() {
|
|
103
|
-
this.logger.info("Shutting down MultiWorkflowPool...");
|
|
104
|
-
if (this.monitoringInterval) {
|
|
105
|
-
clearInterval(this.monitoringInterval);
|
|
106
|
-
}
|
|
107
|
-
// Disconnect all clients
|
|
108
|
-
const disconnectPromises = [];
|
|
109
|
-
for (const client of this.clientRegistry.clients.values()) {
|
|
110
|
-
disconnectPromises.push(new Promise(async (resolve) => {
|
|
111
|
-
try {
|
|
112
|
-
client.api.destroy();
|
|
113
|
-
this.logger.debug(`Disconnected from client ${client.url}`);
|
|
114
|
-
}
|
|
115
|
-
catch (e) {
|
|
116
|
-
this.logger.error(`Error disconnecting from client ${client.url}:`, e);
|
|
117
|
-
}
|
|
118
|
-
finally {
|
|
119
|
-
resolve();
|
|
120
|
-
}
|
|
121
|
-
}));
|
|
122
|
-
}
|
|
123
|
-
await Promise.allSettled(disconnectPromises);
|
|
124
|
-
}
|
|
125
|
-
addClient(clientUrl, options) {
|
|
126
|
-
this.clientRegistry.addClient(clientUrl, options);
|
|
127
|
-
}
|
|
128
|
-
removeClient(clientUrl) {
|
|
129
|
-
this.clientRegistry.removeClient(clientUrl);
|
|
130
|
-
}
|
|
131
|
-
async submitJob(workflow) {
|
|
132
|
-
let workflowHash = workflow.structureHash;
|
|
133
|
-
if (!workflowHash) {
|
|
134
|
-
workflow.updateHash();
|
|
135
|
-
workflowHash = workflow.structureHash;
|
|
136
|
-
}
|
|
137
|
-
// check if there are clients with affinity for this workflow
|
|
138
|
-
let queue;
|
|
139
|
-
if (workflowHash && this.clientRegistry.hasClientsForWorkflow(workflowHash)) {
|
|
140
|
-
queue = this.assertQueue(workflowHash);
|
|
141
|
-
}
|
|
142
|
-
else {
|
|
143
|
-
queue = this.queues.get("general");
|
|
144
|
-
this.logger.debug(`No clients with affinity for workflow hash ${workflowHash}, using general queue.`);
|
|
145
|
-
}
|
|
146
|
-
if (!queue) {
|
|
147
|
-
throw new Error("Failed to create or retrieve job queue for workflow.");
|
|
148
|
-
}
|
|
149
|
-
const newJobId = this.jobRegistry.addJob(workflow);
|
|
150
|
-
await queue.enqueueJob(newJobId, workflow);
|
|
151
|
-
return newJobId;
|
|
152
|
-
}
|
|
153
|
-
getJobStatus(jobId) {
|
|
154
|
-
return this.jobRegistry.getJobStatus(jobId);
|
|
155
|
-
}
|
|
156
|
-
async cancelJob(jobId) {
|
|
157
|
-
return this.jobRegistry.cancelJob(jobId);
|
|
158
|
-
}
|
|
159
|
-
attachEventHook(event, listener) {
|
|
160
|
-
if (event && listener) {
|
|
161
|
-
this.events.attachHook(event, listener);
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
// PRIVATE METHODS
|
|
165
|
-
assertQueue(workflowHash) {
|
|
166
|
-
if (!workflowHash) {
|
|
167
|
-
return null;
|
|
168
|
-
}
|
|
169
|
-
let queue = this.queues.get(workflowHash);
|
|
170
|
-
if (!queue) {
|
|
171
|
-
queue = new JobQueueProcessor(this.jobRegistry, this.clientRegistry, workflowHash, this.logger);
|
|
172
|
-
this.queues.set(workflowHash, queue);
|
|
173
|
-
}
|
|
174
|
-
return queue;
|
|
175
|
-
}
|
|
176
|
-
attachHandlersToClient(client) {
|
|
177
|
-
//
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
client.
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
}
|
|
206
|
-
else {
|
|
207
|
-
|
|
208
|
-
}
|
|
209
|
-
});
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
this.
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
this.
|
|
268
|
-
}
|
|
269
|
-
});
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
console.log("\nš CLIENT STATES:
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
console.
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
}
|
|
1
|
+
import { ClientRegistry } from "./client-registry.js";
|
|
2
|
+
import { PoolEventManager } from "./pool-event-manager.js";
|
|
3
|
+
import { JobStateRegistry } from "./job-state-registry.js";
|
|
4
|
+
import { JobQueueProcessor } from "./job-queue-processor.js";
|
|
5
|
+
import { createLogger } from "./logger.js";
|
|
6
|
+
/**
|
|
7
|
+
* MultiWorkflowPool class to manage heterogeneous clusters of ComfyUI workers with different workflow capabilities.
|
|
8
|
+
* Using a fully event driven architecture to handle client connections, job submissions, and failover strategies.
|
|
9
|
+
* Zero polling is used; all operations are event driven. Maximizes responsiveness and scalability.
|
|
10
|
+
*/
|
|
11
|
+
export class MultiWorkflowPool {
|
|
12
|
+
// Event manager for handling pool events
|
|
13
|
+
events;
|
|
14
|
+
// Registry for managing clients in the pool
|
|
15
|
+
clientRegistry;
|
|
16
|
+
// Registry for managing job state
|
|
17
|
+
jobRegistry;
|
|
18
|
+
// Multi queue map, one per workflow based on the workflow hash
|
|
19
|
+
queues = new Map();
|
|
20
|
+
// Pool configuration
|
|
21
|
+
options;
|
|
22
|
+
// Logger instance
|
|
23
|
+
logger;
|
|
24
|
+
monitoringInterval;
|
|
25
|
+
constructor(options) {
|
|
26
|
+
this.options = {
|
|
27
|
+
connectionTimeoutMs: options?.connectionTimeoutMs ?? 10000,
|
|
28
|
+
enableMonitoring: options?.enableMonitoring ?? false,
|
|
29
|
+
monitoringIntervalMs: options?.monitoringIntervalMs ?? 60000,
|
|
30
|
+
logLevel: options?.logLevel ?? "warn",
|
|
31
|
+
enableProfiling: options?.enableProfiling ?? false
|
|
32
|
+
};
|
|
33
|
+
this.logger = createLogger("MultiWorkflowPool", this.options.logLevel);
|
|
34
|
+
this.events = new PoolEventManager(this);
|
|
35
|
+
this.clientRegistry = new ClientRegistry(this, this.logger);
|
|
36
|
+
this.jobRegistry = new JobStateRegistry(this, this.clientRegistry);
|
|
37
|
+
// Create general queue for workflows without specific hashes
|
|
38
|
+
this.queues.set("general", new JobQueueProcessor(this.jobRegistry, this.clientRegistry, "general", this.logger));
|
|
39
|
+
// Monitoring
|
|
40
|
+
if (this.options.enableMonitoring) {
|
|
41
|
+
this.monitoringInterval = setInterval(() => {
|
|
42
|
+
this.printStatusSummary();
|
|
43
|
+
}, this.options.monitoringIntervalMs);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// PUBLIC API
|
|
47
|
+
async init() {
|
|
48
|
+
if (this.clientRegistry.clients.size === 0) {
|
|
49
|
+
throw new Error("No clients registered in the pool. Please add clients before initializing the pool.");
|
|
50
|
+
}
|
|
51
|
+
const connectionPromises = [];
|
|
52
|
+
for (const client of this.clientRegistry.clients.values()) {
|
|
53
|
+
connectionPromises.push(new Promise(async (resolve, reject) => {
|
|
54
|
+
let timeout = setTimeout(() => {
|
|
55
|
+
client.api.abortReconnect();
|
|
56
|
+
reject(new Error(`Connection to client ${client.url} timed out`));
|
|
57
|
+
}, this.options.connectionTimeoutMs);
|
|
58
|
+
try {
|
|
59
|
+
const readyApi = await client.api.init(1);
|
|
60
|
+
clearTimeout(timeout);
|
|
61
|
+
timeout = null;
|
|
62
|
+
this.logger.info(`Connected to ${client.url}`);
|
|
63
|
+
client.api = readyApi;
|
|
64
|
+
this.attachHandlersToClient(client);
|
|
65
|
+
const queueStatus = await client.api.getQueue();
|
|
66
|
+
if (queueStatus.queue_running.length === 0 && queueStatus.queue_pending.length === 0) {
|
|
67
|
+
this.logger.debug(`Client ${client.url} is idle.`);
|
|
68
|
+
client.state = "idle";
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
client.state = "busy";
|
|
72
|
+
}
|
|
73
|
+
resolve();
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
client.state = "offline";
|
|
77
|
+
reject(e);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
if (timeout) {
|
|
81
|
+
clearTimeout(timeout);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}));
|
|
85
|
+
}
|
|
86
|
+
const promiseResults = await Promise.allSettled(connectionPromises);
|
|
87
|
+
const failedConnections = promiseResults.filter(result => result.status === "rejected");
|
|
88
|
+
if (failedConnections.length > 0) {
|
|
89
|
+
this.logger.warn(`Warning: ${failedConnections.length} client(s) failed to connect.`);
|
|
90
|
+
failedConnections.forEach((result) => {
|
|
91
|
+
if (result.status === "rejected") {
|
|
92
|
+
this.logger.error("Connection failed:", result.reason);
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
// Throw error if all connections failed
|
|
97
|
+
if (failedConnections.length === this.clientRegistry.clients.size) {
|
|
98
|
+
throw new Error("All clients failed to connect. Pool initialization failed.");
|
|
99
|
+
}
|
|
100
|
+
this.logger.info(`Initialization complete. ${this.clientRegistry.clients.size - failedConnections.length} client(s) connected successfully.`);
|
|
101
|
+
}
|
|
102
|
+
async shutdown() {
|
|
103
|
+
this.logger.info("Shutting down MultiWorkflowPool...");
|
|
104
|
+
if (this.monitoringInterval) {
|
|
105
|
+
clearInterval(this.monitoringInterval);
|
|
106
|
+
}
|
|
107
|
+
// Disconnect all clients
|
|
108
|
+
const disconnectPromises = [];
|
|
109
|
+
for (const client of this.clientRegistry.clients.values()) {
|
|
110
|
+
disconnectPromises.push(new Promise(async (resolve) => {
|
|
111
|
+
try {
|
|
112
|
+
client.api.destroy();
|
|
113
|
+
this.logger.debug(`Disconnected from client ${client.url}`);
|
|
114
|
+
}
|
|
115
|
+
catch (e) {
|
|
116
|
+
this.logger.error(`Error disconnecting from client ${client.url}:`, e);
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
resolve();
|
|
120
|
+
}
|
|
121
|
+
}));
|
|
122
|
+
}
|
|
123
|
+
await Promise.allSettled(disconnectPromises);
|
|
124
|
+
}
|
|
125
|
+
addClient(clientUrl, options) {
|
|
126
|
+
this.clientRegistry.addClient(clientUrl, options);
|
|
127
|
+
}
|
|
128
|
+
removeClient(clientUrl) {
|
|
129
|
+
this.clientRegistry.removeClient(clientUrl);
|
|
130
|
+
}
|
|
131
|
+
async submitJob(workflow) {
|
|
132
|
+
let workflowHash = workflow.structureHash;
|
|
133
|
+
if (!workflowHash) {
|
|
134
|
+
workflow.updateHash();
|
|
135
|
+
workflowHash = workflow.structureHash;
|
|
136
|
+
}
|
|
137
|
+
// check if there are clients with affinity for this workflow
|
|
138
|
+
let queue;
|
|
139
|
+
if (workflowHash && this.clientRegistry.hasClientsForWorkflow(workflowHash)) {
|
|
140
|
+
queue = this.assertQueue(workflowHash);
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
queue = this.queues.get("general");
|
|
144
|
+
this.logger.debug(`No clients with affinity for workflow hash ${workflowHash}, using general queue.`);
|
|
145
|
+
}
|
|
146
|
+
if (!queue) {
|
|
147
|
+
throw new Error("Failed to create or retrieve job queue for workflow.");
|
|
148
|
+
}
|
|
149
|
+
const newJobId = this.jobRegistry.addJob(workflow);
|
|
150
|
+
await queue.enqueueJob(newJobId, workflow);
|
|
151
|
+
return newJobId;
|
|
152
|
+
}
|
|
153
|
+
getJobStatus(jobId) {
|
|
154
|
+
return this.jobRegistry.getJobStatus(jobId);
|
|
155
|
+
}
|
|
156
|
+
async cancelJob(jobId) {
|
|
157
|
+
return this.jobRegistry.cancelJob(jobId);
|
|
158
|
+
}
|
|
159
|
+
attachEventHook(event, listener) {
|
|
160
|
+
if (event && listener) {
|
|
161
|
+
this.events.attachHook(event, listener);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// PRIVATE METHODS
|
|
165
|
+
assertQueue(workflowHash) {
|
|
166
|
+
if (!workflowHash) {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
let queue = this.queues.get(workflowHash);
|
|
170
|
+
if (!queue) {
|
|
171
|
+
queue = new JobQueueProcessor(this.jobRegistry, this.clientRegistry, workflowHash, this.logger);
|
|
172
|
+
this.queues.set(workflowHash, queue);
|
|
173
|
+
}
|
|
174
|
+
return queue;
|
|
175
|
+
}
|
|
176
|
+
attachHandlersToClient(client) {
|
|
177
|
+
// Forward all client events through the pool event manager
|
|
178
|
+
client.api.on("all", event => {
|
|
179
|
+
const payload = {
|
|
180
|
+
clientUrl: client.url,
|
|
181
|
+
clientName: client.nodeName,
|
|
182
|
+
eventType: event.detail.type,
|
|
183
|
+
eventData: event.detail.data
|
|
184
|
+
};
|
|
185
|
+
this.events.emitEvent({
|
|
186
|
+
type: `client:${event.detail.type}`,
|
|
187
|
+
payload
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
client.api.on("status", event => {
|
|
191
|
+
this.logger.client(client.nodeName, event.type, `Queue Remaining: ${event.detail.status.exec_info.queue_remaining}`);
|
|
192
|
+
// Update client state based on status
|
|
193
|
+
if (event.detail.status.exec_info.queue_remaining === 0) {
|
|
194
|
+
client.state = "idle";
|
|
195
|
+
// Trigger queue processing
|
|
196
|
+
client.workflowAffinity?.forEach(value => {
|
|
197
|
+
this.logger.debug(`Triggering queue processing for workflow hash ${value} due to client ${client.nodeName} becoming idle.`);
|
|
198
|
+
const queue = this.queues.get(value);
|
|
199
|
+
if (queue) {
|
|
200
|
+
queue.processQueue().catch(reason => {
|
|
201
|
+
this.logger.error(`Error processing job queue for workflow hash ${value}:`, reason);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
client.state = "busy";
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
client.api.on("b_preview_meta", event => {
|
|
211
|
+
const prompt_id = event.detail.metadata.prompt_id;
|
|
212
|
+
if (prompt_id) {
|
|
213
|
+
this.jobRegistry.updateJobPreviewMetadata(prompt_id, event.detail.metadata, event.detail.blob);
|
|
214
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] Preview metadata for prompt ID: ${prompt_id} | blob size: ${event.detail.blob.size} (${event.detail.metadata.image_type})`);
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
this.logger.warn(`[${event.type}@${client.nodeName}] Preview metadata received without prompt ID.`);
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
// Handle finished nodes, extract image for prompt_id
|
|
221
|
+
client.api.on("executed", event => {
|
|
222
|
+
const prompt_id = event.detail.prompt_id;
|
|
223
|
+
if (prompt_id) {
|
|
224
|
+
const output = event.detail.output;
|
|
225
|
+
if (output && output.images) {
|
|
226
|
+
this.jobRegistry.addJobImages(prompt_id, output.images);
|
|
227
|
+
}
|
|
228
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] Node executed for prompt ID: ${prompt_id}`, event.detail.output);
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
this.logger.warn(`[${event.type}@${client.nodeName}] Executed event received without prompt ID.`);
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
client.api.on("progress", event => {
|
|
235
|
+
const prompt_id = event.detail.prompt_id;
|
|
236
|
+
if (prompt_id) {
|
|
237
|
+
const nodeId = event.detail.node;
|
|
238
|
+
this.jobRegistry.updateJobProgress(prompt_id, event.detail.value, event.detail.max, nodeId !== null ? nodeId : undefined);
|
|
239
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] Progress for prompt ID: ${prompt_id} | ${Math.round(event.detail.value / event.detail.max * 100)}%`);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
this.logger.warn(`[${event.type}@${client.nodeName}] Progress event received without prompt ID.`);
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
// Track node execution for profiling
|
|
246
|
+
client.api.on("executing", event => {
|
|
247
|
+
const prompt_id = event.detail.prompt_id;
|
|
248
|
+
const nodeId = event.detail.node;
|
|
249
|
+
if (prompt_id) {
|
|
250
|
+
if (nodeId === null) {
|
|
251
|
+
// Execution completed (node: null event)
|
|
252
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] Execution complete for prompt ID: ${prompt_id}`);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
// Node started executing
|
|
256
|
+
this.jobRegistry.onNodeExecuting(prompt_id, String(nodeId));
|
|
257
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] Node ${nodeId} executing for prompt ID: ${prompt_id}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
// Track cached nodes for profiling
|
|
262
|
+
client.api.on("execution_cached", event => {
|
|
263
|
+
const prompt_id = event.detail.prompt_id;
|
|
264
|
+
const nodeIds = event.detail.nodes;
|
|
265
|
+
if (prompt_id && nodeIds && Array.isArray(nodeIds)) {
|
|
266
|
+
this.jobRegistry.onCachedNodes(prompt_id, nodeIds.map(String));
|
|
267
|
+
this.logger.debug(`[${event.type}@${client.nodeName}] ${nodeIds.length} nodes cached for prompt ID: ${prompt_id}`);
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
client.api.on("execution_success", event => {
|
|
271
|
+
const prompt_id = event.detail.prompt_id;
|
|
272
|
+
if (prompt_id) {
|
|
273
|
+
this.logger.client(client.nodeName, event.type, `Execution success for prompt ID: ${prompt_id}`);
|
|
274
|
+
// Mark client as idle first
|
|
275
|
+
client.state = "idle";
|
|
276
|
+
// Mark job as completed, it will trigger queue processing
|
|
277
|
+
this.jobRegistry.completeJob(prompt_id);
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
printStatusSummary() {
|
|
282
|
+
console.log("\n" + "=".repeat(80));
|
|
283
|
+
console.log("MULTI-WORKFLOW POOL STATUS SUMMARY");
|
|
284
|
+
console.log("=".repeat(80));
|
|
285
|
+
// Print client states using console.table
|
|
286
|
+
if (this.clientRegistry.clients.size > 0) {
|
|
287
|
+
console.log("\nš CLIENT STATES:");
|
|
288
|
+
const clientData = Array.from(this.clientRegistry.clients.values()).map(client => ({
|
|
289
|
+
"URL": client.url,
|
|
290
|
+
"Node Name": client.nodeName,
|
|
291
|
+
"State": client.state,
|
|
292
|
+
"Priority": client.priority !== undefined ? client.priority : "N/A"
|
|
293
|
+
}));
|
|
294
|
+
console.table(clientData);
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
console.log("\nš CLIENT STATES: No clients registered");
|
|
298
|
+
}
|
|
299
|
+
// Print queue states using console.table
|
|
300
|
+
if (this.queues.size > 0) {
|
|
301
|
+
console.log("\nš¬ QUEUE STATES:");
|
|
302
|
+
const queueData = Array.from(this.queues.entries()).map(([workflowHash, queue]) => ({
|
|
303
|
+
"Workflow Hash": workflowHash.length > 50 ? workflowHash.substring(0, 47) + "..." : workflowHash,
|
|
304
|
+
"Jobs Pending": queue.queue.length,
|
|
305
|
+
"Type": workflowHash === "general" ? "General" : "Specific"
|
|
306
|
+
}));
|
|
307
|
+
console.table(queueData);
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
console.log("\nš¬ QUEUE STATES: No queues found");
|
|
311
|
+
}
|
|
312
|
+
console.log("");
|
|
313
|
+
}
|
|
314
|
+
async waitForJobCompletion(jobId) {
|
|
315
|
+
return await this.jobRegistry.waitForResults(jobId);
|
|
316
|
+
}
|
|
317
|
+
attachJobProgressListener(jobId, progressListener) {
|
|
318
|
+
this.jobRegistry.attachJobProgressListener(jobId, progressListener);
|
|
319
|
+
}
|
|
320
|
+
attachJobPreviewListener(jobId, previewListener) {
|
|
321
|
+
this.jobRegistry.attachJobPreviewListener(jobId, previewListener);
|
|
322
|
+
}
|
|
323
|
+
// CLIENT REGISTRY ACCESS METHODS
|
|
324
|
+
/**
|
|
325
|
+
* Get a list of all registered clients with their current state
|
|
326
|
+
* @returns Array of client information objects
|
|
327
|
+
*/
|
|
328
|
+
getClients() {
|
|
329
|
+
return Array.from(this.clientRegistry.clients.values()).map(client => ({
|
|
330
|
+
url: client.url,
|
|
331
|
+
nodeName: client.nodeName,
|
|
332
|
+
state: client.state,
|
|
333
|
+
priority: client.priority,
|
|
334
|
+
workflowAffinityHashes: client.workflowAffinity
|
|
335
|
+
? Array.from(client.workflowAffinity)
|
|
336
|
+
: undefined
|
|
337
|
+
}));
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Get information about a specific client by URL
|
|
341
|
+
* @param clientUrl - The URL of the client to query
|
|
342
|
+
* @returns Client information or null if not found
|
|
343
|
+
*/
|
|
344
|
+
getClient(clientUrl) {
|
|
345
|
+
const client = this.clientRegistry.clients.get(clientUrl);
|
|
346
|
+
if (!client) {
|
|
347
|
+
return null;
|
|
348
|
+
}
|
|
349
|
+
return {
|
|
350
|
+
url: client.url,
|
|
351
|
+
nodeName: client.nodeName,
|
|
352
|
+
state: client.state,
|
|
353
|
+
priority: client.priority,
|
|
354
|
+
workflowAffinityHashes: client.workflowAffinity
|
|
355
|
+
? Array.from(client.workflowAffinity)
|
|
356
|
+
: undefined
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Get all clients that have affinity for a specific workflow
|
|
361
|
+
* @param workflow - The workflow to check affinity for
|
|
362
|
+
* @returns Array of client URLs that can handle this workflow
|
|
363
|
+
*/
|
|
364
|
+
getClientsForWorkflow(workflow) {
|
|
365
|
+
let workflowHash = workflow.structureHash;
|
|
366
|
+
if (!workflowHash) {
|
|
367
|
+
workflow.updateHash();
|
|
368
|
+
workflowHash = workflow.structureHash;
|
|
369
|
+
}
|
|
370
|
+
if (!workflowHash) {
|
|
371
|
+
return [];
|
|
372
|
+
}
|
|
373
|
+
const clientSet = this.clientRegistry.workflowAffinityMap.get(workflowHash);
|
|
374
|
+
return clientSet ? Array.from(clientSet) : [];
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Get all idle clients currently available for work
|
|
378
|
+
* @returns Array of idle client information
|
|
379
|
+
*/
|
|
380
|
+
getIdleClients() {
|
|
381
|
+
return Array.from(this.clientRegistry.clients.values())
|
|
382
|
+
.filter(client => client.state === "idle")
|
|
383
|
+
.map(client => ({
|
|
384
|
+
url: client.url,
|
|
385
|
+
nodeName: client.nodeName,
|
|
386
|
+
priority: client.priority
|
|
387
|
+
}));
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Check if there are any clients available for a specific workflow
|
|
391
|
+
* @param workflow - The workflow to check
|
|
392
|
+
* @returns True if at least one client has affinity for this workflow
|
|
393
|
+
*/
|
|
394
|
+
hasClientsForWorkflow(workflow) {
|
|
395
|
+
let workflowHash = workflow.structureHash;
|
|
396
|
+
if (!workflowHash) {
|
|
397
|
+
workflow.updateHash();
|
|
398
|
+
workflowHash = workflow.structureHash;
|
|
399
|
+
}
|
|
400
|
+
if (!workflowHash) {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
return this.clientRegistry.hasClientsForWorkflow(workflowHash);
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Get statistics about the pool's current state
|
|
407
|
+
* @returns Pool statistics including client counts and queue depths
|
|
408
|
+
*/
|
|
409
|
+
getPoolStats() {
|
|
410
|
+
const clients = Array.from(this.clientRegistry.clients.values());
|
|
411
|
+
return {
|
|
412
|
+
totalClients: clients.length,
|
|
413
|
+
idleClients: clients.filter(c => c.state === "idle").length,
|
|
414
|
+
busyClients: clients.filter(c => c.state === "busy").length,
|
|
415
|
+
offlineClients: clients.filter(c => c.state === "offline").length,
|
|
416
|
+
totalQueues: this.queues.size,
|
|
417
|
+
queues: Array.from(this.queues.entries()).map(([hash, queue]) => ({
|
|
418
|
+
workflowHash: hash,
|
|
419
|
+
pendingJobs: queue.queue.length,
|
|
420
|
+
type: hash === "general" ? "general" : "specific"
|
|
421
|
+
}))
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
}
|
|
314
425
|
//# sourceMappingURL=multi-workflow-pool.js.map
|