@cogitator-ai/worker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +97 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/metrics.d.ts +54 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +160 -0
- package/dist/metrics.js.map +1 -0
- package/dist/processors/agent.d.ts +11 -0
- package/dist/processors/agent.d.ts.map +1 -0
- package/dist/processors/agent.js +82 -0
- package/dist/processors/agent.js.map +1 -0
- package/dist/processors/index.d.ts +4 -0
- package/dist/processors/index.d.ts.map +1 -0
- package/dist/processors/index.js +4 -0
- package/dist/processors/index.js.map +1 -0
- package/dist/processors/swarm.d.ts +11 -0
- package/dist/processors/swarm.d.ts.map +1 -0
- package/dist/processors/swarm.js +107 -0
- package/dist/processors/swarm.js.map +1 -0
- package/dist/processors/workflow.d.ts +14 -0
- package/dist/processors/workflow.d.ts.map +1 -0
- package/dist/processors/workflow.js +36 -0
- package/dist/processors/workflow.js.map +1 -0
- package/dist/queue.d.ts +74 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +166 -0
- package/dist/queue.js.map +1 -0
- package/dist/types.d.ts +158 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/dist/worker.d.ts +50 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +125 -0
- package/dist/worker.js.map +1 -0
- package/package.json +51 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Workflow job processor
|
|
3
|
+
*
|
|
4
|
+
* Recreates a Workflow from serialized config and executes it.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Process a workflow job
|
|
8
|
+
*
|
|
9
|
+
* Note: Full workflow execution requires deserializing the workflow graph
|
|
10
|
+
* and recreating node instances. This is a placeholder implementation.
|
|
11
|
+
*/
|
|
12
|
+
export async function processWorkflowJob(payload) {
|
|
13
|
+
const { workflowConfig, input, runId } = payload;
|
|
14
|
+
const startTime = Date.now();
|
|
15
|
+
console.warn(`[worker] Workflow "${workflowConfig.name}" (${runId}) execution not fully implemented`);
|
|
16
|
+
const nodeResults = {};
|
|
17
|
+
for (const node of workflowConfig.nodes) {
|
|
18
|
+
nodeResults[node.id] = {
|
|
19
|
+
status: 'skipped',
|
|
20
|
+
reason: 'Worker workflow execution not fully implemented',
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const duration = Date.now() - startTime;
|
|
24
|
+
return {
|
|
25
|
+
type: 'workflow',
|
|
26
|
+
output: {
|
|
27
|
+
input,
|
|
28
|
+
warning: 'Workflow execution in worker is not fully implemented',
|
|
29
|
+
workflowId: workflowConfig.id,
|
|
30
|
+
workflowName: workflowConfig.name,
|
|
31
|
+
},
|
|
32
|
+
nodeResults,
|
|
33
|
+
duration,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=workflow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow.js","sourceRoot":"","sources":["../../src/processors/workflow.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAA2B;IAE3B,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,CAAC,IAAI,CACV,sBAAsB,cAAc,CAAC,IAAI,MAAM,KAAK,mCAAmC,CACxF,CAAC;IAEF,MAAM,WAAW,GAA4B,EAAE,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC;QACxC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG;YACrB,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,iDAAiD;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAExC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE;YACN,KAAK;YACL,OAAO,EAAE,uDAAuD;YAChE,UAAU,EAAE,cAAc,CAAC,EAAE;YAC7B,YAAY,EAAE,cAAc,CAAC,IAAI;SAClC;QACD,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
package/dist/queue.d.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job Queue for distributed agent execution
|
|
3
|
+
*
|
|
4
|
+
* Uses BullMQ with Redis for reliable job processing with:
|
|
5
|
+
* - Automatic retries with exponential backoff
|
|
6
|
+
* - Job priorities
|
|
7
|
+
* - Delayed execution
|
|
8
|
+
* - Rate limiting
|
|
9
|
+
*/
|
|
10
|
+
import { Queue, type Job } from 'bullmq';
|
|
11
|
+
import type { QueueConfig, QueueMetrics, JobPayload, AgentJobPayload, WorkflowJobPayload, SwarmJobPayload, SerializedAgent, SerializedWorkflow, SerializedSwarm } from './types';
|
|
12
|
+
export declare class JobQueue {
|
|
13
|
+
private queue;
|
|
14
|
+
constructor(config: QueueConfig);
|
|
15
|
+
/**
|
|
16
|
+
* Add an agent execution job to the queue
|
|
17
|
+
*/
|
|
18
|
+
addAgentJob(agentConfig: SerializedAgent, input: string, options?: {
|
|
19
|
+
threadId?: string;
|
|
20
|
+
priority?: number;
|
|
21
|
+
delay?: number;
|
|
22
|
+
metadata?: Record<string, unknown>;
|
|
23
|
+
}): Promise<Job<AgentJobPayload>>;
|
|
24
|
+
/**
|
|
25
|
+
* Add a workflow execution job to the queue
|
|
26
|
+
*/
|
|
27
|
+
addWorkflowJob(workflowConfig: SerializedWorkflow, input: Record<string, unknown>, options?: {
|
|
28
|
+
runId?: string;
|
|
29
|
+
priority?: number;
|
|
30
|
+
delay?: number;
|
|
31
|
+
metadata?: Record<string, unknown>;
|
|
32
|
+
}): Promise<Job<WorkflowJobPayload>>;
|
|
33
|
+
/**
|
|
34
|
+
* Add a swarm execution job to the queue
|
|
35
|
+
*/
|
|
36
|
+
addSwarmJob(swarmConfig: SerializedSwarm, input: string, options?: {
|
|
37
|
+
priority?: number;
|
|
38
|
+
delay?: number;
|
|
39
|
+
metadata?: Record<string, unknown>;
|
|
40
|
+
}): Promise<Job<SwarmJobPayload>>;
|
|
41
|
+
/**
|
|
42
|
+
* Get job by ID
|
|
43
|
+
*/
|
|
44
|
+
getJob(jobId: string): Promise<Job<JobPayload> | undefined>;
|
|
45
|
+
/**
|
|
46
|
+
* Get job state
|
|
47
|
+
*/
|
|
48
|
+
getJobState(jobId: string): Promise<string>;
|
|
49
|
+
/**
|
|
50
|
+
* Get queue metrics for monitoring and HPA
|
|
51
|
+
*/
|
|
52
|
+
getMetrics(): Promise<QueueMetrics>;
|
|
53
|
+
/**
|
|
54
|
+
* Pause the queue
|
|
55
|
+
*/
|
|
56
|
+
pause(): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* Resume the queue
|
|
59
|
+
*/
|
|
60
|
+
resume(): Promise<void>;
|
|
61
|
+
/**
|
|
62
|
+
* Clean old jobs
|
|
63
|
+
*/
|
|
64
|
+
clean(grace: number, limit: number, type: 'completed' | 'failed' | 'delayed' | 'active' | 'wait'): Promise<string[]>;
|
|
65
|
+
/**
|
|
66
|
+
* Get the underlying BullMQ queue (for advanced usage)
|
|
67
|
+
*/
|
|
68
|
+
getQueue(): Queue<JobPayload>;
|
|
69
|
+
/**
|
|
70
|
+
* Close the queue connection
|
|
71
|
+
*/
|
|
72
|
+
close(): Promise<void>;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,EAAE,MAAM,QAAQ,CAAC;AAEzC,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,UAAU,EACV,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,eAAe,EAChB,MAAM,SAAS,CAAC;AAIjB,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAoB;gBAErB,MAAM,EAAE,WAAW;IA4B/B;;OAEG;IACG,WAAW,CACf,WAAW,EAAE,eAAe,EAC5B,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GACA,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAoBhC;;OAEG;IACG,cAAc,CAClB,cAAc,EAAE,kBAAkB,EAClC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GACA,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAoBnC;;OAEG;IACG,WAAW,CACf,WAAW,EAAE,eAAe,EAC5B,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GACA,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAkBhC;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC;IAIjE;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAMjD;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAoBzC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B;;OAEG;IACG,KAAK,CACT,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,MAAM,GAC3D,OAAO,CAAC,MAAM,EAAE,CAAC;IAIpB;;OAEG;IACH,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC;IAI7B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
package/dist/queue.js
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job Queue for distributed agent execution
|
|
3
|
+
*
|
|
4
|
+
* Uses BullMQ with Redis for reliable job processing with:
|
|
5
|
+
* - Automatic retries with exponential backoff
|
|
6
|
+
* - Job priorities
|
|
7
|
+
* - Delayed execution
|
|
8
|
+
* - Rate limiting
|
|
9
|
+
*/
|
|
10
|
+
import { Queue } from 'bullmq';
|
|
11
|
+
import { nanoid } from 'nanoid';
|
|
12
|
+
const DEFAULT_QUEUE_NAME = 'cogitator-jobs';
|
|
13
|
+
export class JobQueue {
|
|
14
|
+
queue;
|
|
15
|
+
constructor(config) {
|
|
16
|
+
const connection = config.redis.cluster
|
|
17
|
+
? {
|
|
18
|
+
host: config.redis.cluster.nodes[0]?.host ?? 'localhost',
|
|
19
|
+
port: config.redis.cluster.nodes[0]?.port ?? 6379,
|
|
20
|
+
password: config.redis.password,
|
|
21
|
+
}
|
|
22
|
+
: {
|
|
23
|
+
host: config.redis.host ?? 'localhost',
|
|
24
|
+
port: config.redis.port ?? 6379,
|
|
25
|
+
password: config.redis.password,
|
|
26
|
+
};
|
|
27
|
+
this.queue = new Queue(config.name ?? DEFAULT_QUEUE_NAME, {
|
|
28
|
+
connection,
|
|
29
|
+
prefix: config.redis.cluster ? '{cogitator}' : 'cogitator',
|
|
30
|
+
defaultJobOptions: {
|
|
31
|
+
attempts: config.defaultJobOptions?.attempts ?? 3,
|
|
32
|
+
backoff: config.defaultJobOptions?.backoff ?? {
|
|
33
|
+
type: 'exponential',
|
|
34
|
+
delay: 1000,
|
|
35
|
+
},
|
|
36
|
+
removeOnComplete: config.defaultJobOptions?.removeOnComplete ?? 100,
|
|
37
|
+
removeOnFail: config.defaultJobOptions?.removeOnFail ?? 500,
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Add an agent execution job to the queue
|
|
43
|
+
*/
|
|
44
|
+
async addAgentJob(agentConfig, input, options) {
|
|
45
|
+
const jobId = nanoid();
|
|
46
|
+
const threadId = options?.threadId ?? nanoid();
|
|
47
|
+
const payload = {
|
|
48
|
+
type: 'agent',
|
|
49
|
+
jobId,
|
|
50
|
+
agentConfig,
|
|
51
|
+
input,
|
|
52
|
+
threadId,
|
|
53
|
+
metadata: options?.metadata,
|
|
54
|
+
};
|
|
55
|
+
return this.queue.add('agent', payload, {
|
|
56
|
+
jobId,
|
|
57
|
+
priority: options?.priority,
|
|
58
|
+
delay: options?.delay,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Add a workflow execution job to the queue
|
|
63
|
+
*/
|
|
64
|
+
async addWorkflowJob(workflowConfig, input, options) {
|
|
65
|
+
const jobId = nanoid();
|
|
66
|
+
const runId = options?.runId ?? nanoid();
|
|
67
|
+
const payload = {
|
|
68
|
+
type: 'workflow',
|
|
69
|
+
jobId,
|
|
70
|
+
workflowConfig,
|
|
71
|
+
input,
|
|
72
|
+
runId,
|
|
73
|
+
metadata: options?.metadata,
|
|
74
|
+
};
|
|
75
|
+
return this.queue.add('workflow', payload, {
|
|
76
|
+
jobId,
|
|
77
|
+
priority: options?.priority,
|
|
78
|
+
delay: options?.delay,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Add a swarm execution job to the queue
|
|
83
|
+
*/
|
|
84
|
+
async addSwarmJob(swarmConfig, input, options) {
|
|
85
|
+
const jobId = nanoid();
|
|
86
|
+
const payload = {
|
|
87
|
+
type: 'swarm',
|
|
88
|
+
jobId,
|
|
89
|
+
swarmConfig,
|
|
90
|
+
input,
|
|
91
|
+
metadata: options?.metadata,
|
|
92
|
+
};
|
|
93
|
+
return this.queue.add('swarm', payload, {
|
|
94
|
+
jobId,
|
|
95
|
+
priority: options?.priority,
|
|
96
|
+
delay: options?.delay,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Get job by ID
|
|
101
|
+
*/
|
|
102
|
+
async getJob(jobId) {
|
|
103
|
+
return this.queue.getJob(jobId);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Get job state
|
|
107
|
+
*/
|
|
108
|
+
async getJobState(jobId) {
|
|
109
|
+
const job = await this.queue.getJob(jobId);
|
|
110
|
+
if (!job)
|
|
111
|
+
return 'unknown';
|
|
112
|
+
return job.getState();
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Get queue metrics for monitoring and HPA
|
|
116
|
+
*/
|
|
117
|
+
async getMetrics() {
|
|
118
|
+
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
|
119
|
+
this.queue.getWaitingCount(),
|
|
120
|
+
this.queue.getActiveCount(),
|
|
121
|
+
this.queue.getCompletedCount(),
|
|
122
|
+
this.queue.getFailedCount(),
|
|
123
|
+
this.queue.getDelayedCount(),
|
|
124
|
+
]);
|
|
125
|
+
return {
|
|
126
|
+
waiting,
|
|
127
|
+
active,
|
|
128
|
+
completed,
|
|
129
|
+
failed,
|
|
130
|
+
delayed,
|
|
131
|
+
depth: waiting + delayed,
|
|
132
|
+
workerCount: 0,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Pause the queue
|
|
137
|
+
*/
|
|
138
|
+
async pause() {
|
|
139
|
+
await this.queue.pause();
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Resume the queue
|
|
143
|
+
*/
|
|
144
|
+
async resume() {
|
|
145
|
+
await this.queue.resume();
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Clean old jobs
|
|
149
|
+
*/
|
|
150
|
+
async clean(grace, limit, type) {
|
|
151
|
+
return this.queue.clean(grace, limit, type);
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Get the underlying BullMQ queue (for advanced usage)
|
|
155
|
+
*/
|
|
156
|
+
getQueue() {
|
|
157
|
+
return this.queue;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Close the queue connection
|
|
161
|
+
*/
|
|
162
|
+
async close() {
|
|
163
|
+
await this.queue.close();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,KAAK,EAAY,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAahC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAE5C,MAAM,OAAO,QAAQ;IACX,KAAK,CAAoB;IAEjC,YAAY,MAAmB;QAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO;YACrC,CAAC,CAAC;gBACE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,WAAW;gBACxD,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI;gBACjD,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;aAChC;YACH,CAAC,CAAC;gBACE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,WAAW;gBACtC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI;gBAC/B,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;aAChC,CAAC;QAEN,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EAAE;YACxD,UAAU;YACV,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW;YAC1D,iBAAiB,EAAE;gBACjB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,EAAE,QAAQ,IAAI,CAAC;gBACjD,OAAO,EAAE,MAAM,CAAC,iBAAiB,EAAE,OAAO,IAAI;oBAC5C,IAAI,EAAE,aAAa;oBACnB,KAAK,EAAE,IAAI;iBACZ;gBACD,gBAAgB,EAAE,MAAM,CAAC,iBAAiB,EAAE,gBAAgB,IAAI,GAAG;gBACnE,YAAY,EAAE,MAAM,CAAC,iBAAiB,EAAE,YAAY,IAAI,GAAG;aAC5D;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAKC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,EAAE,CAAC;QAE/C,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ;YACR,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACtC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAkC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,cAAkC,EAClC,KAA8B,EAC9B,OAKC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAEzC,MAAM,OAAO,GAAuB;YAClC,IAAI,EAAE,UAAU;YAChB,KAAK;YACL,cAAc;YACd,KAAK;YACL,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE;YACzC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAqC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAIC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACtC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAkC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE;YAC5B,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE;YAC9B,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO;YACP,MAAM;YACN,SAAS;YACT,MAAM;YACN,OAAO;YACP,KAAK,EAAE,OAAO,GAAG,OAAO;YACxB,WAAW,EAAE,CAAC;SACf,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,KAAa,EACb,KAAa,EACb,IAA4D;QAE5D,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;CACF"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker types for distributed job processing
|
|
3
|
+
*/
|
|
4
|
+
import type { ToolSchema } from '@cogitator-ai/types';
|
|
5
|
+
/**
|
|
6
|
+
* Serialized agent configuration for queue transport
|
|
7
|
+
* Tools are stored as schemas, recreated on worker side
|
|
8
|
+
*/
|
|
9
|
+
export interface SerializedAgent {
|
|
10
|
+
name: string;
|
|
11
|
+
instructions: string;
|
|
12
|
+
model: string;
|
|
13
|
+
provider: 'ollama' | 'openai' | 'anthropic';
|
|
14
|
+
temperature?: number;
|
|
15
|
+
maxTokens?: number;
|
|
16
|
+
tools: ToolSchema[];
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Serialized workflow configuration
|
|
20
|
+
*/
|
|
21
|
+
export interface SerializedWorkflow {
|
|
22
|
+
id: string;
|
|
23
|
+
name: string;
|
|
24
|
+
nodes: SerializedWorkflowNode[];
|
|
25
|
+
edges: SerializedWorkflowEdge[];
|
|
26
|
+
}
|
|
27
|
+
export interface SerializedWorkflowNode {
|
|
28
|
+
id: string;
|
|
29
|
+
type: 'agent' | 'transform' | 'condition' | 'parallel';
|
|
30
|
+
config: Record<string, unknown>;
|
|
31
|
+
}
|
|
32
|
+
export interface SerializedWorkflowEdge {
|
|
33
|
+
from: string;
|
|
34
|
+
to: string;
|
|
35
|
+
condition?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Serialized swarm configuration
|
|
39
|
+
*/
|
|
40
|
+
export interface SerializedSwarm {
|
|
41
|
+
topology: 'sequential' | 'hierarchical' | 'collaborative' | 'debate' | 'voting';
|
|
42
|
+
agents: SerializedAgent[];
|
|
43
|
+
coordinator?: SerializedAgent;
|
|
44
|
+
maxRounds?: number;
|
|
45
|
+
consensusThreshold?: number;
|
|
46
|
+
}
|
|
47
|
+
export interface AgentJobPayload {
|
|
48
|
+
type: 'agent';
|
|
49
|
+
jobId: string;
|
|
50
|
+
agentConfig: SerializedAgent;
|
|
51
|
+
input: string;
|
|
52
|
+
threadId: string;
|
|
53
|
+
metadata?: Record<string, unknown>;
|
|
54
|
+
}
|
|
55
|
+
export interface WorkflowJobPayload {
|
|
56
|
+
type: 'workflow';
|
|
57
|
+
jobId: string;
|
|
58
|
+
workflowConfig: SerializedWorkflow;
|
|
59
|
+
input: Record<string, unknown>;
|
|
60
|
+
runId: string;
|
|
61
|
+
metadata?: Record<string, unknown>;
|
|
62
|
+
}
|
|
63
|
+
export interface SwarmJobPayload {
|
|
64
|
+
type: 'swarm';
|
|
65
|
+
jobId: string;
|
|
66
|
+
swarmConfig: SerializedSwarm;
|
|
67
|
+
input: string;
|
|
68
|
+
metadata?: Record<string, unknown>;
|
|
69
|
+
}
|
|
70
|
+
export type JobPayload = AgentJobPayload | WorkflowJobPayload | SwarmJobPayload;
|
|
71
|
+
export interface AgentJobResult {
|
|
72
|
+
type: 'agent';
|
|
73
|
+
output: string;
|
|
74
|
+
toolCalls: {
|
|
75
|
+
name: string;
|
|
76
|
+
input: unknown;
|
|
77
|
+
output: unknown;
|
|
78
|
+
}[];
|
|
79
|
+
tokenUsage?: {
|
|
80
|
+
prompt: number;
|
|
81
|
+
completion: number;
|
|
82
|
+
total: number;
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
export interface WorkflowJobResult {
|
|
86
|
+
type: 'workflow';
|
|
87
|
+
output: Record<string, unknown>;
|
|
88
|
+
nodeResults: Record<string, unknown>;
|
|
89
|
+
duration: number;
|
|
90
|
+
}
|
|
91
|
+
export interface SwarmJobResult {
|
|
92
|
+
type: 'swarm';
|
|
93
|
+
output: string;
|
|
94
|
+
rounds: number;
|
|
95
|
+
agentOutputs: {
|
|
96
|
+
agent: string;
|
|
97
|
+
output: string;
|
|
98
|
+
}[];
|
|
99
|
+
}
|
|
100
|
+
export type JobResult = AgentJobResult | WorkflowJobResult | SwarmJobResult;
|
|
101
|
+
export interface QueueConfig {
|
|
102
|
+
/** Queue name (default: 'cogitator-jobs') */
|
|
103
|
+
name?: string;
|
|
104
|
+
/** Redis connection config */
|
|
105
|
+
redis: {
|
|
106
|
+
host?: string;
|
|
107
|
+
port?: number;
|
|
108
|
+
password?: string;
|
|
109
|
+
/** For cluster mode */
|
|
110
|
+
cluster?: {
|
|
111
|
+
nodes: {
|
|
112
|
+
host: string;
|
|
113
|
+
port: number;
|
|
114
|
+
}[];
|
|
115
|
+
};
|
|
116
|
+
};
|
|
117
|
+
/** Default job options */
|
|
118
|
+
defaultJobOptions?: {
|
|
119
|
+
/** Max attempts before failing */
|
|
120
|
+
attempts?: number;
|
|
121
|
+
/** Backoff strategy */
|
|
122
|
+
backoff?: {
|
|
123
|
+
type: 'exponential' | 'fixed';
|
|
124
|
+
delay: number;
|
|
125
|
+
};
|
|
126
|
+
/** Remove job after completion */
|
|
127
|
+
removeOnComplete?: boolean | number;
|
|
128
|
+
/** Remove job after failure */
|
|
129
|
+
removeOnFail?: boolean | number;
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
export interface WorkerConfig extends QueueConfig {
|
|
133
|
+
/** Number of worker instances */
|
|
134
|
+
workerCount?: number;
|
|
135
|
+
/** Concurrent jobs per worker */
|
|
136
|
+
concurrency?: number;
|
|
137
|
+
/** Lock duration in ms */
|
|
138
|
+
lockDuration?: number;
|
|
139
|
+
/** Stalled job check interval */
|
|
140
|
+
stalledInterval?: number;
|
|
141
|
+
}
|
|
142
|
+
export interface QueueMetrics {
|
|
143
|
+
/** Jobs waiting to be processed */
|
|
144
|
+
waiting: number;
|
|
145
|
+
/** Jobs currently being processed */
|
|
146
|
+
active: number;
|
|
147
|
+
/** Jobs completed successfully */
|
|
148
|
+
completed: number;
|
|
149
|
+
/** Jobs that failed */
|
|
150
|
+
failed: number;
|
|
151
|
+
/** Jobs scheduled for later */
|
|
152
|
+
delayed: number;
|
|
153
|
+
/** Total queue depth (waiting + delayed) */
|
|
154
|
+
depth: number;
|
|
155
|
+
/** Number of active workers */
|
|
156
|
+
workerCount: number;
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,KAAK,EAAE,sBAAsB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,OAAO,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,CAAC;IACvD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,YAAY,GAAG,cAAc,GAAG,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAChF,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,kBAAkB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,kBAAkB,GAAG,eAAe,CAAC;AAEhF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,OAAO,CAAC;KACjB,EAAE,CAAC;IACJ,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,iBAAiB,GAAG,cAAc,CAAC;AAE5E,MAAM,WAAW,WAAW;IAC1B,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,KAAK,EAAE;QACL,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,KAAK,EAAE;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;SACzC,CAAC;KACH,CAAC;IACF,0BAA0B;IAC1B,iBAAiB,CAAC,EAAE;QAClB,kCAAkC;QAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC;YAC9B,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;QACF,kCAAkC;QAClC,gBAAgB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;QACpC,+BAA+B;QAC/B,YAAY,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;KACjC,CAAC;CACH;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
package/dist/worker.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Pool for distributed job processing
|
|
3
|
+
*
|
|
4
|
+
* Manages multiple BullMQ workers that process agent, workflow, and swarm jobs.
|
|
5
|
+
* Supports graceful shutdown and health monitoring.
|
|
6
|
+
*/
|
|
7
|
+
import type { WorkerConfig, JobPayload, JobResult, QueueMetrics } from './types';
|
|
8
|
+
export interface WorkerPoolEvents {
|
|
9
|
+
onJobStarted?: (jobId: string, type: JobPayload['type']) => void;
|
|
10
|
+
onJobCompleted?: (jobId: string, result: JobResult) => void;
|
|
11
|
+
onJobFailed?: (jobId: string, error: Error) => void;
|
|
12
|
+
onWorkerError?: (error: Error) => void;
|
|
13
|
+
}
|
|
14
|
+
export declare class WorkerPool {
|
|
15
|
+
private workers;
|
|
16
|
+
private readonly config;
|
|
17
|
+
private readonly events;
|
|
18
|
+
private isRunning;
|
|
19
|
+
constructor(config: WorkerConfig, events?: WorkerPoolEvents);
|
|
20
|
+
/**
|
|
21
|
+
* Start the worker pool
|
|
22
|
+
*/
|
|
23
|
+
start(): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Process a job based on its type
|
|
26
|
+
*/
|
|
27
|
+
private processJob;
|
|
28
|
+
/**
|
|
29
|
+
* Get current worker count
|
|
30
|
+
*/
|
|
31
|
+
getWorkerCount(): number;
|
|
32
|
+
/**
|
|
33
|
+
* Check if pool is running
|
|
34
|
+
*/
|
|
35
|
+
isPoolRunning(): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Get metrics including worker count
|
|
38
|
+
*/
|
|
39
|
+
getMetrics(baseMetrics: Omit<QueueMetrics, 'workerCount'>): Promise<QueueMetrics>;
|
|
40
|
+
/**
|
|
41
|
+
* Graceful shutdown
|
|
42
|
+
* Waits for active jobs to complete before closing
|
|
43
|
+
*/
|
|
44
|
+
stop(timeout?: number): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Force shutdown without waiting for jobs
|
|
47
|
+
*/
|
|
48
|
+
forceStop(): Promise<void>;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,SAAS,EACT,YAAY,EACb,MAAM,SAAS,CAAC;AAOjB,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC;IACjE,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IAC5D,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACxC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,YAAY,EAAE,MAAM,GAAE,gBAAqB;IAK/D;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmD5B;;OAEG;YACW,UAAU;IAiBxB;;OAEG;IACH,cAAc,IAAI,MAAM;IAIxB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAOvF;;;OAGG;IACG,IAAI,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1C;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAKjC"}
|
package/dist/worker.js
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Pool for distributed job processing
|
|
3
|
+
*
|
|
4
|
+
* Manages multiple BullMQ workers that process agent, workflow, and swarm jobs.
|
|
5
|
+
* Supports graceful shutdown and health monitoring.
|
|
6
|
+
*/
|
|
7
|
+
import { Worker } from 'bullmq';
|
|
8
|
+
import { processAgentJob } from './processors/agent';
|
|
9
|
+
import { processWorkflowJob } from './processors/workflow';
|
|
10
|
+
import { processSwarmJob } from './processors/swarm';
|
|
11
|
+
const DEFAULT_QUEUE_NAME = 'cogitator-jobs';
|
|
12
|
+
export class WorkerPool {
|
|
13
|
+
workers = [];
|
|
14
|
+
config;
|
|
15
|
+
events;
|
|
16
|
+
isRunning = false;
|
|
17
|
+
constructor(config, events = {}) {
|
|
18
|
+
this.config = config;
|
|
19
|
+
this.events = events;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Start the worker pool
|
|
23
|
+
*/
|
|
24
|
+
async start() {
|
|
25
|
+
if (this.isRunning)
|
|
26
|
+
return;
|
|
27
|
+
const workerCount = this.config.workerCount ?? 1;
|
|
28
|
+
const concurrency = this.config.concurrency ?? 5;
|
|
29
|
+
const connection = this.config.redis.cluster
|
|
30
|
+
? {
|
|
31
|
+
host: this.config.redis.cluster.nodes[0]?.host ?? 'localhost',
|
|
32
|
+
port: this.config.redis.cluster.nodes[0]?.port ?? 6379,
|
|
33
|
+
password: this.config.redis.password,
|
|
34
|
+
}
|
|
35
|
+
: {
|
|
36
|
+
host: this.config.redis.host ?? 'localhost',
|
|
37
|
+
port: this.config.redis.port ?? 6379,
|
|
38
|
+
password: this.config.redis.password,
|
|
39
|
+
};
|
|
40
|
+
for (let i = 0; i < workerCount; i++) {
|
|
41
|
+
const worker = new Worker(this.config.name ?? DEFAULT_QUEUE_NAME, async (job) => this.processJob(job), {
|
|
42
|
+
connection,
|
|
43
|
+
prefix: this.config.redis.cluster ? '{cogitator}' : 'cogitator',
|
|
44
|
+
concurrency,
|
|
45
|
+
lockDuration: this.config.lockDuration ?? 30000,
|
|
46
|
+
stalledInterval: this.config.stalledInterval ?? 30000,
|
|
47
|
+
});
|
|
48
|
+
worker.on('completed', (job, result) => {
|
|
49
|
+
this.events.onJobCompleted?.(job.id ?? job.data.jobId, result);
|
|
50
|
+
});
|
|
51
|
+
worker.on('failed', (job, error) => {
|
|
52
|
+
if (job) {
|
|
53
|
+
this.events.onJobFailed?.(job.id ?? job.data.jobId, error);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
worker.on('error', (error) => {
|
|
57
|
+
this.events.onWorkerError?.(error);
|
|
58
|
+
});
|
|
59
|
+
this.workers.push(worker);
|
|
60
|
+
}
|
|
61
|
+
this.isRunning = true;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Process a job based on its type
|
|
65
|
+
*/
|
|
66
|
+
async processJob(job) {
|
|
67
|
+
this.events.onJobStarted?.(job.id ?? job.data.jobId, job.data.type);
|
|
68
|
+
switch (job.data.type) {
|
|
69
|
+
case 'agent':
|
|
70
|
+
return processAgentJob(job.data);
|
|
71
|
+
case 'workflow':
|
|
72
|
+
return processWorkflowJob(job.data);
|
|
73
|
+
case 'swarm':
|
|
74
|
+
return processSwarmJob(job.data);
|
|
75
|
+
default: {
|
|
76
|
+
const _exhaustive = job.data;
|
|
77
|
+
throw new Error(`Unknown job type: ${_exhaustive.type}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get current worker count
|
|
83
|
+
*/
|
|
84
|
+
getWorkerCount() {
|
|
85
|
+
return this.workers.length;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Check if pool is running
|
|
89
|
+
*/
|
|
90
|
+
isPoolRunning() {
|
|
91
|
+
return this.isRunning;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Get metrics including worker count
|
|
95
|
+
*/
|
|
96
|
+
async getMetrics(baseMetrics) {
|
|
97
|
+
return {
|
|
98
|
+
...baseMetrics,
|
|
99
|
+
workerCount: this.workers.length,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Graceful shutdown
|
|
104
|
+
* Waits for active jobs to complete before closing
|
|
105
|
+
*/
|
|
106
|
+
async stop(timeout = 30000) {
|
|
107
|
+
if (!this.isRunning)
|
|
108
|
+
return;
|
|
109
|
+
this.isRunning = false;
|
|
110
|
+
await Promise.race([
|
|
111
|
+
Promise.all(this.workers.map((w) => w.close())),
|
|
112
|
+
new Promise((resolve) => setTimeout(resolve, timeout)),
|
|
113
|
+
]);
|
|
114
|
+
this.workers = [];
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Force shutdown without waiting for jobs
|
|
118
|
+
*/
|
|
119
|
+
async forceStop() {
|
|
120
|
+
this.isRunning = false;
|
|
121
|
+
await Promise.all(this.workers.map((w) => w.close(true)));
|
|
122
|
+
this.workers = [];
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,EAAY,MAAM,QAAQ,CAAC;AAO1C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAS5C,MAAM,OAAO,UAAU;IACb,OAAO,GAAoC,EAAE,CAAC;IACrC,MAAM,CAAe;IACrB,MAAM,CAAmB;IAClC,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,MAAoB,EAAE,SAA2B,EAAE;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAEjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;YAC1C,CAAC,CAAC;gBACE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,WAAW;gBAC7D,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI;gBACtD,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ;aACrC;YACH,CAAC,CAAC;gBACE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,WAAW;gBAC3C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI;gBACpC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ;aACrC,CAAC;QAEN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EACtC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EACnC;gBACE,UAAU;gBACV,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW;gBAC/D,WAAW;gBACX,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK;gBAC/C,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,KAAK;aACtD,CACF,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBACrC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACjC,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAoB;QAC3C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpE,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtB,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,KAAK,UAAU;gBACb,OAAO,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACtC,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,WAAW,GAAU,GAAG,CAAC,IAAI,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,qBAAsB,WAA0B,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAA8C;QAC7D,OAAO;YACL,GAAG,WAAW;YACd,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SACjC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK;QACxB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/C,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;SAC7D,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;CACF"}
|