@supaku/agentfactory-cli 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/analyze-logs.d.ts +2 -2
- package/dist/src/analyze-logs.js +23 -194
- package/dist/src/cleanup.d.ts +2 -6
- package/dist/src/cleanup.d.ts.map +1 -1
- package/dist/src/cleanup.js +24 -225
- package/dist/src/lib/analyze-logs-runner.d.ts +47 -0
- package/dist/src/lib/analyze-logs-runner.d.ts.map +1 -0
- package/dist/src/lib/analyze-logs-runner.js +216 -0
- package/dist/src/lib/cleanup-runner.d.ts +28 -0
- package/dist/src/lib/cleanup-runner.d.ts.map +1 -0
- package/dist/src/lib/cleanup-runner.js +224 -0
- package/dist/src/lib/orchestrator-runner.d.ts +45 -0
- package/dist/src/lib/orchestrator-runner.d.ts.map +1 -0
- package/dist/src/lib/orchestrator-runner.js +144 -0
- package/dist/src/lib/queue-admin-runner.d.ts +30 -0
- package/dist/src/lib/queue-admin-runner.d.ts.map +1 -0
- package/dist/src/lib/queue-admin-runner.js +378 -0
- package/dist/src/lib/worker-fleet-runner.d.ts +28 -0
- package/dist/src/lib/worker-fleet-runner.d.ts.map +1 -0
- package/dist/src/lib/worker-fleet-runner.js +224 -0
- package/dist/src/lib/worker-runner.d.ts +31 -0
- package/dist/src/lib/worker-runner.d.ts.map +1 -0
- package/dist/src/lib/worker-runner.js +735 -0
- package/dist/src/orchestrator.d.ts +1 -1
- package/dist/src/orchestrator.js +42 -106
- package/dist/src/queue-admin.d.ts +3 -2
- package/dist/src/queue-admin.d.ts.map +1 -1
- package/dist/src/queue-admin.js +38 -360
- package/dist/src/worker-fleet.d.ts +1 -1
- package/dist/src/worker-fleet.js +23 -162
- package/dist/src/worker.d.ts +1 -0
- package/dist/src/worker.d.ts.map +1 -1
- package/dist/src/worker.js +33 -702
- package/package.json +28 -4
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue Admin Runner -- Programmatic API for the queue admin CLI.
|
|
3
|
+
*
|
|
4
|
+
* Extracts ALL command handlers from the queue-admin bin script so they can be
|
|
5
|
+
* invoked programmatically (e.g. from a Next.js route handler or test) without
|
|
6
|
+
* process.exit / dotenv / argv coupling.
|
|
7
|
+
*/
|
|
8
|
+
export type QueueAdminCommand = 'list' | 'sessions' | 'workers' | 'clear-claims' | 'clear-queue' | 'clear-all' | 'reset' | 'remove';
|
|
9
|
+
export interface QueueAdminRunnerConfig {
|
|
10
|
+
/** Command to execute */
|
|
11
|
+
command: QueueAdminCommand;
|
|
12
|
+
/** Session ID for 'remove' command (partial match) */
|
|
13
|
+
sessionId?: string;
|
|
14
|
+
}
|
|
15
|
+
export declare const C: {
|
|
16
|
+
readonly reset: "\u001B[0m";
|
|
17
|
+
readonly red: "\u001B[31m";
|
|
18
|
+
readonly green: "\u001B[32m";
|
|
19
|
+
readonly yellow: "\u001B[33m";
|
|
20
|
+
readonly cyan: "\u001B[36m";
|
|
21
|
+
readonly gray: "\u001B[90m";
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Run a queue admin command programmatically.
|
|
25
|
+
*
|
|
26
|
+
* Throws if REDIS_URL is not set or if the 'remove' command is called without
|
|
27
|
+
* a sessionId.
|
|
28
|
+
*/
|
|
29
|
+
export declare function runQueueAdmin(config: QueueAdminRunnerConfig): Promise<void>;
|
|
30
|
+
//# sourceMappingURL=queue-admin-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue-admin-runner.d.ts","sourceRoot":"","sources":["../../../src/lib/queue-admin-runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAkBH,MAAM,MAAM,iBAAiB,GACzB,MAAM,GACN,UAAU,GACV,SAAS,GACT,cAAc,GACd,aAAa,GACb,WAAW,GACX,OAAO,GACP,QAAQ,CAAA;AAEZ,MAAM,WAAW,sBAAsB;IACrC,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAA;IAC1B,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAgBD,eAAO,MAAM,CAAC;;;;;;;CAOJ,CAAA;AAgZV;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BjF"}
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue Admin Runner -- Programmatic API for the queue admin CLI.
|
|
3
|
+
*
|
|
4
|
+
* Extracts ALL command handlers from the queue-admin bin script so they can be
|
|
5
|
+
* invoked programmatically (e.g. from a Next.js route handler or test) without
|
|
6
|
+
* process.exit / dotenv / argv coupling.
|
|
7
|
+
*/
|
|
8
|
+
import { getRedisClient, redisKeys, redisDel, redisGet, redisSet, redisZRangeByScore, redisZRem, redisHGetAll, disconnectRedis, } from '@supaku/agentfactory-server';
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// Redis key constants
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
const WORK_QUEUE_KEY = 'work:queue';
|
|
13
|
+
const WORK_ITEMS_KEY = 'work:items';
|
|
14
|
+
const WORK_CLAIM_PREFIX = 'work:claim:';
|
|
15
|
+
const SESSION_KEY_PREFIX = 'agent:session:';
|
|
16
|
+
const WORKER_PREFIX = 'work:worker:';
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// ANSI colors
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
export const C = {
|
|
21
|
+
reset: '\x1b[0m',
|
|
22
|
+
red: '\x1b[31m',
|
|
23
|
+
green: '\x1b[32m',
|
|
24
|
+
yellow: '\x1b[33m',
|
|
25
|
+
cyan: '\x1b[36m',
|
|
26
|
+
gray: '\x1b[90m',
|
|
27
|
+
};
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
// Helpers
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
function ensureRedis() {
|
|
32
|
+
if (!process.env.REDIS_URL) {
|
|
33
|
+
throw new Error('REDIS_URL environment variable is not set');
|
|
34
|
+
}
|
|
35
|
+
// Initialize the Redis client
|
|
36
|
+
getRedisClient();
|
|
37
|
+
}
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
// Command handlers
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
async function listQueue() {
|
|
42
|
+
ensureRedis();
|
|
43
|
+
// Get items from sorted set (current queue format)
|
|
44
|
+
const queuedSessionIds = await redisZRangeByScore(WORK_QUEUE_KEY, '-inf', '+inf');
|
|
45
|
+
const workItems = await redisHGetAll(WORK_ITEMS_KEY);
|
|
46
|
+
const workItemCount = Object.keys(workItems).length;
|
|
47
|
+
console.log(`\n${C.cyan}Work Queue${C.reset} (${Math.max(queuedSessionIds.length, workItemCount)} items):`);
|
|
48
|
+
console.log('='.repeat(60));
|
|
49
|
+
if (workItemCount === 0 && queuedSessionIds.length === 0) {
|
|
50
|
+
console.log('(empty)');
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
for (const [sessionId, itemJson] of Object.entries(workItems)) {
|
|
54
|
+
try {
|
|
55
|
+
const work = JSON.parse(itemJson);
|
|
56
|
+
console.log(`- ${work.issueIdentifier ?? sessionId.slice(0, 8)} (session: ${sessionId.slice(0, 8)}...)`);
|
|
57
|
+
console.log(` Priority: ${work.priority ?? 'none'}, WorkType: ${work.workType ?? 'development'}`);
|
|
58
|
+
if (work.queuedAt) {
|
|
59
|
+
console.log(` Queued: ${new Date(work.queuedAt).toISOString()}`);
|
|
60
|
+
}
|
|
61
|
+
if (work.claudeSessionId) {
|
|
62
|
+
console.log(` ${C.yellow}Has claudeSessionId: ${work.claudeSessionId.substring(0, 12)}${C.reset}`);
|
|
63
|
+
}
|
|
64
|
+
if (work.prompt) {
|
|
65
|
+
console.log(` Prompt: "${work.prompt.slice(0, 50)}..."`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
console.log(`- [invalid JSON]: ${sessionId}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
await disconnectRedis();
|
|
74
|
+
}
|
|
75
|
+
async function listSessions() {
|
|
76
|
+
ensureRedis();
|
|
77
|
+
const keys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
78
|
+
console.log(`\n${C.cyan}Sessions${C.reset} (${keys.length} total):`);
|
|
79
|
+
console.log('='.repeat(60));
|
|
80
|
+
if (keys.length === 0) {
|
|
81
|
+
console.log('(none)');
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
for (const key of keys) {
|
|
85
|
+
const session = await redisGet(key);
|
|
86
|
+
if (session) {
|
|
87
|
+
const statusColors = {
|
|
88
|
+
pending: C.yellow,
|
|
89
|
+
claimed: C.cyan,
|
|
90
|
+
running: C.cyan,
|
|
91
|
+
completed: C.green,
|
|
92
|
+
failed: C.red,
|
|
93
|
+
stopped: C.yellow,
|
|
94
|
+
};
|
|
95
|
+
const statusColor = statusColors[session.status] || '';
|
|
96
|
+
console.log(`- ${session.issueIdentifier || session.issueId.slice(0, 8)} [${statusColor}${session.status}${C.reset}]`);
|
|
97
|
+
console.log(` Session: ${session.linearSessionId.slice(0, 12)}...`);
|
|
98
|
+
if (session.workType) {
|
|
99
|
+
console.log(` WorkType: ${session.workType}`);
|
|
100
|
+
}
|
|
101
|
+
console.log(` Updated: ${new Date(session.updatedAt * 1000).toISOString()}`);
|
|
102
|
+
if (session.workerId) {
|
|
103
|
+
console.log(` Worker: ${session.workerId}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
await disconnectRedis();
|
|
109
|
+
}
|
|
110
|
+
async function listWorkersFn() {
|
|
111
|
+
ensureRedis();
|
|
112
|
+
const keys = await redisKeys(`${WORKER_PREFIX}*`);
|
|
113
|
+
console.log(`\n${C.cyan}Workers${C.reset} (${keys.length} total):`);
|
|
114
|
+
console.log('='.repeat(60));
|
|
115
|
+
if (keys.length === 0) {
|
|
116
|
+
console.log('(none)');
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
for (const key of keys) {
|
|
120
|
+
const worker = await redisGet(key);
|
|
121
|
+
if (worker) {
|
|
122
|
+
const statusColor = worker.status === 'active' ? C.green : C.yellow;
|
|
123
|
+
console.log(`- ${worker.id.slice(0, 12)} [${statusColor}${worker.status}${C.reset}]`);
|
|
124
|
+
if (worker.hostname) {
|
|
125
|
+
console.log(` Hostname: ${worker.hostname}`);
|
|
126
|
+
}
|
|
127
|
+
console.log(` Capacity: ${worker.activeCount ?? 0}/${worker.capacity ?? '?'}`);
|
|
128
|
+
if (worker.lastHeartbeat) {
|
|
129
|
+
const ago = Math.round((Date.now() - worker.lastHeartbeat) / 1000);
|
|
130
|
+
console.log(` Last heartbeat: ${ago}s ago`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
await disconnectRedis();
|
|
136
|
+
}
|
|
137
|
+
async function clearClaims() {
|
|
138
|
+
ensureRedis();
|
|
139
|
+
console.log('Clearing work claims...');
|
|
140
|
+
const claimKeys = await redisKeys(`${WORK_CLAIM_PREFIX}*`);
|
|
141
|
+
console.log(`Found ${claimKeys.length} claim(s)`);
|
|
142
|
+
if (claimKeys.length === 0) {
|
|
143
|
+
console.log('No claims to clear');
|
|
144
|
+
await disconnectRedis();
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
let deleted = 0;
|
|
148
|
+
for (const key of claimKeys) {
|
|
149
|
+
const result = await redisDel(key);
|
|
150
|
+
if (result > 0) {
|
|
151
|
+
console.log(` Deleted: ${key}`);
|
|
152
|
+
deleted++;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
console.log(`\nCleared ${deleted} claim(s)`);
|
|
156
|
+
await disconnectRedis();
|
|
157
|
+
}
|
|
158
|
+
async function clearQueue() {
|
|
159
|
+
ensureRedis();
|
|
160
|
+
console.log('Clearing work queue...');
|
|
161
|
+
const queuedSessionIds = await redisZRangeByScore(WORK_QUEUE_KEY, '-inf', '+inf');
|
|
162
|
+
console.log(`Found ${queuedSessionIds.length} item(s) in queue sorted set`);
|
|
163
|
+
const workItems = await redisHGetAll(WORK_ITEMS_KEY);
|
|
164
|
+
const workItemCount = Object.keys(workItems).length;
|
|
165
|
+
console.log(`Found ${workItemCount} item(s) in work items hash`);
|
|
166
|
+
// Show what we're clearing
|
|
167
|
+
for (const [sessionId, itemJson] of Object.entries(workItems)) {
|
|
168
|
+
try {
|
|
169
|
+
const work = JSON.parse(itemJson);
|
|
170
|
+
console.log(` - ${work.issueIdentifier ?? sessionId.slice(0, 8)} (workType: ${work.workType || 'development'})`);
|
|
171
|
+
if (work.claudeSessionId) {
|
|
172
|
+
console.log(` ${C.yellow}Has claudeSessionId: ${work.claudeSessionId.substring(0, 12)}${C.reset}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
catch {
|
|
176
|
+
console.log(` - [invalid JSON]: ${sessionId}`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
let cleared = 0;
|
|
180
|
+
if (queuedSessionIds.length > 0) {
|
|
181
|
+
await redisDel(WORK_QUEUE_KEY);
|
|
182
|
+
cleared++;
|
|
183
|
+
}
|
|
184
|
+
if (workItemCount > 0) {
|
|
185
|
+
await redisDel(WORK_ITEMS_KEY);
|
|
186
|
+
cleared++;
|
|
187
|
+
}
|
|
188
|
+
const totalItems = Math.max(queuedSessionIds.length, workItemCount);
|
|
189
|
+
if (cleared > 0) {
|
|
190
|
+
console.log(`\nCleared ${totalItems} item(s) from work queue`);
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
console.log('\nQueue was already empty');
|
|
194
|
+
}
|
|
195
|
+
await disconnectRedis();
|
|
196
|
+
}
|
|
197
|
+
async function clearAll() {
|
|
198
|
+
ensureRedis();
|
|
199
|
+
console.log('Clearing ALL state...\n');
|
|
200
|
+
// Clear work queue
|
|
201
|
+
const queuedSessionIds = await redisZRangeByScore(WORK_QUEUE_KEY, '-inf', '+inf');
|
|
202
|
+
const workItems = await redisHGetAll(WORK_ITEMS_KEY);
|
|
203
|
+
if (queuedSessionIds.length > 0)
|
|
204
|
+
await redisDel(WORK_QUEUE_KEY);
|
|
205
|
+
if (Object.keys(workItems).length > 0)
|
|
206
|
+
await redisDel(WORK_ITEMS_KEY);
|
|
207
|
+
console.log(`Cleared ${Math.max(queuedSessionIds.length, Object.keys(workItems).length)} queue items`);
|
|
208
|
+
// Clear all sessions
|
|
209
|
+
const sessionKeys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
210
|
+
for (const key of sessionKeys) {
|
|
211
|
+
await redisDel(key);
|
|
212
|
+
}
|
|
213
|
+
console.log(`Cleared ${sessionKeys.length} sessions`);
|
|
214
|
+
// Clear all claims
|
|
215
|
+
const claimKeys = await redisKeys(`${WORK_CLAIM_PREFIX}*`);
|
|
216
|
+
for (const key of claimKeys) {
|
|
217
|
+
await redisDel(key);
|
|
218
|
+
}
|
|
219
|
+
console.log(`Cleared ${claimKeys.length} claims`);
|
|
220
|
+
// Clear all workers
|
|
221
|
+
const workerKeys = await redisKeys(`${WORKER_PREFIX}*`);
|
|
222
|
+
for (const key of workerKeys) {
|
|
223
|
+
await redisDel(key);
|
|
224
|
+
}
|
|
225
|
+
console.log(`Cleared ${workerKeys.length} worker registrations`);
|
|
226
|
+
console.log('\nAll cleared!');
|
|
227
|
+
await disconnectRedis();
|
|
228
|
+
}
|
|
229
|
+
async function resetWorkState() {
|
|
230
|
+
ensureRedis();
|
|
231
|
+
console.log('Resetting work state...');
|
|
232
|
+
console.log('-'.repeat(60));
|
|
233
|
+
let totalCleared = 0;
|
|
234
|
+
// 1. Clear work claims
|
|
235
|
+
console.log('\nClearing work claims...');
|
|
236
|
+
const claimKeys = await redisKeys(`${WORK_CLAIM_PREFIX}*`);
|
|
237
|
+
console.log(` Found ${claimKeys.length} claim(s)`);
|
|
238
|
+
for (const key of claimKeys) {
|
|
239
|
+
const result = await redisDel(key);
|
|
240
|
+
if (result > 0) {
|
|
241
|
+
console.log(` Deleted: ${key}`);
|
|
242
|
+
totalCleared++;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
// 2. Clear work queue
|
|
246
|
+
console.log('\nClearing work queue...');
|
|
247
|
+
const queuedSessionIds = await redisZRangeByScore(WORK_QUEUE_KEY, '-inf', '+inf');
|
|
248
|
+
console.log(` Found ${queuedSessionIds.length} queued item(s) in sorted set`);
|
|
249
|
+
const workItems = await redisHGetAll(WORK_ITEMS_KEY);
|
|
250
|
+
const workItemCount = Object.keys(workItems).length;
|
|
251
|
+
console.log(` Found ${workItemCount} item(s) in work items hash`);
|
|
252
|
+
for (const [sessionId, itemJson] of Object.entries(workItems)) {
|
|
253
|
+
try {
|
|
254
|
+
const work = JSON.parse(itemJson);
|
|
255
|
+
console.log(` - ${work.issueIdentifier ?? sessionId.slice(0, 8)} (workType: ${work.workType || 'development'})`);
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
console.log(` - [invalid item: ${sessionId}]`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (queuedSessionIds.length > 0 || workItemCount > 0) {
|
|
262
|
+
await redisDel(WORK_QUEUE_KEY);
|
|
263
|
+
await redisDel(WORK_ITEMS_KEY);
|
|
264
|
+
totalCleared += Math.max(queuedSessionIds.length, workItemCount);
|
|
265
|
+
console.log(` Cleared queue and items hash`);
|
|
266
|
+
}
|
|
267
|
+
// 3. Reset stuck sessions
|
|
268
|
+
console.log('\nResetting stuck sessions...');
|
|
269
|
+
const sessionKeys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
270
|
+
console.log(` Found ${sessionKeys.length} session(s)`);
|
|
271
|
+
let sessionsReset = 0;
|
|
272
|
+
for (const key of sessionKeys) {
|
|
273
|
+
const session = await redisGet(key);
|
|
274
|
+
if (!session)
|
|
275
|
+
continue;
|
|
276
|
+
if (session.status === 'running' || session.status === 'claimed') {
|
|
277
|
+
console.log(` Resetting: ${session.issueIdentifier || session.linearSessionId}`);
|
|
278
|
+
console.log(` Status: ${session.status}, WorkerId: ${session.workerId || 'none'}`);
|
|
279
|
+
const updated = {
|
|
280
|
+
...session,
|
|
281
|
+
status: 'pending',
|
|
282
|
+
workerId: undefined,
|
|
283
|
+
claimedAt: undefined,
|
|
284
|
+
claudeSessionId: undefined,
|
|
285
|
+
updatedAt: Math.floor(Date.now() / 1000),
|
|
286
|
+
};
|
|
287
|
+
await redisSet(key, updated, 24 * 60 * 60);
|
|
288
|
+
sessionsReset++;
|
|
289
|
+
console.log(` Reset to pending`);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
console.log('\n' + '-'.repeat(60));
|
|
293
|
+
console.log(`\nReset complete:`);
|
|
294
|
+
console.log(` - Claims cleared: ${claimKeys.length}`);
|
|
295
|
+
console.log(` - Queue items cleared: ${Math.max(queuedSessionIds.length, workItemCount)}`);
|
|
296
|
+
console.log(` - Sessions reset: ${sessionsReset}`);
|
|
297
|
+
await disconnectRedis();
|
|
298
|
+
}
|
|
299
|
+
async function removeSession(sessionId) {
|
|
300
|
+
ensureRedis();
|
|
301
|
+
let found = false;
|
|
302
|
+
// Find session by partial ID match
|
|
303
|
+
const keys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
304
|
+
for (const key of keys) {
|
|
305
|
+
if (key.includes(sessionId)) {
|
|
306
|
+
await redisDel(key);
|
|
307
|
+
console.log(`Removed session: ${key.replace(SESSION_KEY_PREFIX, '')}`);
|
|
308
|
+
found = true;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
// Also remove from queue if present
|
|
312
|
+
const workItems = await redisHGetAll(WORK_ITEMS_KEY);
|
|
313
|
+
for (const [sid, itemJson] of Object.entries(workItems)) {
|
|
314
|
+
if (sid.includes(sessionId)) {
|
|
315
|
+
// Remove from hash via direct Redis command
|
|
316
|
+
const redis = getRedisClient();
|
|
317
|
+
await redis.hdel(WORK_ITEMS_KEY, sid);
|
|
318
|
+
// Remove from sorted set
|
|
319
|
+
await redisZRem(WORK_QUEUE_KEY, sid);
|
|
320
|
+
const work = JSON.parse(itemJson);
|
|
321
|
+
console.log(`Removed from queue: ${work.issueIdentifier ?? sid.slice(0, 8)}`);
|
|
322
|
+
found = true;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
// Remove claim if present
|
|
326
|
+
const claimKeys = await redisKeys(`${WORK_CLAIM_PREFIX}*`);
|
|
327
|
+
for (const key of claimKeys) {
|
|
328
|
+
if (key.includes(sessionId)) {
|
|
329
|
+
await redisDel(key);
|
|
330
|
+
console.log(`Removed claim: ${key}`);
|
|
331
|
+
found = true;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (!found) {
|
|
335
|
+
console.log(`No session found matching: ${sessionId}`);
|
|
336
|
+
}
|
|
337
|
+
await disconnectRedis();
|
|
338
|
+
}
|
|
339
|
+
// ---------------------------------------------------------------------------
|
|
340
|
+
// Public entry point
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
/**
|
|
343
|
+
* Run a queue admin command programmatically.
|
|
344
|
+
*
|
|
345
|
+
* Throws if REDIS_URL is not set or if the 'remove' command is called without
|
|
346
|
+
* a sessionId.
|
|
347
|
+
*/
|
|
348
|
+
export async function runQueueAdmin(config) {
|
|
349
|
+
switch (config.command) {
|
|
350
|
+
case 'list':
|
|
351
|
+
await listQueue();
|
|
352
|
+
break;
|
|
353
|
+
case 'sessions':
|
|
354
|
+
await listSessions();
|
|
355
|
+
break;
|
|
356
|
+
case 'workers':
|
|
357
|
+
await listWorkersFn();
|
|
358
|
+
break;
|
|
359
|
+
case 'clear-claims':
|
|
360
|
+
await clearClaims();
|
|
361
|
+
break;
|
|
362
|
+
case 'clear-queue':
|
|
363
|
+
await clearQueue();
|
|
364
|
+
break;
|
|
365
|
+
case 'clear-all':
|
|
366
|
+
await clearAll();
|
|
367
|
+
break;
|
|
368
|
+
case 'reset':
|
|
369
|
+
await resetWorkState();
|
|
370
|
+
break;
|
|
371
|
+
case 'remove':
|
|
372
|
+
if (!config.sessionId) {
|
|
373
|
+
throw new Error('remove command requires a sessionId');
|
|
374
|
+
}
|
|
375
|
+
await removeSession(config.sessionId);
|
|
376
|
+
break;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Fleet Runner — Programmatic API for the worker fleet manager CLI.
|
|
3
|
+
*
|
|
4
|
+
* Spawns and manages multiple worker processes for parallel agent execution.
|
|
5
|
+
* Each worker runs as a separate OS process with its own resources.
|
|
6
|
+
*/
|
|
7
|
+
export interface FleetRunnerConfig {
|
|
8
|
+
/** Number of worker processes (default: CPU cores / 2) */
|
|
9
|
+
workers?: number;
|
|
10
|
+
/** Agents per worker (default: 3) */
|
|
11
|
+
capacity?: number;
|
|
12
|
+
/** Show configuration without starting workers (default: false) */
|
|
13
|
+
dryRun?: boolean;
|
|
14
|
+
/** Coordinator API URL (required) */
|
|
15
|
+
apiUrl: string;
|
|
16
|
+
/** API key for authentication (required) */
|
|
17
|
+
apiKey: string;
|
|
18
|
+
/** Path to the worker script/binary (default: auto-detect from this package) */
|
|
19
|
+
workerScript?: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Run a fleet of worker processes.
|
|
23
|
+
*
|
|
24
|
+
* The caller can cancel via the optional {@link AbortSignal}. The function
|
|
25
|
+
* returns once all workers have been stopped.
|
|
26
|
+
*/
|
|
27
|
+
export declare function runWorkerFleet(config: FleetRunnerConfig, signal?: AbortSignal): Promise<void>;
|
|
28
|
+
//# sourceMappingURL=worker-fleet-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-fleet-runner.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-fleet-runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,MAAM,WAAW,iBAAiB;IAChC,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAA;IACd,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAqSD;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,iBAAiB,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,IAAI,CAAC,CAmBf"}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Fleet Runner — Programmatic API for the worker fleet manager CLI.
|
|
3
|
+
*
|
|
4
|
+
* Spawns and manages multiple worker processes for parallel agent execution.
|
|
5
|
+
* Each worker runs as a separate OS process with its own resources.
|
|
6
|
+
*/
|
|
7
|
+
import { spawn } from 'child_process';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import { fileURLToPath } from 'url';
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// ANSI colors
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
const colors = {
|
|
15
|
+
reset: '\x1b[0m',
|
|
16
|
+
red: '\x1b[31m',
|
|
17
|
+
green: '\x1b[32m',
|
|
18
|
+
yellow: '\x1b[33m',
|
|
19
|
+
blue: '\x1b[34m',
|
|
20
|
+
magenta: '\x1b[35m',
|
|
21
|
+
cyan: '\x1b[36m',
|
|
22
|
+
gray: '\x1b[90m',
|
|
23
|
+
};
|
|
24
|
+
const workerColors = [
|
|
25
|
+
colors.cyan,
|
|
26
|
+
colors.magenta,
|
|
27
|
+
colors.yellow,
|
|
28
|
+
colors.green,
|
|
29
|
+
colors.blue,
|
|
30
|
+
];
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
// Helpers
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
function timestamp() {
|
|
35
|
+
return new Date().toLocaleTimeString('en-US', { hour12: false });
|
|
36
|
+
}
|
|
37
|
+
function fleetLog(workerId, color, level, message) {
|
|
38
|
+
const prefix = workerId !== null
|
|
39
|
+
? `[W${workerId.toString().padStart(2, '0')}]`
|
|
40
|
+
: '[FLEET]';
|
|
41
|
+
const levelColor = level === 'ERR' ? colors.red : level === 'WRN' ? colors.yellow : colors.gray;
|
|
42
|
+
console.log(`${colors.gray}${timestamp()}${colors.reset} ${color}${prefix}${colors.reset} ${levelColor}${level}${colors.reset} ${message}`);
|
|
43
|
+
}
|
|
44
|
+
function getDefaultWorkerScript() {
|
|
45
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
46
|
+
const __dirname = path.dirname(__filename);
|
|
47
|
+
// Runner lives in lib/, worker.js is one level up
|
|
48
|
+
return path.resolve(__dirname, '..', 'worker.js');
|
|
49
|
+
}
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
// WorkerFleet class (internal)
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
class WorkerFleet {
|
|
54
|
+
workers = new Map();
|
|
55
|
+
fleetConfig;
|
|
56
|
+
shuttingDown = false;
|
|
57
|
+
workerScript;
|
|
58
|
+
resolveRunning = null;
|
|
59
|
+
constructor(fleetConfig, workerScript) {
|
|
60
|
+
this.fleetConfig = fleetConfig;
|
|
61
|
+
this.workerScript = workerScript;
|
|
62
|
+
}
|
|
63
|
+
async start(signal) {
|
|
64
|
+
const { workers, capacity, dryRun } = this.fleetConfig;
|
|
65
|
+
const totalCapacity = workers * capacity;
|
|
66
|
+
console.log(`
|
|
67
|
+
${colors.cyan}================================================================${colors.reset}
|
|
68
|
+
${colors.cyan} AgentFactory Worker Fleet Manager${colors.reset}
|
|
69
|
+
${colors.cyan}================================================================${colors.reset}
|
|
70
|
+
Workers: ${colors.green}${workers}${colors.reset}
|
|
71
|
+
Capacity/Worker: ${colors.green}${capacity}${colors.reset}
|
|
72
|
+
Total Capacity: ${colors.green}${totalCapacity}${colors.reset} concurrent agents
|
|
73
|
+
|
|
74
|
+
System:
|
|
75
|
+
CPU Cores: ${os.cpus().length}
|
|
76
|
+
Total RAM: ${Math.round(os.totalmem() / 1024 / 1024 / 1024)} GB
|
|
77
|
+
Free RAM: ${Math.round(os.freemem() / 1024 / 1024 / 1024)} GB
|
|
78
|
+
${colors.cyan}================================================================${colors.reset}
|
|
79
|
+
`);
|
|
80
|
+
if (dryRun) {
|
|
81
|
+
console.log(`${colors.yellow}Dry run mode - not starting workers${colors.reset}`);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Wire up AbortSignal for graceful shutdown
|
|
85
|
+
const onAbort = () => this.shutdown('AbortSignal');
|
|
86
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
87
|
+
try {
|
|
88
|
+
// Spawn workers with staggered start to avoid thundering herd
|
|
89
|
+
for (let i = 0; i < workers; i++) {
|
|
90
|
+
if (signal?.aborted)
|
|
91
|
+
break;
|
|
92
|
+
await this.spawnWorker(i);
|
|
93
|
+
if (i < workers - 1) {
|
|
94
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (signal?.aborted)
|
|
98
|
+
return;
|
|
99
|
+
fleetLog(null, colors.green, 'INF', `All ${workers} workers started`);
|
|
100
|
+
// Keep the fleet manager running until shutdown
|
|
101
|
+
await new Promise((resolve) => {
|
|
102
|
+
this.resolveRunning = resolve;
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
signal?.removeEventListener('abort', onAbort);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async spawnWorker(id) {
|
|
110
|
+
const color = workerColors[id % workerColors.length];
|
|
111
|
+
const existingWorker = this.workers.get(id);
|
|
112
|
+
const restartCount = existingWorker?.restartCount ?? 0;
|
|
113
|
+
fleetLog(id, color, 'INF', `Starting worker (capacity: ${this.fleetConfig.capacity})${restartCount > 0 ? ` [restart #${restartCount}]` : ''}`);
|
|
114
|
+
const workerProcess = spawn('node', [
|
|
115
|
+
this.workerScript,
|
|
116
|
+
'--capacity',
|
|
117
|
+
String(this.fleetConfig.capacity),
|
|
118
|
+
'--api-url',
|
|
119
|
+
this.fleetConfig.apiUrl,
|
|
120
|
+
'--api-key',
|
|
121
|
+
this.fleetConfig.apiKey,
|
|
122
|
+
], {
|
|
123
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
124
|
+
env: {
|
|
125
|
+
...process.env,
|
|
126
|
+
WORKER_FLEET_ID: String(id),
|
|
127
|
+
},
|
|
128
|
+
cwd: process.cwd(),
|
|
129
|
+
});
|
|
130
|
+
const workerInfo = {
|
|
131
|
+
id,
|
|
132
|
+
process: workerProcess,
|
|
133
|
+
color,
|
|
134
|
+
startedAt: new Date(),
|
|
135
|
+
restartCount,
|
|
136
|
+
};
|
|
137
|
+
this.workers.set(id, workerInfo);
|
|
138
|
+
// Handle stdout — prefix with worker ID
|
|
139
|
+
workerProcess.stdout?.on('data', (data) => {
|
|
140
|
+
const lines = data.toString().trim().split('\n');
|
|
141
|
+
for (const line of lines) {
|
|
142
|
+
if (line.trim()) {
|
|
143
|
+
console.log(`${color}[W${id.toString().padStart(2, '0')}]${colors.reset} ${line}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
// Handle stderr
|
|
148
|
+
workerProcess.stderr?.on('data', (data) => {
|
|
149
|
+
const lines = data.toString().trim().split('\n');
|
|
150
|
+
for (const line of lines) {
|
|
151
|
+
if (line.trim()) {
|
|
152
|
+
console.log(`${color}[W${id.toString().padStart(2, '0')}]${colors.reset} ${colors.red}${line}${colors.reset}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
// Handle worker exit
|
|
157
|
+
workerProcess.on('exit', (code, sig) => {
|
|
158
|
+
if (this.shuttingDown) {
|
|
159
|
+
fleetLog(id, color, 'INF', `Worker stopped (code: ${code}, signal: ${sig})`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
fleetLog(id, color, 'WRN', `Worker exited unexpectedly (code: ${code}, signal: ${sig}) - restarting in 5s`);
|
|
163
|
+
const worker = this.workers.get(id);
|
|
164
|
+
if (worker) {
|
|
165
|
+
worker.restartCount++;
|
|
166
|
+
}
|
|
167
|
+
setTimeout(() => {
|
|
168
|
+
if (!this.shuttingDown) {
|
|
169
|
+
this.spawnWorker(id);
|
|
170
|
+
}
|
|
171
|
+
}, 5000);
|
|
172
|
+
});
|
|
173
|
+
workerProcess.on('error', (err) => {
|
|
174
|
+
fleetLog(id, color, 'ERR', `Worker error: ${err.message}`);
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
async shutdown(reason) {
|
|
178
|
+
if (this.shuttingDown)
|
|
179
|
+
return;
|
|
180
|
+
this.shuttingDown = true;
|
|
181
|
+
console.log(`\n${colors.yellow}Received ${reason} - shutting down fleet...${colors.reset}`);
|
|
182
|
+
for (const [id, worker] of this.workers) {
|
|
183
|
+
fleetLog(id, worker.color, 'INF', 'Stopping worker...');
|
|
184
|
+
worker.process.kill('SIGTERM');
|
|
185
|
+
}
|
|
186
|
+
// Wait for workers to exit (max 30 seconds)
|
|
187
|
+
const forceKillTimeout = setTimeout(() => {
|
|
188
|
+
console.log(`${colors.red}Timeout waiting for workers - force killing${colors.reset}`);
|
|
189
|
+
for (const worker of this.workers.values()) {
|
|
190
|
+
worker.process.kill('SIGKILL');
|
|
191
|
+
}
|
|
192
|
+
}, 30000);
|
|
193
|
+
await Promise.all(Array.from(this.workers.values()).map((worker) => new Promise((resolve) => {
|
|
194
|
+
worker.process.on('exit', () => resolve());
|
|
195
|
+
})));
|
|
196
|
+
clearTimeout(forceKillTimeout);
|
|
197
|
+
console.log(`${colors.green}All workers stopped${colors.reset}`);
|
|
198
|
+
// Resolve the running promise so start() returns
|
|
199
|
+
this.resolveRunning?.();
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
// Runner
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
/**
|
|
206
|
+
* Run a fleet of worker processes.
|
|
207
|
+
*
|
|
208
|
+
* The caller can cancel via the optional {@link AbortSignal}. The function
|
|
209
|
+
* returns once all workers have been stopped.
|
|
210
|
+
*/
|
|
211
|
+
export async function runWorkerFleet(config, signal) {
|
|
212
|
+
const workers = config.workers ?? Math.max(1, Math.floor(os.cpus().length / 2));
|
|
213
|
+
const capacity = config.capacity ?? 3;
|
|
214
|
+
const dryRun = config.dryRun ?? false;
|
|
215
|
+
const workerScript = config.workerScript ?? getDefaultWorkerScript();
|
|
216
|
+
const fleet = new WorkerFleet({
|
|
217
|
+
workers,
|
|
218
|
+
capacity,
|
|
219
|
+
dryRun,
|
|
220
|
+
apiUrl: config.apiUrl,
|
|
221
|
+
apiKey: config.apiKey,
|
|
222
|
+
}, workerScript);
|
|
223
|
+
await fleet.start(signal);
|
|
224
|
+
}
|