@supaku/agentfactory-server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/src/agent-tracking.d.ts +98 -0
- package/dist/src/agent-tracking.d.ts.map +1 -0
- package/dist/src/agent-tracking.js +185 -0
- package/dist/src/index.d.ts +9 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +16 -0
- package/dist/src/issue-lock.d.ts +126 -0
- package/dist/src/issue-lock.d.ts.map +1 -0
- package/dist/src/issue-lock.js +499 -0
- package/dist/src/redis.d.ts +146 -0
- package/dist/src/redis.d.ts.map +1 -0
- package/dist/src/redis.js +343 -0
- package/dist/src/session-storage.d.ts +144 -0
- package/dist/src/session-storage.d.ts.map +1 -0
- package/dist/src/session-storage.js +349 -0
- package/dist/src/types.d.ts +11 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +7 -0
- package/dist/src/webhook-idempotency.d.ts +44 -0
- package/dist/src/webhook-idempotency.d.ts.map +1 -0
- package/dist/src/webhook-idempotency.js +148 -0
- package/dist/src/work-queue.d.ts +119 -0
- package/dist/src/work-queue.d.ts.map +1 -0
- package/dist/src/work-queue.js +366 -0
- package/dist/src/worker-storage.d.ts +100 -0
- package/dist/src/worker-storage.d.ts.map +1 -0
- package/dist/src/worker-storage.js +283 -0
- package/package.json +61 -0
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue Lock Module
|
|
3
|
+
*
|
|
4
|
+
* Prevents overlapping agents for the same issue by providing:
|
|
5
|
+
* - Per-issue mutex (Redis SET NX) that gates work dispatch
|
|
6
|
+
* - Per-issue pending queue for parking incoming work while locked
|
|
7
|
+
* - Automatic promotion: releasing a lock dispatches the next pending item
|
|
8
|
+
*
|
|
9
|
+
* Redis Keys:
|
|
10
|
+
* - issue:lock:{issueId} -- String (JSON IssueLock), 2hr TTL
|
|
11
|
+
* - issue:pending:{issueId} -- Sorted Set (priority-ordered session IDs)
|
|
12
|
+
* - issue:pending:items:{issueId} -- Hash (sessionId -> JSON QueuedWork)
|
|
13
|
+
*/
|
|
14
|
+
import { redisSetNX, redisGet, redisDel, redisExpire, redisZAdd, redisZRem, redisZPopMin, redisZCard, redisHSet, redisHGet, redisHDel, redisHGetAll, isRedisConfigured, redisKeys, } from './redis';
|
|
15
|
+
import { queueWork } from './work-queue';
|
|
16
|
+
import { getSessionState } from './session-storage';
|
|
17
|
+
const log = {
|
|
18
|
+
info: (msg, data) => console.log(`[issue-lock] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
19
|
+
warn: (msg, data) => console.warn(`[issue-lock] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
20
|
+
error: (msg, data) => console.error(`[issue-lock] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
21
|
+
debug: (_msg, _data) => { },
|
|
22
|
+
};
|
|
23
|
+
// Redis key prefixes
|
|
24
|
+
const LOCK_PREFIX = 'issue:lock:';
|
|
25
|
+
const PENDING_PREFIX = 'issue:pending:';
|
|
26
|
+
const PENDING_ITEMS_PREFIX = 'issue:pending:items:';
|
|
27
|
+
// Default lock TTL: 2 hours
|
|
28
|
+
const LOCK_TTL_SECONDS = 2 * 60 * 60;
|
|
29
|
+
// Pending queue TTL: 24 hours
|
|
30
|
+
const PENDING_TTL_SECONDS = 24 * 60 * 60;
|
|
31
|
+
/**
|
|
32
|
+
* Acquire an issue-level lock.
|
|
33
|
+
* Uses SET NX for atomicity -- only one caller wins.
|
|
34
|
+
*
|
|
35
|
+
* @returns true if lock was acquired
|
|
36
|
+
*/
|
|
37
|
+
export async function acquireIssueLock(issueId, lock) {
|
|
38
|
+
if (!isRedisConfigured()) {
|
|
39
|
+
return true; // No Redis = no locking, pass through
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
const key = `${LOCK_PREFIX}${issueId}`;
|
|
43
|
+
const acquired = await redisSetNX(key, JSON.stringify(lock), LOCK_TTL_SECONDS);
|
|
44
|
+
if (acquired) {
|
|
45
|
+
log.info('Issue lock acquired', {
|
|
46
|
+
issueId,
|
|
47
|
+
sessionId: lock.sessionId,
|
|
48
|
+
workType: lock.workType,
|
|
49
|
+
issueIdentifier: lock.issueIdentifier,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
log.debug('Issue lock not acquired (already held)', {
|
|
54
|
+
issueId,
|
|
55
|
+
sessionId: lock.sessionId,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return acquired;
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
log.error('Failed to acquire issue lock', { error, issueId });
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Read the current lock for an issue.
|
|
67
|
+
*/
|
|
68
|
+
export async function getIssueLock(issueId) {
|
|
69
|
+
if (!isRedisConfigured())
|
|
70
|
+
return null;
|
|
71
|
+
try {
|
|
72
|
+
const key = `${LOCK_PREFIX}${issueId}`;
|
|
73
|
+
// redisSetNX stores the raw JSON string; redisGet parses it back
|
|
74
|
+
// Since redisSetNX stores JSON.stringify(lock), redisGet returns the parsed lock directly
|
|
75
|
+
return await redisGet(key);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
log.error('Failed to get issue lock', { error, issueId });
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Release an issue lock. Idempotent.
|
|
84
|
+
*/
|
|
85
|
+
export async function releaseIssueLock(issueId) {
|
|
86
|
+
if (!isRedisConfigured())
|
|
87
|
+
return;
|
|
88
|
+
try {
|
|
89
|
+
const key = `${LOCK_PREFIX}${issueId}`;
|
|
90
|
+
await redisDel(key);
|
|
91
|
+
log.info('Issue lock released', { issueId });
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
log.error('Failed to release issue lock', { error, issueId });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Refresh the TTL on an issue lock (extend while agent is alive).
|
|
99
|
+
*/
|
|
100
|
+
export async function refreshIssueLockTTL(issueId, ttlSeconds = LOCK_TTL_SECONDS) {
|
|
101
|
+
if (!isRedisConfigured())
|
|
102
|
+
return false;
|
|
103
|
+
try {
|
|
104
|
+
const key = `${LOCK_PREFIX}${issueId}`;
|
|
105
|
+
return await redisExpire(key, ttlSeconds);
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
log.error('Failed to refresh issue lock TTL', { error, issueId });
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Park work for a locked issue.
|
|
114
|
+
*
|
|
115
|
+
* Deduplication: at most one parked item per workType per issue.
|
|
116
|
+
* If a parked item with the same workType already exists, it's replaced
|
|
117
|
+
* (the latest webhook wins). Different workTypes can coexist.
|
|
118
|
+
*/
|
|
119
|
+
export async function parkWorkForIssue(issueId, work) {
|
|
120
|
+
if (!isRedisConfigured()) {
|
|
121
|
+
return { parked: false, replaced: false };
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
const pendingKey = `${PENDING_PREFIX}${issueId}`;
|
|
125
|
+
const itemsKey = `${PENDING_ITEMS_PREFIX}${issueId}`;
|
|
126
|
+
const workType = work.workType || 'development';
|
|
127
|
+
// Dedup key: use workType as the sorted set member
|
|
128
|
+
// This means at most one pending item per workType
|
|
129
|
+
const dedupMember = workType;
|
|
130
|
+
// Check if there's already a parked item with this workType
|
|
131
|
+
const existing = await redisHGet(itemsKey, dedupMember);
|
|
132
|
+
const replaced = !!existing;
|
|
133
|
+
if (replaced) {
|
|
134
|
+
// Remove old entry from sorted set before adding new one
|
|
135
|
+
await redisZRem(pendingKey, dedupMember);
|
|
136
|
+
log.info('Replacing existing parked work', {
|
|
137
|
+
issueId,
|
|
138
|
+
workType,
|
|
139
|
+
sessionId: work.sessionId,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
// Score = priority (lower = higher priority)
|
|
143
|
+
const score = work.priority;
|
|
144
|
+
// Add to sorted set and hash
|
|
145
|
+
await redisZAdd(pendingKey, score, dedupMember);
|
|
146
|
+
await redisHSet(itemsKey, dedupMember, JSON.stringify(work));
|
|
147
|
+
// Set TTL on both keys
|
|
148
|
+
await redisExpire(pendingKey, PENDING_TTL_SECONDS);
|
|
149
|
+
await redisExpire(itemsKey, PENDING_TTL_SECONDS);
|
|
150
|
+
log.info('Work parked for issue', {
|
|
151
|
+
issueId,
|
|
152
|
+
workType,
|
|
153
|
+
sessionId: work.sessionId,
|
|
154
|
+
priority: work.priority,
|
|
155
|
+
replaced,
|
|
156
|
+
});
|
|
157
|
+
return { parked: true, replaced };
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
log.error('Failed to park work for issue', { error, issueId });
|
|
161
|
+
return { parked: false, replaced: false };
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Promote the next pending work item for an issue.
|
|
166
|
+
* Pops the highest-priority item, acquires the issue lock for it,
|
|
167
|
+
* and queues it in the global work queue.
|
|
168
|
+
*
|
|
169
|
+
* @returns The promoted work item, or null if nothing to promote
|
|
170
|
+
*/
|
|
171
|
+
export async function promoteNextPendingWork(issueId) {
|
|
172
|
+
if (!isRedisConfigured())
|
|
173
|
+
return null;
|
|
174
|
+
try {
|
|
175
|
+
const pendingKey = `${PENDING_PREFIX}${issueId}`;
|
|
176
|
+
const itemsKey = `${PENDING_ITEMS_PREFIX}${issueId}`;
|
|
177
|
+
// Pop the highest-priority (lowest score) member
|
|
178
|
+
const popped = await redisZPopMin(pendingKey);
|
|
179
|
+
if (!popped) {
|
|
180
|
+
log.debug('No pending work to promote', { issueId });
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
const dedupMember = popped.member;
|
|
184
|
+
// Get the work item from the hash
|
|
185
|
+
const workJson = await redisHGet(itemsKey, dedupMember);
|
|
186
|
+
if (!workJson) {
|
|
187
|
+
log.warn('Pending work item not found in hash', { issueId, dedupMember });
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
// Remove from hash
|
|
191
|
+
await redisHDel(itemsKey, dedupMember);
|
|
192
|
+
const work = JSON.parse(workJson);
|
|
193
|
+
// Acquire the issue lock for this promoted work
|
|
194
|
+
const lock = {
|
|
195
|
+
sessionId: work.sessionId,
|
|
196
|
+
workType: work.workType || 'development',
|
|
197
|
+
workerId: null,
|
|
198
|
+
lockedAt: Date.now(),
|
|
199
|
+
issueIdentifier: work.issueIdentifier,
|
|
200
|
+
};
|
|
201
|
+
const acquired = await acquireIssueLock(issueId, lock);
|
|
202
|
+
if (!acquired) {
|
|
203
|
+
log.warn('Failed to acquire lock for promoted work -- another lock appeared', {
|
|
204
|
+
issueId,
|
|
205
|
+
sessionId: work.sessionId,
|
|
206
|
+
});
|
|
207
|
+
// Re-park the work since we couldn't acquire the lock
|
|
208
|
+
await parkWorkForIssue(issueId, work);
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
// Queue in the global work queue
|
|
212
|
+
const queued = await queueWork(work);
|
|
213
|
+
if (!queued) {
|
|
214
|
+
log.error('Failed to queue promoted work', { issueId, sessionId: work.sessionId });
|
|
215
|
+
// Release the lock since we couldn't queue
|
|
216
|
+
await releaseIssueLock(issueId);
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
log.info('Pending work promoted', {
|
|
220
|
+
issueId,
|
|
221
|
+
sessionId: work.sessionId,
|
|
222
|
+
workType: work.workType,
|
|
223
|
+
issueIdentifier: work.issueIdentifier,
|
|
224
|
+
});
|
|
225
|
+
return work;
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
log.error('Failed to promote pending work', { error, issueId });
|
|
229
|
+
return null;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Get the count of pending work items for an issue.
|
|
234
|
+
*/
|
|
235
|
+
export async function getPendingWorkCount(issueId) {
|
|
236
|
+
if (!isRedisConfigured())
|
|
237
|
+
return 0;
|
|
238
|
+
try {
|
|
239
|
+
const pendingKey = `${PENDING_PREFIX}${issueId}`;
|
|
240
|
+
return await redisZCard(pendingKey);
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
log.error('Failed to get pending work count', { error, issueId });
|
|
244
|
+
return 0;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Main entry point for dispatching work.
|
|
249
|
+
*
|
|
250
|
+
* Try to acquire the issue lock:
|
|
251
|
+
* - If acquired -> queue the work in the global queue
|
|
252
|
+
* - If locked -> park the work in the per-issue pending queue
|
|
253
|
+
*
|
|
254
|
+
* @returns DispatchResult indicating what happened
|
|
255
|
+
*/
|
|
256
|
+
export async function dispatchWork(work) {
|
|
257
|
+
if (!isRedisConfigured()) {
|
|
258
|
+
// No Redis -- fall back to direct queueing (no locking)
|
|
259
|
+
const queued = await queueWork(work);
|
|
260
|
+
return { dispatched: queued, parked: false, replaced: false };
|
|
261
|
+
}
|
|
262
|
+
const issueId = work.issueId;
|
|
263
|
+
// Try to acquire the issue lock
|
|
264
|
+
const lock = {
|
|
265
|
+
sessionId: work.sessionId,
|
|
266
|
+
workType: work.workType || 'development',
|
|
267
|
+
workerId: null,
|
|
268
|
+
lockedAt: Date.now(),
|
|
269
|
+
issueIdentifier: work.issueIdentifier,
|
|
270
|
+
};
|
|
271
|
+
const acquired = await acquireIssueLock(issueId, lock);
|
|
272
|
+
if (acquired) {
|
|
273
|
+
// Lock acquired -- dispatch to global queue
|
|
274
|
+
const queued = await queueWork(work);
|
|
275
|
+
if (!queued) {
|
|
276
|
+
// Failed to queue -- release the lock
|
|
277
|
+
await releaseIssueLock(issueId);
|
|
278
|
+
return { dispatched: false, parked: false, replaced: false };
|
|
279
|
+
}
|
|
280
|
+
log.info('Work dispatched (lock acquired)', {
|
|
281
|
+
issueId,
|
|
282
|
+
sessionId: work.sessionId,
|
|
283
|
+
workType: work.workType,
|
|
284
|
+
issueIdentifier: work.issueIdentifier,
|
|
285
|
+
});
|
|
286
|
+
return { dispatched: true, parked: false, replaced: false };
|
|
287
|
+
}
|
|
288
|
+
// Lock held by another session -- park this work
|
|
289
|
+
const { parked, replaced } = await parkWorkForIssue(issueId, work);
|
|
290
|
+
if (parked) {
|
|
291
|
+
log.info('Work parked (issue locked)', {
|
|
292
|
+
issueId,
|
|
293
|
+
sessionId: work.sessionId,
|
|
294
|
+
workType: work.workType,
|
|
295
|
+
replaced,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
return { dispatched: false, parked, replaced };
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Remove a parked work item by sessionId.
|
|
302
|
+
*
|
|
303
|
+
* The issue-pending hash is keyed by workType, so we scan all entries
|
|
304
|
+
* to find the one matching the given sessionId.
|
|
305
|
+
*
|
|
306
|
+
* @returns true if a matching parked item was found and removed
|
|
307
|
+
*/
|
|
308
|
+
export async function removeParkedWorkBySessionId(issueId, sessionId) {
|
|
309
|
+
if (!isRedisConfigured())
|
|
310
|
+
return false;
|
|
311
|
+
try {
|
|
312
|
+
const pendingKey = `${PENDING_PREFIX}${issueId}`;
|
|
313
|
+
const itemsKey = `${PENDING_ITEMS_PREFIX}${issueId}`;
|
|
314
|
+
// Get all entries in the hash (keyed by workType)
|
|
315
|
+
const entries = await redisHGetAll(itemsKey);
|
|
316
|
+
if (!entries)
|
|
317
|
+
return false;
|
|
318
|
+
for (const [dedupMember, workJson] of Object.entries(entries)) {
|
|
319
|
+
try {
|
|
320
|
+
const work = JSON.parse(workJson);
|
|
321
|
+
if (work.sessionId === sessionId) {
|
|
322
|
+
// Found the matching entry -- remove from both sorted set and hash
|
|
323
|
+
await redisZRem(pendingKey, dedupMember);
|
|
324
|
+
await redisHDel(itemsKey, dedupMember);
|
|
325
|
+
log.info('Removed parked work by sessionId', {
|
|
326
|
+
issueId,
|
|
327
|
+
sessionId,
|
|
328
|
+
workType: dedupMember,
|
|
329
|
+
});
|
|
330
|
+
return true;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
// Skip malformed entries
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
log.debug('No parked work found for sessionId', { issueId, sessionId });
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
log.error('Failed to remove parked work by sessionId', { error, issueId, sessionId });
|
|
343
|
+
return false;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Check if a session is parked in any issue-pending queue.
|
|
348
|
+
*
|
|
349
|
+
* Scans the issue:pending:items:{issueId} hash entries for a matching sessionId.
|
|
350
|
+
*
|
|
351
|
+
* @param issueId - The issue to check
|
|
352
|
+
* @param sessionId - The session to look for
|
|
353
|
+
* @returns true if the session is parked for this issue
|
|
354
|
+
*/
|
|
355
|
+
export async function isSessionParkedForIssue(issueId, sessionId) {
|
|
356
|
+
if (!isRedisConfigured())
|
|
357
|
+
return false;
|
|
358
|
+
try {
|
|
359
|
+
const itemsKey = `${PENDING_ITEMS_PREFIX}${issueId}`;
|
|
360
|
+
const entries = await redisHGetAll(itemsKey);
|
|
361
|
+
if (!entries)
|
|
362
|
+
return false;
|
|
363
|
+
for (const workJson of Object.values(entries)) {
|
|
364
|
+
try {
|
|
365
|
+
const work = JSON.parse(workJson);
|
|
366
|
+
if (work.sessionId === sessionId) {
|
|
367
|
+
return true;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return false;
|
|
375
|
+
}
|
|
376
|
+
catch (error) {
|
|
377
|
+
log.error('Failed to check if session is parked', { error, issueId, sessionId });
|
|
378
|
+
return false;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Scan for expired issue locks that have pending work.
|
|
383
|
+
* If a lock expired naturally (TTL) but pending items remain, promote them.
|
|
384
|
+
*
|
|
385
|
+
* Called from orphan-cleanup to handle crashed workers that didn't release locks.
|
|
386
|
+
*/
|
|
387
|
+
export async function cleanupExpiredLocksWithPendingWork() {
|
|
388
|
+
if (!isRedisConfigured())
|
|
389
|
+
return 0;
|
|
390
|
+
let promoted = 0;
|
|
391
|
+
try {
|
|
392
|
+
// Find all pending queues
|
|
393
|
+
const pendingKeys = await redisKeys(`${PENDING_PREFIX}*`);
|
|
394
|
+
for (const pendingKey of pendingKeys) {
|
|
395
|
+
// Extract issueId from key
|
|
396
|
+
const issueId = pendingKey.replace(PENDING_PREFIX, '');
|
|
397
|
+
// Check if lock still exists
|
|
398
|
+
const lockKey = `${LOCK_PREFIX}${issueId}`;
|
|
399
|
+
const lock = await redisGet(lockKey);
|
|
400
|
+
if (!lock) {
|
|
401
|
+
// Lock expired but pending work exists -- promote
|
|
402
|
+
const count = await redisZCard(pendingKey);
|
|
403
|
+
if (count > 0) {
|
|
404
|
+
log.info('Found expired lock with pending work, promoting', {
|
|
405
|
+
issueId,
|
|
406
|
+
pendingCount: count,
|
|
407
|
+
});
|
|
408
|
+
const work = await promoteNextPendingWork(issueId);
|
|
409
|
+
if (work) {
|
|
410
|
+
promoted++;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (promoted > 0) {
|
|
416
|
+
log.info('Promoted pending work from expired locks', { promoted });
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
catch (error) {
|
|
420
|
+
log.error('Failed to cleanup expired locks', { error });
|
|
421
|
+
}
|
|
422
|
+
return promoted;
|
|
423
|
+
}
|
|
424
|
+
const TERMINAL_STATUSES = new Set(['completed', 'failed', 'stopped']);
|
|
425
|
+
/**
|
|
426
|
+
* Release issue locks held by sessions that have already reached a terminal state.
|
|
427
|
+
*
|
|
428
|
+
* This handles the case where a session completes but the lock release failed
|
|
429
|
+
* (e.g., network error during cleanup). The lock's 2-hour TTL would eventually
|
|
430
|
+
* expire, but this proactively clears it when workers have idle capacity.
|
|
431
|
+
*
|
|
432
|
+
* Only runs when workers are online -- if no workers are available, there's no
|
|
433
|
+
* point promoting parked work since nothing can pick it up.
|
|
434
|
+
*
|
|
435
|
+
* @param hasIdleWorkers - true if at least one worker is online with spare capacity
|
|
436
|
+
* @returns Number of stale locks released and parked work promoted
|
|
437
|
+
*/
|
|
438
|
+
export async function cleanupStaleLocksWithIdleWorkers(hasIdleWorkers) {
|
|
439
|
+
if (!isRedisConfigured())
|
|
440
|
+
return 0;
|
|
441
|
+
if (!hasIdleWorkers)
|
|
442
|
+
return 0;
|
|
443
|
+
let promoted = 0;
|
|
444
|
+
try {
|
|
445
|
+
// Find all issue locks
|
|
446
|
+
const lockKeys = await redisKeys(`${LOCK_PREFIX}*`);
|
|
447
|
+
for (const lockKey of lockKeys) {
|
|
448
|
+
const issueId = lockKey.replace(LOCK_PREFIX, '');
|
|
449
|
+
// Skip keys that look like pending queue keys (contain extra colons)
|
|
450
|
+
if (issueId.includes(':'))
|
|
451
|
+
continue;
|
|
452
|
+
const lock = await redisGet(lockKey);
|
|
453
|
+
if (!lock)
|
|
454
|
+
continue;
|
|
455
|
+
// Check if the lock holder's session is in a terminal state
|
|
456
|
+
const session = await getSessionState(lock.sessionId);
|
|
457
|
+
if (!session) {
|
|
458
|
+
// Session expired from Redis (24h TTL) but lock remains (2h TTL)
|
|
459
|
+
// Safe to release -- the session is long gone
|
|
460
|
+
log.info('Releasing lock for expired session', {
|
|
461
|
+
issueId,
|
|
462
|
+
sessionId: lock.sessionId,
|
|
463
|
+
issueIdentifier: lock.issueIdentifier,
|
|
464
|
+
});
|
|
465
|
+
await releaseIssueLock(issueId);
|
|
466
|
+
const work = await promoteNextPendingWork(issueId);
|
|
467
|
+
if (work)
|
|
468
|
+
promoted++;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
if (TERMINAL_STATUSES.has(session.status)) {
|
|
472
|
+
log.info('Releasing stale lock (session already terminal)', {
|
|
473
|
+
issueId,
|
|
474
|
+
sessionId: lock.sessionId,
|
|
475
|
+
sessionStatus: session.status,
|
|
476
|
+
issueIdentifier: lock.issueIdentifier,
|
|
477
|
+
lockAge: Math.round((Date.now() - lock.lockedAt) / 1000),
|
|
478
|
+
});
|
|
479
|
+
await releaseIssueLock(issueId);
|
|
480
|
+
const work = await promoteNextPendingWork(issueId);
|
|
481
|
+
if (work) {
|
|
482
|
+
promoted++;
|
|
483
|
+
log.info('Promoted parked work after stale lock cleanup', {
|
|
484
|
+
issueId,
|
|
485
|
+
promotedSessionId: work.sessionId,
|
|
486
|
+
promotedWorkType: work.workType,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
if (promoted > 0) {
|
|
492
|
+
log.info('Promoted parked work from stale locks', { promoted });
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
log.error('Failed to cleanup stale locks', { error });
|
|
497
|
+
}
|
|
498
|
+
return promoted;
|
|
499
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import Redis from 'ioredis';
|
|
2
|
+
/**
|
|
3
|
+
* Check if Redis is configured via REDIS_URL
|
|
4
|
+
*/
|
|
5
|
+
export declare function isRedisConfigured(): boolean;
|
|
6
|
+
/**
|
|
7
|
+
* Get the shared Redis client instance
|
|
8
|
+
* Lazily initialized to avoid errors during build
|
|
9
|
+
*/
|
|
10
|
+
export declare function getRedisClient(): Redis;
|
|
11
|
+
/**
|
|
12
|
+
* Disconnect Redis client (for graceful shutdown)
|
|
13
|
+
*/
|
|
14
|
+
export declare function disconnectRedis(): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Set a value with optional TTL (seconds)
|
|
17
|
+
*/
|
|
18
|
+
export declare function redisSet<T>(key: string, value: T, ttlSeconds?: number): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Get a typed value
|
|
21
|
+
*/
|
|
22
|
+
export declare function redisGet<T>(key: string): Promise<T | null>;
|
|
23
|
+
/**
|
|
24
|
+
* Delete a key
|
|
25
|
+
* @returns number of keys deleted (0 or 1)
|
|
26
|
+
*/
|
|
27
|
+
export declare function redisDel(key: string): Promise<number>;
|
|
28
|
+
/**
|
|
29
|
+
* Check if a key exists
|
|
30
|
+
*/
|
|
31
|
+
export declare function redisExists(key: string): Promise<boolean>;
|
|
32
|
+
/**
|
|
33
|
+
* Get keys matching a pattern
|
|
34
|
+
*/
|
|
35
|
+
export declare function redisKeys(pattern: string): Promise<string[]>;
|
|
36
|
+
/**
|
|
37
|
+
* Push value to the right of a list (RPUSH)
|
|
38
|
+
* @returns length of list after push
|
|
39
|
+
*/
|
|
40
|
+
export declare function redisRPush(key: string, value: string): Promise<number>;
|
|
41
|
+
/**
|
|
42
|
+
* Pop value from the left of a list (LPOP)
|
|
43
|
+
* @returns the popped value or null if list is empty
|
|
44
|
+
*/
|
|
45
|
+
export declare function redisLPop(key: string): Promise<string | null>;
|
|
46
|
+
/**
|
|
47
|
+
* Get a range of elements from a list (LRANGE)
|
|
48
|
+
* @param start - Start index (0-based, inclusive)
|
|
49
|
+
* @param stop - Stop index (inclusive, -1 for end)
|
|
50
|
+
*/
|
|
51
|
+
export declare function redisLRange(key: string, start: number, stop: number): Promise<string[]>;
|
|
52
|
+
/**
|
|
53
|
+
* Get the length of a list (LLEN)
|
|
54
|
+
*/
|
|
55
|
+
export declare function redisLLen(key: string): Promise<number>;
|
|
56
|
+
/**
|
|
57
|
+
* Remove elements from a list (LREM)
|
|
58
|
+
* @param count - Number of occurrences to remove (0 = all)
|
|
59
|
+
* @returns number of elements removed
|
|
60
|
+
*/
|
|
61
|
+
export declare function redisLRem(key: string, count: number, value: string): Promise<number>;
|
|
62
|
+
/**
|
|
63
|
+
* Add member to a set (SADD)
|
|
64
|
+
* @returns number of elements added (0 if already exists)
|
|
65
|
+
*/
|
|
66
|
+
export declare function redisSAdd(key: string, member: string): Promise<number>;
|
|
67
|
+
/**
|
|
68
|
+
* Remove member from a set (SREM)
|
|
69
|
+
* @returns number of elements removed
|
|
70
|
+
*/
|
|
71
|
+
export declare function redisSRem(key: string, member: string): Promise<number>;
|
|
72
|
+
/**
|
|
73
|
+
* Get all members of a set (SMEMBERS)
|
|
74
|
+
*/
|
|
75
|
+
export declare function redisSMembers(key: string): Promise<string[]>;
|
|
76
|
+
/**
|
|
77
|
+
* Get the number of members in a set (SCARD)
|
|
78
|
+
*/
|
|
79
|
+
export declare function redisSCard(key: string): Promise<number>;
|
|
80
|
+
/**
|
|
81
|
+
* Set a value only if key does not exist (SETNX)
|
|
82
|
+
* @returns true if key was set, false if it already existed
|
|
83
|
+
*/
|
|
84
|
+
export declare function redisSetNX(key: string, value: string, ttlSeconds?: number): Promise<boolean>;
|
|
85
|
+
/**
|
|
86
|
+
* Set TTL on an existing key (EXPIRE)
|
|
87
|
+
* @returns true if TTL was set, false if key doesn't exist
|
|
88
|
+
*/
|
|
89
|
+
export declare function redisExpire(key: string, ttlSeconds: number): Promise<boolean>;
|
|
90
|
+
/**
|
|
91
|
+
* Add member to a sorted set with score (ZADD)
|
|
92
|
+
* @returns number of elements added (0 if already exists, updates score)
|
|
93
|
+
*/
|
|
94
|
+
export declare function redisZAdd(key: string, score: number, member: string): Promise<number>;
|
|
95
|
+
/**
|
|
96
|
+
* Remove member from a sorted set (ZREM)
|
|
97
|
+
* @returns number of elements removed
|
|
98
|
+
*/
|
|
99
|
+
export declare function redisZRem(key: string, member: string): Promise<number>;
|
|
100
|
+
/**
|
|
101
|
+
* Get members from sorted set by score range (ZRANGEBYSCORE)
|
|
102
|
+
* Returns members with lowest scores first (highest priority)
|
|
103
|
+
* @param min - Minimum score (use '-inf' for no minimum)
|
|
104
|
+
* @param max - Maximum score (use '+inf' for no maximum)
|
|
105
|
+
* @param limit - Maximum number of results
|
|
106
|
+
*/
|
|
107
|
+
export declare function redisZRangeByScore(key: string, min: number | string, max: number | string, limit?: number): Promise<string[]>;
|
|
108
|
+
/**
|
|
109
|
+
* Get the number of members in a sorted set (ZCARD)
|
|
110
|
+
*/
|
|
111
|
+
export declare function redisZCard(key: string): Promise<number>;
|
|
112
|
+
/**
|
|
113
|
+
* Pop the member with the lowest score (ZPOPMIN)
|
|
114
|
+
* @returns [member, score] or null if set is empty
|
|
115
|
+
*/
|
|
116
|
+
export declare function redisZPopMin(key: string): Promise<{
|
|
117
|
+
member: string;
|
|
118
|
+
score: number;
|
|
119
|
+
} | null>;
|
|
120
|
+
/**
|
|
121
|
+
* Set a field in a hash (HSET)
|
|
122
|
+
* @returns 1 if field is new, 0 if field existed
|
|
123
|
+
*/
|
|
124
|
+
export declare function redisHSet(key: string, field: string, value: string): Promise<number>;
|
|
125
|
+
/**
|
|
126
|
+
* Get a field from a hash (HGET)
|
|
127
|
+
*/
|
|
128
|
+
export declare function redisHGet(key: string, field: string): Promise<string | null>;
|
|
129
|
+
/**
|
|
130
|
+
* Delete a field from a hash (HDEL)
|
|
131
|
+
* @returns number of fields removed
|
|
132
|
+
*/
|
|
133
|
+
export declare function redisHDel(key: string, field: string): Promise<number>;
|
|
134
|
+
/**
|
|
135
|
+
* Get multiple fields from a hash (HMGET)
|
|
136
|
+
*/
|
|
137
|
+
export declare function redisHMGet(key: string, fields: string[]): Promise<(string | null)[]>;
|
|
138
|
+
/**
|
|
139
|
+
* Get all fields and values from a hash (HGETALL)
|
|
140
|
+
*/
|
|
141
|
+
export declare function redisHGetAll(key: string): Promise<Record<string, string>>;
|
|
142
|
+
/**
|
|
143
|
+
* Get the number of fields in a hash (HLEN)
|
|
144
|
+
*/
|
|
145
|
+
export declare function redisHLen(key: string): Promise<number>;
|
|
146
|
+
//# sourceMappingURL=redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../../src/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAuB,MAAM,SAAS,CAAA;AA6D7C;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,KAAK,CA0BtC;AAED;;GAEG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAMrD;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAC9B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,CAAC,EACR,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAShE;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG3D;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI/D;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGlE;AAMD;;;GAGG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5E;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5D;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC,CAGjB;AAMD;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5E;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5E;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGlE;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG7D;AAMD;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,OAAO,CAAC,CAIlB;AAMD;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5E;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,EAAE,CAAC,CAMnB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG7D;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAOnD;AAMD;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGxB;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG3E;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAI5B;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAGjC;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5D"}
|