@supaku/agentfactory-server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/src/agent-tracking.d.ts +98 -0
- package/dist/src/agent-tracking.d.ts.map +1 -0
- package/dist/src/agent-tracking.js +185 -0
- package/dist/src/index.d.ts +9 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +16 -0
- package/dist/src/issue-lock.d.ts +126 -0
- package/dist/src/issue-lock.d.ts.map +1 -0
- package/dist/src/issue-lock.js +499 -0
- package/dist/src/redis.d.ts +146 -0
- package/dist/src/redis.d.ts.map +1 -0
- package/dist/src/redis.js +343 -0
- package/dist/src/session-storage.d.ts +144 -0
- package/dist/src/session-storage.d.ts.map +1 -0
- package/dist/src/session-storage.js +349 -0
- package/dist/src/types.d.ts +11 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +7 -0
- package/dist/src/webhook-idempotency.d.ts +44 -0
- package/dist/src/webhook-idempotency.d.ts.map +1 -0
- package/dist/src/webhook-idempotency.js +148 -0
- package/dist/src/work-queue.d.ts +119 -0
- package/dist/src/work-queue.d.ts.map +1 -0
- package/dist/src/work-queue.js +366 -0
- package/dist/src/worker-storage.d.ts +100 -0
- package/dist/src/worker-storage.d.ts.map +1 -0
- package/dist/src/worker-storage.js +283 -0
- package/package.json +61 -0
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
import { isRedisConfigured, redisSet, redisGet, redisDel, redisKeys } from './redis';
|
|
2
|
+
const log = {
|
|
3
|
+
info: (msg, data) => console.log(`[session] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
4
|
+
warn: (msg, data) => console.warn(`[session] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
5
|
+
error: (msg, data) => console.error(`[session] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
6
|
+
debug: (_msg, _data) => { },
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Key prefix for session state in KV
|
|
10
|
+
*/
|
|
11
|
+
const SESSION_KEY_PREFIX = 'agent:session:';
|
|
12
|
+
/**
|
|
13
|
+
* Session state TTL in seconds (24 hours)
|
|
14
|
+
* Sessions older than this are automatically cleaned up by KV
|
|
15
|
+
*/
|
|
16
|
+
const SESSION_TTL_SECONDS = 24 * 60 * 60;
|
|
17
|
+
/**
|
|
18
|
+
* Build the KV key for a session
|
|
19
|
+
*/
|
|
20
|
+
function buildSessionKey(linearSessionId) {
|
|
21
|
+
return `${SESSION_KEY_PREFIX}${linearSessionId}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Store agent session state in Redis
|
|
25
|
+
*
|
|
26
|
+
* @param linearSessionId - The Linear session ID from webhook
|
|
27
|
+
* @param state - The session state to store
|
|
28
|
+
*/
|
|
29
|
+
export async function storeSessionState(linearSessionId, state) {
|
|
30
|
+
if (!isRedisConfigured()) {
|
|
31
|
+
log.warn('Redis not configured, session state will not be persisted');
|
|
32
|
+
const now = Math.floor(Date.now() / 1000);
|
|
33
|
+
return {
|
|
34
|
+
...state,
|
|
35
|
+
linearSessionId,
|
|
36
|
+
createdAt: now,
|
|
37
|
+
updatedAt: now,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
const now = Math.floor(Date.now() / 1000);
|
|
41
|
+
const key = buildSessionKey(linearSessionId);
|
|
42
|
+
// Check for existing session to preserve createdAt
|
|
43
|
+
const existing = await redisGet(key);
|
|
44
|
+
const sessionState = {
|
|
45
|
+
...state,
|
|
46
|
+
linearSessionId,
|
|
47
|
+
createdAt: existing?.createdAt ?? now,
|
|
48
|
+
updatedAt: now,
|
|
49
|
+
};
|
|
50
|
+
await redisSet(key, sessionState, SESSION_TTL_SECONDS);
|
|
51
|
+
log.info('Stored session state', {
|
|
52
|
+
linearSessionId,
|
|
53
|
+
issueId: state.issueId,
|
|
54
|
+
status: state.status,
|
|
55
|
+
hasClaudeSessionId: !!state.claudeSessionId,
|
|
56
|
+
});
|
|
57
|
+
return sessionState;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Retrieve agent session state from Redis
|
|
61
|
+
*
|
|
62
|
+
* @param linearSessionId - The Linear session ID
|
|
63
|
+
* @returns The session state or null if not found
|
|
64
|
+
*/
|
|
65
|
+
export async function getSessionState(linearSessionId) {
|
|
66
|
+
if (!isRedisConfigured()) {
|
|
67
|
+
log.debug('Redis not configured, cannot retrieve session state');
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
const key = buildSessionKey(linearSessionId);
|
|
71
|
+
const state = await redisGet(key);
|
|
72
|
+
if (state) {
|
|
73
|
+
log.debug('Retrieved session state', {
|
|
74
|
+
linearSessionId,
|
|
75
|
+
issueId: state.issueId,
|
|
76
|
+
status: state.status,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return state;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Update the Claude session ID for a session
|
|
83
|
+
* Called when the Claude init event is received with the session ID
|
|
84
|
+
*
|
|
85
|
+
* @param linearSessionId - The Linear session ID
|
|
86
|
+
* @param claudeSessionId - The Claude CLI session ID
|
|
87
|
+
*/
|
|
88
|
+
export async function updateClaudeSessionId(linearSessionId, claudeSessionId) {
|
|
89
|
+
if (!isRedisConfigured()) {
|
|
90
|
+
log.warn('Redis not configured, cannot update Claude session ID');
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
const existing = await getSessionState(linearSessionId);
|
|
94
|
+
if (!existing) {
|
|
95
|
+
log.warn('Session not found for Claude session ID update', { linearSessionId });
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
const key = buildSessionKey(linearSessionId);
|
|
99
|
+
const now = Math.floor(Date.now() / 1000);
|
|
100
|
+
const updated = {
|
|
101
|
+
...existing,
|
|
102
|
+
claudeSessionId,
|
|
103
|
+
updatedAt: now,
|
|
104
|
+
};
|
|
105
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
106
|
+
log.info('Updated Claude session ID', { linearSessionId, claudeSessionId });
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Update session status
|
|
111
|
+
*
|
|
112
|
+
* @param linearSessionId - The Linear session ID
|
|
113
|
+
* @param status - The new status
|
|
114
|
+
*/
|
|
115
|
+
export async function updateSessionStatus(linearSessionId, status) {
|
|
116
|
+
if (!isRedisConfigured()) {
|
|
117
|
+
log.warn('Redis not configured, cannot update session status');
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
const existing = await getSessionState(linearSessionId);
|
|
121
|
+
if (!existing) {
|
|
122
|
+
log.warn('Session not found for status update', { linearSessionId });
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
const key = buildSessionKey(linearSessionId);
|
|
126
|
+
const now = Math.floor(Date.now() / 1000);
|
|
127
|
+
const updated = {
|
|
128
|
+
...existing,
|
|
129
|
+
status,
|
|
130
|
+
updatedAt: now,
|
|
131
|
+
};
|
|
132
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
133
|
+
log.info('Updated session status', { linearSessionId, status });
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Reset a session for re-queuing after orphan cleanup
|
|
138
|
+
* Clears workerId and resets status to pending so a new worker can claim it
|
|
139
|
+
*
|
|
140
|
+
* @param linearSessionId - The Linear session ID
|
|
141
|
+
*/
|
|
142
|
+
export async function resetSessionForRequeue(linearSessionId) {
|
|
143
|
+
if (!isRedisConfigured()) {
|
|
144
|
+
log.warn('Redis not configured, cannot reset session');
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
const existing = await getSessionState(linearSessionId);
|
|
148
|
+
if (!existing) {
|
|
149
|
+
log.warn('Session not found for reset', { linearSessionId });
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
const key = buildSessionKey(linearSessionId);
|
|
153
|
+
const now = Math.floor(Date.now() / 1000);
|
|
154
|
+
const updated = {
|
|
155
|
+
...existing,
|
|
156
|
+
status: 'pending',
|
|
157
|
+
workerId: undefined, // Clear workerId so new worker can claim
|
|
158
|
+
claimedAt: undefined,
|
|
159
|
+
updatedAt: now,
|
|
160
|
+
};
|
|
161
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
162
|
+
log.info('Reset session for requeue', {
|
|
163
|
+
linearSessionId,
|
|
164
|
+
previousWorkerId: existing.workerId,
|
|
165
|
+
});
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Delete session state from KV
|
|
170
|
+
*
|
|
171
|
+
* @param linearSessionId - The Linear session ID
|
|
172
|
+
* @returns Whether the deletion was successful
|
|
173
|
+
*/
|
|
174
|
+
export async function deleteSessionState(linearSessionId) {
|
|
175
|
+
if (!isRedisConfigured()) {
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
const key = buildSessionKey(linearSessionId);
|
|
179
|
+
const result = await redisDel(key);
|
|
180
|
+
log.info('Deleted session state', { linearSessionId });
|
|
181
|
+
return result > 0;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Get session state by issue ID
|
|
185
|
+
* Useful when we have the issue but not the session ID
|
|
186
|
+
*
|
|
187
|
+
* @param issueId - The Linear issue ID
|
|
188
|
+
* @returns The most recent session state for this issue or null
|
|
189
|
+
*/
|
|
190
|
+
export async function getSessionStateByIssue(issueId) {
|
|
191
|
+
if (!isRedisConfigured()) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
// Scan for sessions with this issue ID
|
|
195
|
+
// Note: This is less efficient than direct lookup, use sparingly
|
|
196
|
+
const keys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
197
|
+
for (const key of keys) {
|
|
198
|
+
const state = await redisGet(key);
|
|
199
|
+
if (state?.issueId === issueId) {
|
|
200
|
+
return state;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
// ============================================
|
|
206
|
+
// Worker Pool Operations
|
|
207
|
+
// ============================================
|
|
208
|
+
/**
|
|
209
|
+
* Mark a session as claimed by a worker
|
|
210
|
+
*
|
|
211
|
+
* @param linearSessionId - The Linear session ID
|
|
212
|
+
* @param workerId - The worker claiming the session
|
|
213
|
+
*/
|
|
214
|
+
export async function claimSession(linearSessionId, workerId) {
|
|
215
|
+
if (!isRedisConfigured()) {
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
const existing = await getSessionState(linearSessionId);
|
|
219
|
+
if (!existing) {
|
|
220
|
+
log.warn('Session not found for claim', { linearSessionId });
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
if (existing.status !== 'pending') {
|
|
224
|
+
log.warn('Session not in pending status', {
|
|
225
|
+
linearSessionId,
|
|
226
|
+
status: existing.status,
|
|
227
|
+
});
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
const key = buildSessionKey(linearSessionId);
|
|
231
|
+
const now = Math.floor(Date.now() / 1000);
|
|
232
|
+
const updated = {
|
|
233
|
+
...existing,
|
|
234
|
+
status: 'claimed',
|
|
235
|
+
workerId,
|
|
236
|
+
claimedAt: now,
|
|
237
|
+
updatedAt: now,
|
|
238
|
+
};
|
|
239
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
240
|
+
log.info('Session claimed', { linearSessionId, workerId });
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Update session with worker info when work starts
|
|
245
|
+
*
|
|
246
|
+
* @param linearSessionId - The Linear session ID
|
|
247
|
+
* @param workerId - The worker processing the session
|
|
248
|
+
* @param worktreePath - Path to the git worktree
|
|
249
|
+
*/
|
|
250
|
+
export async function startSession(linearSessionId, workerId, worktreePath) {
|
|
251
|
+
if (!isRedisConfigured()) {
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
const existing = await getSessionState(linearSessionId);
|
|
255
|
+
if (!existing) {
|
|
256
|
+
log.warn('Session not found for start', { linearSessionId });
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
const key = buildSessionKey(linearSessionId);
|
|
260
|
+
const now = Math.floor(Date.now() / 1000);
|
|
261
|
+
const updated = {
|
|
262
|
+
...existing,
|
|
263
|
+
status: 'running',
|
|
264
|
+
workerId,
|
|
265
|
+
worktreePath,
|
|
266
|
+
updatedAt: now,
|
|
267
|
+
};
|
|
268
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
269
|
+
log.info('Session started', { linearSessionId, workerId, worktreePath });
|
|
270
|
+
return true;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Get all sessions from Redis
|
|
274
|
+
* For dashboard display
|
|
275
|
+
*/
|
|
276
|
+
export async function getAllSessions() {
|
|
277
|
+
if (!isRedisConfigured()) {
|
|
278
|
+
return [];
|
|
279
|
+
}
|
|
280
|
+
try {
|
|
281
|
+
const keys = await redisKeys(`${SESSION_KEY_PREFIX}*`);
|
|
282
|
+
const sessions = [];
|
|
283
|
+
for (const key of keys) {
|
|
284
|
+
const state = await redisGet(key);
|
|
285
|
+
if (state) {
|
|
286
|
+
sessions.push(state);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
// Sort by updatedAt descending (most recent first)
|
|
290
|
+
sessions.sort((a, b) => b.updatedAt - a.updatedAt);
|
|
291
|
+
return sessions;
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
log.error('Failed to get all sessions', { error });
|
|
295
|
+
return [];
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Get sessions by status
|
|
300
|
+
*/
|
|
301
|
+
export async function getSessionsByStatus(status) {
|
|
302
|
+
const allSessions = await getAllSessions();
|
|
303
|
+
const statusArray = Array.isArray(status) ? status : [status];
|
|
304
|
+
return allSessions.filter((s) => statusArray.includes(s.status));
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Transfer session ownership to a new worker
|
|
308
|
+
* Used when a worker re-registers after disconnection and gets a new ID
|
|
309
|
+
*
|
|
310
|
+
* @param linearSessionId - The Linear session ID
|
|
311
|
+
* @param newWorkerId - The new worker ID to assign
|
|
312
|
+
* @param oldWorkerId - The previous worker ID (for validation)
|
|
313
|
+
* @returns Whether the transfer was successful
|
|
314
|
+
*/
|
|
315
|
+
export async function transferSessionOwnership(linearSessionId, newWorkerId, oldWorkerId) {
|
|
316
|
+
if (!isRedisConfigured()) {
|
|
317
|
+
return { transferred: false, reason: 'Redis not configured' };
|
|
318
|
+
}
|
|
319
|
+
const existing = await getSessionState(linearSessionId);
|
|
320
|
+
if (!existing) {
|
|
321
|
+
return { transferred: false, reason: 'Session not found' };
|
|
322
|
+
}
|
|
323
|
+
// Validate that the old worker ID matches (security check)
|
|
324
|
+
if (existing.workerId && existing.workerId !== oldWorkerId) {
|
|
325
|
+
log.warn('Session ownership transfer rejected - worker ID mismatch', {
|
|
326
|
+
linearSessionId,
|
|
327
|
+
expectedWorkerId: oldWorkerId,
|
|
328
|
+
actualWorkerId: existing.workerId,
|
|
329
|
+
});
|
|
330
|
+
return {
|
|
331
|
+
transferred: false,
|
|
332
|
+
reason: `Session owned by different worker: ${existing.workerId}`,
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
const key = buildSessionKey(linearSessionId);
|
|
336
|
+
const now = Math.floor(Date.now() / 1000);
|
|
337
|
+
const updated = {
|
|
338
|
+
...existing,
|
|
339
|
+
workerId: newWorkerId,
|
|
340
|
+
updatedAt: now,
|
|
341
|
+
};
|
|
342
|
+
await redisSet(key, updated, SESSION_TTL_SECONDS);
|
|
343
|
+
log.info('Session ownership transferred', {
|
|
344
|
+
linearSessionId,
|
|
345
|
+
oldWorkerId,
|
|
346
|
+
newWorkerId,
|
|
347
|
+
});
|
|
348
|
+
return { transferred: true };
|
|
349
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the server package
|
|
3
|
+
*
|
|
4
|
+
* AgentWorkType is re-exported from here until @supaku/agentfactory-linear
|
|
5
|
+
* provides it. Consumers should import from this module.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Type of agent work being performed based on issue status
|
|
9
|
+
*/
|
|
10
|
+
export type AgentWorkType = 'research' | 'backlog-creation' | 'development' | 'inflight' | 'qa' | 'acceptance' | 'refinement' | 'coordination' | 'qa-coordination' | 'acceptance-coordination';
|
|
11
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,UAAU,GACV,kBAAkB,GAClB,aAAa,GACb,UAAU,GACV,IAAI,GACJ,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,iBAAiB,GACjB,yBAAyB,CAAA"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webhook Idempotency Module
|
|
3
|
+
*
|
|
4
|
+
* Prevents duplicate webhook processing using a two-layer approach:
|
|
5
|
+
* 1. In-memory Set for fast local checks (avoids network latency)
|
|
6
|
+
* 2. Redis for distributed/persistent storage (survives restarts)
|
|
7
|
+
*
|
|
8
|
+
* Uses webhookId (unique per delivery) as the primary key, falling back
|
|
9
|
+
* to sessionId if webhookId is not available.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Generate an idempotency key from webhook data
|
|
13
|
+
* Prefers webhookId (unique per delivery), falls back to sessionId
|
|
14
|
+
*/
|
|
15
|
+
export declare function generateIdempotencyKey(webhookId: string | undefined, sessionId: string): string;
|
|
16
|
+
/**
|
|
17
|
+
* Check if a webhook has already been processed
|
|
18
|
+
* First checks in-memory cache, then falls back to KV
|
|
19
|
+
*
|
|
20
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
21
|
+
* @returns Whether the webhook was already processed
|
|
22
|
+
*/
|
|
23
|
+
export declare function isWebhookProcessed(idempotencyKey: string): Promise<boolean>;
|
|
24
|
+
/**
|
|
25
|
+
* Mark a webhook as processed in both memory and KV
|
|
26
|
+
*
|
|
27
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
28
|
+
*/
|
|
29
|
+
export declare function markWebhookProcessed(idempotencyKey: string): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Remove a webhook from processed state (for cleanup after failed spawn)
|
|
32
|
+
*
|
|
33
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
34
|
+
*/
|
|
35
|
+
export declare function unmarkWebhookProcessed(idempotencyKey: string): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Get current cache statistics (for monitoring)
|
|
38
|
+
*/
|
|
39
|
+
export declare function getCacheStats(): {
|
|
40
|
+
memorySize: number;
|
|
41
|
+
memoryExpiryMs: number;
|
|
42
|
+
kvExpirySeconds: number;
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=webhook-idempotency.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"webhook-idempotency.d.ts","sourceRoot":"","sources":["../../src/webhook-idempotency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAyCH;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAChB,MAAM,CAOR;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,OAAO,CAAC,CA4BlB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAcf;AAWD;;GAEG;AACH,wBAAgB,aAAa,IAAI;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;CACxB,CAMA"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webhook Idempotency Module
|
|
3
|
+
*
|
|
4
|
+
* Prevents duplicate webhook processing using a two-layer approach:
|
|
5
|
+
* 1. In-memory Set for fast local checks (avoids network latency)
|
|
6
|
+
* 2. Redis for distributed/persistent storage (survives restarts)
|
|
7
|
+
*
|
|
8
|
+
* Uses webhookId (unique per delivery) as the primary key, falling back
|
|
9
|
+
* to sessionId if webhookId is not available.
|
|
10
|
+
*/
|
|
11
|
+
import { isRedisConfigured, redisSet, redisExists, redisDel } from './redis';
|
|
12
|
+
const log = {
|
|
13
|
+
info: (msg, data) => console.log(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
14
|
+
warn: (msg, data) => console.warn(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
15
|
+
error: (msg, data) => console.error(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
16
|
+
debug: (_msg, _data) => { },
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Key prefix for webhook idempotency keys in KV
|
|
20
|
+
*/
|
|
21
|
+
const WEBHOOK_KEY_PREFIX = 'webhook:processed:';
|
|
22
|
+
/**
|
|
23
|
+
* Time window for deduplication (24 hours)
|
|
24
|
+
* Linear's retry window is typically 24-48 hours
|
|
25
|
+
*/
|
|
26
|
+
const DEDUP_WINDOW_SECONDS = 24 * 60 * 60;
|
|
27
|
+
/**
|
|
28
|
+
* In-memory expiry for local cache (5 minutes)
|
|
29
|
+
* Shorter than KV to prevent memory growth
|
|
30
|
+
*/
|
|
31
|
+
const MEMORY_EXPIRY_MS = 5 * 60 * 1000;
|
|
32
|
+
/**
|
|
33
|
+
* In-memory cache for fast local checks
|
|
34
|
+
* Maps idempotency key to timestamp when it was added
|
|
35
|
+
*/
|
|
36
|
+
const processedWebhooks = new Map();
|
|
37
|
+
/**
|
|
38
|
+
* Build the KV key for a webhook idempotency entry
|
|
39
|
+
*/
|
|
40
|
+
function buildWebhookKey(idempotencyKey) {
|
|
41
|
+
return `${WEBHOOK_KEY_PREFIX}${idempotencyKey}`;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Generate an idempotency key from webhook data
|
|
45
|
+
* Prefers webhookId (unique per delivery), falls back to sessionId
|
|
46
|
+
*/
|
|
47
|
+
export function generateIdempotencyKey(webhookId, sessionId) {
|
|
48
|
+
// webhookId is unique per delivery attempt - best for idempotency
|
|
49
|
+
if (webhookId) {
|
|
50
|
+
return `wh:${webhookId}`;
|
|
51
|
+
}
|
|
52
|
+
// Fallback to sessionId if webhookId not available
|
|
53
|
+
return `session:${sessionId}`;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Check if a webhook has already been processed
|
|
57
|
+
* First checks in-memory cache, then falls back to KV
|
|
58
|
+
*
|
|
59
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
60
|
+
* @returns Whether the webhook was already processed
|
|
61
|
+
*/
|
|
62
|
+
export async function isWebhookProcessed(idempotencyKey) {
|
|
63
|
+
// Fast path: check in-memory cache first
|
|
64
|
+
if (processedWebhooks.has(idempotencyKey)) {
|
|
65
|
+
log.info(`Cache hit (memory): ${idempotencyKey}`);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
// Slow path: check Redis for distributed/persistent state
|
|
69
|
+
if (isRedisConfigured()) {
|
|
70
|
+
try {
|
|
71
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
72
|
+
const exists = await redisExists(key);
|
|
73
|
+
if (exists) {
|
|
74
|
+
log.info(`Cache hit (Redis): ${idempotencyKey}`);
|
|
75
|
+
// Warm up memory cache for subsequent checks
|
|
76
|
+
processedWebhooks.set(idempotencyKey, Date.now());
|
|
77
|
+
scheduleMemoryCleanup(idempotencyKey);
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
// Log but don't fail - better to potentially double-process
|
|
83
|
+
// than to block legitimate webhooks
|
|
84
|
+
log.error('KV check failed', { error: err });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Mark a webhook as processed in both memory and KV
|
|
91
|
+
*
|
|
92
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
93
|
+
*/
|
|
94
|
+
export async function markWebhookProcessed(idempotencyKey) {
|
|
95
|
+
// Always update memory cache
|
|
96
|
+
processedWebhooks.set(idempotencyKey, Date.now());
|
|
97
|
+
scheduleMemoryCleanup(idempotencyKey);
|
|
98
|
+
// Persist to Redis for distributed state
|
|
99
|
+
if (isRedisConfigured()) {
|
|
100
|
+
try {
|
|
101
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
102
|
+
await redisSet(key, Date.now(), DEDUP_WINDOW_SECONDS);
|
|
103
|
+
log.info(`Marked processed in Redis: ${idempotencyKey}`);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
// Log but don't fail - memory cache provides some protection
|
|
107
|
+
log.error('Redis write failed', { error: err });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Remove a webhook from processed state (for cleanup after failed spawn)
|
|
113
|
+
*
|
|
114
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
115
|
+
*/
|
|
116
|
+
export async function unmarkWebhookProcessed(idempotencyKey) {
|
|
117
|
+
// Remove from memory
|
|
118
|
+
processedWebhooks.delete(idempotencyKey);
|
|
119
|
+
// Remove from Redis
|
|
120
|
+
if (isRedisConfigured()) {
|
|
121
|
+
try {
|
|
122
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
123
|
+
await redisDel(key);
|
|
124
|
+
log.info(`Removed from Redis: ${idempotencyKey}`);
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
log.error('Redis delete failed', { error: err });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Schedule cleanup of memory cache entry after expiry
|
|
133
|
+
*/
|
|
134
|
+
function scheduleMemoryCleanup(idempotencyKey) {
|
|
135
|
+
setTimeout(() => {
|
|
136
|
+
processedWebhooks.delete(idempotencyKey);
|
|
137
|
+
}, MEMORY_EXPIRY_MS);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Get current cache statistics (for monitoring)
|
|
141
|
+
*/
|
|
142
|
+
export function getCacheStats() {
|
|
143
|
+
return {
|
|
144
|
+
memorySize: processedWebhooks.size,
|
|
145
|
+
memoryExpiryMs: MEMORY_EXPIRY_MS,
|
|
146
|
+
kvExpirySeconds: DEDUP_WINDOW_SECONDS,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Work Queue Module (Optimized)
|
|
3
|
+
*
|
|
4
|
+
* Manages the queue of pending agent work items in Redis.
|
|
5
|
+
* Workers poll this queue to claim and process work.
|
|
6
|
+
*
|
|
7
|
+
* Data Structures (optimized for high concurrency):
|
|
8
|
+
* - work:items (Hash): sessionId -> JSON work item - O(1) lookup
|
|
9
|
+
* - work:queue (Sorted Set): score = priority, member = sessionId - O(log n) operations
|
|
10
|
+
* - work:claim:{sessionId} (String): workerId with TTL - atomic claims
|
|
11
|
+
*
|
|
12
|
+
* Performance:
|
|
13
|
+
* - queueWork: O(log n) - HSET + ZADD
|
|
14
|
+
* - claimWork: O(log n) - SETNX + HGET + ZREM
|
|
15
|
+
* - peekWork: O(log n + k) - ZRANGEBYSCORE + HMGET where k = limit
|
|
16
|
+
* - getQueueLength: O(1) - ZCARD
|
|
17
|
+
*/
|
|
18
|
+
import type { AgentWorkType } from './types';
|
|
19
|
+
/**
|
|
20
|
+
* Type of work being performed
|
|
21
|
+
* @deprecated Use AgentWorkType from './types' instead
|
|
22
|
+
*/
|
|
23
|
+
export type WorkType = AgentWorkType;
|
|
24
|
+
/**
|
|
25
|
+
* Work item stored in the queue
|
|
26
|
+
*/
|
|
27
|
+
export interface QueuedWork {
|
|
28
|
+
sessionId: string;
|
|
29
|
+
issueId: string;
|
|
30
|
+
issueIdentifier: string;
|
|
31
|
+
priority: number;
|
|
32
|
+
queuedAt: number;
|
|
33
|
+
prompt?: string;
|
|
34
|
+
claudeSessionId?: string;
|
|
35
|
+
workType?: AgentWorkType;
|
|
36
|
+
sourceSessionId?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Add work to the queue
|
|
40
|
+
*
|
|
41
|
+
* @param work - Work item to queue
|
|
42
|
+
* @returns true if queued successfully
|
|
43
|
+
*/
|
|
44
|
+
export declare function queueWork(work: QueuedWork): Promise<boolean>;
|
|
45
|
+
/**
|
|
46
|
+
* Peek at pending work without removing from queue
|
|
47
|
+
* Returns items sorted by priority (lowest number = highest priority)
|
|
48
|
+
*
|
|
49
|
+
* @param limit - Maximum number of items to return
|
|
50
|
+
* @returns Array of work items sorted by priority
|
|
51
|
+
*/
|
|
52
|
+
export declare function peekWork(limit?: number): Promise<QueuedWork[]>;
|
|
53
|
+
/**
|
|
54
|
+
* Get the number of items in the queue
|
|
55
|
+
*/
|
|
56
|
+
export declare function getQueueLength(): Promise<number>;
|
|
57
|
+
/**
|
|
58
|
+
* Claim a work item for processing
|
|
59
|
+
*
|
|
60
|
+
* Uses SETNX for atomic claim to prevent race conditions.
|
|
61
|
+
* O(log n) complexity for claim + remove operations.
|
|
62
|
+
*
|
|
63
|
+
* @param sessionId - Session ID to claim
|
|
64
|
+
* @param workerId - Worker claiming the work
|
|
65
|
+
* @returns The work item if claimed successfully, null otherwise
|
|
66
|
+
*/
|
|
67
|
+
export declare function claimWork(sessionId: string, workerId: string): Promise<QueuedWork | null>;
|
|
68
|
+
/**
|
|
69
|
+
* Release a work claim (e.g., on failure or cancellation)
|
|
70
|
+
*
|
|
71
|
+
* @param sessionId - Session ID to release
|
|
72
|
+
* @returns true if released successfully
|
|
73
|
+
*/
|
|
74
|
+
export declare function releaseClaim(sessionId: string): Promise<boolean>;
|
|
75
|
+
/**
|
|
76
|
+
* Check which worker has claimed a session
|
|
77
|
+
*
|
|
78
|
+
* @param sessionId - Session ID to check
|
|
79
|
+
* @returns Worker ID if claimed, null otherwise
|
|
80
|
+
*/
|
|
81
|
+
export declare function getClaimOwner(sessionId: string): Promise<string | null>;
|
|
82
|
+
/**
|
|
83
|
+
* Check if a session has an entry in the work queue.
|
|
84
|
+
* O(1) check via the work items hash.
|
|
85
|
+
*
|
|
86
|
+
* @param sessionId - Session ID to check
|
|
87
|
+
* @returns true if the session is present in the work queue
|
|
88
|
+
*/
|
|
89
|
+
export declare function isSessionInQueue(sessionId: string): Promise<boolean>;
|
|
90
|
+
/**
|
|
91
|
+
* Re-queue work that failed or was abandoned
|
|
92
|
+
*
|
|
93
|
+
* @param work - Work item to re-queue
|
|
94
|
+
* @param priorityBoost - Decrease priority number (higher priority) by this amount
|
|
95
|
+
* @returns true if re-queued successfully
|
|
96
|
+
*/
|
|
97
|
+
export declare function requeueWork(work: QueuedWork, priorityBoost?: number): Promise<boolean>;
|
|
98
|
+
/**
|
|
99
|
+
* Get all pending work items (for dashboard/monitoring)
|
|
100
|
+
* Returns items sorted by priority
|
|
101
|
+
*/
|
|
102
|
+
export declare function getAllPendingWork(): Promise<QueuedWork[]>;
|
|
103
|
+
/**
|
|
104
|
+
* Remove a work item from queue (without claiming)
|
|
105
|
+
* Used for cleanup operations
|
|
106
|
+
*
|
|
107
|
+
* @param sessionId - Session ID to remove
|
|
108
|
+
* @returns true if removed
|
|
109
|
+
*/
|
|
110
|
+
export declare function removeFromQueue(sessionId: string): Promise<boolean>;
|
|
111
|
+
/**
|
|
112
|
+
* Migrate data from legacy list-based queue to new sorted set/hash structure
|
|
113
|
+
* Run this once after deployment to migrate existing data
|
|
114
|
+
*/
|
|
115
|
+
export declare function migrateFromLegacyQueue(): Promise<{
|
|
116
|
+
migrated: number;
|
|
117
|
+
failed: number;
|
|
118
|
+
}>;
|
|
119
|
+
//# sourceMappingURL=work-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"work-queue.d.ts","sourceRoot":"","sources":["../../src/work-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAqBH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAoB5C;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG,aAAa,CAAA;AAEpC;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,eAAe,EAAE,MAAM,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAA;IACxB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAeD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CA4BlE;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuCxE;AAED;;GAEG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAWtD;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CA6C5B;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAatE;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAY7E;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAY1E;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,EAChB,aAAa,GAAE,MAAU,GACxB,OAAO,CAAC,OAAO,CAAC,CAwBlB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAgC/D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAczE;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,IAAI,OAAO,CAAC;IACtD,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf,CAAC,CA8CD"}
|