@renseiai/agentfactory-server 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -0
- package/dist/src/a2a-server.d.ts +88 -0
- package/dist/src/a2a-server.d.ts.map +1 -0
- package/dist/src/a2a-server.integration.test.d.ts +9 -0
- package/dist/src/a2a-server.integration.test.d.ts.map +1 -0
- package/dist/src/a2a-server.integration.test.js +397 -0
- package/dist/src/a2a-server.js +235 -0
- package/dist/src/a2a-server.test.d.ts +2 -0
- package/dist/src/a2a-server.test.d.ts.map +1 -0
- package/dist/src/a2a-server.test.js +311 -0
- package/dist/src/a2a-types.d.ts +125 -0
- package/dist/src/a2a-types.d.ts.map +1 -0
- package/dist/src/a2a-types.js +8 -0
- package/dist/src/agent-tracking.d.ts +201 -0
- package/dist/src/agent-tracking.d.ts.map +1 -0
- package/dist/src/agent-tracking.js +349 -0
- package/dist/src/env-validation.d.ts +65 -0
- package/dist/src/env-validation.d.ts.map +1 -0
- package/dist/src/env-validation.js +134 -0
- package/dist/src/governor-dedup.d.ts +15 -0
- package/dist/src/governor-dedup.d.ts.map +1 -0
- package/dist/src/governor-dedup.js +31 -0
- package/dist/src/governor-event-bus.d.ts +54 -0
- package/dist/src/governor-event-bus.d.ts.map +1 -0
- package/dist/src/governor-event-bus.js +152 -0
- package/dist/src/governor-storage.d.ts +28 -0
- package/dist/src/governor-storage.d.ts.map +1 -0
- package/dist/src/governor-storage.js +52 -0
- package/dist/src/index.d.ts +26 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +50 -0
- package/dist/src/issue-lock.d.ts +129 -0
- package/dist/src/issue-lock.d.ts.map +1 -0
- package/dist/src/issue-lock.js +508 -0
- package/dist/src/logger.d.ts +76 -0
- package/dist/src/logger.d.ts.map +1 -0
- package/dist/src/logger.js +218 -0
- package/dist/src/orphan-cleanup.d.ts +64 -0
- package/dist/src/orphan-cleanup.d.ts.map +1 -0
- package/dist/src/orphan-cleanup.js +369 -0
- package/dist/src/pending-prompts.d.ts +67 -0
- package/dist/src/pending-prompts.d.ts.map +1 -0
- package/dist/src/pending-prompts.js +176 -0
- package/dist/src/processing-state-storage.d.ts +38 -0
- package/dist/src/processing-state-storage.d.ts.map +1 -0
- package/dist/src/processing-state-storage.js +61 -0
- package/dist/src/quota-tracker.d.ts +62 -0
- package/dist/src/quota-tracker.d.ts.map +1 -0
- package/dist/src/quota-tracker.js +155 -0
- package/dist/src/rate-limit.d.ts +111 -0
- package/dist/src/rate-limit.d.ts.map +1 -0
- package/dist/src/rate-limit.js +171 -0
- package/dist/src/redis-circuit-breaker.d.ts +67 -0
- package/dist/src/redis-circuit-breaker.d.ts.map +1 -0
- package/dist/src/redis-circuit-breaker.js +290 -0
- package/dist/src/redis-rate-limiter.d.ts +51 -0
- package/dist/src/redis-rate-limiter.d.ts.map +1 -0
- package/dist/src/redis-rate-limiter.js +168 -0
- package/dist/src/redis.d.ts +146 -0
- package/dist/src/redis.d.ts.map +1 -0
- package/dist/src/redis.js +343 -0
- package/dist/src/session-hash.d.ts +48 -0
- package/dist/src/session-hash.d.ts.map +1 -0
- package/dist/src/session-hash.js +80 -0
- package/dist/src/session-storage.d.ts +166 -0
- package/dist/src/session-storage.d.ts.map +1 -0
- package/dist/src/session-storage.js +397 -0
- package/dist/src/token-storage.d.ts +118 -0
- package/dist/src/token-storage.d.ts.map +1 -0
- package/dist/src/token-storage.js +263 -0
- package/dist/src/types.d.ts +11 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +7 -0
- package/dist/src/webhook-idempotency.d.ts +44 -0
- package/dist/src/webhook-idempotency.d.ts.map +1 -0
- package/dist/src/webhook-idempotency.js +148 -0
- package/dist/src/work-queue.d.ts +120 -0
- package/dist/src/work-queue.d.ts.map +1 -0
- package/dist/src/work-queue.js +384 -0
- package/dist/src/worker-auth.d.ts +29 -0
- package/dist/src/worker-auth.d.ts.map +1 -0
- package/dist/src/worker-auth.js +49 -0
- package/dist/src/worker-storage.d.ts +108 -0
- package/dist/src/worker-storage.d.ts.map +1 -0
- package/dist/src/worker-storage.js +295 -0
- package/dist/src/workflow-state-integration.test.d.ts +2 -0
- package/dist/src/workflow-state-integration.test.d.ts.map +1 -0
- package/dist/src/workflow-state-integration.test.js +342 -0
- package/dist/src/workflow-state.test.d.ts +2 -0
- package/dist/src/workflow-state.test.d.ts.map +1 -0
- package/dist/src/workflow-state.test.js +113 -0
- package/package.json +72 -0
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OAuth Token Storage Module
|
|
3
|
+
*
|
|
4
|
+
* Manages Linear OAuth token lifecycle in Redis:
|
|
5
|
+
* - Store, retrieve, refresh, and revoke tokens
|
|
6
|
+
* - Automatic token refresh before expiration
|
|
7
|
+
* - Multi-workspace token support
|
|
8
|
+
*/
|
|
9
|
+
import { createLogger } from './logger.js';
|
|
10
|
+
import { isRedisConfigured, redisSet, redisGet, redisDel, redisKeys } from './redis.js';
|
|
11
|
+
const log = createLogger('token-storage');
|
|
12
|
+
/**
|
|
13
|
+
* Key prefix for workspace tokens in KV
|
|
14
|
+
*/
|
|
15
|
+
const TOKEN_KEY_PREFIX = 'oauth:workspace:';
|
|
16
|
+
/**
|
|
17
|
+
* Buffer time (in seconds) before expiration to trigger refresh
|
|
18
|
+
* Refresh tokens 5 minutes before they expire
|
|
19
|
+
*/
|
|
20
|
+
const REFRESH_BUFFER_SECONDS = 5 * 60;
|
|
21
|
+
/**
|
|
22
|
+
* Build the KV key for a workspace token
|
|
23
|
+
*/
|
|
24
|
+
function buildTokenKey(workspaceId) {
|
|
25
|
+
return `${TOKEN_KEY_PREFIX}${workspaceId}`;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Store OAuth token for a workspace in Redis
|
|
29
|
+
*
|
|
30
|
+
* @param workspaceId - The Linear organization ID
|
|
31
|
+
* @param tokenResponse - The token data from OAuth exchange
|
|
32
|
+
* @param workspaceName - Optional workspace name for display
|
|
33
|
+
*/
|
|
34
|
+
export async function storeToken(workspaceId, tokenResponse, workspaceName) {
|
|
35
|
+
const now = Math.floor(Date.now() / 1000);
|
|
36
|
+
const storedToken = {
|
|
37
|
+
accessToken: tokenResponse.access_token,
|
|
38
|
+
refreshToken: tokenResponse.refresh_token,
|
|
39
|
+
tokenType: tokenResponse.token_type,
|
|
40
|
+
scope: tokenResponse.scope,
|
|
41
|
+
expiresAt: tokenResponse.expires_in
|
|
42
|
+
? now + tokenResponse.expires_in
|
|
43
|
+
: undefined,
|
|
44
|
+
storedAt: now,
|
|
45
|
+
workspaceId,
|
|
46
|
+
workspaceName,
|
|
47
|
+
};
|
|
48
|
+
const key = buildTokenKey(workspaceId);
|
|
49
|
+
await redisSet(key, storedToken);
|
|
50
|
+
log.info('Stored OAuth token', { workspaceId, workspaceName });
|
|
51
|
+
return storedToken;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Retrieve OAuth token for a workspace from Redis
|
|
55
|
+
*
|
|
56
|
+
* @param workspaceId - The Linear organization ID
|
|
57
|
+
* @returns The stored token or null if not found
|
|
58
|
+
*/
|
|
59
|
+
export async function getToken(workspaceId) {
|
|
60
|
+
if (!isRedisConfigured()) {
|
|
61
|
+
log.warn('Redis not configured, cannot retrieve token');
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const key = buildTokenKey(workspaceId);
|
|
65
|
+
const token = await redisGet(key);
|
|
66
|
+
return token;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Check if a token needs to be refreshed
|
|
70
|
+
* Returns true if token expires within the buffer period
|
|
71
|
+
*
|
|
72
|
+
* @param token - The stored token to check
|
|
73
|
+
* @returns Whether the token should be refreshed
|
|
74
|
+
*/
|
|
75
|
+
export function shouldRefreshToken(token) {
|
|
76
|
+
// No expiration means token doesn't expire (Linear API tokens typically don't)
|
|
77
|
+
if (!token.expiresAt) {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
const now = Math.floor(Date.now() / 1000);
|
|
81
|
+
const timeUntilExpiry = token.expiresAt - now;
|
|
82
|
+
return timeUntilExpiry <= REFRESH_BUFFER_SECONDS;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Refresh an OAuth token using the refresh token
|
|
86
|
+
*
|
|
87
|
+
* @param token - The current stored token with refresh token
|
|
88
|
+
* @param clientId - The Linear OAuth client ID
|
|
89
|
+
* @param clientSecret - The Linear OAuth client secret
|
|
90
|
+
* @returns The new stored token or null if refresh failed
|
|
91
|
+
*/
|
|
92
|
+
export async function refreshToken(token, clientId, clientSecret) {
|
|
93
|
+
const workspaceId = token.workspaceId;
|
|
94
|
+
if (!token.refreshToken) {
|
|
95
|
+
log.warn('No refresh token available', { workspaceId });
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const response = await fetch('https://api.linear.app/oauth/token', {
|
|
100
|
+
method: 'POST',
|
|
101
|
+
headers: {
|
|
102
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
103
|
+
},
|
|
104
|
+
body: new URLSearchParams({
|
|
105
|
+
grant_type: 'refresh_token',
|
|
106
|
+
client_id: clientId,
|
|
107
|
+
client_secret: clientSecret,
|
|
108
|
+
refresh_token: token.refreshToken,
|
|
109
|
+
}),
|
|
110
|
+
});
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
const errorText = await response.text();
|
|
113
|
+
log.error('Token refresh failed', {
|
|
114
|
+
workspaceId,
|
|
115
|
+
statusCode: response.status,
|
|
116
|
+
errorDetails: errorText,
|
|
117
|
+
});
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
const tokenResponse = (await response.json());
|
|
121
|
+
// Store the new token
|
|
122
|
+
const newToken = await storeToken(token.workspaceId, tokenResponse, token.workspaceName);
|
|
123
|
+
log.info('Refreshed OAuth token', { workspaceId });
|
|
124
|
+
return newToken;
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
log.error('Token refresh error', { workspaceId, error: err });
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Get a valid access token for a workspace, refreshing if necessary
|
|
133
|
+
*
|
|
134
|
+
* @param workspaceId - The Linear organization ID
|
|
135
|
+
* @param clientId - Optional OAuth client ID for refresh (defaults to env var)
|
|
136
|
+
* @param clientSecret - Optional OAuth client secret for refresh (defaults to env var)
|
|
137
|
+
* @returns The access token or null if not available
|
|
138
|
+
*/
|
|
139
|
+
export async function getAccessToken(workspaceId, clientId, clientSecret) {
|
|
140
|
+
const token = await getToken(workspaceId);
|
|
141
|
+
if (!token) {
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
// Check if token needs refresh
|
|
145
|
+
if (shouldRefreshToken(token)) {
|
|
146
|
+
const cid = clientId ?? process.env.LINEAR_CLIENT_ID;
|
|
147
|
+
const csecret = clientSecret ?? process.env.LINEAR_CLIENT_SECRET;
|
|
148
|
+
if (!cid || !csecret) {
|
|
149
|
+
log.warn('OAuth credentials not configured, cannot refresh token', { workspaceId });
|
|
150
|
+
// Return existing token even if it might be expiring soon
|
|
151
|
+
return token.accessToken;
|
|
152
|
+
}
|
|
153
|
+
const refreshedToken = await refreshToken(token, cid, csecret);
|
|
154
|
+
if (refreshedToken) {
|
|
155
|
+
return refreshedToken.accessToken;
|
|
156
|
+
}
|
|
157
|
+
// Refresh failed, return existing token
|
|
158
|
+
log.warn('Token refresh failed, using existing token', { workspaceId });
|
|
159
|
+
return token.accessToken;
|
|
160
|
+
}
|
|
161
|
+
return token.accessToken;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Delete a token from Redis (for cleanup or revocation)
|
|
165
|
+
*
|
|
166
|
+
* @param workspaceId - The Linear organization ID
|
|
167
|
+
* @returns Whether the deletion was successful
|
|
168
|
+
*/
|
|
169
|
+
export async function deleteToken(workspaceId) {
|
|
170
|
+
if (!isRedisConfigured()) {
|
|
171
|
+
log.warn('Redis not configured, cannot delete token');
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
const key = buildTokenKey(workspaceId);
|
|
175
|
+
const result = await redisDel(key);
|
|
176
|
+
log.info('Deleted OAuth token', { workspaceId });
|
|
177
|
+
return result > 0;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* List all stored workspace tokens (for admin purposes)
|
|
181
|
+
* Note: This scans all keys with the token prefix
|
|
182
|
+
*
|
|
183
|
+
* @returns Array of workspace IDs with stored tokens
|
|
184
|
+
*/
|
|
185
|
+
export async function listStoredWorkspaces() {
|
|
186
|
+
if (!isRedisConfigured()) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
const keys = await redisKeys(`${TOKEN_KEY_PREFIX}*`);
|
|
190
|
+
return keys.map((key) => key.replace(TOKEN_KEY_PREFIX, ''));
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Clean up expired tokens from Redis storage
|
|
194
|
+
* Should be called periodically (e.g., via cron job)
|
|
195
|
+
*
|
|
196
|
+
* @returns Number of tokens cleaned up
|
|
197
|
+
*/
|
|
198
|
+
export async function cleanupExpiredTokens() {
|
|
199
|
+
if (!isRedisConfigured()) {
|
|
200
|
+
return 0;
|
|
201
|
+
}
|
|
202
|
+
const workspaces = await listStoredWorkspaces();
|
|
203
|
+
const now = Math.floor(Date.now() / 1000);
|
|
204
|
+
let cleanedCount = 0;
|
|
205
|
+
for (const workspaceId of workspaces) {
|
|
206
|
+
const token = await getToken(workspaceId);
|
|
207
|
+
// Remove tokens that have expired (with some grace period)
|
|
208
|
+
// We add 1 hour grace period to avoid removing tokens that might still be usable
|
|
209
|
+
if (token?.expiresAt && token.expiresAt + 3600 < now) {
|
|
210
|
+
await deleteToken(workspaceId);
|
|
211
|
+
cleanedCount++;
|
|
212
|
+
log.info('Cleaned up expired token', { workspaceId });
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return cleanedCount;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Fetch the current user's organization from Linear API
|
|
219
|
+
* Used after OAuth to determine which workspace the token belongs to
|
|
220
|
+
*
|
|
221
|
+
* @param accessToken - The OAuth access token
|
|
222
|
+
* @returns Organization info or null if fetch failed
|
|
223
|
+
*/
|
|
224
|
+
export async function fetchOrganization(accessToken) {
|
|
225
|
+
try {
|
|
226
|
+
const response = await fetch('https://api.linear.app/graphql', {
|
|
227
|
+
method: 'POST',
|
|
228
|
+
headers: {
|
|
229
|
+
'Content-Type': 'application/json',
|
|
230
|
+
Authorization: `Bearer ${accessToken}`,
|
|
231
|
+
},
|
|
232
|
+
body: JSON.stringify({
|
|
233
|
+
query: `
|
|
234
|
+
query {
|
|
235
|
+
organization {
|
|
236
|
+
id
|
|
237
|
+
name
|
|
238
|
+
urlKey
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
`,
|
|
242
|
+
}),
|
|
243
|
+
});
|
|
244
|
+
if (!response.ok) {
|
|
245
|
+
const errorText = await response.text();
|
|
246
|
+
log.error('Failed to fetch organization', {
|
|
247
|
+
statusCode: response.status,
|
|
248
|
+
errorDetails: errorText,
|
|
249
|
+
});
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
const data = (await response.json());
|
|
253
|
+
if (data.errors) {
|
|
254
|
+
log.error('GraphQL errors fetching organization', { errors: data.errors });
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
return data.data?.organization ?? null;
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
log.error('Error fetching organization', { error: err });
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the server package
|
|
3
|
+
*
|
|
4
|
+
* AgentWorkType is re-exported from here until @renseiai/agentfactory-linear
|
|
5
|
+
* provides it. Consumers should import from this module.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Type of agent work being performed based on issue status
|
|
9
|
+
*/
|
|
10
|
+
export type AgentWorkType = 'research' | 'backlog-creation' | 'development' | 'inflight' | 'qa' | 'acceptance' | 'refinement' | 'refinement-coordination' | 'coordination' | 'qa-coordination' | 'acceptance-coordination';
|
|
11
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,UAAU,GACV,kBAAkB,GAClB,aAAa,GACb,UAAU,GACV,IAAI,GACJ,YAAY,GACZ,YAAY,GACZ,yBAAyB,GACzB,cAAc,GACd,iBAAiB,GACjB,yBAAyB,CAAA"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webhook Idempotency Module
|
|
3
|
+
*
|
|
4
|
+
* Prevents duplicate webhook processing using a two-layer approach:
|
|
5
|
+
* 1. In-memory Set for fast local checks (avoids network latency)
|
|
6
|
+
* 2. Redis for distributed/persistent storage (survives restarts)
|
|
7
|
+
*
|
|
8
|
+
* Uses webhookId (unique per delivery) as the primary key, falling back
|
|
9
|
+
* to sessionId if webhookId is not available.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Generate an idempotency key from webhook data
|
|
13
|
+
* Prefers webhookId (unique per delivery), falls back to sessionId
|
|
14
|
+
*/
|
|
15
|
+
export declare function generateIdempotencyKey(webhookId: string | undefined, sessionId: string): string;
|
|
16
|
+
/**
|
|
17
|
+
* Check if a webhook has already been processed
|
|
18
|
+
* First checks in-memory cache, then falls back to KV
|
|
19
|
+
*
|
|
20
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
21
|
+
* @returns Whether the webhook was already processed
|
|
22
|
+
*/
|
|
23
|
+
export declare function isWebhookProcessed(idempotencyKey: string): Promise<boolean>;
|
|
24
|
+
/**
|
|
25
|
+
* Mark a webhook as processed in both memory and KV
|
|
26
|
+
*
|
|
27
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
28
|
+
*/
|
|
29
|
+
export declare function markWebhookProcessed(idempotencyKey: string): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Remove a webhook from processed state (for cleanup after failed spawn)
|
|
32
|
+
*
|
|
33
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
34
|
+
*/
|
|
35
|
+
export declare function unmarkWebhookProcessed(idempotencyKey: string): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Get current cache statistics (for monitoring)
|
|
38
|
+
*/
|
|
39
|
+
export declare function getCacheStats(): {
|
|
40
|
+
memorySize: number;
|
|
41
|
+
memoryExpiryMs: number;
|
|
42
|
+
kvExpirySeconds: number;
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=webhook-idempotency.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"webhook-idempotency.d.ts","sourceRoot":"","sources":["../../src/webhook-idempotency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAyCH;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAChB,MAAM,CAOR;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,OAAO,CAAC,CA4BlB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAcf;AAWD;;GAEG;AACH,wBAAgB,aAAa,IAAI;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;CACxB,CAMA"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webhook Idempotency Module
|
|
3
|
+
*
|
|
4
|
+
* Prevents duplicate webhook processing using a two-layer approach:
|
|
5
|
+
* 1. In-memory Set for fast local checks (avoids network latency)
|
|
6
|
+
* 2. Redis for distributed/persistent storage (survives restarts)
|
|
7
|
+
*
|
|
8
|
+
* Uses webhookId (unique per delivery) as the primary key, falling back
|
|
9
|
+
* to sessionId if webhookId is not available.
|
|
10
|
+
*/
|
|
11
|
+
import { isRedisConfigured, redisSet, redisExists, redisDel } from './redis.js';
|
|
12
|
+
const log = {
|
|
13
|
+
info: (msg, data) => console.log(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
14
|
+
warn: (msg, data) => console.warn(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
15
|
+
error: (msg, data) => console.error(`[idempotency] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
16
|
+
debug: (_msg, _data) => { },
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Key prefix for webhook idempotency keys in KV
|
|
20
|
+
*/
|
|
21
|
+
const WEBHOOK_KEY_PREFIX = 'webhook:processed:';
|
|
22
|
+
/**
|
|
23
|
+
* Time window for deduplication (24 hours)
|
|
24
|
+
* Linear's retry window is typically 24-48 hours
|
|
25
|
+
*/
|
|
26
|
+
const DEDUP_WINDOW_SECONDS = 24 * 60 * 60;
|
|
27
|
+
/**
|
|
28
|
+
* In-memory expiry for local cache (5 minutes)
|
|
29
|
+
* Shorter than KV to prevent memory growth
|
|
30
|
+
*/
|
|
31
|
+
const MEMORY_EXPIRY_MS = 5 * 60 * 1000;
|
|
32
|
+
/**
|
|
33
|
+
* In-memory cache for fast local checks
|
|
34
|
+
* Maps idempotency key to timestamp when it was added
|
|
35
|
+
*/
|
|
36
|
+
const processedWebhooks = new Map();
|
|
37
|
+
/**
|
|
38
|
+
* Build the KV key for a webhook idempotency entry
|
|
39
|
+
*/
|
|
40
|
+
function buildWebhookKey(idempotencyKey) {
|
|
41
|
+
return `${WEBHOOK_KEY_PREFIX}${idempotencyKey}`;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Generate an idempotency key from webhook data
|
|
45
|
+
* Prefers webhookId (unique per delivery), falls back to sessionId
|
|
46
|
+
*/
|
|
47
|
+
export function generateIdempotencyKey(webhookId, sessionId) {
|
|
48
|
+
// webhookId is unique per delivery attempt - best for idempotency
|
|
49
|
+
if (webhookId) {
|
|
50
|
+
return `wh:${webhookId}`;
|
|
51
|
+
}
|
|
52
|
+
// Fallback to sessionId if webhookId not available
|
|
53
|
+
return `session:${sessionId}`;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Check if a webhook has already been processed
|
|
57
|
+
* First checks in-memory cache, then falls back to KV
|
|
58
|
+
*
|
|
59
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
60
|
+
* @returns Whether the webhook was already processed
|
|
61
|
+
*/
|
|
62
|
+
export async function isWebhookProcessed(idempotencyKey) {
|
|
63
|
+
// Fast path: check in-memory cache first
|
|
64
|
+
if (processedWebhooks.has(idempotencyKey)) {
|
|
65
|
+
log.info(`Cache hit (memory): ${idempotencyKey}`);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
// Slow path: check Redis for distributed/persistent state
|
|
69
|
+
if (isRedisConfigured()) {
|
|
70
|
+
try {
|
|
71
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
72
|
+
const exists = await redisExists(key);
|
|
73
|
+
if (exists) {
|
|
74
|
+
log.info(`Cache hit (Redis): ${idempotencyKey}`);
|
|
75
|
+
// Warm up memory cache for subsequent checks
|
|
76
|
+
processedWebhooks.set(idempotencyKey, Date.now());
|
|
77
|
+
scheduleMemoryCleanup(idempotencyKey);
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
// Log but don't fail - better to potentially double-process
|
|
83
|
+
// than to block legitimate webhooks
|
|
84
|
+
log.error('KV check failed', { error: err });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Mark a webhook as processed in both memory and KV
|
|
91
|
+
*
|
|
92
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
93
|
+
*/
|
|
94
|
+
export async function markWebhookProcessed(idempotencyKey) {
|
|
95
|
+
// Always update memory cache
|
|
96
|
+
processedWebhooks.set(idempotencyKey, Date.now());
|
|
97
|
+
scheduleMemoryCleanup(idempotencyKey);
|
|
98
|
+
// Persist to Redis for distributed state
|
|
99
|
+
if (isRedisConfigured()) {
|
|
100
|
+
try {
|
|
101
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
102
|
+
await redisSet(key, Date.now(), DEDUP_WINDOW_SECONDS);
|
|
103
|
+
log.info(`Marked processed in Redis: ${idempotencyKey}`);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
// Log but don't fail - memory cache provides some protection
|
|
107
|
+
log.error('Redis write failed', { error: err });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Remove a webhook from processed state (for cleanup after failed spawn)
|
|
113
|
+
*
|
|
114
|
+
* @param idempotencyKey - The key generated from generateIdempotencyKey
|
|
115
|
+
*/
|
|
116
|
+
export async function unmarkWebhookProcessed(idempotencyKey) {
|
|
117
|
+
// Remove from memory
|
|
118
|
+
processedWebhooks.delete(idempotencyKey);
|
|
119
|
+
// Remove from Redis
|
|
120
|
+
if (isRedisConfigured()) {
|
|
121
|
+
try {
|
|
122
|
+
const key = buildWebhookKey(idempotencyKey);
|
|
123
|
+
await redisDel(key);
|
|
124
|
+
log.info(`Removed from Redis: ${idempotencyKey}`);
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
log.error('Redis delete failed', { error: err });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Schedule cleanup of memory cache entry after expiry
|
|
133
|
+
*/
|
|
134
|
+
function scheduleMemoryCleanup(idempotencyKey) {
|
|
135
|
+
setTimeout(() => {
|
|
136
|
+
processedWebhooks.delete(idempotencyKey);
|
|
137
|
+
}, MEMORY_EXPIRY_MS);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Get current cache statistics (for monitoring)
|
|
141
|
+
*/
|
|
142
|
+
export function getCacheStats() {
|
|
143
|
+
return {
|
|
144
|
+
memorySize: processedWebhooks.size,
|
|
145
|
+
memoryExpiryMs: MEMORY_EXPIRY_MS,
|
|
146
|
+
kvExpirySeconds: DEDUP_WINDOW_SECONDS,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Work Queue Module (Optimized)
|
|
3
|
+
*
|
|
4
|
+
* Manages the queue of pending agent work items in Redis.
|
|
5
|
+
* Workers poll this queue to claim and process work.
|
|
6
|
+
*
|
|
7
|
+
* Data Structures (optimized for high concurrency):
|
|
8
|
+
* - work:items (Hash): sessionId -> JSON work item - O(1) lookup
|
|
9
|
+
* - work:queue (Sorted Set): score = priority, member = sessionId - O(log n) operations
|
|
10
|
+
* - work:claim:{sessionId} (String): workerId with TTL - atomic claims
|
|
11
|
+
*
|
|
12
|
+
* Performance:
|
|
13
|
+
* - queueWork: O(log n) - HSET + ZADD
|
|
14
|
+
* - claimWork: O(log n) - SETNX + HGET + ZREM
|
|
15
|
+
* - peekWork: O(log n + k) - ZRANGEBYSCORE + HMGET where k = limit
|
|
16
|
+
* - getQueueLength: O(1) - ZCARD
|
|
17
|
+
*/
|
|
18
|
+
import type { AgentWorkType } from './types.js';
|
|
19
|
+
/**
|
|
20
|
+
* Type of work being performed
|
|
21
|
+
* @deprecated Use AgentWorkType from './types.js' instead
|
|
22
|
+
*/
|
|
23
|
+
export type WorkType = AgentWorkType;
|
|
24
|
+
/**
|
|
25
|
+
* Work item stored in the queue
|
|
26
|
+
*/
|
|
27
|
+
export interface QueuedWork {
|
|
28
|
+
sessionId: string;
|
|
29
|
+
issueId: string;
|
|
30
|
+
issueIdentifier: string;
|
|
31
|
+
priority: number;
|
|
32
|
+
queuedAt: number;
|
|
33
|
+
prompt?: string;
|
|
34
|
+
providerSessionId?: string;
|
|
35
|
+
workType?: AgentWorkType;
|
|
36
|
+
sourceSessionId?: string;
|
|
37
|
+
projectName?: string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Add work to the queue
|
|
41
|
+
*
|
|
42
|
+
* @param work - Work item to queue
|
|
43
|
+
* @returns true if queued successfully
|
|
44
|
+
*/
|
|
45
|
+
export declare function queueWork(work: QueuedWork): Promise<boolean>;
|
|
46
|
+
/**
|
|
47
|
+
* Peek at pending work without removing from queue
|
|
48
|
+
* Returns items sorted by priority (lowest number = highest priority)
|
|
49
|
+
*
|
|
50
|
+
* @param limit - Maximum number of items to return
|
|
51
|
+
* @returns Array of work items sorted by priority
|
|
52
|
+
*/
|
|
53
|
+
export declare function peekWork(limit?: number): Promise<QueuedWork[]>;
|
|
54
|
+
/**
|
|
55
|
+
* Get the number of items in the queue
|
|
56
|
+
*/
|
|
57
|
+
export declare function getQueueLength(): Promise<number>;
|
|
58
|
+
/**
|
|
59
|
+
* Claim a work item for processing
|
|
60
|
+
*
|
|
61
|
+
* Uses SETNX for atomic claim to prevent race conditions.
|
|
62
|
+
* O(log n) complexity for claim + remove operations.
|
|
63
|
+
*
|
|
64
|
+
* @param sessionId - Session ID to claim
|
|
65
|
+
* @param workerId - Worker claiming the work
|
|
66
|
+
* @returns The work item if claimed successfully, null otherwise
|
|
67
|
+
*/
|
|
68
|
+
export declare function claimWork(sessionId: string, workerId: string): Promise<QueuedWork | null>;
|
|
69
|
+
/**
|
|
70
|
+
* Release a work claim (e.g., on failure or cancellation)
|
|
71
|
+
*
|
|
72
|
+
* @param sessionId - Session ID to release
|
|
73
|
+
* @returns true if released successfully
|
|
74
|
+
*/
|
|
75
|
+
export declare function releaseClaim(sessionId: string): Promise<boolean>;
|
|
76
|
+
/**
|
|
77
|
+
* Check which worker has claimed a session
|
|
78
|
+
*
|
|
79
|
+
* @param sessionId - Session ID to check
|
|
80
|
+
* @returns Worker ID if claimed, null otherwise
|
|
81
|
+
*/
|
|
82
|
+
export declare function getClaimOwner(sessionId: string): Promise<string | null>;
|
|
83
|
+
/**
|
|
84
|
+
* Check if a session has an entry in the work queue.
|
|
85
|
+
* O(1) check via the work items hash.
|
|
86
|
+
*
|
|
87
|
+
* @param sessionId - Session ID to check
|
|
88
|
+
* @returns true if the session is present in the work queue
|
|
89
|
+
*/
|
|
90
|
+
export declare function isSessionInQueue(sessionId: string): Promise<boolean>;
|
|
91
|
+
/**
|
|
92
|
+
* Re-queue work that failed or was abandoned
|
|
93
|
+
*
|
|
94
|
+
* @param work - Work item to re-queue
|
|
95
|
+
* @param priorityBoost - Decrease priority number (higher priority) by this amount
|
|
96
|
+
* @returns true if re-queued successfully
|
|
97
|
+
*/
|
|
98
|
+
export declare function requeueWork(work: QueuedWork, priorityBoost?: number): Promise<boolean>;
|
|
99
|
+
/**
|
|
100
|
+
* Get all pending work items (for dashboard/monitoring)
|
|
101
|
+
* Returns items sorted by priority
|
|
102
|
+
*/
|
|
103
|
+
export declare function getAllPendingWork(): Promise<QueuedWork[]>;
|
|
104
|
+
/**
|
|
105
|
+
* Remove a work item from queue (without claiming)
|
|
106
|
+
* Used for cleanup operations
|
|
107
|
+
*
|
|
108
|
+
* @param sessionId - Session ID to remove
|
|
109
|
+
* @returns true if removed
|
|
110
|
+
*/
|
|
111
|
+
export declare function removeFromQueue(sessionId: string): Promise<boolean>;
|
|
112
|
+
/**
|
|
113
|
+
* Migrate data from legacy list-based queue to new sorted set/hash structure
|
|
114
|
+
* Run this once after deployment to migrate existing data
|
|
115
|
+
*/
|
|
116
|
+
export declare function migrateFromLegacyQueue(): Promise<{
|
|
117
|
+
migrated: number;
|
|
118
|
+
failed: number;
|
|
119
|
+
}>;
|
|
120
|
+
//# sourceMappingURL=work-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"work-queue.d.ts","sourceRoot":"","sources":["../../src/work-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAqBH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAoB/C;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG,aAAa,CAAA;AAEpC;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,eAAe,EAAE,MAAM,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,EAAE,aAAa,CAAA;IACxB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAeD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CA4BlE;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuCxE;AAED;;GAEG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAWtD;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAgE5B;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAatE;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAY7E;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAY1E;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,EAChB,aAAa,GAAE,MAAU,GACxB,OAAO,CAAC,OAAO,CAAC,CAwBlB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAgC/D;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAczE;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,IAAI,OAAO,CAAC;IACtD,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf,CAAC,CA8CD"}
|