@renseiai/agentfactory-server 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -0
- package/dist/src/a2a-server.d.ts +88 -0
- package/dist/src/a2a-server.d.ts.map +1 -0
- package/dist/src/a2a-server.integration.test.d.ts +9 -0
- package/dist/src/a2a-server.integration.test.d.ts.map +1 -0
- package/dist/src/a2a-server.integration.test.js +397 -0
- package/dist/src/a2a-server.js +235 -0
- package/dist/src/a2a-server.test.d.ts +2 -0
- package/dist/src/a2a-server.test.d.ts.map +1 -0
- package/dist/src/a2a-server.test.js +311 -0
- package/dist/src/a2a-types.d.ts +125 -0
- package/dist/src/a2a-types.d.ts.map +1 -0
- package/dist/src/a2a-types.js +8 -0
- package/dist/src/agent-tracking.d.ts +201 -0
- package/dist/src/agent-tracking.d.ts.map +1 -0
- package/dist/src/agent-tracking.js +349 -0
- package/dist/src/env-validation.d.ts +65 -0
- package/dist/src/env-validation.d.ts.map +1 -0
- package/dist/src/env-validation.js +134 -0
- package/dist/src/governor-dedup.d.ts +15 -0
- package/dist/src/governor-dedup.d.ts.map +1 -0
- package/dist/src/governor-dedup.js +31 -0
- package/dist/src/governor-event-bus.d.ts +54 -0
- package/dist/src/governor-event-bus.d.ts.map +1 -0
- package/dist/src/governor-event-bus.js +152 -0
- package/dist/src/governor-storage.d.ts +28 -0
- package/dist/src/governor-storage.d.ts.map +1 -0
- package/dist/src/governor-storage.js +52 -0
- package/dist/src/index.d.ts +26 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +50 -0
- package/dist/src/issue-lock.d.ts +129 -0
- package/dist/src/issue-lock.d.ts.map +1 -0
- package/dist/src/issue-lock.js +508 -0
- package/dist/src/logger.d.ts +76 -0
- package/dist/src/logger.d.ts.map +1 -0
- package/dist/src/logger.js +218 -0
- package/dist/src/orphan-cleanup.d.ts +64 -0
- package/dist/src/orphan-cleanup.d.ts.map +1 -0
- package/dist/src/orphan-cleanup.js +369 -0
- package/dist/src/pending-prompts.d.ts +67 -0
- package/dist/src/pending-prompts.d.ts.map +1 -0
- package/dist/src/pending-prompts.js +176 -0
- package/dist/src/processing-state-storage.d.ts +38 -0
- package/dist/src/processing-state-storage.d.ts.map +1 -0
- package/dist/src/processing-state-storage.js +61 -0
- package/dist/src/quota-tracker.d.ts +62 -0
- package/dist/src/quota-tracker.d.ts.map +1 -0
- package/dist/src/quota-tracker.js +155 -0
- package/dist/src/rate-limit.d.ts +111 -0
- package/dist/src/rate-limit.d.ts.map +1 -0
- package/dist/src/rate-limit.js +171 -0
- package/dist/src/redis-circuit-breaker.d.ts +67 -0
- package/dist/src/redis-circuit-breaker.d.ts.map +1 -0
- package/dist/src/redis-circuit-breaker.js +290 -0
- package/dist/src/redis-rate-limiter.d.ts +51 -0
- package/dist/src/redis-rate-limiter.d.ts.map +1 -0
- package/dist/src/redis-rate-limiter.js +168 -0
- package/dist/src/redis.d.ts +146 -0
- package/dist/src/redis.d.ts.map +1 -0
- package/dist/src/redis.js +343 -0
- package/dist/src/session-hash.d.ts +48 -0
- package/dist/src/session-hash.d.ts.map +1 -0
- package/dist/src/session-hash.js +80 -0
- package/dist/src/session-storage.d.ts +166 -0
- package/dist/src/session-storage.d.ts.map +1 -0
- package/dist/src/session-storage.js +397 -0
- package/dist/src/token-storage.d.ts +118 -0
- package/dist/src/token-storage.d.ts.map +1 -0
- package/dist/src/token-storage.js +263 -0
- package/dist/src/types.d.ts +11 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +7 -0
- package/dist/src/webhook-idempotency.d.ts +44 -0
- package/dist/src/webhook-idempotency.d.ts.map +1 -0
- package/dist/src/webhook-idempotency.js +148 -0
- package/dist/src/work-queue.d.ts +120 -0
- package/dist/src/work-queue.d.ts.map +1 -0
- package/dist/src/work-queue.js +384 -0
- package/dist/src/worker-auth.d.ts +29 -0
- package/dist/src/worker-auth.d.ts.map +1 -0
- package/dist/src/worker-auth.js +49 -0
- package/dist/src/worker-storage.d.ts +108 -0
- package/dist/src/worker-storage.d.ts.map +1 -0
- package/dist/src/worker-storage.js +295 -0
- package/dist/src/workflow-state-integration.test.d.ts +2 -0
- package/dist/src/workflow-state-integration.test.d.ts.map +1 -0
- package/dist/src/workflow-state-integration.test.js +342 -0
- package/dist/src/workflow-state.test.d.ts +2 -0
- package/dist/src/workflow-state.test.d.ts.map +1 -0
- package/dist/src/workflow-state.test.js +113 -0
- package/package.json +72 -0
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Storage Module
|
|
3
|
+
*
|
|
4
|
+
* Manages worker registration and tracking in Redis.
|
|
5
|
+
* Workers register on startup, send periodic heartbeats,
|
|
6
|
+
* and deregister on shutdown.
|
|
7
|
+
*/
|
|
8
|
+
import crypto from 'crypto';
|
|
9
|
+
import { redisSet, redisGet, redisDel, redisKeys, redisSAdd, redisSRem, redisSMembers, isRedisConfigured, } from './redis.js';
|
|
10
|
+
const log = {
|
|
11
|
+
info: (msg, data) => console.log(`[worker] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
12
|
+
warn: (msg, data) => console.warn(`[worker] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
13
|
+
error: (msg, data) => console.error(`[worker] ${msg}`, data ? JSON.stringify(data) : ''),
|
|
14
|
+
debug: (_msg, _data) => { },
|
|
15
|
+
};
|
|
16
|
+
// Redis key constants
|
|
17
|
+
const WORKER_PREFIX = 'work:worker:';
|
|
18
|
+
const WORKER_SESSIONS_SUFFIX = ':sessions';
|
|
19
|
+
// Default TTL for worker registration (300 seconds = 5 minutes)
|
|
20
|
+
// Worker must send heartbeat within this time or be considered offline.
|
|
21
|
+
// 120s was too tight — busy workers processing long agents can miss heartbeats,
|
|
22
|
+
// causing Redis key expiry, 404s, and re-registration cascades.
|
|
23
|
+
const WORKER_TTL = parseInt(process.env.WORKER_TTL ?? '300', 10);
|
|
24
|
+
// Heartbeat timeout (180 seconds = 6 missed 30-second heartbeats)
|
|
25
|
+
// Increased from 90s to match the higher WORKER_TTL and prevent spurious offline detection.
|
|
26
|
+
const HEARTBEAT_TIMEOUT = parseInt(process.env.WORKER_HEARTBEAT_TIMEOUT ?? '180000', 10);
|
|
27
|
+
// Configurable intervals (in milliseconds)
|
|
28
|
+
const HEARTBEAT_INTERVAL = parseInt(process.env.WORKER_HEARTBEAT_INTERVAL ?? '30000', 10);
|
|
29
|
+
const POLL_INTERVAL = parseInt(process.env.WORKER_POLL_INTERVAL ?? '5000', 10);
|
|
30
|
+
/**
|
|
31
|
+
* Register a new worker
|
|
32
|
+
*
|
|
33
|
+
* @param hostname - Worker's hostname
|
|
34
|
+
* @param capacity - Maximum concurrent agents the worker can handle
|
|
35
|
+
* @param version - Optional worker software version
|
|
36
|
+
* @returns Worker ID and configuration
|
|
37
|
+
*/
|
|
38
|
+
export async function registerWorker(hostname, capacity, version, projects) {
|
|
39
|
+
if (!isRedisConfigured()) {
|
|
40
|
+
log.warn('Redis not configured, cannot register worker');
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
const workerId = `wkr_${crypto.randomUUID().replace(/-/g, '').slice(0, 16)}`;
|
|
45
|
+
const now = Date.now();
|
|
46
|
+
const workerData = {
|
|
47
|
+
id: workerId,
|
|
48
|
+
hostname,
|
|
49
|
+
capacity,
|
|
50
|
+
activeCount: 0,
|
|
51
|
+
registeredAt: now,
|
|
52
|
+
lastHeartbeat: now,
|
|
53
|
+
status: 'active',
|
|
54
|
+
version,
|
|
55
|
+
projects: projects?.length ? projects : undefined,
|
|
56
|
+
};
|
|
57
|
+
const key = `${WORKER_PREFIX}${workerId}`;
|
|
58
|
+
await redisSet(key, workerData, WORKER_TTL);
|
|
59
|
+
log.info('Worker registered', {
|
|
60
|
+
workerId,
|
|
61
|
+
hostname,
|
|
62
|
+
capacity,
|
|
63
|
+
projects: projects?.length ? projects : 'all',
|
|
64
|
+
});
|
|
65
|
+
return {
|
|
66
|
+
workerId,
|
|
67
|
+
heartbeatInterval: HEARTBEAT_INTERVAL,
|
|
68
|
+
pollInterval: POLL_INTERVAL,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
log.error('Failed to register worker', { error, hostname });
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Update worker heartbeat
|
|
78
|
+
*
|
|
79
|
+
* @param workerId - Worker ID
|
|
80
|
+
* @param activeCount - Current number of active agents
|
|
81
|
+
* @param load - Optional system load metrics
|
|
82
|
+
* @returns Heartbeat acknowledgment or null on failure
|
|
83
|
+
*/
|
|
84
|
+
export async function updateHeartbeat(workerId, activeCount, load) {
|
|
85
|
+
if (!isRedisConfigured()) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
const key = `${WORKER_PREFIX}${workerId}`;
|
|
90
|
+
const worker = await redisGet(key);
|
|
91
|
+
if (!worker) {
|
|
92
|
+
log.warn('Heartbeat for unknown worker', { workerId });
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
// Update worker data
|
|
96
|
+
const updatedWorker = {
|
|
97
|
+
...worker,
|
|
98
|
+
activeCount,
|
|
99
|
+
lastHeartbeat: Date.now(),
|
|
100
|
+
status: 'active',
|
|
101
|
+
};
|
|
102
|
+
// Reset TTL on heartbeat
|
|
103
|
+
await redisSet(key, updatedWorker, WORKER_TTL);
|
|
104
|
+
// Get pending work count (import dynamically to avoid circular dep)
|
|
105
|
+
const { getQueueLength } = await import('./work-queue');
|
|
106
|
+
const pendingWorkCount = await getQueueLength();
|
|
107
|
+
return {
|
|
108
|
+
acknowledged: true,
|
|
109
|
+
serverTime: new Date().toISOString(),
|
|
110
|
+
pendingWorkCount,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
log.error('Failed to update heartbeat', { error, workerId });
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Get worker by ID
|
|
120
|
+
*/
|
|
121
|
+
export async function getWorker(workerId) {
|
|
122
|
+
if (!isRedisConfigured()) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
const key = `${WORKER_PREFIX}${workerId}`;
|
|
127
|
+
const worker = await redisGet(key);
|
|
128
|
+
if (!worker) {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
// Get active sessions for this worker
|
|
132
|
+
const sessionsKey = `${key}${WORKER_SESSIONS_SUFFIX}`;
|
|
133
|
+
const activeSessions = await redisSMembers(sessionsKey);
|
|
134
|
+
return {
|
|
135
|
+
...worker,
|
|
136
|
+
activeSessions,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
log.error('Failed to get worker', { error, workerId });
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Deregister a worker
|
|
146
|
+
*
|
|
147
|
+
* @param workerId - Worker ID to deregister
|
|
148
|
+
* @returns List of session IDs that need to be re-queued
|
|
149
|
+
*/
|
|
150
|
+
export async function deregisterWorker(workerId) {
|
|
151
|
+
if (!isRedisConfigured()) {
|
|
152
|
+
return { deregistered: false, unclaimedSessions: [] };
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
const key = `${WORKER_PREFIX}${workerId}`;
|
|
156
|
+
const sessionsKey = `${key}${WORKER_SESSIONS_SUFFIX}`;
|
|
157
|
+
// Get sessions that need to be re-queued
|
|
158
|
+
const unclaimedSessions = await redisSMembers(sessionsKey);
|
|
159
|
+
// Delete worker registration and sessions set
|
|
160
|
+
await redisDel(key);
|
|
161
|
+
await redisDel(sessionsKey);
|
|
162
|
+
log.info('Worker deregistered', {
|
|
163
|
+
workerId,
|
|
164
|
+
unclaimedSessions: unclaimedSessions.length,
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
deregistered: true,
|
|
168
|
+
unclaimedSessions,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
log.error('Failed to deregister worker', { error, workerId });
|
|
173
|
+
return { deregistered: false, unclaimedSessions: [] };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* List all registered workers
|
|
178
|
+
*/
|
|
179
|
+
export async function listWorkers() {
|
|
180
|
+
if (!isRedisConfigured()) {
|
|
181
|
+
return [];
|
|
182
|
+
}
|
|
183
|
+
try {
|
|
184
|
+
const keys = await redisKeys(`${WORKER_PREFIX}*`);
|
|
185
|
+
// Filter out session keys
|
|
186
|
+
const workerKeys = keys.filter((k) => !k.endsWith(WORKER_SESSIONS_SUFFIX));
|
|
187
|
+
const workers = [];
|
|
188
|
+
for (const key of workerKeys) {
|
|
189
|
+
const worker = await redisGet(key);
|
|
190
|
+
if (worker) {
|
|
191
|
+
const sessionsKey = `${key}${WORKER_SESSIONS_SUFFIX}`;
|
|
192
|
+
const activeSessions = await redisSMembers(sessionsKey);
|
|
193
|
+
// Check if worker is stale
|
|
194
|
+
const isStale = Date.now() - worker.lastHeartbeat > HEARTBEAT_TIMEOUT;
|
|
195
|
+
const status = isStale ? 'offline' : worker.status;
|
|
196
|
+
workers.push({
|
|
197
|
+
...worker,
|
|
198
|
+
status,
|
|
199
|
+
activeSessions,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return workers;
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
log.error('Failed to list workers', { error });
|
|
207
|
+
return [];
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Get workers that have missed heartbeats (stale workers)
|
|
212
|
+
*/
|
|
213
|
+
export async function getStaleWorkers() {
|
|
214
|
+
if (!isRedisConfigured()) {
|
|
215
|
+
return [];
|
|
216
|
+
}
|
|
217
|
+
try {
|
|
218
|
+
const workers = await listWorkers();
|
|
219
|
+
return workers.filter((w) => w.status === 'offline');
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
log.error('Failed to get stale workers', { error });
|
|
223
|
+
return [];
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Add a session to a worker's active sessions
|
|
228
|
+
*
|
|
229
|
+
* @param workerId - Worker ID
|
|
230
|
+
* @param sessionId - Session ID being processed
|
|
231
|
+
*/
|
|
232
|
+
export async function addWorkerSession(workerId, sessionId) {
|
|
233
|
+
if (!isRedisConfigured()) {
|
|
234
|
+
return false;
|
|
235
|
+
}
|
|
236
|
+
try {
|
|
237
|
+
const sessionsKey = `${WORKER_PREFIX}${workerId}${WORKER_SESSIONS_SUFFIX}`;
|
|
238
|
+
await redisSAdd(sessionsKey, sessionId);
|
|
239
|
+
return true;
|
|
240
|
+
}
|
|
241
|
+
catch (error) {
|
|
242
|
+
log.error('Failed to add worker session', { error, workerId, sessionId });
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Remove a session from a worker's active sessions
|
|
248
|
+
*
|
|
249
|
+
* @param workerId - Worker ID
|
|
250
|
+
* @param sessionId - Session ID to remove
|
|
251
|
+
*/
|
|
252
|
+
export async function removeWorkerSession(workerId, sessionId) {
|
|
253
|
+
if (!isRedisConfigured()) {
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
try {
|
|
257
|
+
const sessionsKey = `${WORKER_PREFIX}${workerId}${WORKER_SESSIONS_SUFFIX}`;
|
|
258
|
+
await redisSRem(sessionsKey, sessionId);
|
|
259
|
+
return true;
|
|
260
|
+
}
|
|
261
|
+
catch (error) {
|
|
262
|
+
log.error('Failed to remove worker session', { error, workerId, sessionId });
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Get total capacity across all active workers.
|
|
268
|
+
*
|
|
269
|
+
* Accepts an optional pre-fetched workers list to avoid redundant Redis scans
|
|
270
|
+
* (callers like the stats handler already call listWorkers()).
|
|
271
|
+
*
|
|
272
|
+
* Uses activeSessions.length (authoritative Redis set) instead of the
|
|
273
|
+
* heartbeat-reported activeCount, which can be stale after re-registration
|
|
274
|
+
* or between heartbeat intervals.
|
|
275
|
+
*/
|
|
276
|
+
export async function getTotalCapacity(prefetchedWorkers) {
|
|
277
|
+
if (!isRedisConfigured()) {
|
|
278
|
+
return { totalCapacity: 0, totalActive: 0, availableCapacity: 0 };
|
|
279
|
+
}
|
|
280
|
+
try {
|
|
281
|
+
const workers = prefetchedWorkers ?? await listWorkers();
|
|
282
|
+
const activeWorkers = workers.filter((w) => w.status === 'active');
|
|
283
|
+
const totalCapacity = activeWorkers.reduce((sum, w) => sum + w.capacity, 0);
|
|
284
|
+
const totalActive = activeWorkers.reduce((sum, w) => sum + w.activeSessions.length, 0);
|
|
285
|
+
return {
|
|
286
|
+
totalCapacity,
|
|
287
|
+
totalActive,
|
|
288
|
+
availableCapacity: totalCapacity - totalActive,
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
catch (error) {
|
|
292
|
+
log.error('Failed to get total capacity', { error });
|
|
293
|
+
return { totalCapacity: 0, totalActive: 0, availableCapacity: 0 };
|
|
294
|
+
}
|
|
295
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow-state-integration.test.d.ts","sourceRoot":"","sources":["../../src/workflow-state-integration.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration tests for the 4-cycle workflow state progression
|
|
3
|
+
* and escalate-human circuit breaker.
|
|
4
|
+
*
|
|
5
|
+
* Mocks Redis to test the full lifecycle:
|
|
6
|
+
* Cycle 1 (normal) → Cycle 2 (context-enriched) → Cycle 3 (decompose) → Cycle 4 (escalate-human)
|
|
7
|
+
*/
|
|
8
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
9
|
+
// In-memory Redis store for testing
|
|
10
|
+
const store = new Map();
|
|
11
|
+
vi.mock('./redis.js', () => ({
|
|
12
|
+
redisSet: vi.fn(async (key, value, _ttl) => {
|
|
13
|
+
store.set(key, JSON.stringify(value));
|
|
14
|
+
}),
|
|
15
|
+
redisGet: vi.fn(async (key) => {
|
|
16
|
+
const val = store.get(key);
|
|
17
|
+
return val ? JSON.parse(val) : null;
|
|
18
|
+
}),
|
|
19
|
+
redisDel: vi.fn(async (key) => {
|
|
20
|
+
const existed = store.has(key) ? 1 : 0;
|
|
21
|
+
store.delete(key);
|
|
22
|
+
return existed;
|
|
23
|
+
}),
|
|
24
|
+
redisExists: vi.fn(async (key) => store.has(key)),
|
|
25
|
+
}));
|
|
26
|
+
import { computeStrategy, getWorkflowState, updateWorkflowState, recordPhaseAttempt, incrementCycleCount, appendFailureSummary, clearWorkflowState, extractFailureReason, getTotalSessionCount, MAX_TOTAL_SESSIONS, markAcceptanceCompleted, didAcceptanceJustComplete, clearAcceptanceCompleted, } from './agent-tracking.js';
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
store.clear();
|
|
29
|
+
});
|
|
30
|
+
/**
|
|
31
|
+
* Simulate what the webhook-orchestrator onAgentComplete handler does
|
|
32
|
+
* when a QA/acceptance agent fails.
|
|
33
|
+
*/
|
|
34
|
+
async function simulateQAFailure(issueId, failureMessage) {
|
|
35
|
+
const state = await incrementCycleCount(issueId);
|
|
36
|
+
const failureReason = extractFailureReason(failureMessage);
|
|
37
|
+
const formattedFailure = `--- Cycle ${state.cycleCount}, qa (${new Date().toISOString()}) ---\n${failureReason}`;
|
|
38
|
+
await appendFailureSummary(issueId, formattedFailure);
|
|
39
|
+
return await getWorkflowState(issueId);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Simulate what the webhook-orchestrator onAgentComplete handler does
|
|
43
|
+
* when recording a phase attempt.
|
|
44
|
+
*/
|
|
45
|
+
async function simulatePhaseCompletion(issueId, phase, result, costUsd = 0.50) {
|
|
46
|
+
await recordPhaseAttempt(issueId, phase, {
|
|
47
|
+
attempt: 1,
|
|
48
|
+
sessionId: `session-${phase}-${Date.now()}`,
|
|
49
|
+
startedAt: Date.now(),
|
|
50
|
+
completedAt: Date.now(),
|
|
51
|
+
result,
|
|
52
|
+
costUsd,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
describe('4-cycle workflow progression integration', () => {
|
|
56
|
+
const issueId = 'test-issue-001';
|
|
57
|
+
const issueIdentifier = 'TEST-42';
|
|
58
|
+
it('progresses through all 4 escalation tiers across full dev-QA-rejected cycles', async () => {
|
|
59
|
+
// Initialize workflow state (happens on first agent completion)
|
|
60
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
61
|
+
// === Cycle 1: normal ===
|
|
62
|
+
// Development completes, QA fails
|
|
63
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed');
|
|
64
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed');
|
|
65
|
+
let state = await simulateQAFailure(issueId, '## QA Failed\n\nTests failed: TypeError in handleUserLogin — null check missing');
|
|
66
|
+
expect(state).not.toBeNull();
|
|
67
|
+
expect(state.cycleCount).toBe(1);
|
|
68
|
+
expect(state.strategy).toBe('normal');
|
|
69
|
+
expect(state.failureSummary).toContain('TypeError in handleUserLogin');
|
|
70
|
+
// At cycle 1 (normal), strategy would provide retry context for development
|
|
71
|
+
expect(computeStrategy(1)).toBe('normal');
|
|
72
|
+
// === Cycle 2: context-enriched ===
|
|
73
|
+
// Refinement completes, development re-runs, QA fails again
|
|
74
|
+
await simulatePhaseCompletion(issueId, 'refinement', 'passed');
|
|
75
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed');
|
|
76
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed');
|
|
77
|
+
state = await simulateQAFailure(issueId, '## QA Failed\n\nBuild passes but integration test for /api/users returns 500. Missing database migration.');
|
|
78
|
+
expect(state.cycleCount).toBe(2);
|
|
79
|
+
expect(state.strategy).toBe('context-enriched');
|
|
80
|
+
expect(state.failureSummary).toContain('Cycle 1');
|
|
81
|
+
expect(state.failureSummary).toContain('Cycle 2');
|
|
82
|
+
expect(state.failureSummary).toContain('database migration');
|
|
83
|
+
// At cycle 2 (context-enriched), refinement gets full failure history
|
|
84
|
+
expect(computeStrategy(2)).toBe('context-enriched');
|
|
85
|
+
// === Cycle 3: decompose ===
|
|
86
|
+
await simulatePhaseCompletion(issueId, 'refinement', 'passed');
|
|
87
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed');
|
|
88
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed');
|
|
89
|
+
state = await simulateQAFailure(issueId, 'Failure Reason: The same /api/users endpoint still returns 500. Root cause is the missing foreign key constraint in the migration.');
|
|
90
|
+
expect(state.cycleCount).toBe(3);
|
|
91
|
+
expect(state.strategy).toBe('decompose');
|
|
92
|
+
expect(state.failureSummary).toContain('Cycle 3');
|
|
93
|
+
// At cycle 3 (decompose), refinement gets decomposition instructions
|
|
94
|
+
expect(computeStrategy(3)).toBe('decompose');
|
|
95
|
+
// === Cycle 4: escalate-human ===
|
|
96
|
+
await simulatePhaseCompletion(issueId, 'refinement', 'passed');
|
|
97
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed');
|
|
98
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed');
|
|
99
|
+
state = await simulateQAFailure(issueId, 'Issues Found:\n1. API endpoint still broken\n2. Migration file has syntax error\n3. Schema mismatch between ORM and SQL');
|
|
100
|
+
expect(state.cycleCount).toBe(4);
|
|
101
|
+
expect(state.strategy).toBe('escalate-human');
|
|
102
|
+
expect(state.failureSummary).toContain('Cycle 4');
|
|
103
|
+
// At cycle 4+ (escalate-human), the loop should stop entirely
|
|
104
|
+
expect(computeStrategy(state.cycleCount)).toBe('escalate-human');
|
|
105
|
+
expect(computeStrategy(state.cycleCount + 1)).toBe('escalate-human');
|
|
106
|
+
// Verify all 4 phase records tracked for development and QA
|
|
107
|
+
expect(state.phases.development).toHaveLength(4);
|
|
108
|
+
expect(state.phases.qa).toHaveLength(4);
|
|
109
|
+
expect(state.phases.refinement).toHaveLength(3);
|
|
110
|
+
});
|
|
111
|
+
it('accumulates failure summaries across all cycles with correct formatting', async () => {
|
|
112
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
113
|
+
// Simulate 4 failures with distinct messages
|
|
114
|
+
const failures = [
|
|
115
|
+
'## QA Failed\n\nNull pointer in UserService.getById',
|
|
116
|
+
'Failure Reason: Missing validation on email field causes 500',
|
|
117
|
+
'Issues Found:\n1. Race condition in session manager\n2. Stale cache returns wrong user',
|
|
118
|
+
'## QA Failed\n\nDatabase connection pool exhausted after 10 concurrent requests',
|
|
119
|
+
];
|
|
120
|
+
for (const msg of failures) {
|
|
121
|
+
await simulateQAFailure(issueId, msg);
|
|
122
|
+
}
|
|
123
|
+
const state = await getWorkflowState(issueId);
|
|
124
|
+
expect(state).not.toBeNull();
|
|
125
|
+
expect(state.cycleCount).toBe(4);
|
|
126
|
+
expect(state.strategy).toBe('escalate-human');
|
|
127
|
+
// All 4 cycle markers should be present
|
|
128
|
+
expect(state.failureSummary).toContain('Cycle 1');
|
|
129
|
+
expect(state.failureSummary).toContain('Cycle 2');
|
|
130
|
+
expect(state.failureSummary).toContain('Cycle 3');
|
|
131
|
+
expect(state.failureSummary).toContain('Cycle 4');
|
|
132
|
+
// Failure details from each cycle should be present
|
|
133
|
+
expect(state.failureSummary).toContain('Null pointer in UserService');
|
|
134
|
+
expect(state.failureSummary).toContain('Missing validation on email');
|
|
135
|
+
expect(state.failureSummary).toContain('Race condition in session manager');
|
|
136
|
+
expect(state.failureSummary).toContain('Database connection pool exhausted');
|
|
137
|
+
});
|
|
138
|
+
it('tracks per-phase attempt records and costs across cycles', async () => {
|
|
139
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
140
|
+
// Cycle 1: dev → qa(fail)
|
|
141
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed', 1.50);
|
|
142
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed', 0.75);
|
|
143
|
+
await simulateQAFailure(issueId, 'Tests failed');
|
|
144
|
+
// Cycle 2: refinement → dev → qa(fail)
|
|
145
|
+
await simulatePhaseCompletion(issueId, 'refinement', 'passed', 0.50);
|
|
146
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed', 2.00);
|
|
147
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed', 0.80);
|
|
148
|
+
await simulateQAFailure(issueId, 'Build failed');
|
|
149
|
+
const state = await getWorkflowState(issueId);
|
|
150
|
+
expect(state).not.toBeNull();
|
|
151
|
+
// Check phase records exist
|
|
152
|
+
expect(state.phases.development).toHaveLength(2);
|
|
153
|
+
expect(state.phases.qa).toHaveLength(2);
|
|
154
|
+
expect(state.phases.refinement).toHaveLength(1);
|
|
155
|
+
expect(state.phases.acceptance).toHaveLength(0);
|
|
156
|
+
// Calculate total cost (simulates what the escalation comment builder does)
|
|
157
|
+
const allPhases = [
|
|
158
|
+
...state.phases.development,
|
|
159
|
+
...state.phases.qa,
|
|
160
|
+
...state.phases.refinement,
|
|
161
|
+
...state.phases.acceptance,
|
|
162
|
+
];
|
|
163
|
+
const totalCost = allPhases.reduce((sum, p) => sum + (p.costUsd ?? 0), 0);
|
|
164
|
+
expect(totalCost).toBeCloseTo(5.55, 2);
|
|
165
|
+
});
|
|
166
|
+
it('clears workflow state on acceptance pass', async () => {
|
|
167
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
168
|
+
await simulateQAFailure(issueId, 'Tests failed');
|
|
169
|
+
let state = await getWorkflowState(issueId);
|
|
170
|
+
expect(state).not.toBeNull();
|
|
171
|
+
expect(state.cycleCount).toBe(1);
|
|
172
|
+
// Acceptance pass clears everything
|
|
173
|
+
await clearWorkflowState(issueId);
|
|
174
|
+
state = await getWorkflowState(issueId);
|
|
175
|
+
expect(state).toBeNull();
|
|
176
|
+
});
|
|
177
|
+
it('strategy-based circuit breaker blocks at cycle 4+', async () => {
|
|
178
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
179
|
+
// Progress to escalate-human (cycle 4)
|
|
180
|
+
for (let i = 0; i < 4; i++) {
|
|
181
|
+
await simulateQAFailure(issueId, `Failure in cycle ${i + 1}`);
|
|
182
|
+
}
|
|
183
|
+
const state = await getWorkflowState(issueId);
|
|
184
|
+
expect(state.strategy).toBe('escalate-human');
|
|
185
|
+
// Circuit breaker check: this is what issue-updated.ts does
|
|
186
|
+
const shouldBlock = state.strategy === 'escalate-human';
|
|
187
|
+
expect(shouldBlock).toBe(true);
|
|
188
|
+
// Verify further increments stay at escalate-human
|
|
189
|
+
await simulateQAFailure(issueId, 'Yet another failure');
|
|
190
|
+
const state2 = await getWorkflowState(issueId);
|
|
191
|
+
expect(state2.cycleCount).toBe(5);
|
|
192
|
+
expect(state2.strategy).toBe('escalate-human');
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
describe('escalate-human blocker creation data', () => {
|
|
196
|
+
const issueId = 'blocker-test-001';
|
|
197
|
+
const issueIdentifier = 'TEST-99';
|
|
198
|
+
it('provides correct data for blocker issue creation at cycle 4', async () => {
|
|
199
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
200
|
+
// Simulate 4 cycles with realistic failures
|
|
201
|
+
const qaFailures = [
|
|
202
|
+
'## QA Failed\n\nUnit tests in packages/core fail with TypeError: Cannot read property of undefined',
|
|
203
|
+
'Failure Reason: Integration test for webhook handler times out after 30s',
|
|
204
|
+
'Issues Found:\n1. Missing mock for Redis in test setup\n2. Race condition in async handler',
|
|
205
|
+
'## QA Failed\n\nTypecheck fails: Property "strategy" does not exist on type "WorkflowState"',
|
|
206
|
+
];
|
|
207
|
+
for (const msg of qaFailures) {
|
|
208
|
+
await simulatePhaseCompletion(issueId, 'development', 'passed', 1.00);
|
|
209
|
+
await simulatePhaseCompletion(issueId, 'qa', 'failed', 0.50);
|
|
210
|
+
await simulateQAFailure(issueId, msg);
|
|
211
|
+
}
|
|
212
|
+
const state = await getWorkflowState(issueId);
|
|
213
|
+
expect(state).not.toBeNull();
|
|
214
|
+
expect(state.strategy).toBe('escalate-human');
|
|
215
|
+
expect(state.cycleCount).toBe(4);
|
|
216
|
+
// Verify the data that would be used to create the blocker issue
|
|
217
|
+
const { cycleCount, failureSummary, phases } = state;
|
|
218
|
+
// Blocker title matches what issue-updated.ts generates
|
|
219
|
+
const blockerTitle = `Human review needed: ${issueIdentifier} failed ${cycleCount} automated cycles`;
|
|
220
|
+
expect(blockerTitle).toBe('Human review needed: TEST-99 failed 4 automated cycles');
|
|
221
|
+
// Blocker description includes failure summary
|
|
222
|
+
const blockerDescription = [
|
|
223
|
+
`This issue has failed **${cycleCount} automated dev-QA-rejected cycles** and requires human intervention.`,
|
|
224
|
+
'',
|
|
225
|
+
'### Failure History',
|
|
226
|
+
failureSummary ?? 'No failure details recorded.',
|
|
227
|
+
'',
|
|
228
|
+
'---',
|
|
229
|
+
`*Source issue: ${issueIdentifier}*`,
|
|
230
|
+
].join('\n');
|
|
231
|
+
expect(blockerDescription).toContain('4 automated dev-QA-rejected cycles');
|
|
232
|
+
expect(blockerDescription).toContain('TypeError: Cannot read property');
|
|
233
|
+
expect(blockerDescription).toContain('Race condition in async handler');
|
|
234
|
+
expect(blockerDescription).toContain('Source issue: TEST-99');
|
|
235
|
+
// Escalation comment includes total cost
|
|
236
|
+
const allPhases = [
|
|
237
|
+
...phases.development,
|
|
238
|
+
...phases.qa,
|
|
239
|
+
...phases.refinement,
|
|
240
|
+
...phases.acceptance,
|
|
241
|
+
];
|
|
242
|
+
const totalCost = allPhases.reduce((sum, p) => sum + (p.costUsd ?? 0), 0);
|
|
243
|
+
expect(totalCost).toBeCloseTo(6.00, 2); // 4 × (1.00 + 0.50)
|
|
244
|
+
const costLine = totalCost > 0 ? `\n**Total cost across all attempts:** $${totalCost.toFixed(2)}` : '';
|
|
245
|
+
expect(costLine).toContain('$6.00');
|
|
246
|
+
// Escalation comment content
|
|
247
|
+
const escalationComment = `## Circuit Breaker: Human Intervention Required\n\n` +
|
|
248
|
+
`This issue has gone through **${cycleCount} dev-QA-rejected cycles** without passing.\n` +
|
|
249
|
+
`The automated system is stopping further attempts.\n` +
|
|
250
|
+
costLine +
|
|
251
|
+
`\n\n### Failure History\n\n${failureSummary ?? 'No failure details recorded.'}\n\n` +
|
|
252
|
+
`### Recommended Actions\n` +
|
|
253
|
+
`1. Review the failure patterns above\n` +
|
|
254
|
+
`2. Consider if the acceptance criteria need clarification\n` +
|
|
255
|
+
`3. Investigate whether there's an architectural issue\n` +
|
|
256
|
+
`4. Manually fix or decompose the issue before re-enabling automation`;
|
|
257
|
+
expect(escalationComment).toContain('4 dev-QA-rejected cycles');
|
|
258
|
+
expect(escalationComment).toContain('$6.00');
|
|
259
|
+
expect(escalationComment).toContain('Cycle 1');
|
|
260
|
+
expect(escalationComment).toContain('Cycle 4');
|
|
261
|
+
expect(escalationComment).toContain('Recommended Actions');
|
|
262
|
+
});
|
|
263
|
+
it('verifies escalation strategy is deterministic and never regresses', async () => {
|
|
264
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
265
|
+
const expectedProgression = [
|
|
266
|
+
{ cycle: 1, strategy: 'normal' },
|
|
267
|
+
{ cycle: 2, strategy: 'context-enriched' },
|
|
268
|
+
{ cycle: 3, strategy: 'decompose' },
|
|
269
|
+
{ cycle: 4, strategy: 'escalate-human' },
|
|
270
|
+
{ cycle: 5, strategy: 'escalate-human' },
|
|
271
|
+
{ cycle: 6, strategy: 'escalate-human' },
|
|
272
|
+
];
|
|
273
|
+
for (const { cycle, strategy } of expectedProgression) {
|
|
274
|
+
const state = await incrementCycleCount(issueId);
|
|
275
|
+
expect(state.cycleCount).toBe(cycle);
|
|
276
|
+
expect(state.strategy).toBe(strategy);
|
|
277
|
+
// Verify computeStrategy matches what's stored
|
|
278
|
+
expect(computeStrategy(state.cycleCount)).toBe(state.strategy);
|
|
279
|
+
}
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
describe('getTotalSessionCount', () => {
|
|
283
|
+
const issueId = 'session-count-test-001';
|
|
284
|
+
const issueIdentifier = 'TEST-SC';
|
|
285
|
+
it('returns 0 for nonexistent issue', async () => {
|
|
286
|
+
const count = await getTotalSessionCount('nonexistent-issue-id');
|
|
287
|
+
expect(count).toBe(0);
|
|
288
|
+
});
|
|
289
|
+
it('correctly sums across all phases', async () => {
|
|
290
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
291
|
+
// Add 2 development, 2 qa, 1 refinement, 1 acceptance
|
|
292
|
+
await recordPhaseAttempt(issueId, 'development', {
|
|
293
|
+
attempt: 1, sessionId: 'dev-1', startedAt: Date.now(), result: 'passed', costUsd: 1.0,
|
|
294
|
+
});
|
|
295
|
+
await recordPhaseAttempt(issueId, 'development', {
|
|
296
|
+
attempt: 2, sessionId: 'dev-2', startedAt: Date.now(), result: 'passed', costUsd: 1.0,
|
|
297
|
+
});
|
|
298
|
+
await recordPhaseAttempt(issueId, 'qa', {
|
|
299
|
+
attempt: 1, sessionId: 'qa-1', startedAt: Date.now(), result: 'failed', costUsd: 0.5,
|
|
300
|
+
});
|
|
301
|
+
await recordPhaseAttempt(issueId, 'qa', {
|
|
302
|
+
attempt: 2, sessionId: 'qa-2', startedAt: Date.now(), result: 'passed', costUsd: 0.5,
|
|
303
|
+
});
|
|
304
|
+
await recordPhaseAttempt(issueId, 'refinement', {
|
|
305
|
+
attempt: 1, sessionId: 'ref-1', startedAt: Date.now(), result: 'passed', costUsd: 0.3,
|
|
306
|
+
});
|
|
307
|
+
await recordPhaseAttempt(issueId, 'acceptance', {
|
|
308
|
+
attempt: 1, sessionId: 'acc-1', startedAt: Date.now(), result: 'passed', costUsd: 0.2,
|
|
309
|
+
});
|
|
310
|
+
const count = await getTotalSessionCount(issueId);
|
|
311
|
+
expect(count).toBe(6); // 2 + 2 + 1 + 1
|
|
312
|
+
});
|
|
313
|
+
it('returns count that would hit MAX_TOTAL_SESSIONS', async () => {
|
|
314
|
+
await updateWorkflowState(issueId, { issueIdentifier });
|
|
315
|
+
// Add exactly MAX_TOTAL_SESSIONS phase records
|
|
316
|
+
for (let i = 0; i < MAX_TOTAL_SESSIONS; i++) {
|
|
317
|
+
const phase = i % 2 === 0 ? 'development' : 'qa';
|
|
318
|
+
await recordPhaseAttempt(issueId, phase, {
|
|
319
|
+
attempt: i + 1, sessionId: `session-${i}`, startedAt: Date.now(), result: 'passed',
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
const count = await getTotalSessionCount(issueId);
|
|
323
|
+
expect(count).toBe(MAX_TOTAL_SESSIONS);
|
|
324
|
+
expect(count >= MAX_TOTAL_SESSIONS).toBe(true);
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
describe('acceptance completion lock', () => {
|
|
328
|
+
const issueId = 'acceptance-lock-test-001';
|
|
329
|
+
it('returns false before marking, true after marking', async () => {
|
|
330
|
+
const beforeMark = await didAcceptanceJustComplete(issueId);
|
|
331
|
+
expect(beforeMark).toBe(false);
|
|
332
|
+
await markAcceptanceCompleted(issueId);
|
|
333
|
+
const afterMark = await didAcceptanceJustComplete(issueId);
|
|
334
|
+
expect(afterMark).toBe(true);
|
|
335
|
+
});
|
|
336
|
+
it('returns false after clearing', async () => {
|
|
337
|
+
await markAcceptanceCompleted(issueId);
|
|
338
|
+
expect(await didAcceptanceJustComplete(issueId)).toBe(true);
|
|
339
|
+
await clearAcceptanceCompleted(issueId);
|
|
340
|
+
expect(await didAcceptanceJustComplete(issueId)).toBe(false);
|
|
341
|
+
});
|
|
342
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow-state.test.d.ts","sourceRoot":"","sources":["../../src/workflow-state.test.ts"],"names":[],"mappings":""}
|