agents-relay 1.0.142 → 1.0.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blockers.js +11 -2
- package/dist/dashboard.js +6 -3
- package/dist/pool.js +159 -49
- package/dist/reconcile-coordinator.js +90 -0
- package/dist/reconciler-review.js +4 -5
- package/dist/reconciler-runtime.js +57 -26
- package/dist/relayd.js +3 -13
- package/dist/store-github.js +7 -4
- package/package.json +1 -1
package/dist/blockers.js
CHANGED
|
@@ -62,9 +62,10 @@ export async function resolveBlockerDependencies(options) {
|
|
|
62
62
|
return { resumed: 0, obsolete: 0, waiting };
|
|
63
63
|
}
|
|
64
64
|
const now = options.now ?? new Date().toISOString();
|
|
65
|
+
const relations = blocker.unblocks ?? [];
|
|
65
66
|
let resumed = 0;
|
|
66
67
|
let obsolete = 0;
|
|
67
|
-
for (const relation of
|
|
68
|
+
for (const relation of relations) {
|
|
68
69
|
if (relation.state !== 'waiting')
|
|
69
70
|
continue;
|
|
70
71
|
const store = options.dependentStore(relation.repository, relation.prNumber);
|
|
@@ -110,8 +111,16 @@ export async function resolveBlockerDependencies(options) {
|
|
|
110
111
|
relation.note = 'dependent task requeued';
|
|
111
112
|
resumed += 1;
|
|
112
113
|
}
|
|
114
|
+
const remaining = relations.filter(relation => relation.state === 'waiting');
|
|
115
|
+
const consumed = remaining.length !== relations.length;
|
|
116
|
+
if (!consumed)
|
|
117
|
+
return { resumed, obsolete, waiting: remaining.length };
|
|
118
|
+
if (remaining.length > 0)
|
|
119
|
+
blocker.unblocks = remaining;
|
|
120
|
+
else
|
|
121
|
+
delete blocker.unblocks;
|
|
113
122
|
blocker.blockerResolvedAt = now;
|
|
114
123
|
blocker.updatedAt = now;
|
|
115
124
|
await options.blockerStore.saveJob(blocker);
|
|
116
|
-
return { resumed, obsolete, waiting:
|
|
125
|
+
return { resumed, obsolete, waiting: remaining.length };
|
|
117
126
|
}
|
package/dist/dashboard.js
CHANGED
|
@@ -184,9 +184,12 @@ export function serveDashboard(store, port = 8787, bus, refreshMs = 300000, jobI
|
|
|
184
184
|
res.writeHead(202);
|
|
185
185
|
res.end('Accepted');
|
|
186
186
|
queueMicrotask(() => {
|
|
187
|
-
|
|
188
|
-
.
|
|
189
|
-
|
|
187
|
+
const refreshDashboard = webhook.refresh
|
|
188
|
+
? webhook.refresh(event)
|
|
189
|
+
.then(jobIds => broadcastSnapshots(jobIds))
|
|
190
|
+
.catch(error => process.stderr.write(`warning: webhook dashboard snapshot unavailable: ${error instanceof Error ? error.message : String(error)}\n`))
|
|
191
|
+
: Promise.resolve();
|
|
192
|
+
void refreshDashboard
|
|
190
193
|
.then(() => webhook.wake(event))
|
|
191
194
|
.catch(error => process.stderr.write(`warning: webhook wake unavailable: ${error instanceof Error ? error.message : String(error)}\n`));
|
|
192
195
|
});
|
package/dist/pool.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { GitHubStore, listManagedGitHubJobs, withGitHubRequestContext } from './store.js';
|
|
2
2
|
import { eventFor } from './events.js';
|
|
3
|
+
import { ReconcileCoordinator } from './reconcile-coordinator.js';
|
|
3
4
|
const RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
4
5
|
export function compareJobPriority(a, b) { return RANK[a.priority ?? 'P2'] - RANK[b.priority ?? 'P2'] || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
|
|
5
6
|
export function runnableManagedJobs(items) { return items.filter(item => item.githubState === 'OPEN' && item.job.state === 'ACTIVE').map(item => item.job).sort(compareJobPriority); }
|
|
@@ -23,19 +24,28 @@ export class RepositoryWorkerPool {
|
|
|
23
24
|
concurrency;
|
|
24
25
|
makeReconciler;
|
|
25
26
|
isRateLimited;
|
|
27
|
+
refreshWebhookSnapshot;
|
|
26
28
|
running = false;
|
|
27
29
|
pending = false;
|
|
28
30
|
reconcileLocks = new Map();
|
|
29
|
-
|
|
31
|
+
coordinator;
|
|
32
|
+
githubSignalCoordinator;
|
|
33
|
+
watchedBus;
|
|
30
34
|
jobTargets = new Map();
|
|
31
35
|
targetSeeds = new Map();
|
|
32
|
-
constructor(client, repository, trustedAuthors, concurrency, makeReconciler, isRateLimited = () => false) {
|
|
36
|
+
constructor(client, repository, trustedAuthors, concurrency, makeReconciler, isRateLimited = () => false, refreshWebhookSnapshot) {
|
|
33
37
|
this.client = client;
|
|
34
38
|
this.trustedAuthors = trustedAuthors;
|
|
35
39
|
this.concurrency = concurrency;
|
|
36
40
|
this.makeReconciler = makeReconciler;
|
|
37
41
|
this.isRateLimited = isRateLimited;
|
|
42
|
+
this.refreshWebhookSnapshot = refreshWebhookSnapshot;
|
|
38
43
|
this.repositories = [...new Set(typeof repository === 'string' ? [repository] : repository)];
|
|
44
|
+
const reportCoordinatorError = (error, signal) => {
|
|
45
|
+
process.stderr.write(`warning: reconcile coordinator failed for ${signal.jobId} (${signal.reason}): ${error instanceof Error ? error.message : String(error)}\n`);
|
|
46
|
+
};
|
|
47
|
+
this.coordinator = new ReconcileCoordinator((jobId, signals) => this.consumeSignals(jobId, signals), reportCoordinatorError);
|
|
48
|
+
this.githubSignalCoordinator = new ReconcileCoordinator((_key, signals) => this.consumeGitHubSignals(signals), reportCoordinatorError);
|
|
39
49
|
}
|
|
40
50
|
repositories;
|
|
41
51
|
async reconcileExclusive(jobId, work) {
|
|
@@ -53,6 +63,91 @@ export class RepositoryWorkerPool {
|
|
|
53
63
|
release();
|
|
54
64
|
}
|
|
55
65
|
}
|
|
66
|
+
signalReason(event) {
|
|
67
|
+
if (event.type === 'github.webhook') {
|
|
68
|
+
const webhookEvent = typeof event.data?.event === 'string' ? event.data.event : 'unknown';
|
|
69
|
+
const action = typeof event.data?.action === 'string' && event.data.action ? `:${event.data.action}` : '';
|
|
70
|
+
return `github.webhook:${webhookEvent}${action}`;
|
|
71
|
+
}
|
|
72
|
+
return `event.${event.type}`;
|
|
73
|
+
}
|
|
74
|
+
async consumeGitHubSignals(signals) {
|
|
75
|
+
if (this.isRateLimited() || signals.length === 0)
|
|
76
|
+
return;
|
|
77
|
+
const repository = signals.find(signal => signal.repository)?.repository;
|
|
78
|
+
const prNumber = signals.find(signal => signal.prNumber)?.prNumber;
|
|
79
|
+
if (!repository || !prNumber || !this.repositories.includes(repository))
|
|
80
|
+
return;
|
|
81
|
+
if (!this.refreshWebhookSnapshot) {
|
|
82
|
+
process.stderr.write(`warning: GitHub webhook signal refresh unavailable for ${repository}#${prNumber}\n`);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const reasons = [...new Set(signals.map(signal => signal.reason))];
|
|
86
|
+
const trigger = reasons.join('+') || 'github.webhook';
|
|
87
|
+
const snapshot = await this.refreshWebhookSnapshot(repository, prNumber, trigger);
|
|
88
|
+
const sourceEvents = signals.map(signal => signal.event).filter((event) => Boolean(event));
|
|
89
|
+
for (const item of snapshot.overviews) {
|
|
90
|
+
this.jobTargets.set(item.job.id, { repository, prNumber });
|
|
91
|
+
this.seedTargetSnapshot(repository, prNumber, item.job.id, snapshot.seed);
|
|
92
|
+
if (this.watchedBus) {
|
|
93
|
+
for (const event of sourceEvents) {
|
|
94
|
+
const publication = event.data?.publication;
|
|
95
|
+
if (publication && typeof publication === 'object') {
|
|
96
|
+
const value = publication;
|
|
97
|
+
await this.watchedBus.publish(eventFor(item.job.id, 'release', null, 'package.published', 'succeeded', `Published ${String(value.package ?? 'package')}@${String(value.version ?? 'unknown')}`, 'orchestrator', {
|
|
98
|
+
deliveryId: event.data?.deliveryId,
|
|
99
|
+
repository,
|
|
100
|
+
prNumber,
|
|
101
|
+
package: value.package,
|
|
102
|
+
version: value.version,
|
|
103
|
+
registry: value.registry,
|
|
104
|
+
sha: value.sha,
|
|
105
|
+
}, event.source));
|
|
106
|
+
}
|
|
107
|
+
await this.watchedBus.publish(eventFor(item.job.id, 'github', null, 'github.webhook', 'queued', event.message, 'orchestrator', { ...(event.data ?? {}), coordinatorIngress: false, coordinatorRouted: true }, event.source));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
await this.coordinator.accept({ jobId: item.job.id, reason: trigger, repository, prNumber });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async consumeSignals(jobId, signals) {
|
|
114
|
+
if (this.isRateLimited())
|
|
115
|
+
return;
|
|
116
|
+
for (const signal of signals) {
|
|
117
|
+
if (signal.repository && signal.prNumber && this.repositories.includes(signal.repository)) {
|
|
118
|
+
this.jobTargets.set(jobId, { repository: signal.repository, prNumber: signal.prNumber });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const target = this.jobTargets.get(jobId);
|
|
122
|
+
if (!target || !this.repositories.includes(target.repository)) {
|
|
123
|
+
process.stderr.write(`warning: reconcile signal target unknown for job ${jobId}\n`);
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const seedKey = `${target.repository}#${target.prNumber}:${jobId}`;
|
|
127
|
+
const mayConsumeSeed = signals.some(signal => signal.reason === 'target.request' || signal.reason.startsWith('github.webhook:'));
|
|
128
|
+
const seed = mayConsumeSeed ? this.targetSeeds.get(seedKey) ?? null : null;
|
|
129
|
+
if (mayConsumeSeed)
|
|
130
|
+
this.targetSeeds.delete(seedKey);
|
|
131
|
+
const store = new GitHubStore(this.client, target.repository, target.prNumber, this.trustedAuthors, seed);
|
|
132
|
+
const reconciler = this.makeReconciler(store, Math.max(1, this.concurrency));
|
|
133
|
+
let shouldReconcile = false;
|
|
134
|
+
const reasons = [];
|
|
135
|
+
for (const signal of signals) {
|
|
136
|
+
reasons.push(signal.reason);
|
|
137
|
+
if (!signal.event) {
|
|
138
|
+
shouldReconcile = true;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const result = await withGitHubRequestContext(this.client, `pool.coordinator-apply:${signal.reason}`, () => reconciler.applyEvent(signal.event));
|
|
142
|
+
shouldReconcile ||= result.reconcile;
|
|
143
|
+
}
|
|
144
|
+
if (!shouldReconcile)
|
|
145
|
+
return;
|
|
146
|
+
const reason = [...new Set(reasons)].join('+') || 'unknown';
|
|
147
|
+
await this.reconcileExclusive(jobId, async () => {
|
|
148
|
+
await withGitHubRequestContext(this.client, `pool.coordinator-reconcile:${reason}`, () => reconciler.reconcile(jobId));
|
|
149
|
+
});
|
|
150
|
+
}
|
|
56
151
|
async reconcileDiscovered(discovered) {
|
|
57
152
|
for (const item of discovered)
|
|
58
153
|
this.jobTargets.set(item.job.id, { repository: item.job.repository, prNumber: item.job.prNumber });
|
|
@@ -105,60 +200,75 @@ export class RepositoryWorkerPool {
|
|
|
105
200
|
return;
|
|
106
201
|
this.targetSeeds.set(`${repository}#${prNumber}:${jobId}`, seed);
|
|
107
202
|
}
|
|
108
|
-
async reconcileTarget(repository, prNumber, jobId) {
|
|
203
|
+
async reconcileTarget(repository, prNumber, jobId, reason = 'target.request') {
|
|
109
204
|
if (this.isRateLimited() || !this.repositories.includes(repository))
|
|
110
205
|
return;
|
|
111
206
|
this.jobTargets.set(jobId, { repository, prNumber });
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
const
|
|
117
|
-
const
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
const
|
|
121
|
-
|
|
207
|
+
await this.coordinator.accept({ jobId, reason, repository, prNumber });
|
|
208
|
+
}
|
|
209
|
+
async watch(bus) {
|
|
210
|
+
this.watchedBus = bus;
|
|
211
|
+
const unsubscribe = await bus.subscribeAll(async (event) => {
|
|
212
|
+
const repository = typeof event.data?.repository === 'string' ? event.data.repository : undefined;
|
|
213
|
+
const prValue = Number(event.data?.prNumber);
|
|
214
|
+
const prNumber = Number.isInteger(prValue) && prValue > 0 ? prValue : undefined;
|
|
215
|
+
const routedWebhook = event.type === 'github.webhook' && event.data?.coordinatorRouted === true;
|
|
216
|
+
const ingressWebhook = event.type === 'github.webhook' && event.data?.coordinatorIngress === true && !routedWebhook;
|
|
217
|
+
if (ingressWebhook && repository && prNumber) {
|
|
218
|
+
await this.githubSignalCoordinator.acceptTransport({
|
|
219
|
+
jobId: `${repository}#${prNumber}`,
|
|
220
|
+
reason: this.signalReason(event),
|
|
221
|
+
event,
|
|
222
|
+
repository,
|
|
223
|
+
prNumber,
|
|
224
|
+
});
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
if (routedWebhook)
|
|
228
|
+
return;
|
|
229
|
+
const lifecycle = ['task.started', 'task.completed', 'task.failed', 'task.blocked', 'task.cancelled', 'review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type);
|
|
230
|
+
const wake = event.type === 'job.wake' || event.type === 'github.webhook';
|
|
231
|
+
if (!lifecycle && !wake)
|
|
232
|
+
return;
|
|
233
|
+
if (repository && prNumber)
|
|
234
|
+
this.jobTargets.set(event.job_id, { repository, prNumber });
|
|
235
|
+
await this.coordinator.acceptTransport({
|
|
236
|
+
jobId: event.job_id,
|
|
237
|
+
reason: this.signalReason(event),
|
|
238
|
+
event: lifecycle ? event : undefined,
|
|
239
|
+
repository,
|
|
240
|
+
prNumber,
|
|
241
|
+
});
|
|
122
242
|
});
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
if (this.targetedReconcileFlights.get(key) === flight)
|
|
129
|
-
this.targetedReconcileFlights.delete(key);
|
|
130
|
-
}
|
|
243
|
+
return async () => {
|
|
244
|
+
await unsubscribe();
|
|
245
|
+
if (this.watchedBus === bus)
|
|
246
|
+
this.watchedBus = undefined;
|
|
247
|
+
};
|
|
131
248
|
}
|
|
132
|
-
async
|
|
133
|
-
if (
|
|
249
|
+
async webhookSignal(bus, wake) {
|
|
250
|
+
if (!wake.repository || !wake.pullRequest)
|
|
134
251
|
return;
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
252
|
+
const source = { id: `github/${wake.repository}/delivery/${wake.deliveryId}`, component: 'github-webhook' };
|
|
253
|
+
const event = eventFor('github-webhook', 'github', null, 'github.webhook', 'queued', `GitHub webhook ${wake.event}${wake.action ? ':' + wake.action : ''}`, 'orchestrator', {
|
|
254
|
+
deliveryId: wake.deliveryId,
|
|
255
|
+
event: wake.event,
|
|
256
|
+
action: wake.action,
|
|
257
|
+
repository: wake.repository,
|
|
258
|
+
prNumber: wake.pullRequest,
|
|
259
|
+
publication: wake.publication ?? null,
|
|
260
|
+
coordinatorIngress: true,
|
|
261
|
+
}, source);
|
|
262
|
+
if (bus) {
|
|
263
|
+
await bus.publish(event);
|
|
138
264
|
return;
|
|
139
265
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
async watch(bus) {
|
|
147
|
-
return bus.subscribeAll(async (event) => {
|
|
148
|
-
if (['task.started', 'task.completed', 'task.failed', 'task.blocked', 'task.cancelled', 'review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type)) {
|
|
149
|
-
await this.handleLifecycleEvent(event);
|
|
150
|
-
return;
|
|
151
|
-
}
|
|
152
|
-
if (event.type === 'job.wake' || event.type === 'github.webhook') {
|
|
153
|
-
const repository = typeof event.data?.repository === 'string' ? event.data.repository : '';
|
|
154
|
-
const prNumber = Number(event.data?.prNumber);
|
|
155
|
-
if (repository && Number.isInteger(prNumber) && prNumber > 0) {
|
|
156
|
-
await this.reconcileTarget(repository, prNumber, event.job_id);
|
|
157
|
-
return;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
if (event.type === 'job.wake' || event.type === 'github.webhook')
|
|
161
|
-
await this.reconcile();
|
|
266
|
+
await this.githubSignalCoordinator.accept({
|
|
267
|
+
jobId: `${wake.repository}#${wake.pullRequest}`,
|
|
268
|
+
reason: this.signalReason(event),
|
|
269
|
+
event,
|
|
270
|
+
repository: wake.repository,
|
|
271
|
+
prNumber: wake.pullRequest,
|
|
162
272
|
});
|
|
163
273
|
}
|
|
164
274
|
async webhookWake(bus, job, wake) {
|
|
@@ -178,6 +288,6 @@ export class RepositoryWorkerPool {
|
|
|
178
288
|
await bus.publish(eventFor(job.id, 'github', null, 'github.webhook', 'queued', `GitHub webhook ${wake.event}${wake.action ? ':' + wake.action : ''}`, 'orchestrator', { deliveryId: wake.deliveryId, event: wake.event, action: wake.action, repository: job.repository, prNumber: job.prNumber }, source));
|
|
179
289
|
return;
|
|
180
290
|
}
|
|
181
|
-
await this.reconcileTarget(job.repository, job.prNumber, job.id);
|
|
291
|
+
await this.reconcileTarget(job.repository, job.prNumber, job.id, `github.webhook:${wake.event}${wake.action ? ':' + wake.action : ''}`);
|
|
182
292
|
}
|
|
183
293
|
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
export class ReconcileCoordinator {
|
|
2
|
+
consumeBatch;
|
|
3
|
+
reportError;
|
|
4
|
+
mailboxes = new Map();
|
|
5
|
+
idleWaiters = [];
|
|
6
|
+
constructor(consumeBatch, reportError) {
|
|
7
|
+
this.consumeBatch = consumeBatch;
|
|
8
|
+
this.reportError = reportError;
|
|
9
|
+
}
|
|
10
|
+
accept(signal) {
|
|
11
|
+
return this.enqueue(signal, true);
|
|
12
|
+
}
|
|
13
|
+
acceptTransport(signal) {
|
|
14
|
+
return this.enqueue(signal, false);
|
|
15
|
+
}
|
|
16
|
+
acceptObserved(signal) {
|
|
17
|
+
const running = this.mailboxes.get(signal.jobId)?.running === true;
|
|
18
|
+
return this.enqueue(signal, !running);
|
|
19
|
+
}
|
|
20
|
+
isIdle() {
|
|
21
|
+
return [...this.mailboxes.values()].every(mailbox => !mailbox.running && !mailbox.scheduled && mailbox.pending.length === 0);
|
|
22
|
+
}
|
|
23
|
+
settle() {
|
|
24
|
+
if (this.isIdle())
|
|
25
|
+
return Promise.resolve();
|
|
26
|
+
return new Promise(resolve => this.idleWaiters.push(resolve));
|
|
27
|
+
}
|
|
28
|
+
notifyIdle() {
|
|
29
|
+
if (!this.isIdle())
|
|
30
|
+
return;
|
|
31
|
+
for (const resolve of this.idleWaiters.splice(0))
|
|
32
|
+
resolve();
|
|
33
|
+
}
|
|
34
|
+
enqueue(signal, waitForProcessing) {
|
|
35
|
+
const mailbox = this.mailboxes.get(signal.jobId) ?? { pending: [], running: false, scheduled: false };
|
|
36
|
+
this.mailboxes.set(signal.jobId, mailbox);
|
|
37
|
+
let accepted;
|
|
38
|
+
if (waitForProcessing) {
|
|
39
|
+
accepted = new Promise((resolve, reject) => {
|
|
40
|
+
mailbox.pending.push({ signal, resolve, reject });
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
mailbox.pending.push({
|
|
45
|
+
signal,
|
|
46
|
+
resolve: () => { },
|
|
47
|
+
reject: error => this.reportError?.(error, signal),
|
|
48
|
+
});
|
|
49
|
+
accepted = Promise.resolve();
|
|
50
|
+
}
|
|
51
|
+
if (!mailbox.running && !mailbox.scheduled) {
|
|
52
|
+
mailbox.scheduled = true;
|
|
53
|
+
queueMicrotask(() => { void this.drain(signal.jobId); });
|
|
54
|
+
}
|
|
55
|
+
return accepted;
|
|
56
|
+
}
|
|
57
|
+
async drain(jobId) {
|
|
58
|
+
const mailbox = this.mailboxes.get(jobId);
|
|
59
|
+
if (!mailbox || mailbox.running)
|
|
60
|
+
return;
|
|
61
|
+
mailbox.running = true;
|
|
62
|
+
mailbox.scheduled = false;
|
|
63
|
+
try {
|
|
64
|
+
while (mailbox.pending.length > 0) {
|
|
65
|
+
const batch = mailbox.pending.splice(0);
|
|
66
|
+
try {
|
|
67
|
+
await this.consumeBatch(jobId, batch.map(item => item.signal));
|
|
68
|
+
for (const item of batch)
|
|
69
|
+
item.resolve();
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
for (const item of batch)
|
|
73
|
+
item.reject(error);
|
|
74
|
+
}
|
|
75
|
+
await Promise.resolve();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
mailbox.running = false;
|
|
80
|
+
if (mailbox.pending.length > 0 && !mailbox.scheduled) {
|
|
81
|
+
mailbox.scheduled = true;
|
|
82
|
+
queueMicrotask(() => { void this.drain(jobId); });
|
|
83
|
+
}
|
|
84
|
+
else if (mailbox.pending.length === 0) {
|
|
85
|
+
this.mailboxes.delete(jobId);
|
|
86
|
+
}
|
|
87
|
+
this.notifyIdle();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -149,14 +149,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
149
149
|
const reviewData = event.data?.review && typeof event.data.review === 'object' ? event.data.review : undefined;
|
|
150
150
|
const headSha = typeof reviewData?.headSha === 'string' ? reviewData.headSha : typeof event.data?.headSha === 'string' ? event.data.headSha : task?.reviewHeadSha;
|
|
151
151
|
if (!task || !headSha || task.reviewHeadSha !== headSha)
|
|
152
|
-
return;
|
|
152
|
+
return { applied: false, reconcile: false, reason: `${event.type}.ignored` };
|
|
153
153
|
const outcome = event.type === 'review.started' ? 'started' : event.type === 'review.approved' ? 'approved' : event.type === 'review.changes_required' ? 'changes_required' : 'blocked';
|
|
154
154
|
const currentHeadSha = await this.store.headSha?.();
|
|
155
155
|
if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
|
|
156
|
-
return;
|
|
156
|
+
return { applied: false, reconcile: false, reason: `${event.type}.stale` };
|
|
157
157
|
const taskDecision = tryTransitionTaskEvent(task, event.type);
|
|
158
158
|
if (!taskDecision.accepted || !taskDecision.target)
|
|
159
|
-
return;
|
|
159
|
+
return { applied: false, reconcile: false, reason: `${event.type}.ignored` };
|
|
160
160
|
task.error = null;
|
|
161
161
|
if (taskDecision.target !== 'RUNNING') {
|
|
162
162
|
task.leaseOwner = null;
|
|
@@ -199,8 +199,6 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
199
199
|
}
|
|
200
200
|
job.updatedAt = job.review.updatedAt;
|
|
201
201
|
await this.store.saveJob(job);
|
|
202
|
-
if (outcome === 'changes_required')
|
|
203
|
-
this.detach(this.reconcile(job.id), job.id, task.id, 'review-fix reconciliation');
|
|
204
202
|
if (outcome === 'approved' && job.autoMerge !== false && this.store.mergePullRequest) {
|
|
205
203
|
const decision = await transitionJobEvent(job, 'review.approved', { autoMerge: true });
|
|
206
204
|
await this.store.saveJob(job);
|
|
@@ -210,5 +208,6 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
210
208
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'review.merged', 'succeeded', 'Reviewer-approved PR #' + job.prNumber + ' merge requested', 'user', { headSha, autoMerge: true }, { agent: 'reviewer', component: 'agents-relay' }));
|
|
211
209
|
}
|
|
212
210
|
}
|
|
211
|
+
return { applied: true, reconcile: outcome === 'changes_required', reason: `review.${outcome}`, reconcileMode: outcome === 'changes_required' ? 'detached' : undefined };
|
|
213
212
|
}
|
|
214
213
|
}
|
|
@@ -4,6 +4,7 @@ import { StartAckTimeout } from './adapters.js';
|
|
|
4
4
|
import { eventFor } from './events.js';
|
|
5
5
|
import { WorkflowEffectExecutor } from './workflow/effects.js';
|
|
6
6
|
import { workflowEffectFailureEvent } from './workflow/contracts.js';
|
|
7
|
+
import { ReconcileCoordinator } from './reconcile-coordinator.js';
|
|
7
8
|
export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
|
|
8
9
|
function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
|
|
9
10
|
const terminalExecutionClaims = new Set();
|
|
@@ -26,12 +27,43 @@ export class ReconcilerRuntimeBase {
|
|
|
26
27
|
launchFailureClaims = new Set();
|
|
27
28
|
terminalWaiters = new Map();
|
|
28
29
|
taskTransitions;
|
|
30
|
+
eventCoordinator;
|
|
31
|
+
detachedReconciles = new Set();
|
|
29
32
|
workflowEffects = new WorkflowEffectExecutor();
|
|
30
33
|
reconciling = false;
|
|
31
34
|
constructor(store, options) {
|
|
32
35
|
this.store = store;
|
|
33
36
|
this.options = options;
|
|
34
37
|
this.taskTransitions = new DurableTaskTransitionRuntime(store);
|
|
38
|
+
this.eventCoordinator = new ReconcileCoordinator(async (jobId, signals) => {
|
|
39
|
+
let shouldReconcile = false;
|
|
40
|
+
let detached = false;
|
|
41
|
+
let taskId = 'job';
|
|
42
|
+
const reasons = [];
|
|
43
|
+
for (const signal of signals) {
|
|
44
|
+
reasons.push(signal.reason);
|
|
45
|
+
if (!signal.event) {
|
|
46
|
+
shouldReconcile = true;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
taskId = signal.event.task_id || taskId;
|
|
50
|
+
const result = await this.applyEvent(signal.event);
|
|
51
|
+
shouldReconcile ||= result.reconcile;
|
|
52
|
+
detached ||= result.reconcile && result.reconcileMode === 'detached';
|
|
53
|
+
}
|
|
54
|
+
if (!shouldReconcile)
|
|
55
|
+
return;
|
|
56
|
+
const reconciliation = this.reconcile(jobId);
|
|
57
|
+
if (detached) {
|
|
58
|
+
let tracked;
|
|
59
|
+
tracked = reconciliation.finally(() => this.detachedReconciles.delete(tracked));
|
|
60
|
+
this.detachedReconciles.add(tracked);
|
|
61
|
+
this.detach(tracked, jobId, taskId, `coordinator reconciliation: ${[...new Set(reasons)].join('+')}`);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
await reconciliation;
|
|
65
|
+
}
|
|
66
|
+
});
|
|
35
67
|
}
|
|
36
68
|
async applyJobEffectOutcomes(job, outcomes) {
|
|
37
69
|
for (const outcome of outcomes.filter(item => item.status === 'failed')) {
|
|
@@ -60,42 +92,41 @@ export class ReconcilerRuntimeBase {
|
|
|
60
92
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
|
|
61
93
|
}
|
|
62
94
|
}
|
|
63
|
-
async
|
|
95
|
+
async applyEvent(event) {
|
|
64
96
|
const sourceId = eventSourceId(event);
|
|
65
97
|
const activeOwner = sourceId ? activeExecutionOwners.get(sourceId) : undefined;
|
|
66
|
-
if (activeOwner && activeOwner !== this)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if (['task.completed', 'task.failed', 'task.blocked', 'task.cancelled'].includes(event.type)) {
|
|
75
|
-
await this.applyTerminalEvent(event);
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
if (['review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type)) {
|
|
79
|
-
await this.applyReviewEvent(event);
|
|
80
|
-
return;
|
|
81
|
-
}
|
|
98
|
+
if (activeOwner && activeOwner !== this)
|
|
99
|
+
return activeOwner.applyEvent(event);
|
|
100
|
+
if (event.type === 'task.started')
|
|
101
|
+
return this.markWorkerStarted(event);
|
|
102
|
+
if (['task.completed', 'task.failed', 'task.blocked', 'task.cancelled'].includes(event.type))
|
|
103
|
+
return this.applyTerminalEvent(event);
|
|
104
|
+
if (['review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type))
|
|
105
|
+
return this.applyReviewEvent(event);
|
|
82
106
|
if (event.type === 'job.wake' || event.type === 'github.webhook')
|
|
83
|
-
|
|
107
|
+
return { applied: true, reconcile: true, reason: event.type };
|
|
108
|
+
return { applied: false, reconcile: false, reason: event.type };
|
|
109
|
+
}
|
|
110
|
+
async handleEvent(event) {
|
|
111
|
+
await this.eventCoordinator.accept({ jobId: event.job_id, reason: `event.${event.type}`, event });
|
|
84
112
|
}
|
|
85
113
|
async watch(jobId) {
|
|
86
114
|
if (!this.options.eventBus)
|
|
87
115
|
return async () => { };
|
|
88
116
|
try {
|
|
89
|
-
return await this.options.eventBus.subscribe(jobId, event => this.
|
|
117
|
+
return await this.options.eventBus.subscribe(jobId, event => this.eventCoordinator.acceptObserved({ jobId: event.job_id, reason: `event.${event.type}`, event }));
|
|
90
118
|
}
|
|
91
119
|
catch (error) {
|
|
92
120
|
await this.reportTransportFailure(error);
|
|
93
121
|
return async () => { };
|
|
94
122
|
}
|
|
95
123
|
}
|
|
96
|
-
async idle() { while ([...this.live.values()].some(item => !item.processSettled) || this.plannerLive.size > 0 || this.continuationLive.size > 0 || this.reconciling)
|
|
124
|
+
async idle() { while ([...this.live.values()].some(item => !item.processSettled) || this.plannerLive.size > 0 || this.continuationLive.size > 0 || this.reconciling || !this.eventCoordinator.isIdle())
|
|
97
125
|
await new Promise(resolve => setTimeout(resolve, 25)); }
|
|
98
126
|
async close() {
|
|
127
|
+
await this.eventCoordinator.settle();
|
|
128
|
+
if (this.detachedReconciles.size > 0)
|
|
129
|
+
await Promise.allSettled([...this.detachedReconciles]);
|
|
99
130
|
for (const live of this.live.values()) {
|
|
100
131
|
const sourceId = executionEventSource(live.jobId, live.taskId, live.executionId);
|
|
101
132
|
if (activeExecutionOwners.get(sourceId) === this)
|
|
@@ -180,7 +211,7 @@ export class ReconcilerRuntimeBase {
|
|
|
180
211
|
async markWorkerStarted(event) {
|
|
181
212
|
const live = this.live.get(event.task_id);
|
|
182
213
|
if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
|
|
183
|
-
return;
|
|
214
|
+
return { applied: false, reconcile: false, reason: 'task.started.stale' };
|
|
184
215
|
const transition = await this.taskTransitions.apply(event.job_id, event.task_id, 'task.started', {
|
|
185
216
|
eligible: (job, task) => {
|
|
186
217
|
if (!task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
|
|
@@ -198,11 +229,10 @@ export class ReconcilerRuntimeBase {
|
|
|
198
229
|
},
|
|
199
230
|
});
|
|
200
231
|
if (!transition?.decision.accepted)
|
|
201
|
-
return;
|
|
232
|
+
return { applied: false, reconcile: false, reason: 'task.started.ignored' };
|
|
202
233
|
if (live?.executionId === transition.task.executionId)
|
|
203
234
|
live.workerStarted = true;
|
|
204
|
-
|
|
205
|
-
this.detach(this.reconcile(event.job_id), event.job_id, event.task_id, 'late worker-start reconciliation');
|
|
235
|
+
return { applied: true, reconcile: transition.from === 'FAILED', reason: transition.from === 'FAILED' ? 'task.started.recovery' : 'task.started', reconcileMode: transition.from === 'FAILED' ? 'detached' : undefined };
|
|
206
236
|
}
|
|
207
237
|
isRecoverableWatchdogFailure(task) {
|
|
208
238
|
if (['worker_start_timeout', 'terminal_event_timeout', 'worker_exit_before_start', 'worker_exit_without_terminal_event', 'worker_delivery_after_start'].includes(task.failureReason ?? ''))
|
|
@@ -326,10 +356,11 @@ export class ReconcilerRuntimeBase {
|
|
|
326
356
|
},
|
|
327
357
|
});
|
|
328
358
|
if (!transition?.decision.accepted)
|
|
329
|
-
return;
|
|
359
|
+
return { applied: false, reconcile: false, reason: `${event.type}.ignored` };
|
|
330
360
|
if (event.type === 'task.completed')
|
|
331
361
|
await this.deliverContinuation(transition.job, transition.task);
|
|
332
|
-
|
|
362
|
+
const reviewWorker = transition.task.agentName === 'reviewer';
|
|
363
|
+
return { applied: true, reconcile: !reviewWorker, reason: reviewWorker ? `${event.type}.review-worker` : event.type, reconcileMode: reviewWorker ? undefined : 'detached' };
|
|
333
364
|
}
|
|
334
365
|
terminalWait(taskId, executionId, launchBarrier) {
|
|
335
366
|
let resolve;
|
package/dist/relayd.js
CHANGED
|
@@ -106,7 +106,6 @@ export async function runDaemon(argv) {
|
|
|
106
106
|
const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), runtimeSummaryProvider: () => ({ chatgptRunning: summarizeRuntime(cachedOverviews.map(item => item.job)).workers.chatgpt }), decisionEngine: new LayaCliDecisionEngine(), decisionMode: decisionMode(), decisionAuditStore, eventBus: bus, reviewer: runtimeReviewer(), projectPathForJob: async (job) => resolveRepositoryProjectPath(repositoryPaths.get(job.repository) ?? await resolveManagedProjectPath(job.repository, explicitProjectPath, workspaceRoot), job.projectPath) });
|
|
107
107
|
if (repositories.length === 0)
|
|
108
108
|
throw new Error(`No Git repositories with GitHub origins found under ${normalized.workspaceRoot ?? defaultWorkspaceRoot()}`);
|
|
109
|
-
const pool = new RepositoryWorkerPool(client, repositories, trusted, concurrency, makeReconciler, () => client.rateLimitStatus().limited);
|
|
110
109
|
const cacheDirectory = join(homedir(), '.agents-relay');
|
|
111
110
|
const cacheFile = join(cacheDirectory, 'dashboard-jobs.json');
|
|
112
111
|
try {
|
|
@@ -186,6 +185,7 @@ export async function runDaemon(argv) {
|
|
|
186
185
|
}
|
|
187
186
|
};
|
|
188
187
|
const refreshPullRequest = async (repository, prNumber, trigger, options) => (await refreshPullRequestSnapshot(repository, prNumber, trigger, options)).overviews;
|
|
188
|
+
const pool = new RepositoryWorkerPool(client, repositories, trusted, concurrency, makeReconciler, () => client.rateLimitStatus().limited, (repository, prNumber, trigger) => refreshPullRequestSnapshot(repository, prNumber, trigger));
|
|
189
189
|
const initial = cachedOverviews;
|
|
190
190
|
const fallback = initial[0]?.job;
|
|
191
191
|
const dashboardStore = fallback ? new GitHubStore(client, fallback.repository, fallback.prNumber, trusted) : new InMemoryStore();
|
|
@@ -221,21 +221,11 @@ export async function runDaemon(argv) {
|
|
|
221
221
|
const webhook = secret ? {
|
|
222
222
|
secret,
|
|
223
223
|
repositories: new Set(repositories),
|
|
224
|
-
refresh: async (event) => {
|
|
225
|
-
if (!event.repository || !event.pullRequest)
|
|
226
|
-
return [];
|
|
227
|
-
const snapshot = await refreshPullRequestSnapshot(event.repository, event.pullRequest, `github.webhook:${event.event}${event.action ? ':' + event.action : ''}`);
|
|
228
|
-
for (const item of snapshot.overviews)
|
|
229
|
-
pool.seedTargetSnapshot(event.repository, event.pullRequest, item.job.id, snapshot.seed);
|
|
230
|
-
return snapshot.overviews.map(item => item.job.id);
|
|
231
|
-
},
|
|
232
224
|
wake: async (event) => {
|
|
233
225
|
if (!event.repository || !event.pullRequest)
|
|
234
226
|
return;
|
|
235
227
|
try {
|
|
236
|
-
|
|
237
|
-
for (const item of refreshed)
|
|
238
|
-
await pool.webhookWake(bus, item.job, event);
|
|
228
|
+
await pool.webhookSignal(bus, event);
|
|
239
229
|
}
|
|
240
230
|
catch (error) {
|
|
241
231
|
process.stderr.write(`warning: webhook wake unavailable: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
@@ -260,6 +250,7 @@ export async function runDaemon(argv) {
|
|
|
260
250
|
return cachedJob();
|
|
261
251
|
}
|
|
262
252
|
};
|
|
253
|
+
const stopWatch = bus ? await pool.watch(bus) : async () => { };
|
|
263
254
|
const server = serveDashboard(dashboardStore, Number(value(argv, '--port', String(SERVICE_DEFAULTS.port))), bus, Number(value(argv, '--refresh-ms', String(SERVICE_DEFAULTS.dashboardRefreshMs))), fallback?.id, usageRegistry, overviews, webhook, () => ({ ...client.rateLimitStatus(), readPaths: { ...githubReadPaths } }), () => client.rateLimitSnapshot(), exactJob, {
|
|
264
255
|
start: workspaceJobAction('start'),
|
|
265
256
|
reconcile: workspaceJobAction('reconcile'),
|
|
@@ -282,7 +273,6 @@ export async function runDaemon(argv) {
|
|
|
282
273
|
const watchdogDefault = webhook ? SERVICE_DEFAULTS.webhookWatchdogMs : SERVICE_DEFAULTS.watchdogMs;
|
|
283
274
|
const interval = setInterval(() => void reconcileSafely(), Number(value(argv, '--interval', String(watchdogDefault))));
|
|
284
275
|
interval.unref();
|
|
285
|
-
const stopWatch = bus ? await pool.watch(bus) : async () => { };
|
|
286
276
|
const stop = () => { clearInterval(interval); void stopWatch().finally(() => server.close(() => process.exit(0))); };
|
|
287
277
|
process.once('SIGINT', stop);
|
|
288
278
|
process.once('SIGTERM', stop);
|
package/dist/store-github.js
CHANGED
|
@@ -275,10 +275,13 @@ export class GitHubStore {
|
|
|
275
275
|
const conflicts = jobComments.filter(comment => parseJob(comment.body)?.id !== job.id);
|
|
276
276
|
if (conflicts.length > 0)
|
|
277
277
|
throw new Error(`PR #${this.prNumber} already has a different ${JOB_MARKER} marker`);
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
278
|
+
const nextBody = renderJobBody(job);
|
|
279
|
+
if (currentBody !== nextBody) {
|
|
280
|
+
await this.updatePullRequestBody(nextBody);
|
|
281
|
+
const persisted = parseJob(await this.pullRequestBody());
|
|
282
|
+
if (!persisted || persisted.id !== job.id)
|
|
283
|
+
throw new Error(`Failed to persist ${JOB_MARKER} marker in PR #${this.prNumber} body for job ${job.id}`);
|
|
284
|
+
}
|
|
282
285
|
for (const legacy of jobComments.filter(comment => parseJob(comment.body)?.id === job.id))
|
|
283
286
|
await this.deleteComment(legacy);
|
|
284
287
|
}
|