@agent-relay/factory 0.1.35 → 0.1.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +9 -4
- package/dist/cli/fleet.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +507 -114
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/state.d.ts +2 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -62,6 +62,17 @@ const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
|
62
62
|
const INJECTION_MAX_ATTEMPTS = 6;
|
|
63
63
|
const BABYSITTER_EVENT_COALESCE_MS = 750;
|
|
64
64
|
const BABYSITTER_EVENT_RETRY_MS = 1_000;
|
|
65
|
+
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
66
|
+
// recipient unavailable) both when a freshly spawned agent has not finished
|
|
67
|
+
// enrolling AND when an agent is up but its relay identity never becomes
|
|
68
|
+
// resolvable (e.g. a resumed agent whose relay enrollment silently dropped).
|
|
69
|
+
// The two are indistinguishable per-attempt, so treating every such failure as
|
|
70
|
+
// transient produced an unbounded 1s retry loop that never recovered. Once the
|
|
71
|
+
// same wake has been failing this long, stop the tight loop: back off to a slow
|
|
72
|
+
// cadence and flag the stuck babysitter once for human attention. Genuine
|
|
73
|
+
// startup lag clears well within this window.
|
|
74
|
+
const BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS = 120_000;
|
|
75
|
+
const BABYSITTER_WAKE_UNREACHABLE_RETRY_MS = 60_000;
|
|
65
76
|
const CLARIFICATION_WAKE_LEASE_MS = 60_000;
|
|
66
77
|
const CLARIFICATION_WAKE_RETRY_MS = 1_000;
|
|
67
78
|
const CLARIFICATION_PARK_RETRY_MS = 5_000;
|
|
@@ -108,6 +119,7 @@ export class FactoryLoop {
|
|
|
108
119
|
#probeCloser;
|
|
109
120
|
#probePrResolver;
|
|
110
121
|
#customProbePrResolver;
|
|
122
|
+
#hasProbePrGhRunner;
|
|
111
123
|
#probePrGhRunner;
|
|
112
124
|
#logger;
|
|
113
125
|
#clock;
|
|
@@ -116,6 +128,8 @@ export class FactoryLoop {
|
|
|
116
128
|
#kill;
|
|
117
129
|
#readChildPids;
|
|
118
130
|
#terminationGraceMs;
|
|
131
|
+
#babysitterWakeUnreachableEscalateMs;
|
|
132
|
+
#babysitterWakeUnreachableRetryMs;
|
|
119
133
|
#state;
|
|
120
134
|
#workspaceId;
|
|
121
135
|
#relayflows;
|
|
@@ -132,6 +146,7 @@ export class FactoryLoop {
|
|
|
132
146
|
#githubIssueCommentWatchers = new Map();
|
|
133
147
|
#githubIssueCommentWatchStates = new Map();
|
|
134
148
|
#githubIssueCommentQueues = new Map();
|
|
149
|
+
#githubIssueCommentReplays = new Map();
|
|
135
150
|
#githubIssueAuthors = new Map();
|
|
136
151
|
#githubIssueAuthorLookups = new Map();
|
|
137
152
|
#githubIssuePreferredPaths = new Map();
|
|
@@ -197,15 +212,19 @@ export class FactoryLoop {
|
|
|
197
212
|
#completionSweepTimer;
|
|
198
213
|
#completionSweepActive = false;
|
|
199
214
|
#completionInFlight = new Set();
|
|
215
|
+
#agentExitsInFlight = new Map();
|
|
200
216
|
#agentLifecycleSignalsInFlight = new Map();
|
|
201
|
-
|
|
202
|
-
//
|
|
217
|
+
#startupAgentAdoptionActive = false;
|
|
218
|
+
// Composite issue + PR identities for which a babysitter has already been spawned, so repeated
|
|
219
|
+
// webhooks / agent-exit safety nets don't respawn it while multi-repository issues retain one
|
|
220
|
+
// owner per PR.
|
|
203
221
|
#babysitterSpawned = new Set();
|
|
204
222
|
#babysitterSpawnInFlight = new Map();
|
|
205
|
-
// Composite issue identity -> the open PR the babysitter is shepherding, including the
|
|
223
|
+
// Composite issue + PR identity -> the open PR the babysitter is shepherding, including the
|
|
206
224
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
207
225
|
#babysitterPr = new Map();
|
|
208
226
|
#babysitterIssueRefs = new Map();
|
|
227
|
+
#babysitterReady = new Set();
|
|
209
228
|
#babysitterWakeStates = new Map();
|
|
210
229
|
// A babysitter announces this fence before invoking destructive git tooling
|
|
211
230
|
// and clears it afterward. Event text can be broker-delivered while a prompt
|
|
@@ -252,6 +271,7 @@ export class FactoryLoop {
|
|
|
252
271
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
253
272
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
254
273
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
274
|
+
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
255
275
|
this.#probePrGhRunner = ports.probePrGhRunner ?? failClosedGhRunner;
|
|
256
276
|
this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
|
|
257
277
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
@@ -264,6 +284,8 @@ export class FactoryLoop {
|
|
|
264
284
|
this.#kill = ports.kill ?? process.kill;
|
|
265
285
|
this.#readChildPids = ports.readChildPids;
|
|
266
286
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
287
|
+
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
288
|
+
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
267
289
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
268
290
|
this.#relayflows = ports.relayflows;
|
|
269
291
|
this.#worktrees = ports.worktrees;
|
|
@@ -444,9 +466,28 @@ export class FactoryLoop {
|
|
|
444
466
|
this.#error(new Error(`${GITHUB_ISSUE_ROOT} sub-root is not mounted`));
|
|
445
467
|
return;
|
|
446
468
|
}
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
469
|
+
const live = (opts.mode ?? 'live') === 'live';
|
|
470
|
+
// Capture the legacy registry before the first live heartbeat rewrites it.
|
|
471
|
+
// Durable lifecycle rows are authoritative, but this fallback is still
|
|
472
|
+
// required to adopt workers started by pre-lifecycle Factory versions.
|
|
473
|
+
const legacyRegistry = live
|
|
474
|
+
? await readFactoryInFlightRegistry(this.#config.loop.registryPath)
|
|
475
|
+
: undefined;
|
|
476
|
+
if (live)
|
|
477
|
+
await this.#startLiveHeartbeat();
|
|
478
|
+
this.#startupAgentAdoptionActive = true;
|
|
479
|
+
try {
|
|
480
|
+
this.#wireFleetEvents();
|
|
481
|
+
await this.#adoptInFlightAgents(legacyRegistry);
|
|
482
|
+
this.#startupAgentAdoptionActive = false;
|
|
483
|
+
await this.#restoreBabysitterOwnership();
|
|
484
|
+
}
|
|
485
|
+
catch (error) {
|
|
486
|
+
this.#startupAgentAdoptionActive = false;
|
|
487
|
+
if (live)
|
|
488
|
+
await this.#stopLiveHeartbeat('stopping');
|
|
489
|
+
throw error;
|
|
490
|
+
}
|
|
450
491
|
if (opts.mode === 'dispatch-owner') {
|
|
451
492
|
this.#started = true;
|
|
452
493
|
this.#scheduleDispatchLifecycleRenewal();
|
|
@@ -458,7 +499,7 @@ export class FactoryLoop {
|
|
|
458
499
|
await this.#rearmGithubIssueCommentWatchers();
|
|
459
500
|
return;
|
|
460
501
|
}
|
|
461
|
-
if (
|
|
502
|
+
if (live) {
|
|
462
503
|
this.#started = true;
|
|
463
504
|
try {
|
|
464
505
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
@@ -534,6 +575,7 @@ export class FactoryLoop {
|
|
|
534
575
|
await this.#drainClarificationWakesForStop();
|
|
535
576
|
this.#clarificationIntents.clear();
|
|
536
577
|
await this.#drainBabysitterWakesForStop();
|
|
578
|
+
await this.#drainAgentExitsInFlight();
|
|
537
579
|
// Durable relay placements must survive an owner restart so a successor
|
|
538
580
|
// can adopt them. The one-shot/daemon stop path releases only
|
|
539
581
|
// non-durable (local/internal) records; terminal completion performs the
|
|
@@ -552,6 +594,7 @@ export class FactoryLoop {
|
|
|
552
594
|
this.#babysitterSpawned.clear();
|
|
553
595
|
this.#babysitterPr.clear();
|
|
554
596
|
this.#babysitterIssueRefs.clear();
|
|
597
|
+
this.#babysitterReady.clear();
|
|
555
598
|
this.#babysitterCriticalAgents.clear();
|
|
556
599
|
const subscription = this.#subscription;
|
|
557
600
|
this.#subscription = undefined;
|
|
@@ -623,7 +666,6 @@ export class FactoryLoop {
|
|
|
623
666
|
}
|
|
624
667
|
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
625
668
|
const options = this.#liveOptions(overrides);
|
|
626
|
-
await this.#startLiveHeartbeat();
|
|
627
669
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
628
670
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
629
671
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1080,6 +1122,10 @@ export class FactoryLoop {
|
|
|
1080
1122
|
this.#completionSweepTimer.unref?.();
|
|
1081
1123
|
}
|
|
1082
1124
|
async #sweepPrStateCompletions(reason) {
|
|
1125
|
+
// This timer is also the durable safety net for fleet exit events missed
|
|
1126
|
+
// while the event loop was busy (for example during a large startup pull).
|
|
1127
|
+
// Keep reconciliation active even when babysitters own PR completion.
|
|
1128
|
+
await this.#fleet.reconcileTrackedAgents?.();
|
|
1083
1129
|
// When the babysitter owns PR-open, completion is driven by PR webhooks +
|
|
1084
1130
|
// the babysitter's readiness signal (see #handlePrChange / #handleAgentExit),
|
|
1085
1131
|
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
@@ -1115,6 +1161,10 @@ export class FactoryLoop {
|
|
|
1115
1161
|
this.#probePrGhBackoffUntilMs.set(issueStateKey(issueRef(issue)), this.#clock.now() + PROBE_PR_GH_BACKOFF_MS);
|
|
1116
1162
|
return undefined;
|
|
1117
1163
|
}
|
|
1164
|
+
if (record.decision.implementers.length > 1 && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
1165
|
+
this.#increment('completionSweepMissingPr');
|
|
1166
|
+
return undefined;
|
|
1167
|
+
}
|
|
1118
1168
|
return { record, pr };
|
|
1119
1169
|
}));
|
|
1120
1170
|
for (const candidate of candidates) {
|
|
@@ -1226,6 +1276,7 @@ export class FactoryLoop {
|
|
|
1226
1276
|
await this.#recordCanonicalIssueState(issue);
|
|
1227
1277
|
}
|
|
1228
1278
|
issueEntries.push({ path, issue });
|
|
1279
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
1229
1280
|
}
|
|
1230
1281
|
if (issueSource === 'github') {
|
|
1231
1282
|
// New ready work must not sit behind a long sequence of stale
|
|
@@ -1243,6 +1294,7 @@ export class FactoryLoop {
|
|
|
1243
1294
|
});
|
|
1244
1295
|
}
|
|
1245
1296
|
for (const { issue } of issueEntries) {
|
|
1297
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
1246
1298
|
if (!issue) {
|
|
1247
1299
|
continue;
|
|
1248
1300
|
}
|
|
@@ -1891,15 +1943,9 @@ export class FactoryLoop {
|
|
|
1891
1943
|
this.#error(error, decision.issue);
|
|
1892
1944
|
throw error;
|
|
1893
1945
|
}
|
|
1894
|
-
const
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
this.#recordTriageEscalation(decision, escalationReason);
|
|
1898
|
-
return replayedResult ?? { issue: decision.issue, agents: [], dryRun };
|
|
1899
|
-
}
|
|
1900
|
-
// TODO(AR-274 follow-up): short-circuit LLM triage once label-derived
|
|
1901
|
-
// routes are authoritative for dispatch identity.
|
|
1902
|
-
const labelDispatch = labelDerivedDispatchDecision(liveIssue, decision, this.#config);
|
|
1946
|
+
const labelDispatch = opts.labelsValidated
|
|
1947
|
+
? { ok: true, decision }
|
|
1948
|
+
: labelDerivedDispatchDecision(liveIssue, decision, this.#config);
|
|
1903
1949
|
if (!labelDispatch.ok) {
|
|
1904
1950
|
const comment = labelDispatchFailureComment(decision.issue, labelDispatch);
|
|
1905
1951
|
this.#logger.warn?.('[factory] skipped dispatch due to invalid repo labels', {
|
|
@@ -1925,10 +1971,16 @@ export class FactoryLoop {
|
|
|
1925
1971
|
}
|
|
1926
1972
|
return { issue: decision.issue, agents: [], comments: [comment], dryRun };
|
|
1927
1973
|
}
|
|
1928
|
-
let dispatchDecision = labelDispatch.decision;
|
|
1974
|
+
let dispatchDecision = authoritativeRoutedDecision(decision, labelDispatch.decision);
|
|
1929
1975
|
// A valid label resolution clears any prior failure notice so a later
|
|
1930
1976
|
// regression posts a fresh, actionable comment instead of being deduped.
|
|
1931
1977
|
this.#labelDispatchFailures.delete(issueStateKey(dispatchDecision.issue));
|
|
1978
|
+
const escalationReason = triageEscalationReason(dispatchDecision);
|
|
1979
|
+
if (escalationReason) {
|
|
1980
|
+
const replayedResult = await this.#escalateTriage(dispatchDecision, escalationReason, dryRun);
|
|
1981
|
+
this.#recordTriageEscalation(dispatchDecision, escalationReason);
|
|
1982
|
+
return replayedResult ?? { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
1983
|
+
}
|
|
1932
1984
|
// Full task rendering is part of the durable spawn specification. It must
|
|
1933
1985
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1934
1986
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
@@ -2119,7 +2171,25 @@ export class FactoryLoop {
|
|
|
2119
2171
|
#wireFleetEvents() {
|
|
2120
2172
|
if (!this.#offAgentExit) {
|
|
2121
2173
|
this.#offAgentExit = this.#fleet.onAgentExit((name, reason) => {
|
|
2122
|
-
|
|
2174
|
+
// Internal broker subscriptions replay historical exits immediately.
|
|
2175
|
+
// Ignore that pre-hydration history; the roster reconcile below runs
|
|
2176
|
+
// after durable records are restored and is the authoritative signal.
|
|
2177
|
+
if (this.#startupAgentAdoptionActive)
|
|
2178
|
+
return;
|
|
2179
|
+
// Broker replay can deliver an old exit immediately when the listener
|
|
2180
|
+
// is installed, before durable agents are restored. Queue a later
|
|
2181
|
+
// roster-reconciled exit behind it instead of dropping the newer event.
|
|
2182
|
+
const previous = this.#agentExitsInFlight.get(name) ?? Promise.resolve();
|
|
2183
|
+
const handling = previous
|
|
2184
|
+
.catch(() => undefined)
|
|
2185
|
+
.then(async () => await this.#handleAgentExit(name, reason))
|
|
2186
|
+
.catch((error) => this.#error(error))
|
|
2187
|
+
.finally(() => {
|
|
2188
|
+
if (this.#agentExitsInFlight.get(name) === handling) {
|
|
2189
|
+
this.#agentExitsInFlight.delete(name);
|
|
2190
|
+
}
|
|
2191
|
+
});
|
|
2192
|
+
this.#agentExitsInFlight.set(name, handling);
|
|
2123
2193
|
});
|
|
2124
2194
|
}
|
|
2125
2195
|
if (!this.#offDeliveryFailed) {
|
|
@@ -2152,12 +2222,16 @@ export class FactoryLoop {
|
|
|
2152
2222
|
// in the durable lifecycle store, restore their full batch/spec association,
|
|
2153
2223
|
// then reconcile once so exits that happened while this process was down are
|
|
2154
2224
|
// handled instead of being dropped as unknown agents.
|
|
2155
|
-
async #adoptInFlightAgents() {
|
|
2225
|
+
async #adoptInFlightAgents(legacyRegistry) {
|
|
2156
2226
|
try {
|
|
2157
2227
|
const batch = await this.#batch();
|
|
2158
2228
|
const agents = [];
|
|
2159
2229
|
let hasNonterminalDurableLifecycle = false;
|
|
2160
|
-
|
|
2230
|
+
const durableLifecycles = await this.#state.listDispatchLifecycles(this.#workspaceId);
|
|
2231
|
+
this.#logger.info?.('[factory] durable startup adoption loaded', {
|
|
2232
|
+
lifecycles: durableLifecycles.length,
|
|
2233
|
+
});
|
|
2234
|
+
for (const [key, lifecycle] of durableLifecycles) {
|
|
2161
2235
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
2162
2236
|
continue;
|
|
2163
2237
|
hasNonterminalDurableLifecycle = true;
|
|
@@ -2197,7 +2271,7 @@ export class FactoryLoop {
|
|
|
2197
2271
|
// records existed. It preserves observation, but only new lifecycle rows
|
|
2198
2272
|
// carry enough decision/spec state to process the reconciled exit.
|
|
2199
2273
|
if (agents.length === 0 && !hasNonterminalDurableLifecycle) {
|
|
2200
|
-
const registry = await readFactoryInFlightRegistry(this.#config.loop.registryPath);
|
|
2274
|
+
const registry = legacyRegistry ?? await readFactoryInFlightRegistry(this.#config.loop.registryPath);
|
|
2201
2275
|
agents.push(...(registry?.agents ?? [])
|
|
2202
2276
|
.filter((agent) => agent.invocationId || agent.node)
|
|
2203
2277
|
.map((agent) => ({ name: agent.name, invocationId: agent.invocationId, node: agent.node })));
|
|
@@ -2206,13 +2280,37 @@ export class FactoryLoop {
|
|
|
2206
2280
|
this.#fleet.hydrateTracked(agents);
|
|
2207
2281
|
}
|
|
2208
2282
|
this.#scheduleDispatchLifecycleRenewal();
|
|
2209
|
-
if (this.#fleet.hydrateTracked)
|
|
2283
|
+
if (this.#fleet.hydrateTracked) {
|
|
2284
|
+
this.#startupAgentAdoptionActive = false;
|
|
2285
|
+
this.#logger.info?.('[factory] durable startup roster reconciliation started', {
|
|
2286
|
+
agents: agents.map((agent) => agent.name),
|
|
2287
|
+
});
|
|
2210
2288
|
await this.#fleet.reconcileTrackedAgents?.();
|
|
2289
|
+
this.#logger.info?.('[factory] durable startup roster reconciliation completed', {
|
|
2290
|
+
pendingExits: [...this.#agentExitsInFlight.keys()].filter((name) => agents.some((agent) => agent.name === name)),
|
|
2291
|
+
});
|
|
2292
|
+
// Fleet callbacks are intentionally synchronous at the port boundary,
|
|
2293
|
+
// but recovery work is asynchronous (issue reads, worktree restore,
|
|
2294
|
+
// PR publication). Finish exits discovered by the startup reconcile
|
|
2295
|
+
// before the full ready-issue backfill can monopolize mount I/O.
|
|
2296
|
+
await this.#drainAgentExitsInFlight(new Set(agents.map((agent) => agent.name)));
|
|
2297
|
+
this.#logger.info?.('[factory] durable startup reconciled exits drained');
|
|
2298
|
+
}
|
|
2211
2299
|
}
|
|
2212
2300
|
catch (error) {
|
|
2213
2301
|
this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
|
|
2214
2302
|
}
|
|
2215
2303
|
}
|
|
2304
|
+
async #drainAgentExitsInFlight(names) {
|
|
2305
|
+
for (;;) {
|
|
2306
|
+
const pending = [...this.#agentExitsInFlight]
|
|
2307
|
+
.filter(([name]) => !names || names.has(name))
|
|
2308
|
+
.map(([, handling]) => handling);
|
|
2309
|
+
if (pending.length === 0)
|
|
2310
|
+
return;
|
|
2311
|
+
await Promise.allSettled(pending);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2216
2314
|
#scheduleDispatchLifecycleRenewal() {
|
|
2217
2315
|
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
2218
2316
|
return;
|
|
@@ -2358,7 +2456,9 @@ export class FactoryLoop {
|
|
|
2358
2456
|
return false;
|
|
2359
2457
|
}
|
|
2360
2458
|
const previous = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2361
|
-
const
|
|
2459
|
+
const pullRequests = mergePublishedPullRequests(previous, pullRequest);
|
|
2460
|
+
const primaryPullRequest = previous?.pullRequest ?? pullRequest ?? pullRequests[0];
|
|
2461
|
+
const lifecycle = lifecycleFromInFlightRecord(record, previous?.runId ?? randomUUID(), phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason);
|
|
2362
2462
|
for (const agent of lifecycle.agents) {
|
|
2363
2463
|
const previouslyReleasedAtMs = previous?.agents.find((candidate) => candidate.name === agent.name)?.releasedAtMs;
|
|
2364
2464
|
if (previouslyReleasedAtMs !== undefined)
|
|
@@ -2516,31 +2616,46 @@ export class FactoryLoop {
|
|
|
2516
2616
|
return;
|
|
2517
2617
|
}
|
|
2518
2618
|
if (lifecycle.phase === 'publishing') {
|
|
2519
|
-
const
|
|
2520
|
-
if (
|
|
2619
|
+
const implementers = [...record.agents.values()].filter((agent) => agent.spec.role === 'implementer');
|
|
2620
|
+
if (implementers.length === 0)
|
|
2521
2621
|
throw new Error(`durable dispatch ${record.issue.key} has no implementer to publish`);
|
|
2522
|
-
const
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2622
|
+
const publishedReceipts = [];
|
|
2623
|
+
for (const implementer of implementers) {
|
|
2624
|
+
const published = await this.#publishImplementerPullRequest(record, implementer, { reconcileExisting: true });
|
|
2625
|
+
if (!published)
|
|
2626
|
+
throw new Error(`durable dispatch ${record.issue.key} did not produce a pull request for ${implementer.spec.repo}`);
|
|
2627
|
+
publishedReceipts.push(published);
|
|
2628
|
+
if (!await this.#saveDispatchLifecycle(record, 'publishing', published))
|
|
2629
|
+
return;
|
|
2630
|
+
}
|
|
2631
|
+
if (!await this.#saveDispatchLifecycle(record, 'published'))
|
|
2526
2632
|
return;
|
|
2527
2633
|
if (this.#config.babysitter.enabled) {
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2634
|
+
for (const receipt of publishedReceipts) {
|
|
2635
|
+
await this.#ensureBabysitter(record, {
|
|
2636
|
+
repo: receipt.repo,
|
|
2637
|
+
prNumber: receipt.number,
|
|
2638
|
+
url: receipt.url,
|
|
2639
|
+
headRef: receipt.headRef,
|
|
2640
|
+
});
|
|
2641
|
+
}
|
|
2533
2642
|
return;
|
|
2534
2643
|
}
|
|
2535
2644
|
await this.#completeIssue(record);
|
|
2536
2645
|
return;
|
|
2537
2646
|
}
|
|
2538
2647
|
if (lifecycle.phase === 'published' && this.#config.babysitter.enabled && lifecycle.pullRequest) {
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2648
|
+
for (const receipt of lifecycle.pullRequests ?? [lifecycle.pullRequest]) {
|
|
2649
|
+
await this.#ensureBabysitter(record, {
|
|
2650
|
+
repo: receipt.repo,
|
|
2651
|
+
prNumber: receipt.number,
|
|
2652
|
+
url: receipt.url,
|
|
2653
|
+
headRef: receipt.headRef,
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
if (lifecycle.phase === 'published' && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
2544
2659
|
return;
|
|
2545
2660
|
}
|
|
2546
2661
|
if (lifecycle.phase === 'published' || lifecycle.phase === 'writeback-applied') {
|
|
@@ -2711,7 +2826,7 @@ export class FactoryLoop {
|
|
|
2711
2826
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
2712
2827
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
2713
2828
|
if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
|
|
2714
|
-
await this.#
|
|
2829
|
+
await this.#cancelBabysittersForIssue(record.issue);
|
|
2715
2830
|
}
|
|
2716
2831
|
if (!await this.#saveDispatchLifecycle(record, 'complete'))
|
|
2717
2832
|
return false;
|
|
@@ -2781,18 +2896,20 @@ export class FactoryLoop {
|
|
|
2781
2896
|
return;
|
|
2782
2897
|
}
|
|
2783
2898
|
const decision = await this.triageIssue(issue);
|
|
2784
|
-
const
|
|
2899
|
+
const routed = labelDerivedDispatchDecision(issue, decision, this.#config);
|
|
2900
|
+
const escalationDecision = routed.ok ? authoritativeRoutedDecision(decision, routed.decision) : decision;
|
|
2901
|
+
const escalationReason = triageEscalationReason(escalationDecision);
|
|
2785
2902
|
if (escalationReason) {
|
|
2786
|
-
await this.#escalateTriage(
|
|
2787
|
-
this.#recordTriageEscalation(
|
|
2903
|
+
await this.#escalateTriage(escalationDecision, escalationReason, this.#config.dryRun);
|
|
2904
|
+
this.#recordTriageEscalation(escalationDecision, escalationReason);
|
|
2788
2905
|
return;
|
|
2789
2906
|
}
|
|
2790
2907
|
if (batch.canStart()) {
|
|
2791
|
-
await this.dispatch(
|
|
2908
|
+
await this.dispatch(escalationDecision, { dryRun: this.#config.dryRun, labelsValidated: routed.ok });
|
|
2792
2909
|
}
|
|
2793
2910
|
else {
|
|
2794
|
-
if (batch.queue(
|
|
2795
|
-
this.#emit('issue-queued', { issue:
|
|
2911
|
+
if (batch.queue(escalationDecision, this.#config.dryRun)) {
|
|
2912
|
+
this.#emit('issue-queued', { issue: escalationDecision.issue });
|
|
2796
2913
|
}
|
|
2797
2914
|
}
|
|
2798
2915
|
}
|
|
@@ -2874,6 +2991,7 @@ export class FactoryLoop {
|
|
|
2874
2991
|
await this.#handleGithubIssueChange(path, { ...opts, candidates });
|
|
2875
2992
|
processed += 1;
|
|
2876
2993
|
lastProgressAtMs = this.#logTimedProgress('[factory] GitHub issue ingestion progress', startedAtMs, lastProgressAtMs, { processed, total: paths.length, path });
|
|
2994
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2877
2995
|
}
|
|
2878
2996
|
this.#logger.info?.('[factory] GitHub issue ingestion completed', {
|
|
2879
2997
|
dryRun: opts.dryRun ?? false,
|
|
@@ -2901,6 +3019,7 @@ export class FactoryLoop {
|
|
|
2901
3019
|
let scanned = 0;
|
|
2902
3020
|
let lastProgressAtMs = startedAtMs;
|
|
2903
3021
|
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading')) {
|
|
3022
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2904
3023
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
2905
3024
|
continue;
|
|
2906
3025
|
}
|
|
@@ -2923,7 +3042,8 @@ export class FactoryLoop {
|
|
|
2923
3042
|
const issuePaths = new Map();
|
|
2924
3043
|
for (const root of githubIssueScanRoots(this.#config)) {
|
|
2925
3044
|
const paths = await this.#listRelayfileTree(root, 'GitHub issue ingestion');
|
|
2926
|
-
for (
|
|
3045
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
3046
|
+
const path = paths[index];
|
|
2927
3047
|
const parts = githubIssuePathParts(path);
|
|
2928
3048
|
if (parts) {
|
|
2929
3049
|
const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
|
|
@@ -2948,7 +3068,12 @@ export class FactoryLoop {
|
|
|
2948
3068
|
else if (isGithubIssueTreePath(path)) {
|
|
2949
3069
|
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
2950
3070
|
}
|
|
3071
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
3072
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
3073
|
+
await liveEventYield();
|
|
3074
|
+
}
|
|
2951
3075
|
}
|
|
3076
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2952
3077
|
}
|
|
2953
3078
|
for (const [identity, path] of issuePaths) {
|
|
2954
3079
|
this.#githubIssuePreferredPaths.set(identity, path);
|
|
@@ -3800,6 +3925,13 @@ export class FactoryLoop {
|
|
|
3800
3925
|
}
|
|
3801
3926
|
return;
|
|
3802
3927
|
}
|
|
3928
|
+
const tracingReconciledExit = reason === 'reconciled-missing';
|
|
3929
|
+
if (tracingReconciledExit) {
|
|
3930
|
+
this.#logger.info?.('[factory] reconciled agent exit recovery started', {
|
|
3931
|
+
issue: record.issue.key,
|
|
3932
|
+
name,
|
|
3933
|
+
});
|
|
3934
|
+
}
|
|
3803
3935
|
if (!await this.#assertDispatchLifecycleOwner(record)) {
|
|
3804
3936
|
this.#logger.warn?.('[factory] ignored agent exit after durable lifecycle ownership was lost', {
|
|
3805
3937
|
issue: record.issue.key,
|
|
@@ -3807,6 +3939,8 @@ export class FactoryLoop {
|
|
|
3807
3939
|
});
|
|
3808
3940
|
return;
|
|
3809
3941
|
}
|
|
3942
|
+
if (tracingReconciledExit)
|
|
3943
|
+
this.#logger.info?.('[factory] reconciled agent exit ownership confirmed', { issue: record.issue.key, name });
|
|
3810
3944
|
// The issue-comment subscription and the fleet exit callback are separate
|
|
3811
3945
|
// event streams. Reconcile comments that are already durable in the mount
|
|
3812
3946
|
// before interpreting a clean exit as task completion, so an agent that
|
|
@@ -3815,9 +3949,13 @@ export class FactoryLoop {
|
|
|
3815
3949
|
this.#increment('githubQuestionExitsSuppressed');
|
|
3816
3950
|
return;
|
|
3817
3951
|
}
|
|
3952
|
+
if (tracingReconciledExit)
|
|
3953
|
+
this.#logger.info?.('[factory] reconciled agent exit question replay completed', { issue: record.issue.key, name });
|
|
3818
3954
|
const exiting = record.agents.get(name);
|
|
3819
3955
|
if (exiting)
|
|
3820
3956
|
await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
|
|
3957
|
+
if (tracingReconciledExit)
|
|
3958
|
+
this.#logger.info?.('[factory] reconciled agent exit telemetry completed', { issue: record.issue.key, name });
|
|
3821
3959
|
if (this.#usesDurableDispatchLifecycle()) {
|
|
3822
3960
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
3823
3961
|
if (lifecycle?.phase === 'parking') {
|
|
@@ -3828,10 +3966,10 @@ export class FactoryLoop {
|
|
|
3828
3966
|
if (isCompletionReason(reason)) {
|
|
3829
3967
|
if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
|
|
3830
3968
|
openOnly: this.#config.babysitter.enabled,
|
|
3831
|
-
})) {
|
|
3969
|
+
}, exiting)) {
|
|
3832
3970
|
if (this.#config.babysitter.enabled)
|
|
3833
3971
|
await this.#ensureBabysitterForIssue(record);
|
|
3834
|
-
else
|
|
3972
|
+
else if (await this.#allImplementersHaveCompletionPr(record))
|
|
3835
3973
|
await this.#completeIssue(record);
|
|
3836
3974
|
return;
|
|
3837
3975
|
}
|
|
@@ -3858,7 +3996,7 @@ export class FactoryLoop {
|
|
|
3858
3996
|
// itself finishing means it believes the PR is ready, so re-check and
|
|
3859
3997
|
// advance to Human Review.
|
|
3860
3998
|
if (exiting?.spec.role === 'babysitter') {
|
|
3861
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
3999
|
+
await this.#maybeAdvanceToHumanReview(record, name);
|
|
3862
4000
|
}
|
|
3863
4001
|
else if (publishedPr) {
|
|
3864
4002
|
await this.#ensureBabysitter(record, {
|
|
@@ -3872,7 +4010,8 @@ export class FactoryLoop {
|
|
|
3872
4010
|
}
|
|
3873
4011
|
return;
|
|
3874
4012
|
}
|
|
3875
|
-
await this.#
|
|
4013
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4014
|
+
await this.#completeIssue(record);
|
|
3876
4015
|
return;
|
|
3877
4016
|
}
|
|
3878
4017
|
const tracked = exiting;
|
|
@@ -3880,14 +4019,25 @@ export class FactoryLoop {
|
|
|
3880
4019
|
return;
|
|
3881
4020
|
}
|
|
3882
4021
|
try {
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
4022
|
+
const hasCompletionPr = tracked.spec.role === 'implementer'
|
|
4023
|
+
? await this.#issueHasCompletionPr(record, {
|
|
4024
|
+
openOnly: this.#config.babysitter.enabled,
|
|
4025
|
+
}, tracked)
|
|
4026
|
+
: false;
|
|
4027
|
+
if (tracingReconciledExit) {
|
|
4028
|
+
this.#logger.info?.('[factory] reconciled agent exit completion PR lookup completed', {
|
|
4029
|
+
issue: record.issue.key,
|
|
4030
|
+
name,
|
|
4031
|
+
hasCompletionPr,
|
|
4032
|
+
});
|
|
4033
|
+
}
|
|
4034
|
+
if (hasCompletionPr) {
|
|
3886
4035
|
if (this.#config.babysitter.enabled) {
|
|
3887
4036
|
await this.#ensureBabysitterForIssue(record);
|
|
3888
4037
|
return;
|
|
3889
4038
|
}
|
|
3890
|
-
await this.#
|
|
4039
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4040
|
+
await this.#completeIssue(record);
|
|
3891
4041
|
return;
|
|
3892
4042
|
}
|
|
3893
4043
|
// The implementer's turn ended without a PR of record. Agents reliably
|
|
@@ -3900,6 +4050,8 @@ export class FactoryLoop {
|
|
|
3900
4050
|
// ahead of base, clone gone) it returns undefined and we fall through.
|
|
3901
4051
|
if (tracked.spec.role === 'implementer') {
|
|
3902
4052
|
await this.#saveDispatchLifecycle(record, 'publishing');
|
|
4053
|
+
if (tracingReconciledExit)
|
|
4054
|
+
this.#logger.info?.('[factory] reconciled agent exit PR publication started', { issue: record.issue.key, name });
|
|
3903
4055
|
const publishedPr = await this.#tryPublishImplementerPr(record, tracked);
|
|
3904
4056
|
if (publishedPr) {
|
|
3905
4057
|
await this.#saveDispatchLifecycle(record, 'published', publishedPr);
|
|
@@ -3911,7 +4063,8 @@ export class FactoryLoop {
|
|
|
3911
4063
|
});
|
|
3912
4064
|
}
|
|
3913
4065
|
else {
|
|
3914
|
-
await this.#
|
|
4066
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4067
|
+
await this.#completeIssue(record);
|
|
3915
4068
|
}
|
|
3916
4069
|
return;
|
|
3917
4070
|
}
|
|
@@ -4030,6 +4183,10 @@ export class FactoryLoop {
|
|
|
4030
4183
|
return undefined;
|
|
4031
4184
|
}
|
|
4032
4185
|
try {
|
|
4186
|
+
// A missed exit can be reconciled after the worker checkout was pruned.
|
|
4187
|
+
// Re-create its deterministic worktree from the retained local branch so
|
|
4188
|
+
// publication can still push/read the completed commit.
|
|
4189
|
+
await this.#prepareAgentWorktree(record, implementer.spec);
|
|
4033
4190
|
const published = await this.#publishImplementerPullRequest(record, implementer);
|
|
4034
4191
|
if (published) {
|
|
4035
4192
|
this.#increment('implementerPrsPublishedOnExit');
|
|
@@ -4054,8 +4211,6 @@ export class FactoryLoop {
|
|
|
4054
4211
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
4055
4212
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
4056
4213
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
4057
|
-
if (durable?.pullRequest)
|
|
4058
|
-
return durable.pullRequest;
|
|
4059
4214
|
const cached = this.#publishedPullRequests.get(key);
|
|
4060
4215
|
if (cached)
|
|
4061
4216
|
return cached;
|
|
@@ -4080,6 +4235,9 @@ export class FactoryLoop {
|
|
|
4080
4235
|
? sourceRepoParts.owner
|
|
4081
4236
|
: undefined;
|
|
4082
4237
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
4238
|
+
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
4239
|
+
if (durableReceipt)
|
|
4240
|
+
return durableReceipt;
|
|
4083
4241
|
const expectedHeadRef = implementer.spec.branch ?? remoteBranch;
|
|
4084
4242
|
if (opts.reconcileExisting && expectedHeadRef) {
|
|
4085
4243
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
@@ -4124,6 +4282,44 @@ export class FactoryLoop {
|
|
|
4124
4282
|
return result;
|
|
4125
4283
|
}
|
|
4126
4284
|
async #openPullRequestByHead(repo, expectedHeadRef) {
|
|
4285
|
+
if (this.#hasProbePrGhRunner) {
|
|
4286
|
+
try {
|
|
4287
|
+
const result = await this.#probePrGhRunner([
|
|
4288
|
+
'pr',
|
|
4289
|
+
'list',
|
|
4290
|
+
'--repo',
|
|
4291
|
+
repo,
|
|
4292
|
+
'--head',
|
|
4293
|
+
expectedHeadRef,
|
|
4294
|
+
'--state',
|
|
4295
|
+
'open',
|
|
4296
|
+
'--json',
|
|
4297
|
+
'number,url,headRefName,isDraft',
|
|
4298
|
+
'--limit',
|
|
4299
|
+
'10',
|
|
4300
|
+
]);
|
|
4301
|
+
const payload = parseJsonContent(result.stdout);
|
|
4302
|
+
if (Array.isArray(payload)) {
|
|
4303
|
+
const candidates = payload.flatMap((entry) => {
|
|
4304
|
+
const candidate = asRecord(entry);
|
|
4305
|
+
const number = numberValue(candidate?.number);
|
|
4306
|
+
const url = stringValue(candidate?.url);
|
|
4307
|
+
const headRef = stringValue(candidate?.headRefName);
|
|
4308
|
+
if (!number || !url || headRef !== expectedHeadRef || candidate?.isDraft !== false)
|
|
4309
|
+
return [];
|
|
4310
|
+
return [{ repo, number, url, headRef }];
|
|
4311
|
+
});
|
|
4312
|
+
return candidates.sort((a, b) => b.number - a.number)[0];
|
|
4313
|
+
}
|
|
4314
|
+
}
|
|
4315
|
+
catch (error) {
|
|
4316
|
+
this.#logger.warn?.('[factory] exact-head gh PR lookup failed; falling back to mounted metadata', {
|
|
4317
|
+
repo,
|
|
4318
|
+
headRef: expectedHeadRef,
|
|
4319
|
+
error: describeError(error).errorMessage,
|
|
4320
|
+
});
|
|
4321
|
+
}
|
|
4322
|
+
}
|
|
4127
4323
|
const parts = githubRepoParts(repo);
|
|
4128
4324
|
if (!parts)
|
|
4129
4325
|
return undefined;
|
|
@@ -4435,12 +4631,22 @@ export class FactoryLoop {
|
|
|
4435
4631
|
timer.unref?.();
|
|
4436
4632
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
4437
4633
|
}
|
|
4438
|
-
async #issueHasCompletionPr(record, opts = {}) {
|
|
4634
|
+
async #issueHasCompletionPr(record, opts = {}, implementer) {
|
|
4439
4635
|
try {
|
|
4440
4636
|
const issue = await this.#readIssue(record.issue.path);
|
|
4441
4637
|
if (!issue) {
|
|
4442
4638
|
return false;
|
|
4443
4639
|
}
|
|
4640
|
+
if (implementer?.spec.branch && record.decision.implementers.length > 1) {
|
|
4641
|
+
const sourceOwner = record.issue.path
|
|
4642
|
+
? githubIssuePathParts(record.issue.path)?.owner
|
|
4643
|
+
: undefined;
|
|
4644
|
+
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
4645
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
4646
|
+
if (publishedPullRequests(lifecycle).some((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase()))
|
|
4647
|
+
return true;
|
|
4648
|
+
return Boolean(await this.#openPullRequestByHead(repo, implementer.spec.branch));
|
|
4649
|
+
}
|
|
4444
4650
|
// Only a NON-DRAFT (ready) PR counts as completion. A draft PR means the
|
|
4445
4651
|
// work isn't review-ready, so an implementer exiting with only a draft PR
|
|
4446
4652
|
// must NOT mark the issue done / release agents — mirror the
|
|
@@ -4459,6 +4665,15 @@ export class FactoryLoop {
|
|
|
4459
4665
|
return false;
|
|
4460
4666
|
}
|
|
4461
4667
|
}
|
|
4668
|
+
async #allImplementersHaveCompletionPr(record, opts = {}) {
|
|
4669
|
+
const implementers = [...record.agents.values()].filter((agent) => agent.spec.role === 'implementer');
|
|
4670
|
+
if (implementers.length === 0)
|
|
4671
|
+
return false;
|
|
4672
|
+
if (implementers.length === 1)
|
|
4673
|
+
return true;
|
|
4674
|
+
const completed = await Promise.all(implementers.map(async (implementer) => this.#issueHasCompletionPr(record, opts, implementer)));
|
|
4675
|
+
return completed.every(Boolean);
|
|
4676
|
+
}
|
|
4462
4677
|
async #resumeTrackedAgent(record, name, tracked) {
|
|
4463
4678
|
if (!tracked.sessionRef) {
|
|
4464
4679
|
return;
|
|
@@ -4482,11 +4697,12 @@ export class FactoryLoop {
|
|
|
4482
4697
|
record.agents.set(result.name, tracked);
|
|
4483
4698
|
if (tracked.spec.role === 'babysitter') {
|
|
4484
4699
|
this.#babysitterCriticalAgents.delete(name);
|
|
4485
|
-
const
|
|
4700
|
+
const ownership = [...this.#babysitterPr.entries()].find(([, candidate]) => candidate.agentName === name);
|
|
4701
|
+
const [ownershipKey, ref] = ownership ?? [];
|
|
4486
4702
|
if (ref) {
|
|
4487
4703
|
ref.agentName = result.name;
|
|
4488
4704
|
for (const [wakeKey, state] of this.#babysitterWakeStates) {
|
|
4489
|
-
if (
|
|
4705
|
+
if (state.agentName !== name)
|
|
4490
4706
|
continue;
|
|
4491
4707
|
if (state.timer)
|
|
4492
4708
|
clearTimeout(state.timer);
|
|
@@ -4504,7 +4720,7 @@ export class FactoryLoop {
|
|
|
4504
4720
|
if (state.kinds.size > 0)
|
|
4505
4721
|
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
4506
4722
|
}
|
|
4507
|
-
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
4723
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked, ownershipKey);
|
|
4508
4724
|
}
|
|
4509
4725
|
}
|
|
4510
4726
|
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
@@ -4571,7 +4787,7 @@ export class FactoryLoop {
|
|
|
4571
4787
|
return;
|
|
4572
4788
|
}
|
|
4573
4789
|
this.#increment('agentLifecycleReadySignals');
|
|
4574
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
4790
|
+
await this.#maybeAdvanceToHumanReview(record, signal.name);
|
|
4575
4791
|
return;
|
|
4576
4792
|
}
|
|
4577
4793
|
if (tracked.spec.role === 'babysitter') {
|
|
@@ -4671,7 +4887,7 @@ export class FactoryLoop {
|
|
|
4671
4887
|
this.#increment('prReadySignalsIgnoredIssueMismatch');
|
|
4672
4888
|
return;
|
|
4673
4889
|
}
|
|
4674
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
4890
|
+
await this.#maybeAdvanceToHumanReview(record, ready.agentName);
|
|
4675
4891
|
return;
|
|
4676
4892
|
}
|
|
4677
4893
|
}
|
|
@@ -5489,13 +5705,25 @@ export class FactoryLoop {
|
|
|
5489
5705
|
}
|
|
5490
5706
|
}
|
|
5491
5707
|
async #replayGithubIssueComments(key) {
|
|
5708
|
+
const active = this.#githubIssueCommentReplays.get(key);
|
|
5709
|
+
if (active)
|
|
5710
|
+
return await active;
|
|
5711
|
+
const replay = this.#runGithubIssueCommentReplay(key).finally(() => {
|
|
5712
|
+
if (this.#githubIssueCommentReplays.get(key) === replay) {
|
|
5713
|
+
this.#githubIssueCommentReplays.delete(key);
|
|
5714
|
+
}
|
|
5715
|
+
});
|
|
5716
|
+
this.#githubIssueCommentReplays.set(key, replay);
|
|
5717
|
+
return await replay;
|
|
5718
|
+
}
|
|
5719
|
+
async #runGithubIssueCommentReplay(key) {
|
|
5492
5720
|
const watch = this.#githubIssueCommentWatchStates.get(key);
|
|
5493
5721
|
if (!watch)
|
|
5494
5722
|
return;
|
|
5495
5723
|
const comments = [];
|
|
5496
5724
|
const sinceCommentId = githubCommentNumericId(watch.sinceCommentId ?? watch.lastSeenCommentId);
|
|
5497
5725
|
const processedCommentIds = new Set(watch.processedCommentIds ?? []);
|
|
5498
|
-
for (const path of await this.#githubIssueCommentPaths(watch.source)) {
|
|
5726
|
+
for (const path of await this.#githubIssueCommentPaths(watch.source, watch.issue.path)) {
|
|
5499
5727
|
const parts = githubIssueCommentPathParts(path);
|
|
5500
5728
|
const id = parts ? githubCommentNumericId(parts.commentId) : undefined;
|
|
5501
5729
|
if (id !== undefined && id > sinceCommentId && !processedCommentIds.has(String(id))) {
|
|
@@ -5522,14 +5750,25 @@ export class FactoryLoop {
|
|
|
5522
5750
|
}
|
|
5523
5751
|
}
|
|
5524
5752
|
}
|
|
5525
|
-
async #githubIssueCommentPaths(source) {
|
|
5753
|
+
async #githubIssueCommentPaths(source, issuePath) {
|
|
5526
5754
|
const paths = new Set();
|
|
5527
5755
|
const owner = encodeURIComponent(source.owner);
|
|
5528
5756
|
const repo = encodeURIComponent(source.repo);
|
|
5529
|
-
|
|
5530
|
-
|
|
5531
|
-
|
|
5532
|
-
|
|
5757
|
+
const issueParts = issuePath ? githubIssuePathParts(issuePath) : undefined;
|
|
5758
|
+
const canonicalIssueRoot = issuePath
|
|
5759
|
+
&& issueParts?.owner.toLowerCase() === source.owner.toLowerCase()
|
|
5760
|
+
&& issueParts.repo.toLowerCase() === source.repo.toLowerCase()
|
|
5761
|
+
&& issueParts.number === source.number
|
|
5762
|
+
&& /\/(?:meta|metadata)\.json$/u.test(issuePath)
|
|
5763
|
+
? dirname(issuePath)
|
|
5764
|
+
: undefined;
|
|
5765
|
+
const prefixes = canonicalIssueRoot
|
|
5766
|
+
? [canonicalIssueRoot]
|
|
5767
|
+
: [
|
|
5768
|
+
`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`,
|
|
5769
|
+
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
5770
|
+
];
|
|
5771
|
+
for (const prefix of prefixes) {
|
|
5533
5772
|
try {
|
|
5534
5773
|
for (const path of await this.#mount.listTree(prefix)) {
|
|
5535
5774
|
const parts = githubIssueCommentPathParts(path);
|
|
@@ -5926,7 +6165,8 @@ export class FactoryLoop {
|
|
|
5926
6165
|
async #restoreBabysitterOwnership() {
|
|
5927
6166
|
const batch = await this.#batch();
|
|
5928
6167
|
for (const [persistedKey, session] of await this.#state.listBabysitterSessions(this.#workspaceId)) {
|
|
5929
|
-
|
|
6168
|
+
const ownershipKey = babysitterOwnershipKey(session.issue, session);
|
|
6169
|
+
if ((persistedKey !== issueKey(session.issue) && persistedKey !== ownershipKey) ||
|
|
5930
6170
|
!validGithubRepo(session.repo) ||
|
|
5931
6171
|
!validPrNumber(session.prNumber) ||
|
|
5932
6172
|
!session.agentName) {
|
|
@@ -5956,7 +6196,9 @@ export class FactoryLoop {
|
|
|
5956
6196
|
}
|
|
5957
6197
|
const record = batch.getIssue(session.issue);
|
|
5958
6198
|
const tracked = record?.agents.get(session.agentName)
|
|
5959
|
-
?? [...(record?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter'
|
|
6199
|
+
?? [...(record?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter' &&
|
|
6200
|
+
githubPrIdentity(agent.spec.ownedPullRequest?.repo ?? '', agent.spec.ownedPullRequest?.number ?? 0) ===
|
|
6201
|
+
githubPrIdentity(session.repo, session.prNumber))
|
|
5960
6202
|
?? durableBabysitterTrackedAgent(session, this.#config.agentCapabilities.babysitter);
|
|
5961
6203
|
const ref = {
|
|
5962
6204
|
repo: session.repo,
|
|
@@ -5964,11 +6206,15 @@ export class FactoryLoop {
|
|
|
5964
6206
|
path: session.path,
|
|
5965
6207
|
agentName: session.agentName,
|
|
5966
6208
|
};
|
|
5967
|
-
this.#babysitterPr.set(
|
|
5968
|
-
this.#babysitterIssueRefs.set(
|
|
5969
|
-
this.#babysitterSpawned.add(
|
|
6209
|
+
this.#babysitterPr.set(ownershipKey, ref);
|
|
6210
|
+
this.#babysitterIssueRefs.set(ownershipKey, { ...session.issue });
|
|
6211
|
+
this.#babysitterSpawned.add(ownershipKey);
|
|
5970
6212
|
if (session.critical)
|
|
5971
6213
|
this.#babysitterCriticalAgents.add(session.agentName);
|
|
6214
|
+
if (persistedKey !== ownershipKey) {
|
|
6215
|
+
await this.#state.setBabysitterSession(this.#workspaceId, ownershipKey, session);
|
|
6216
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, persistedKey);
|
|
6217
|
+
}
|
|
5972
6218
|
this.#increment('babysitterOwnershipRestored');
|
|
5973
6219
|
const pendingKinds = session.pendingKinds.filter(isBabysitterWakeKind);
|
|
5974
6220
|
if (pendingKinds.length > 0) {
|
|
@@ -5991,12 +6237,13 @@ export class FactoryLoop {
|
|
|
5991
6237
|
}
|
|
5992
6238
|
this.#babysitterWakeStates.clear();
|
|
5993
6239
|
}
|
|
5994
|
-
async #cancelBabysitterWake(
|
|
5995
|
-
const issue = this.#babysitterIssueRefs.get(
|
|
6240
|
+
async #cancelBabysitterWake(ownershipKey) {
|
|
6241
|
+
const issue = this.#babysitterIssueRefs.get(ownershipKey);
|
|
6242
|
+
const ref = this.#babysitterPr.get(ownershipKey);
|
|
5996
6243
|
const mayClearDurable = !this.#usesDurableDispatchLifecycle()
|
|
5997
6244
|
|| Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
|
|
5998
6245
|
for (const [key, state] of this.#babysitterWakeStates) {
|
|
5999
|
-
if (
|
|
6246
|
+
if (!ref || babysitterOwnershipKey(state.issue, state) !== ownershipKey)
|
|
6000
6247
|
continue;
|
|
6001
6248
|
state.cancelled = true;
|
|
6002
6249
|
delete state.tracked.spec.pendingPullRequestWake;
|
|
@@ -6005,11 +6252,19 @@ export class FactoryLoop {
|
|
|
6005
6252
|
this.#babysitterWakeStates.delete(key);
|
|
6006
6253
|
this.#babysitterCriticalAgents.delete(state.agentName);
|
|
6007
6254
|
}
|
|
6008
|
-
this.#babysitterPr.delete(
|
|
6009
|
-
this.#babysitterIssueRefs.delete(
|
|
6010
|
-
this.#babysitterSpawned.delete(
|
|
6255
|
+
this.#babysitterPr.delete(ownershipKey);
|
|
6256
|
+
this.#babysitterIssueRefs.delete(ownershipKey);
|
|
6257
|
+
this.#babysitterSpawned.delete(ownershipKey);
|
|
6258
|
+
this.#babysitterReady.delete(ownershipKey);
|
|
6011
6259
|
if (mayClearDurable)
|
|
6012
|
-
await this.#state.clearBabysitterSession(this.#workspaceId,
|
|
6260
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, ownershipKey);
|
|
6261
|
+
}
|
|
6262
|
+
async #cancelBabysittersForIssue(issue) {
|
|
6263
|
+
const wanted = issueKey(issue);
|
|
6264
|
+
const keys = [...this.#babysitterIssueRefs.entries()]
|
|
6265
|
+
.filter(([, candidate]) => issueKey(candidate) === wanted)
|
|
6266
|
+
.map(([key]) => key);
|
|
6267
|
+
await Promise.all(keys.map(async (key) => this.#cancelBabysitterWake(key)));
|
|
6013
6268
|
}
|
|
6014
6269
|
async #routeBabysitterEvent(path, extraKinds = []) {
|
|
6015
6270
|
const event = githubBabysitterEventPathParts(path);
|
|
@@ -6071,7 +6326,7 @@ export class FactoryLoop {
|
|
|
6071
6326
|
const tracked = record?.agents.get(ref.agentName)
|
|
6072
6327
|
?? [...(record?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
6073
6328
|
?? durableBabysitterTrackedAgent({ issue, repo: ref.repo, prNumber: ref.prNumber, path: ref.path, agentName: ref.agentName, critical: false, pendingKinds: [] }, this.#config.agentCapabilities.babysitter);
|
|
6074
|
-
return { issue, record, ref, tracked };
|
|
6329
|
+
return { key, issue, record, ref, tracked };
|
|
6075
6330
|
}
|
|
6076
6331
|
}
|
|
6077
6332
|
return undefined;
|
|
@@ -6110,13 +6365,16 @@ export class FactoryLoop {
|
|
|
6110
6365
|
// Owner lookup and queueing straddle async mount/state reads. Revalidate
|
|
6111
6366
|
// the exact composite owner so a concurrent close/merge cancellation can
|
|
6112
6367
|
// never recreate durable state from a stale child event.
|
|
6113
|
-
const
|
|
6368
|
+
const ownershipKey = babysitterOwnershipKey(issue, ref);
|
|
6369
|
+
const current = this.#babysitterPr.get(ownershipKey);
|
|
6114
6370
|
if (!current ||
|
|
6115
6371
|
current.agentName !== ref.agentName ||
|
|
6116
6372
|
githubPrIdentity(current.repo, current.prNumber) !== githubPrIdentity(ref.repo, ref.prNumber)) {
|
|
6117
6373
|
this.#increment('babysitterEventsIgnoredStaleOwner');
|
|
6118
6374
|
return;
|
|
6119
6375
|
}
|
|
6376
|
+
// Any new event invalidates a prior readiness assertion for this exact PR.
|
|
6377
|
+
this.#babysitterReady.delete(ownershipKey);
|
|
6120
6378
|
const key = babysitterWakeKey(issue, ref);
|
|
6121
6379
|
let state = this.#babysitterWakeStates.get(key);
|
|
6122
6380
|
if (!state) {
|
|
@@ -6162,18 +6420,18 @@ export class FactoryLoop {
|
|
|
6162
6420
|
kinds: [...kinds].sort(compareBabysitterWakeKinds),
|
|
6163
6421
|
};
|
|
6164
6422
|
}
|
|
6165
|
-
await this.#persistBabysitterSession(state.issue, this.#babysitterPr.get(
|
|
6423
|
+
await this.#persistBabysitterSession(state.issue, this.#babysitterPr.get(babysitterOwnershipKey(state.issue, state)) ?? {
|
|
6166
6424
|
repo: state.repo,
|
|
6167
6425
|
prNumber: state.prNumber,
|
|
6168
6426
|
agentName: state.agentName,
|
|
6169
6427
|
}, state.tracked);
|
|
6170
6428
|
}
|
|
6171
|
-
async #persistBabysitterSession(issue, ref, tracked) {
|
|
6429
|
+
async #persistBabysitterSession(issue, ref, tracked, ownershipKey = babysitterOwnershipKey(issue, ref)) {
|
|
6172
6430
|
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
6173
6431
|
throw new Error(`Babysitter lifecycle ownership lost for ${issue.key}`);
|
|
6174
6432
|
}
|
|
6175
6433
|
const pending = tracked?.spec.pendingPullRequestWake;
|
|
6176
|
-
await this.#state.setBabysitterSession(this.#workspaceId,
|
|
6434
|
+
await this.#state.setBabysitterSession(this.#workspaceId, ownershipKey, {
|
|
6177
6435
|
issue: { ...issue },
|
|
6178
6436
|
repo: ref.repo,
|
|
6179
6437
|
prNumber: ref.prNumber,
|
|
@@ -6256,6 +6514,8 @@ export class FactoryLoop {
|
|
|
6256
6514
|
let targets;
|
|
6257
6515
|
if (!this.#fleet.waitForInjected) {
|
|
6258
6516
|
await this.#fleet.sendMessage(input);
|
|
6517
|
+
state.unreachableSinceMs = undefined;
|
|
6518
|
+
state.unreachableEscalated = false;
|
|
6259
6519
|
if (this.#stopping || state.cancelled) {
|
|
6260
6520
|
state.deliveringKinds = undefined;
|
|
6261
6521
|
return;
|
|
@@ -6267,6 +6527,10 @@ export class FactoryLoop {
|
|
|
6267
6527
|
}
|
|
6268
6528
|
else {
|
|
6269
6529
|
const ack = await this.#waitForInjectedWithRetry(input);
|
|
6530
|
+
// Delivery confirmed: the target is reachable again, so clear any
|
|
6531
|
+
// registration-lag backoff state accumulated by prior failures.
|
|
6532
|
+
state.unreachableSinceMs = undefined;
|
|
6533
|
+
state.unreachableEscalated = false;
|
|
6270
6534
|
targets = ack.targets.length > 0 ? [...new Set(ack.targets)] : [input.to];
|
|
6271
6535
|
}
|
|
6272
6536
|
if (this.#stopping || state.cancelled)
|
|
@@ -6313,14 +6577,52 @@ export class FactoryLoop {
|
|
|
6313
6577
|
});
|
|
6314
6578
|
}
|
|
6315
6579
|
this.#increment('babysitterEventWakeFailures');
|
|
6316
|
-
|
|
6317
|
-
|
|
6318
|
-
|
|
6319
|
-
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6323
|
-
|
|
6580
|
+
const registrationLag = isRegistrationLagInjectionError(error);
|
|
6581
|
+
if (registrationLag) {
|
|
6582
|
+
state.unreachableSinceMs ??= this.#clock.now();
|
|
6583
|
+
}
|
|
6584
|
+
else {
|
|
6585
|
+
// A different failure mode (not "target unreachable") resets the
|
|
6586
|
+
// unreachable window so a later genuine registration lag starts fresh.
|
|
6587
|
+
state.unreachableSinceMs = undefined;
|
|
6588
|
+
state.unreachableEscalated = false;
|
|
6589
|
+
}
|
|
6590
|
+
const unreachableMs = state.unreachableSinceMs !== undefined
|
|
6591
|
+
? this.#clock.now() - state.unreachableSinceMs
|
|
6592
|
+
: 0;
|
|
6593
|
+
if (registrationLag && unreachableMs >= this.#babysitterWakeUnreachableEscalateMs) {
|
|
6594
|
+
// The agent is up but its relay identity never became resolvable. Stop
|
|
6595
|
+
// the tight 1s loop: back off to a slow cadence (still eventually
|
|
6596
|
+
// recovering if the agent finally enrolls) and flag it once so an
|
|
6597
|
+
// operator can intervene (e.g. re-spawn / restart) instead of the
|
|
6598
|
+
// failure spinning silently forever.
|
|
6599
|
+
state.nextDelayMs = this.#babysitterWakeUnreachableRetryMs;
|
|
6600
|
+
if (!state.unreachableEscalated) {
|
|
6601
|
+
state.unreachableEscalated = true;
|
|
6602
|
+
await this.#fleet.reconcileTrackedAgents?.();
|
|
6603
|
+
this.#increment('babysitterEventWakeUnreachableReconciliations');
|
|
6604
|
+
this.#increment('babysitterEventWakeUnreachableEscalations');
|
|
6605
|
+
this.#logger.warn?.('[factory] babysitter unreachable past grace window; slowing wake retries and flagging for human attention', {
|
|
6606
|
+
issue: state.issue.key,
|
|
6607
|
+
repo: state.repo,
|
|
6608
|
+
prNumber: state.prNumber,
|
|
6609
|
+
babysitter: state.agentName,
|
|
6610
|
+
unreachableMs,
|
|
6611
|
+
retryDelayMs: this.#babysitterWakeUnreachableRetryMs,
|
|
6612
|
+
error: describeError(error).errorMessage,
|
|
6613
|
+
});
|
|
6614
|
+
}
|
|
6615
|
+
}
|
|
6616
|
+
else {
|
|
6617
|
+
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
6618
|
+
this.#logger.warn?.('[factory] babysitter event wake failed; preserving it for retry', {
|
|
6619
|
+
issue: state.issue.key,
|
|
6620
|
+
repo: state.repo,
|
|
6621
|
+
prNumber: state.prNumber,
|
|
6622
|
+
babysitter: state.agentName,
|
|
6623
|
+
error: describeError(error).errorMessage,
|
|
6624
|
+
});
|
|
6625
|
+
}
|
|
6324
6626
|
}
|
|
6325
6627
|
}
|
|
6326
6628
|
async #submitBabysitterWakeTargets(targets) {
|
|
@@ -6397,18 +6699,17 @@ export class FactoryLoop {
|
|
|
6397
6699
|
// only and can never redirect a live babysitter.
|
|
6398
6700
|
const owned = await this.#babysitterOwnerFor(repo, snapshot.number);
|
|
6399
6701
|
if (owned) {
|
|
6400
|
-
const ownedKey = issueKey(owned.issue);
|
|
6401
6702
|
if (prMetaShowsMerged(snapshot)) {
|
|
6402
6703
|
if (owned.record)
|
|
6403
6704
|
await this.#advanceMergedPrToDone(snapshot, owned.record);
|
|
6404
6705
|
else
|
|
6405
|
-
await this.#cancelBabysitterWake(
|
|
6706
|
+
await this.#cancelBabysitterWake(owned.key);
|
|
6406
6707
|
return;
|
|
6407
6708
|
}
|
|
6408
6709
|
if (!this.#config.babysitter.enabled)
|
|
6409
6710
|
return;
|
|
6410
6711
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
6411
|
-
await this.#cancelBabysitterWake(
|
|
6712
|
+
await this.#cancelBabysitterWake(owned.key);
|
|
6412
6713
|
return;
|
|
6413
6714
|
}
|
|
6414
6715
|
if (snapshot.draft)
|
|
@@ -6417,8 +6718,23 @@ export class FactoryLoop {
|
|
|
6417
6718
|
return;
|
|
6418
6719
|
}
|
|
6419
6720
|
const record = this.#inFlightIssueForPrSnapshot(snapshot, await this.#batch(), repo);
|
|
6420
|
-
const babysitterKey = record ?
|
|
6721
|
+
const babysitterKey = record ? babysitterOwnershipKey(record.issue, { repo, prNumber: snapshot.number }) : undefined;
|
|
6421
6722
|
const existing = babysitterKey ? this.#babysitterPr.get(babysitterKey) : undefined;
|
|
6723
|
+
const sameRepoOwner = record
|
|
6724
|
+
? [...this.#babysitterPr.entries()].find(([key, candidate]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue) &&
|
|
6725
|
+
candidate.repo.toLowerCase() === repo.toLowerCase())?.[1]
|
|
6726
|
+
: undefined;
|
|
6727
|
+
if (sameRepoOwner && githubPrIdentity(sameRepoOwner.repo, sameRepoOwner.prNumber) !== githubPrIdentity(repo, snapshot.number)) {
|
|
6728
|
+
this.#increment('babysitterEventsIgnoredOwnershipMismatch');
|
|
6729
|
+
this.#logger.warn?.('[factory] ignored PR event that conflicts with established babysitter ownership', {
|
|
6730
|
+
issue: record?.issue.key,
|
|
6731
|
+
ownedRepo: sameRepoOwner.repo,
|
|
6732
|
+
ownedPrNumber: sameRepoOwner.prNumber,
|
|
6733
|
+
eventRepo: repo,
|
|
6734
|
+
eventPrNumber: snapshot.number,
|
|
6735
|
+
});
|
|
6736
|
+
return;
|
|
6737
|
+
}
|
|
6422
6738
|
if (existing && githubPrIdentity(existing.repo, existing.prNumber) !== githubPrIdentity(repo, snapshot.number)) {
|
|
6423
6739
|
this.#increment('babysitterEventsIgnoredOwnershipMismatch');
|
|
6424
6740
|
this.#logger.warn?.('[factory] ignored PR event that conflicts with established babysitter ownership', {
|
|
@@ -6584,8 +6900,20 @@ export class FactoryLoop {
|
|
|
6584
6900
|
// probe resolver and spawn the babysitter. Triggered by an implementer exiting
|
|
6585
6901
|
// after opening its PR (an event, not a poll).
|
|
6586
6902
|
async #ensureBabysitterForIssue(record) {
|
|
6587
|
-
if (this.#
|
|
6588
|
-
|
|
6903
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
6904
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
6905
|
+
const receipts = lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : []);
|
|
6906
|
+
if (receipts.length > 0) {
|
|
6907
|
+
for (const receipt of receipts) {
|
|
6908
|
+
await this.#ensureBabysitter(record, {
|
|
6909
|
+
repo: receipt.repo,
|
|
6910
|
+
prNumber: receipt.number,
|
|
6911
|
+
url: receipt.url,
|
|
6912
|
+
headRef: receipt.headRef,
|
|
6913
|
+
});
|
|
6914
|
+
}
|
|
6915
|
+
return;
|
|
6916
|
+
}
|
|
6589
6917
|
}
|
|
6590
6918
|
const issue = await this.#readIssue(record.issue.path);
|
|
6591
6919
|
if (!issue) {
|
|
@@ -6598,7 +6926,7 @@ export class FactoryLoop {
|
|
|
6598
6926
|
await this.#ensureBabysitter(record, { repo: pr.repo, prNumber: pr.prNumber });
|
|
6599
6927
|
}
|
|
6600
6928
|
async #ensureBabysitter(record, prRef) {
|
|
6601
|
-
const babysitterKey =
|
|
6929
|
+
const babysitterKey = babysitterOwnershipKey(record.issue, prRef);
|
|
6602
6930
|
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
6603
6931
|
this.#increment('babysitterLifecycleOwnershipRejected');
|
|
6604
6932
|
return;
|
|
@@ -6627,7 +6955,9 @@ export class FactoryLoop {
|
|
|
6627
6955
|
settled.path = prRef.path;
|
|
6628
6956
|
return;
|
|
6629
6957
|
}
|
|
6630
|
-
const
|
|
6958
|
+
const wantedPr = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
6959
|
+
const trackedBabysitter = [...record.agents.entries()].find(([, agent]) => agent.spec.role === 'babysitter' &&
|
|
6960
|
+
githubPrIdentity(agent.spec.ownedPullRequest?.repo ?? '', agent.spec.ownedPullRequest?.number ?? 0) === wantedPr);
|
|
6631
6961
|
if (trackedBabysitter) {
|
|
6632
6962
|
const [trackedName, tracked] = trackedBabysitter;
|
|
6633
6963
|
const owned = tracked.spec.ownedPullRequest;
|
|
@@ -6661,6 +6991,12 @@ export class FactoryLoop {
|
|
|
6661
6991
|
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
6662
6992
|
?? record.decision.routes[0];
|
|
6663
6993
|
const initialSpec = babysitterSpec(issue, this.#config, route);
|
|
6994
|
+
if ([...this.#babysitterIssueRefs.entries()].some(([key, candidate]) => key !== babysitterKey && issueKey(candidate) === issueKey(record.issue))) {
|
|
6995
|
+
initialSpec.name = agentNameForRole(issue, 'babysit', {
|
|
6996
|
+
repo: prRef.repo,
|
|
6997
|
+
discriminator: `${sanitizeAgentSlug(prRef.repo)}-${prRef.prNumber}`,
|
|
6998
|
+
});
|
|
6999
|
+
}
|
|
6664
7000
|
const sharedCheckout = [...record.agents.values()]
|
|
6665
7001
|
.map((agent) => agent.spec)
|
|
6666
7002
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath)
|
|
@@ -6763,18 +7099,21 @@ export class FactoryLoop {
|
|
|
6763
7099
|
// invoking the lifecycle action with `kind: ready`. The orchestrator trusts that signal and only
|
|
6764
7100
|
// guards on the PR's OWN webhook-fed meta (still open, not a draft, not already
|
|
6765
7101
|
// merged) before flipping the issue to Human Review. No `gh` call.
|
|
6766
|
-
async #maybeAdvanceToHumanReview(record) {
|
|
7102
|
+
async #maybeAdvanceToHumanReview(record, agentName) {
|
|
6767
7103
|
if (this.#completionInFlight.has(issueKey(record.issue))) {
|
|
6768
7104
|
return;
|
|
6769
7105
|
}
|
|
6770
|
-
|
|
7106
|
+
const ownership = [...this.#babysitterPr.entries()].find(([key, ref]) => ref.agentName === agentName && issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue));
|
|
7107
|
+
if (!ownership) {
|
|
6771
7108
|
this.#increment('babysitterReadinessGuardBlocked');
|
|
6772
7109
|
this.#logger.info?.('[factory] babysitter ready signal ignored; PR ownership is no longer active', {
|
|
6773
7110
|
issue: record.issue.key,
|
|
7111
|
+
babysitter: agentName,
|
|
6774
7112
|
});
|
|
6775
7113
|
return;
|
|
6776
7114
|
}
|
|
6777
|
-
const
|
|
7115
|
+
const [ownershipKey, ref] = ownership;
|
|
7116
|
+
const snapshot = await this.#readPrSnapshot(ref);
|
|
6778
7117
|
if (!snapshot) {
|
|
6779
7118
|
this.#increment('babysitterReadinessGuardBlocked');
|
|
6780
7119
|
this.#logger.info?.('[factory] babysitter ready signal ignored; authoritative PR meta is unavailable', {
|
|
@@ -6791,6 +7130,25 @@ export class FactoryLoop {
|
|
|
6791
7130
|
});
|
|
6792
7131
|
return;
|
|
6793
7132
|
}
|
|
7133
|
+
this.#babysitterReady.add(ownershipKey);
|
|
7134
|
+
await this.#ensureBabysitterForIssue(record);
|
|
7135
|
+
const owners = [...this.#babysitterIssueRefs.entries()]
|
|
7136
|
+
.filter(([, issue]) => issueKey(issue) === issueKey(record.issue))
|
|
7137
|
+
.map(([key]) => key);
|
|
7138
|
+
const expectedPrOwners = new Set(record.decision.implementers.map((implementer) => implementer.repo)).size;
|
|
7139
|
+
if (owners.length < expectedPrOwners) {
|
|
7140
|
+
this.#increment('babysitterReadinessWaitingForPeers');
|
|
7141
|
+
this.#logger.info?.('[factory] babysitter ready; waiting for remaining repository PRs', {
|
|
7142
|
+
issue: record.issue.key,
|
|
7143
|
+
repo: ref.repo,
|
|
7144
|
+
prNumber: ref.prNumber,
|
|
7145
|
+
});
|
|
7146
|
+
return;
|
|
7147
|
+
}
|
|
7148
|
+
if (owners.length === 0 || owners.some((key) => !this.#babysitterReady.has(key))) {
|
|
7149
|
+
this.#increment('babysitterReadinessWaitingForPeers');
|
|
7150
|
+
return;
|
|
7151
|
+
}
|
|
6794
7152
|
this.#increment('babysitterReadinessReady');
|
|
6795
7153
|
this.#logger.info?.('[factory] babysitter signalled PR ready; advancing to human review', {
|
|
6796
7154
|
issue: record.issue.key,
|
|
@@ -6802,7 +7160,8 @@ export class FactoryLoop {
|
|
|
6802
7160
|
// exact path captured when the babysitter was spawned; otherwise scans the
|
|
6803
7161
|
// repo's pulls subtree for the PR number across known layout shapes.
|
|
6804
7162
|
async #readBabysatPrSnapshot(record) {
|
|
6805
|
-
const ref = this.#babysitterPr.
|
|
7163
|
+
const ref = [...this.#babysitterPr.entries()]
|
|
7164
|
+
.find(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))?.[1];
|
|
6806
7165
|
if (!ref) {
|
|
6807
7166
|
return undefined;
|
|
6808
7167
|
}
|
|
@@ -6852,7 +7211,9 @@ export class FactoryLoop {
|
|
|
6852
7211
|
if (babysatSnapshot && prMetaShowsMerged(babysatSnapshot)) {
|
|
6853
7212
|
return true;
|
|
6854
7213
|
}
|
|
6855
|
-
const pr = this.#babysitterPr.
|
|
7214
|
+
const pr = [...this.#babysitterPr.entries()]
|
|
7215
|
+
.find(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))?.[1]
|
|
7216
|
+
?? await this.#completionPrForIssue(issue);
|
|
6856
7217
|
if (!pr) {
|
|
6857
7218
|
return false;
|
|
6858
7219
|
}
|
|
@@ -6988,9 +7349,7 @@ export class FactoryLoop {
|
|
|
6988
7349
|
const stateKey = issueStateKey(record.issue);
|
|
6989
7350
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
6990
7351
|
this.#probePrResolvedCache.delete(stateKey);
|
|
6991
|
-
this.#
|
|
6992
|
-
this.#babysitterPr.delete(completionKey);
|
|
6993
|
-
await this.#cancelBabysitterWake(completionKey);
|
|
7352
|
+
await this.#cancelBabysittersForIssue(record.issue);
|
|
6994
7353
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
6995
7354
|
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
6996
7355
|
for (const publishedKey of this.#publishedPullRequests.keys()) {
|
|
@@ -8887,6 +9246,19 @@ function dispatchSpecs(decision) {
|
|
|
8887
9246
|
}
|
|
8888
9247
|
return [...decision.implementers, decision.reviewer];
|
|
8889
9248
|
}
|
|
9249
|
+
function authoritativeRoutedDecision(triaged, routed) {
|
|
9250
|
+
if (triaged.confidence !== 'low' || triaged.routes.length > 0 || routed.routes.length === 0) {
|
|
9251
|
+
return routed;
|
|
9252
|
+
}
|
|
9253
|
+
return {
|
|
9254
|
+
...routed,
|
|
9255
|
+
confidence: 'high',
|
|
9256
|
+
rationale: [
|
|
9257
|
+
routed.routes.map((route) => route.rationale).filter(Boolean).join(' '),
|
|
9258
|
+
'Repository identity was resolved authoritatively from the live issue labels or GitHub source repository.',
|
|
9259
|
+
].filter(Boolean).join(' '),
|
|
9260
|
+
};
|
|
9261
|
+
}
|
|
8890
9262
|
function labelDerivedDispatchDecision(liveIssue, decision, config) {
|
|
8891
9263
|
const routesByLabel = labelRoutesForIssue(liveIssue, config);
|
|
8892
9264
|
if (routesByLabel.labels.length === 0) {
|
|
@@ -10015,6 +10387,7 @@ const decodeGithubPathSegment = (value) => {
|
|
|
10015
10387
|
const validGithubRepo = (repo) => /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})\/[A-Za-z0-9_.-]{1,100}$/u.test(repo);
|
|
10016
10388
|
const validPrNumber = (value) => Number.isInteger(value) && value > 0;
|
|
10017
10389
|
const githubPrIdentity = (repo, prNumber) => validGithubRepo(repo) && validPrNumber(prNumber) ? `${repo.toLowerCase()}#${prNumber}` : undefined;
|
|
10390
|
+
const babysitterOwnershipKey = (issue, ref) => `${issueKey(issue)}:${githubPrIdentity(ref.repo, ref.prNumber) ?? 'invalid'}`;
|
|
10018
10391
|
const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
10019
10392
|
if (!validGithubRepo(eventRepo))
|
|
10020
10393
|
return false;
|
|
@@ -10031,7 +10404,7 @@ const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
|
10031
10404
|
}
|
|
10032
10405
|
});
|
|
10033
10406
|
};
|
|
10034
|
-
const babysitterWakeKey = (issue, ref) => `${
|
|
10407
|
+
const babysitterWakeKey = (issue, ref) => `${babysitterOwnershipKey(issue, ref)}:${ref.agentName}`;
|
|
10035
10408
|
const BABYSITTER_WAKE_KIND_ORDER = [
|
|
10036
10409
|
'changes-requested',
|
|
10037
10410
|
'review-comment',
|
|
@@ -10555,7 +10928,26 @@ const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') =>
|
|
|
10555
10928
|
result: { name: session.agentName },
|
|
10556
10929
|
});
|
|
10557
10930
|
const isTerminalDispatchLifecycle = (lifecycle) => lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned';
|
|
10558
|
-
const
|
|
10931
|
+
const publishedPullRequests = (lifecycle) => {
|
|
10932
|
+
const receipts = [
|
|
10933
|
+
...(lifecycle?.pullRequests ?? []).filter(Boolean),
|
|
10934
|
+
...(lifecycle?.pullRequest ? [lifecycle.pullRequest] : []),
|
|
10935
|
+
];
|
|
10936
|
+
return [...new Map(receipts.map((receipt) => [
|
|
10937
|
+
`${receipt.repo.toLowerCase()}#${receipt.number}`,
|
|
10938
|
+
{ ...receipt },
|
|
10939
|
+
])).values()];
|
|
10940
|
+
};
|
|
10941
|
+
const mergePublishedPullRequests = (lifecycle, receipt) => {
|
|
10942
|
+
const receipts = publishedPullRequests(lifecycle);
|
|
10943
|
+
if (receipt)
|
|
10944
|
+
receipts.push(receipt);
|
|
10945
|
+
return [...new Map(receipts.map((candidate) => [
|
|
10946
|
+
candidate.repo.toLowerCase(),
|
|
10947
|
+
{ ...candidate },
|
|
10948
|
+
])).values()];
|
|
10949
|
+
};
|
|
10950
|
+
const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequest, pullRequests = [], releaseReason) => ({
|
|
10559
10951
|
runId,
|
|
10560
10952
|
issue: { ...record.issue },
|
|
10561
10953
|
decision: structuredClone(record.decision),
|
|
@@ -10564,6 +10956,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
10564
10956
|
agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
|
|
10565
10957
|
invocationIds: [...record.invocationIds],
|
|
10566
10958
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
10959
|
+
...(pullRequests.length > 0 ? { pullRequests: pullRequests.map((receipt) => ({ ...receipt })) } : {}),
|
|
10567
10960
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
10568
10961
|
...(releaseReason ? { releaseReason } : {}),
|
|
10569
10962
|
updatedAtMs,
|