@agent-relay/factory 0.1.36 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -6
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +48 -8
- package/dist/cli/fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +13 -0
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +1 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +4 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/github/repo-identity.d.ts +8 -0
- package/dist/github/repo-identity.d.ts.map +1 -0
- package/dist/github/repo-identity.js +28 -0
- package/dist/github/repo-identity.js.map +1 -0
- package/dist/hosted/index.d.ts +5 -0
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +3 -0
- package/dist/hosted/index.js.map +1 -0
- package/dist/hosted/orchestrator.d.ts +12 -0
- package/dist/hosted/orchestrator.d.ts.map +1 -0
- package/dist/hosted/orchestrator.js +513 -0
- package/dist/hosted/orchestrator.js.map +1 -0
- package/dist/hosted/state-store.d.ts +48 -0
- package/dist/hosted/state-store.d.ts.map +1 -0
- package/dist/hosted/state-store.js +210 -0
- package/dist/hosted/state-store.js.map +1 -0
- package/dist/hosted/types.d.ts +173 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/hosted/types.js +2 -0
- package/dist/hosted/types.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +20 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +209 -22
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +17 -1
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +1 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1002 -165
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/state.d.ts +2 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +13 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +13 -1
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/index.d.ts +2 -0
- package/dist/writeback/index.d.ts.map +1 -1
- package/dist/writeback/index.js +1 -0
- package/dist/writeback/index.js.map +1 -1
- package/dist/writeback/mount-health.d.ts +15 -0
- package/dist/writeback/mount-health.d.ts.map +1 -0
- package/dist/writeback/mount-health.js +15 -0
- package/dist/writeback/mount-health.js.map +1 -0
- package/package.json +6 -2
|
@@ -85,6 +85,7 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
85
85
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
86
86
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
87
87
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
88
|
+
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
88
89
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
89
90
|
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
90
91
|
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
@@ -99,6 +100,8 @@ const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
|
99
100
|
const GITHUB_MIRROR_SOURCE_PREFIX = 'Source: ';
|
|
100
101
|
export const DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH = '/tmp/factory-run/factory-loop-heartbeat.json';
|
|
101
102
|
export const DEFAULT_FACTORY_LOOP_REGISTRY_PATH = '/tmp/factory-run/factory-loop-registry.json';
|
|
103
|
+
class DispatchLifecycleCapacityError extends Error {
|
|
104
|
+
}
|
|
102
105
|
const realClock = {
|
|
103
106
|
now: () => Date.now(),
|
|
104
107
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -119,6 +122,7 @@ export class FactoryLoop {
|
|
|
119
122
|
#probeCloser;
|
|
120
123
|
#probePrResolver;
|
|
121
124
|
#customProbePrResolver;
|
|
125
|
+
#hasProbePrGhRunner;
|
|
122
126
|
#probePrGhRunner;
|
|
123
127
|
#logger;
|
|
124
128
|
#clock;
|
|
@@ -129,6 +133,7 @@ export class FactoryLoop {
|
|
|
129
133
|
#terminationGraceMs;
|
|
130
134
|
#babysitterWakeUnreachableEscalateMs;
|
|
131
135
|
#babysitterWakeUnreachableRetryMs;
|
|
136
|
+
#startupAgentExitDrainTimeoutMs;
|
|
132
137
|
#state;
|
|
133
138
|
#workspaceId;
|
|
134
139
|
#relayflows;
|
|
@@ -145,6 +150,7 @@ export class FactoryLoop {
|
|
|
145
150
|
#githubIssueCommentWatchers = new Map();
|
|
146
151
|
#githubIssueCommentWatchStates = new Map();
|
|
147
152
|
#githubIssueCommentQueues = new Map();
|
|
153
|
+
#githubIssueCommentReplays = new Map();
|
|
148
154
|
#githubIssueAuthors = new Map();
|
|
149
155
|
#githubIssueAuthorLookups = new Map();
|
|
150
156
|
#githubIssuePreferredPaths = new Map();
|
|
@@ -174,6 +180,7 @@ export class FactoryLoop {
|
|
|
174
180
|
#dispatchTerminalWaiters = new Map();
|
|
175
181
|
#dispatchLifecycleRetryTimers = new Map();
|
|
176
182
|
#dispatchLifecycleDrives = new Set();
|
|
183
|
+
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
177
184
|
#localReleaseCheckpoints = new Map();
|
|
178
185
|
#dispatchLifecycleRenewTimer;
|
|
179
186
|
#clarificationSweepTimer;
|
|
@@ -210,15 +217,19 @@ export class FactoryLoop {
|
|
|
210
217
|
#completionSweepTimer;
|
|
211
218
|
#completionSweepActive = false;
|
|
212
219
|
#completionInFlight = new Set();
|
|
220
|
+
#agentExitsInFlight = new Map();
|
|
213
221
|
#agentLifecycleSignalsInFlight = new Map();
|
|
214
|
-
|
|
215
|
-
//
|
|
222
|
+
#startupAgentAdoptionActive = false;
|
|
223
|
+
// Composite issue + PR identities for which a babysitter has already been spawned, so repeated
|
|
224
|
+
// webhooks / agent-exit safety nets don't respawn it while multi-repository issues retain one
|
|
225
|
+
// owner per PR.
|
|
216
226
|
#babysitterSpawned = new Set();
|
|
217
227
|
#babysitterSpawnInFlight = new Map();
|
|
218
|
-
// Composite issue identity -> the open PR the babysitter is shepherding, including the
|
|
228
|
+
// Composite issue + PR identity -> the open PR the babysitter is shepherding, including the
|
|
219
229
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
220
230
|
#babysitterPr = new Map();
|
|
221
231
|
#babysitterIssueRefs = new Map();
|
|
232
|
+
#babysitterReady = new Set();
|
|
222
233
|
#babysitterWakeStates = new Map();
|
|
223
234
|
// A babysitter announces this fence before invoking destructive git tooling
|
|
224
235
|
// and clears it afterward. Event text can be broker-delivered while a prompt
|
|
@@ -265,6 +276,7 @@ export class FactoryLoop {
|
|
|
265
276
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
266
277
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
267
278
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
279
|
+
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
268
280
|
this.#probePrGhRunner = ports.probePrGhRunner ?? failClosedGhRunner;
|
|
269
281
|
this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
|
|
270
282
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
@@ -279,6 +291,7 @@ export class FactoryLoop {
|
|
|
279
291
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
280
292
|
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
281
293
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
294
|
+
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
282
295
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
283
296
|
this.#relayflows = ports.relayflows;
|
|
284
297
|
this.#worktrees = ports.worktrees;
|
|
@@ -459,9 +472,33 @@ export class FactoryLoop {
|
|
|
459
472
|
this.#error(new Error(`${GITHUB_ISSUE_ROOT} sub-root is not mounted`));
|
|
460
473
|
return;
|
|
461
474
|
}
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
475
|
+
const live = (opts.mode ?? 'live') === 'live';
|
|
476
|
+
// Capture the legacy registry before the first live heartbeat rewrites it.
|
|
477
|
+
// Durable lifecycle rows are authoritative, but this fallback is still
|
|
478
|
+
// required to adopt workers started by pre-lifecycle Factory versions.
|
|
479
|
+
const legacyRegistry = live
|
|
480
|
+
? await readFactoryInFlightRegistry(this.#config.loop.registryPath)
|
|
481
|
+
: undefined;
|
|
482
|
+
if (live)
|
|
483
|
+
await this.#startLiveHeartbeat();
|
|
484
|
+
this.#startupAgentAdoptionActive = true;
|
|
485
|
+
try {
|
|
486
|
+
this.#wireFleetEvents();
|
|
487
|
+
await this.#adoptInFlightAgents(legacyRegistry);
|
|
488
|
+
this.#startupAgentAdoptionActive = false;
|
|
489
|
+
if (this.#config.babysitter.enabled) {
|
|
490
|
+
// Re-run the idempotent receipt fold after adoption returns. This
|
|
491
|
+
// catches records restored by lifecycle work that completed while the
|
|
492
|
+
// startup roster drain was in progress.
|
|
493
|
+
await this.#reconcileRestoredBabysitterReceipts();
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
catch (error) {
|
|
497
|
+
this.#startupAgentAdoptionActive = false;
|
|
498
|
+
if (live)
|
|
499
|
+
await this.#stopLiveHeartbeat('stopping');
|
|
500
|
+
throw error;
|
|
501
|
+
}
|
|
465
502
|
if (opts.mode === 'dispatch-owner') {
|
|
466
503
|
this.#started = true;
|
|
467
504
|
this.#scheduleDispatchLifecycleRenewal();
|
|
@@ -473,7 +510,7 @@ export class FactoryLoop {
|
|
|
473
510
|
await this.#rearmGithubIssueCommentWatchers();
|
|
474
511
|
return;
|
|
475
512
|
}
|
|
476
|
-
if (
|
|
513
|
+
if (live) {
|
|
477
514
|
this.#started = true;
|
|
478
515
|
try {
|
|
479
516
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
@@ -533,6 +570,12 @@ export class FactoryLoop {
|
|
|
533
570
|
this.#completionSweepTimer = undefined;
|
|
534
571
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
535
572
|
try {
|
|
573
|
+
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
574
|
+
// drives. A slow Relayfile scan must not consume the shutdown deadline
|
|
575
|
+
// while every issue remains fenced to a publisher that is already
|
|
576
|
+
// stopping. The owner/epoch fence makes any late completion from those
|
|
577
|
+
// drives harmless; a second sweep below catches claims racing this one.
|
|
578
|
+
await this.#releaseOwnedDispatchLifecycleLeases();
|
|
536
579
|
await Promise.allSettled([...this.#dispatchLifecycleDrives]);
|
|
537
580
|
// Fence every source of new clarification work before touching the fleet.
|
|
538
581
|
// A wake already past the fence is allowed to unwind, and is awaited
|
|
@@ -549,15 +592,13 @@ export class FactoryLoop {
|
|
|
549
592
|
await this.#drainClarificationWakesForStop();
|
|
550
593
|
this.#clarificationIntents.clear();
|
|
551
594
|
await this.#drainBabysitterWakesForStop();
|
|
595
|
+
await this.#drainAgentExitsInFlight();
|
|
552
596
|
// Durable relay placements must survive an owner restart so a successor
|
|
553
597
|
// can adopt them. The one-shot/daemon stop path releases only
|
|
554
598
|
// non-durable (local/internal) records; terminal completion performs the
|
|
555
599
|
// normal remote release before clearing the lifecycle.
|
|
556
600
|
await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
|
|
557
|
-
|
|
558
|
-
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
559
|
-
}
|
|
560
|
-
this.#dispatchLifecycleEpochs.clear();
|
|
601
|
+
await this.#releaseOwnedDispatchLifecycleLeases();
|
|
561
602
|
if (this.#livePollTimer)
|
|
562
603
|
clearTimeout(this.#livePollTimer);
|
|
563
604
|
this.#livePollTimer = undefined;
|
|
@@ -567,6 +608,7 @@ export class FactoryLoop {
|
|
|
567
608
|
this.#babysitterSpawned.clear();
|
|
568
609
|
this.#babysitterPr.clear();
|
|
569
610
|
this.#babysitterIssueRefs.clear();
|
|
611
|
+
this.#babysitterReady.clear();
|
|
570
612
|
this.#babysitterCriticalAgents.clear();
|
|
571
613
|
const subscription = this.#subscription;
|
|
572
614
|
this.#subscription = undefined;
|
|
@@ -594,6 +636,14 @@ export class FactoryLoop {
|
|
|
594
636
|
this.#stoppingHeartbeatRefreshActive = false;
|
|
595
637
|
}
|
|
596
638
|
}
|
|
639
|
+
async #releaseOwnedDispatchLifecycleLeases() {
|
|
640
|
+
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
641
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
642
|
+
if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
|
|
643
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
597
647
|
async #drainClarificationWakesForStop() {
|
|
598
648
|
// A wake may add its promise just as the sweep that discovered it settles.
|
|
599
649
|
// Re-snapshot until the map is empty rather than assuming one await is a
|
|
@@ -638,7 +688,6 @@ export class FactoryLoop {
|
|
|
638
688
|
}
|
|
639
689
|
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
640
690
|
const options = this.#liveOptions(overrides);
|
|
641
|
-
await this.#startLiveHeartbeat();
|
|
642
691
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
643
692
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
644
693
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1095,6 +1144,10 @@ export class FactoryLoop {
|
|
|
1095
1144
|
this.#completionSweepTimer.unref?.();
|
|
1096
1145
|
}
|
|
1097
1146
|
async #sweepPrStateCompletions(reason) {
|
|
1147
|
+
// This timer is also the durable safety net for fleet exit events missed
|
|
1148
|
+
// while the event loop was busy (for example during a large startup pull).
|
|
1149
|
+
// Keep reconciliation active even when babysitters own PR completion.
|
|
1150
|
+
await this.#fleet.reconcileTrackedAgents?.();
|
|
1098
1151
|
// When the babysitter owns PR-open, completion is driven by PR webhooks +
|
|
1099
1152
|
// the babysitter's readiness signal (see #handlePrChange / #handleAgentExit),
|
|
1100
1153
|
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
@@ -1130,6 +1183,10 @@ export class FactoryLoop {
|
|
|
1130
1183
|
this.#probePrGhBackoffUntilMs.set(issueStateKey(issueRef(issue)), this.#clock.now() + PROBE_PR_GH_BACKOFF_MS);
|
|
1131
1184
|
return undefined;
|
|
1132
1185
|
}
|
|
1186
|
+
if (record.decision.implementers.length > 1 && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
1187
|
+
this.#increment('completionSweepMissingPr');
|
|
1188
|
+
return undefined;
|
|
1189
|
+
}
|
|
1133
1190
|
return { record, pr };
|
|
1134
1191
|
}));
|
|
1135
1192
|
for (const candidate of candidates) {
|
|
@@ -1241,6 +1298,7 @@ export class FactoryLoop {
|
|
|
1241
1298
|
await this.#recordCanonicalIssueState(issue);
|
|
1242
1299
|
}
|
|
1243
1300
|
issueEntries.push({ path, issue });
|
|
1301
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
1244
1302
|
}
|
|
1245
1303
|
if (issueSource === 'github') {
|
|
1246
1304
|
// New ready work must not sit behind a long sequence of stale
|
|
@@ -1258,6 +1316,7 @@ export class FactoryLoop {
|
|
|
1258
1316
|
});
|
|
1259
1317
|
}
|
|
1260
1318
|
for (const { issue } of issueEntries) {
|
|
1319
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
1261
1320
|
if (!issue) {
|
|
1262
1321
|
continue;
|
|
1263
1322
|
}
|
|
@@ -1672,6 +1731,7 @@ export class FactoryLoop {
|
|
|
1672
1731
|
url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
|
|
1673
1732
|
path: pr.path,
|
|
1674
1733
|
headRef,
|
|
1734
|
+
authoritative: true,
|
|
1675
1735
|
});
|
|
1676
1736
|
const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
|
|
1677
1737
|
if (!babysitter) {
|
|
@@ -1906,15 +1966,9 @@ export class FactoryLoop {
|
|
|
1906
1966
|
this.#error(error, decision.issue);
|
|
1907
1967
|
throw error;
|
|
1908
1968
|
}
|
|
1909
|
-
const
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
this.#recordTriageEscalation(decision, escalationReason);
|
|
1913
|
-
return replayedResult ?? { issue: decision.issue, agents: [], dryRun };
|
|
1914
|
-
}
|
|
1915
|
-
// TODO(AR-274 follow-up): short-circuit LLM triage once label-derived
|
|
1916
|
-
// routes are authoritative for dispatch identity.
|
|
1917
|
-
const labelDispatch = labelDerivedDispatchDecision(liveIssue, decision, this.#config);
|
|
1969
|
+
const labelDispatch = opts.labelsValidated
|
|
1970
|
+
? { ok: true, decision }
|
|
1971
|
+
: labelDerivedDispatchDecision(liveIssue, decision, this.#config);
|
|
1918
1972
|
if (!labelDispatch.ok) {
|
|
1919
1973
|
const comment = labelDispatchFailureComment(decision.issue, labelDispatch);
|
|
1920
1974
|
this.#logger.warn?.('[factory] skipped dispatch due to invalid repo labels', {
|
|
@@ -1940,10 +1994,16 @@ export class FactoryLoop {
|
|
|
1940
1994
|
}
|
|
1941
1995
|
return { issue: decision.issue, agents: [], comments: [comment], dryRun };
|
|
1942
1996
|
}
|
|
1943
|
-
let dispatchDecision = labelDispatch.decision;
|
|
1997
|
+
let dispatchDecision = authoritativeRoutedDecision(decision, labelDispatch.decision);
|
|
1944
1998
|
// A valid label resolution clears any prior failure notice so a later
|
|
1945
1999
|
// regression posts a fresh, actionable comment instead of being deduped.
|
|
1946
2000
|
this.#labelDispatchFailures.delete(issueStateKey(dispatchDecision.issue));
|
|
2001
|
+
const escalationReason = triageEscalationReason(dispatchDecision);
|
|
2002
|
+
if (escalationReason) {
|
|
2003
|
+
const replayedResult = await this.#escalateTriage(dispatchDecision, escalationReason, dryRun);
|
|
2004
|
+
this.#recordTriageEscalation(dispatchDecision, escalationReason);
|
|
2005
|
+
return replayedResult ?? { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
2006
|
+
}
|
|
1947
2007
|
// Full task rendering is part of the durable spawn specification. It must
|
|
1948
2008
|
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
1949
2009
|
// recover a persisted minimal triage task after a crash in this gap.
|
|
@@ -2134,7 +2194,25 @@ export class FactoryLoop {
|
|
|
2134
2194
|
#wireFleetEvents() {
|
|
2135
2195
|
if (!this.#offAgentExit) {
|
|
2136
2196
|
this.#offAgentExit = this.#fleet.onAgentExit((name, reason) => {
|
|
2137
|
-
|
|
2197
|
+
// Internal broker subscriptions replay historical exits immediately.
|
|
2198
|
+
// Ignore that pre-hydration history; the roster reconcile below runs
|
|
2199
|
+
// after durable records are restored and is the authoritative signal.
|
|
2200
|
+
if (this.#startupAgentAdoptionActive)
|
|
2201
|
+
return;
|
|
2202
|
+
// Broker replay can deliver an old exit immediately when the listener
|
|
2203
|
+
// is installed, before durable agents are restored. Queue a later
|
|
2204
|
+
// roster-reconciled exit behind it instead of dropping the newer event.
|
|
2205
|
+
const previous = this.#agentExitsInFlight.get(name) ?? Promise.resolve();
|
|
2206
|
+
const handling = previous
|
|
2207
|
+
.catch(() => undefined)
|
|
2208
|
+
.then(async () => await this.#handleAgentExit(name, reason))
|
|
2209
|
+
.catch((error) => this.#error(error))
|
|
2210
|
+
.finally(() => {
|
|
2211
|
+
if (this.#agentExitsInFlight.get(name) === handling) {
|
|
2212
|
+
this.#agentExitsInFlight.delete(name);
|
|
2213
|
+
}
|
|
2214
|
+
});
|
|
2215
|
+
this.#agentExitsInFlight.set(name, handling);
|
|
2138
2216
|
});
|
|
2139
2217
|
}
|
|
2140
2218
|
if (!this.#offDeliveryFailed) {
|
|
@@ -2167,12 +2245,16 @@ export class FactoryLoop {
|
|
|
2167
2245
|
// in the durable lifecycle store, restore their full batch/spec association,
|
|
2168
2246
|
// then reconcile once so exits that happened while this process was down are
|
|
2169
2247
|
// handled instead of being dropped as unknown agents.
|
|
2170
|
-
async #adoptInFlightAgents() {
|
|
2248
|
+
async #adoptInFlightAgents(legacyRegistry) {
|
|
2171
2249
|
try {
|
|
2172
2250
|
const batch = await this.#batch();
|
|
2173
2251
|
const agents = [];
|
|
2174
2252
|
let hasNonterminalDurableLifecycle = false;
|
|
2175
|
-
|
|
2253
|
+
const durableLifecycles = await this.#state.listDispatchLifecycles(this.#workspaceId);
|
|
2254
|
+
this.#logger.info?.('[factory] durable startup adoption loaded', {
|
|
2255
|
+
lifecycles: durableLifecycles.length,
|
|
2256
|
+
});
|
|
2257
|
+
for (const [key, lifecycle] of durableLifecycles) {
|
|
2176
2258
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
2177
2259
|
continue;
|
|
2178
2260
|
hasNonterminalDurableLifecycle = true;
|
|
@@ -2212,7 +2294,7 @@ export class FactoryLoop {
|
|
|
2212
2294
|
// records existed. It preserves observation, but only new lifecycle rows
|
|
2213
2295
|
// carry enough decision/spec state to process the reconciled exit.
|
|
2214
2296
|
if (agents.length === 0 && !hasNonterminalDurableLifecycle) {
|
|
2215
|
-
const registry = await readFactoryInFlightRegistry(this.#config.loop.registryPath);
|
|
2297
|
+
const registry = legacyRegistry ?? await readFactoryInFlightRegistry(this.#config.loop.registryPath);
|
|
2216
2298
|
agents.push(...(registry?.agents ?? [])
|
|
2217
2299
|
.filter((agent) => agent.invocationId || agent.node)
|
|
2218
2300
|
.map((agent) => ({ name: agent.name, invocationId: agent.invocationId, node: agent.node })));
|
|
@@ -2220,14 +2302,73 @@ export class FactoryLoop {
|
|
|
2220
2302
|
if (agents.length > 0 && this.#fleet.hydrateTracked) {
|
|
2221
2303
|
this.#fleet.hydrateTracked(agents);
|
|
2222
2304
|
}
|
|
2305
|
+
if (this.#config.babysitter.enabled) {
|
|
2306
|
+
// Restore and reconcile exact PR ownership before asking the fleet to
|
|
2307
|
+
// report missing agents. Otherwise a stale weak-match babysitter from
|
|
2308
|
+
// the lifecycle can be resumed before its independently durable,
|
|
2309
|
+
// metadata-validated replacement session becomes authoritative.
|
|
2310
|
+
await this.#restoreBabysitterOwnership();
|
|
2311
|
+
await this.#reconcileRestoredBabysitterReceipts();
|
|
2312
|
+
}
|
|
2223
2313
|
this.#scheduleDispatchLifecycleRenewal();
|
|
2224
|
-
if (this.#fleet.hydrateTracked)
|
|
2314
|
+
if (this.#fleet.hydrateTracked) {
|
|
2315
|
+
this.#startupAgentAdoptionActive = false;
|
|
2316
|
+
this.#logger.info?.('[factory] durable startup roster reconciliation started', {
|
|
2317
|
+
agents: agents.map((agent) => agent.name),
|
|
2318
|
+
});
|
|
2225
2319
|
await this.#fleet.reconcileTrackedAgents?.();
|
|
2320
|
+
this.#logger.info?.('[factory] durable startup roster reconciliation completed', {
|
|
2321
|
+
pendingExits: [...this.#agentExitsInFlight.keys()].filter((name) => agents.some((agent) => agent.name === name)),
|
|
2322
|
+
});
|
|
2323
|
+
// Fleet callbacks are intentionally synchronous at the port boundary,
|
|
2324
|
+
// but recovery work is asynchronous (issue reads, worktree restore,
|
|
2325
|
+
// PR publication). Finish exits discovered by the startup reconcile
|
|
2326
|
+
// before the full ready-issue backfill can monopolize mount I/O.
|
|
2327
|
+
const exitNames = new Set(agents.map((agent) => agent.name));
|
|
2328
|
+
const drained = await this.#drainAgentExitsInFlight(exitNames, this.#startMode === 'live' ? this.#startupAgentExitDrainTimeoutMs : undefined);
|
|
2329
|
+
if (drained) {
|
|
2330
|
+
this.#logger.info?.('[factory] durable startup reconciled exits drained');
|
|
2331
|
+
}
|
|
2332
|
+
else {
|
|
2333
|
+
this.#increment('startupAgentExitDrainTimeouts');
|
|
2334
|
+
this.#logger.warn?.('[factory] startup agent exit reconciliation is still running; continuing ready-issue discovery', {
|
|
2335
|
+
timeoutMs: this.#startupAgentExitDrainTimeoutMs,
|
|
2336
|
+
pendingExits: [...this.#agentExitsInFlight.keys()].filter((name) => exitNames.has(name)),
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2226
2340
|
}
|
|
2227
2341
|
catch (error) {
|
|
2228
2342
|
this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
|
|
2229
2343
|
}
|
|
2230
2344
|
}
|
|
2345
|
+
async #drainAgentExitsInFlight(names, timeoutMs) {
|
|
2346
|
+
const drain = async () => {
|
|
2347
|
+
for (;;) {
|
|
2348
|
+
const pending = [...this.#agentExitsInFlight]
|
|
2349
|
+
.filter(([name]) => !names || names.has(name))
|
|
2350
|
+
.map(([, handling]) => handling);
|
|
2351
|
+
if (pending.length === 0)
|
|
2352
|
+
return;
|
|
2353
|
+
await Promise.allSettled(pending);
|
|
2354
|
+
}
|
|
2355
|
+
};
|
|
2356
|
+
if (timeoutMs === undefined) {
|
|
2357
|
+
await drain();
|
|
2358
|
+
return true;
|
|
2359
|
+
}
|
|
2360
|
+
let timer;
|
|
2361
|
+
const completed = await Promise.race([
|
|
2362
|
+
drain().then(() => true),
|
|
2363
|
+
new Promise((resolve) => {
|
|
2364
|
+
timer = setTimeout(() => resolve(false), Math.max(0, timeoutMs));
|
|
2365
|
+
timer.unref?.();
|
|
2366
|
+
}),
|
|
2367
|
+
]);
|
|
2368
|
+
if (timer)
|
|
2369
|
+
clearTimeout(timer);
|
|
2370
|
+
return completed;
|
|
2371
|
+
}
|
|
2231
2372
|
#scheduleDispatchLifecycleRenewal() {
|
|
2232
2373
|
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
2233
2374
|
return;
|
|
@@ -2373,7 +2514,9 @@ export class FactoryLoop {
|
|
|
2373
2514
|
return false;
|
|
2374
2515
|
}
|
|
2375
2516
|
const previous = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2376
|
-
const
|
|
2517
|
+
const pullRequests = mergePublishedPullRequests(previous, pullRequest);
|
|
2518
|
+
const primaryPullRequest = primaryPublishedPullRequest(previous, pullRequest, pullRequests);
|
|
2519
|
+
const lifecycle = lifecycleFromInFlightRecord(record, previous?.runId ?? randomUUID(), phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason);
|
|
2377
2520
|
for (const agent of lifecycle.agents) {
|
|
2378
2521
|
const previouslyReleasedAtMs = previous?.agents.find((candidate) => candidate.name === agent.name)?.releasedAtMs;
|
|
2379
2522
|
if (previouslyReleasedAtMs !== undefined)
|
|
@@ -2417,11 +2560,27 @@ export class FactoryLoop {
|
|
|
2417
2560
|
const timer = setTimeout(() => {
|
|
2418
2561
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2419
2562
|
const drive = this.#driveDispatchLifecycle(key)
|
|
2563
|
+
.then(() => {
|
|
2564
|
+
this.#dispatchLifecycleCapacityWaitLogged.delete(key);
|
|
2565
|
+
})
|
|
2420
2566
|
.catch((error) => {
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2567
|
+
if (error instanceof DispatchLifecycleCapacityError) {
|
|
2568
|
+
if (!this.#dispatchLifecycleCapacityWaitLogged.has(key)) {
|
|
2569
|
+
this.#dispatchLifecycleCapacityWaitLogged.add(key);
|
|
2570
|
+
this.#increment('dispatchLifecycleCapacityWaits');
|
|
2571
|
+
this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
|
|
2572
|
+
issue: record.issue.key,
|
|
2573
|
+
retryMs: DISPATCH_LIFECYCLE_RETRY_MS,
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
else {
|
|
2578
|
+
this.#dispatchLifecycleCapacityWaitLogged.delete(key);
|
|
2579
|
+
this.#logger.warn?.('[factory] durable dispatch lifecycle retry failed', {
|
|
2580
|
+
issue: record.issue.key,
|
|
2581
|
+
error: describeError(error).errorMessage,
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2425
2584
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2426
2585
|
})
|
|
2427
2586
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
@@ -2480,7 +2639,7 @@ export class FactoryLoop {
|
|
|
2480
2639
|
if (lifecycle.phase === 'queued') {
|
|
2481
2640
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
2482
2641
|
if (epoch === undefined || !await this.#state.promoteDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now())) {
|
|
2483
|
-
throw new
|
|
2642
|
+
throw new DispatchLifecycleCapacityError(`durable dispatch ${lifecycle.issue.key} is waiting for batch capacity`);
|
|
2484
2643
|
}
|
|
2485
2644
|
const promoted = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2486
2645
|
if (!promoted || promoted.phase !== 'dispatching') {
|
|
@@ -2516,6 +2675,8 @@ export class FactoryLoop {
|
|
|
2516
2675
|
})));
|
|
2517
2676
|
await this.#fleet.reconcileTrackedAgents?.();
|
|
2518
2677
|
}
|
|
2678
|
+
if (this.#config.babysitter.enabled)
|
|
2679
|
+
await this.#reconcileRestoredBabysitterReceipts(record);
|
|
2519
2680
|
return;
|
|
2520
2681
|
}
|
|
2521
2682
|
if (lifecycle.phase === 'parking') {
|
|
@@ -2531,31 +2692,48 @@ export class FactoryLoop {
|
|
|
2531
2692
|
return;
|
|
2532
2693
|
}
|
|
2533
2694
|
if (lifecycle.phase === 'publishing') {
|
|
2534
|
-
const
|
|
2535
|
-
if (
|
|
2695
|
+
const implementers = [...record.agents.values()].filter((agent) => agent.spec.role === 'implementer');
|
|
2696
|
+
if (implementers.length === 0)
|
|
2536
2697
|
throw new Error(`durable dispatch ${record.issue.key} has no implementer to publish`);
|
|
2537
|
-
const
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2698
|
+
const publishedReceipts = [];
|
|
2699
|
+
for (const implementer of implementers) {
|
|
2700
|
+
const published = await this.#publishImplementerPullRequest(record, implementer, { reconcileExisting: true });
|
|
2701
|
+
if (!published)
|
|
2702
|
+
throw new Error(`durable dispatch ${record.issue.key} did not produce a pull request for ${implementer.spec.repo}`);
|
|
2703
|
+
publishedReceipts.push(published);
|
|
2704
|
+
if (!await this.#saveDispatchLifecycle(record, 'publishing', published))
|
|
2705
|
+
return;
|
|
2706
|
+
}
|
|
2707
|
+
if (!await this.#saveDispatchLifecycle(record, 'published'))
|
|
2541
2708
|
return;
|
|
2542
2709
|
if (this.#config.babysitter.enabled) {
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2710
|
+
for (const receipt of publishedReceipts) {
|
|
2711
|
+
await this.#ensureBabysitter(record, {
|
|
2712
|
+
repo: receipt.repo,
|
|
2713
|
+
prNumber: receipt.number,
|
|
2714
|
+
url: receipt.url,
|
|
2715
|
+
headRef: receipt.headRef,
|
|
2716
|
+
authoritative: true,
|
|
2717
|
+
});
|
|
2718
|
+
}
|
|
2548
2719
|
return;
|
|
2549
2720
|
}
|
|
2550
2721
|
await this.#completeIssue(record);
|
|
2551
2722
|
return;
|
|
2552
2723
|
}
|
|
2553
2724
|
if (lifecycle.phase === 'published' && this.#config.babysitter.enabled && lifecycle.pullRequest) {
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2725
|
+
for (const receipt of lifecycle.pullRequests ?? [lifecycle.pullRequest]) {
|
|
2726
|
+
await this.#ensureBabysitter(record, {
|
|
2727
|
+
repo: receipt.repo,
|
|
2728
|
+
prNumber: receipt.number,
|
|
2729
|
+
url: receipt.url,
|
|
2730
|
+
headRef: receipt.headRef,
|
|
2731
|
+
authoritative: true,
|
|
2732
|
+
});
|
|
2733
|
+
}
|
|
2734
|
+
return;
|
|
2735
|
+
}
|
|
2736
|
+
if (lifecycle.phase === 'published' && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
2559
2737
|
return;
|
|
2560
2738
|
}
|
|
2561
2739
|
if (lifecycle.phase === 'published' || lifecycle.phase === 'writeback-applied') {
|
|
@@ -2726,7 +2904,7 @@ export class FactoryLoop {
|
|
|
2726
2904
|
// the babysitter's durable ownership/wake/critical state while that epoch
|
|
2727
2905
|
// is still valid so a later reopened issue cannot inherit a stale PR owner.
|
|
2728
2906
|
if (this.#usesDurableDispatchLifecycle() && this.#config.babysitter.enabled) {
|
|
2729
|
-
await this.#
|
|
2907
|
+
await this.#cancelBabysittersForIssue(record.issue);
|
|
2730
2908
|
}
|
|
2731
2909
|
if (!await this.#saveDispatchLifecycle(record, 'complete'))
|
|
2732
2910
|
return false;
|
|
@@ -2796,18 +2974,20 @@ export class FactoryLoop {
|
|
|
2796
2974
|
return;
|
|
2797
2975
|
}
|
|
2798
2976
|
const decision = await this.triageIssue(issue);
|
|
2799
|
-
const
|
|
2977
|
+
const routed = labelDerivedDispatchDecision(issue, decision, this.#config);
|
|
2978
|
+
const escalationDecision = routed.ok ? authoritativeRoutedDecision(decision, routed.decision) : decision;
|
|
2979
|
+
const escalationReason = triageEscalationReason(escalationDecision);
|
|
2800
2980
|
if (escalationReason) {
|
|
2801
|
-
await this.#escalateTriage(
|
|
2802
|
-
this.#recordTriageEscalation(
|
|
2981
|
+
await this.#escalateTriage(escalationDecision, escalationReason, this.#config.dryRun);
|
|
2982
|
+
this.#recordTriageEscalation(escalationDecision, escalationReason);
|
|
2803
2983
|
return;
|
|
2804
2984
|
}
|
|
2805
2985
|
if (batch.canStart()) {
|
|
2806
|
-
await this.dispatch(
|
|
2986
|
+
await this.dispatch(escalationDecision, { dryRun: this.#config.dryRun, labelsValidated: routed.ok });
|
|
2807
2987
|
}
|
|
2808
2988
|
else {
|
|
2809
|
-
if (batch.queue(
|
|
2810
|
-
this.#emit('issue-queued', { issue:
|
|
2989
|
+
if (batch.queue(escalationDecision, this.#config.dryRun)) {
|
|
2990
|
+
this.#emit('issue-queued', { issue: escalationDecision.issue });
|
|
2811
2991
|
}
|
|
2812
2992
|
}
|
|
2813
2993
|
}
|
|
@@ -2889,6 +3069,7 @@ export class FactoryLoop {
|
|
|
2889
3069
|
await this.#handleGithubIssueChange(path, { ...opts, candidates });
|
|
2890
3070
|
processed += 1;
|
|
2891
3071
|
lastProgressAtMs = this.#logTimedProgress('[factory] GitHub issue ingestion progress', startedAtMs, lastProgressAtMs, { processed, total: paths.length, path });
|
|
3072
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2892
3073
|
}
|
|
2893
3074
|
this.#logger.info?.('[factory] GitHub issue ingestion completed', {
|
|
2894
3075
|
dryRun: opts.dryRun ?? false,
|
|
@@ -2916,6 +3097,7 @@ export class FactoryLoop {
|
|
|
2916
3097
|
let scanned = 0;
|
|
2917
3098
|
let lastProgressAtMs = startedAtMs;
|
|
2918
3099
|
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading')) {
|
|
3100
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2919
3101
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
2920
3102
|
continue;
|
|
2921
3103
|
}
|
|
@@ -2936,33 +3118,54 @@ export class FactoryLoop {
|
|
|
2936
3118
|
async #githubIssuePaths() {
|
|
2937
3119
|
try {
|
|
2938
3120
|
const issuePaths = new Map();
|
|
2939
|
-
for (const
|
|
2940
|
-
const
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
3121
|
+
for (const { owner, repo } of configuredGithubRepoParts(this.#config)) {
|
|
3122
|
+
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
3123
|
+
const roots = githubIssueRepoRoots(owner, repo);
|
|
3124
|
+
// Keep the fallback roots as separate batches. Flattening a very large
|
|
3125
|
+
// provider result is synchronous work and can starve the durable loop
|
|
3126
|
+
// heartbeat before the bounded scan below gets a chance to yield.
|
|
3127
|
+
const pathBatches = indexedPaths
|
|
3128
|
+
? [indexedPaths]
|
|
3129
|
+
: await Promise.all(roots.map(async (root) => await this.#listRelayfileTree(root, 'GitHub issue ingestion')));
|
|
3130
|
+
if (indexedPaths) {
|
|
3131
|
+
this.#increment('githubIssueIndexReposUsed');
|
|
3132
|
+
}
|
|
3133
|
+
else {
|
|
3134
|
+
this.#increment('githubIssueIndexFallbacks');
|
|
3135
|
+
}
|
|
3136
|
+
for (const paths of pathBatches) {
|
|
3137
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
3138
|
+
const path = paths[index];
|
|
3139
|
+
const parts = githubIssuePathParts(path);
|
|
3140
|
+
if (parts) {
|
|
3141
|
+
const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
|
|
3142
|
+
const existing = issuePaths.get(identity);
|
|
3143
|
+
if (!existing || githubIssuePathPreference(path) < githubIssuePathPreference(existing)) {
|
|
3144
|
+
if (existing)
|
|
3145
|
+
this.#increment('githubIssueAliasPathsSuppressed');
|
|
3146
|
+
issuePaths.set(identity, path);
|
|
3147
|
+
}
|
|
3148
|
+
else {
|
|
2948
3149
|
this.#increment('githubIssueAliasPathsSuppressed');
|
|
2949
|
-
|
|
3150
|
+
}
|
|
2950
3151
|
}
|
|
2951
|
-
else {
|
|
2952
|
-
|
|
3152
|
+
else if (githubIssueDirectoryPathParts(path) !== undefined) {
|
|
3153
|
+
// listTree returns the issue directory entry alongside its
|
|
3154
|
+
// meta.json file; githubIssuePathParts() already collected the
|
|
3155
|
+
// file, so skip the directory to avoid reading the same issue
|
|
3156
|
+
// twice in one backfill pass. Directory paths are only meaningful
|
|
3157
|
+
// for live change events, not the tree scan.
|
|
3158
|
+
continue;
|
|
3159
|
+
}
|
|
3160
|
+
else if (isGithubIssueTreePath(path)) {
|
|
3161
|
+
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
3162
|
+
}
|
|
3163
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
3164
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
3165
|
+
await liveEventYield();
|
|
2953
3166
|
}
|
|
2954
3167
|
}
|
|
2955
|
-
|
|
2956
|
-
// listTree returns the issue directory entry alongside its
|
|
2957
|
-
// meta.json file; githubIssuePathParts() already collected the
|
|
2958
|
-
// file, so skip the directory to avoid reading the same issue
|
|
2959
|
-
// twice in one backfill pass. Directory paths are only meaningful
|
|
2960
|
-
// for live change events, not the tree scan.
|
|
2961
|
-
continue;
|
|
2962
|
-
}
|
|
2963
|
-
else if (isGithubIssueTreePath(path)) {
|
|
2964
|
-
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
2965
|
-
}
|
|
3168
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2966
3169
|
}
|
|
2967
3170
|
}
|
|
2968
3171
|
for (const [identity, path] of issuePaths) {
|
|
@@ -2978,6 +3181,41 @@ export class FactoryLoop {
|
|
|
2978
3181
|
return [];
|
|
2979
3182
|
}
|
|
2980
3183
|
}
|
|
3184
|
+
async #githubIssuePathsFromIndex(owner, repo) {
|
|
3185
|
+
const indexPath = `${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues/_index.json`;
|
|
3186
|
+
let parsed;
|
|
3187
|
+
try {
|
|
3188
|
+
const { content } = await this.#readRelayfileFile(indexPath, 'GitHub issue index discovery');
|
|
3189
|
+
parsed = parseJsonContent(content);
|
|
3190
|
+
}
|
|
3191
|
+
catch {
|
|
3192
|
+
return undefined;
|
|
3193
|
+
}
|
|
3194
|
+
if (!Array.isArray(parsed))
|
|
3195
|
+
return undefined;
|
|
3196
|
+
const requiredLabel = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
3197
|
+
if (!requiredLabel)
|
|
3198
|
+
return undefined;
|
|
3199
|
+
const paths = [];
|
|
3200
|
+
for (const entry of parsed) {
|
|
3201
|
+
const row = asRecord(entry);
|
|
3202
|
+
const number = row?.number;
|
|
3203
|
+
const state = typeof row?.state === 'string' ? row.state.trim().toLowerCase() : undefined;
|
|
3204
|
+
const labels = row?.labels;
|
|
3205
|
+
// Labels were added to the public GitHub issue index contract after the
|
|
3206
|
+
// first index version. Fall back for the entire repository if any row is
|
|
3207
|
+
// legacy or malformed so an eligible issue can never be filtered out.
|
|
3208
|
+
if (!Number.isSafeInteger(number) || Number(number) <= 0 || !state || !Array.isArray(labels) ||
|
|
3209
|
+
!labels.every((label) => typeof label === 'string')) {
|
|
3210
|
+
return undefined;
|
|
3211
|
+
}
|
|
3212
|
+
if (state !== 'open' || !labels.some((label) => label.trim().toLowerCase() === requiredLabel)) {
|
|
3213
|
+
continue;
|
|
3214
|
+
}
|
|
3215
|
+
paths.push(`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues/by-id/${number}.json`);
|
|
3216
|
+
}
|
|
3217
|
+
return paths;
|
|
3218
|
+
}
|
|
2981
3219
|
async #handleGithubIssueChange(path, opts = {}) {
|
|
2982
3220
|
if (this.#githubIngestionEnabled === false || !isGithubIssueFilePath(path)) {
|
|
2983
3221
|
return;
|
|
@@ -3815,6 +4053,13 @@ export class FactoryLoop {
|
|
|
3815
4053
|
}
|
|
3816
4054
|
return;
|
|
3817
4055
|
}
|
|
4056
|
+
const tracingReconciledExit = reason === 'reconciled-missing';
|
|
4057
|
+
if (tracingReconciledExit) {
|
|
4058
|
+
this.#logger.info?.('[factory] reconciled agent exit recovery started', {
|
|
4059
|
+
issue: record.issue.key,
|
|
4060
|
+
name,
|
|
4061
|
+
});
|
|
4062
|
+
}
|
|
3818
4063
|
if (!await this.#assertDispatchLifecycleOwner(record)) {
|
|
3819
4064
|
this.#logger.warn?.('[factory] ignored agent exit after durable lifecycle ownership was lost', {
|
|
3820
4065
|
issue: record.issue.key,
|
|
@@ -3822,6 +4067,8 @@ export class FactoryLoop {
|
|
|
3822
4067
|
});
|
|
3823
4068
|
return;
|
|
3824
4069
|
}
|
|
4070
|
+
if (tracingReconciledExit)
|
|
4071
|
+
this.#logger.info?.('[factory] reconciled agent exit ownership confirmed', { issue: record.issue.key, name });
|
|
3825
4072
|
// The issue-comment subscription and the fleet exit callback are separate
|
|
3826
4073
|
// event streams. Reconcile comments that are already durable in the mount
|
|
3827
4074
|
// before interpreting a clean exit as task completion, so an agent that
|
|
@@ -3830,9 +4077,13 @@ export class FactoryLoop {
|
|
|
3830
4077
|
this.#increment('githubQuestionExitsSuppressed');
|
|
3831
4078
|
return;
|
|
3832
4079
|
}
|
|
4080
|
+
if (tracingReconciledExit)
|
|
4081
|
+
this.#logger.info?.('[factory] reconciled agent exit question replay completed', { issue: record.issue.key, name });
|
|
3833
4082
|
const exiting = record.agents.get(name);
|
|
3834
4083
|
if (exiting)
|
|
3835
4084
|
await this.#reportAgent(record, exiting, 'agent.exited', { releaseReason: reason });
|
|
4085
|
+
if (tracingReconciledExit)
|
|
4086
|
+
this.#logger.info?.('[factory] reconciled agent exit telemetry completed', { issue: record.issue.key, name });
|
|
3836
4087
|
if (this.#usesDurableDispatchLifecycle()) {
|
|
3837
4088
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
3838
4089
|
if (lifecycle?.phase === 'parking') {
|
|
@@ -3843,10 +4094,10 @@ export class FactoryLoop {
|
|
|
3843
4094
|
if (isCompletionReason(reason)) {
|
|
3844
4095
|
if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
|
|
3845
4096
|
openOnly: this.#config.babysitter.enabled,
|
|
3846
|
-
})) {
|
|
4097
|
+
}, exiting)) {
|
|
3847
4098
|
if (this.#config.babysitter.enabled)
|
|
3848
4099
|
await this.#ensureBabysitterForIssue(record);
|
|
3849
|
-
else
|
|
4100
|
+
else if (await this.#allImplementersHaveCompletionPr(record))
|
|
3850
4101
|
await this.#completeIssue(record);
|
|
3851
4102
|
return;
|
|
3852
4103
|
}
|
|
@@ -3873,13 +4124,14 @@ export class FactoryLoop {
|
|
|
3873
4124
|
// itself finishing means it believes the PR is ready, so re-check and
|
|
3874
4125
|
// advance to Human Review.
|
|
3875
4126
|
if (exiting?.spec.role === 'babysitter') {
|
|
3876
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
4127
|
+
await this.#maybeAdvanceToHumanReview(record, name);
|
|
3877
4128
|
}
|
|
3878
4129
|
else if (publishedPr) {
|
|
3879
4130
|
await this.#ensureBabysitter(record, {
|
|
3880
4131
|
repo: publishedPr.repo,
|
|
3881
4132
|
prNumber: publishedPr.number,
|
|
3882
4133
|
url: publishedPr.url,
|
|
4134
|
+
authoritative: true,
|
|
3883
4135
|
});
|
|
3884
4136
|
}
|
|
3885
4137
|
else {
|
|
@@ -3887,7 +4139,8 @@ export class FactoryLoop {
|
|
|
3887
4139
|
}
|
|
3888
4140
|
return;
|
|
3889
4141
|
}
|
|
3890
|
-
await this.#
|
|
4142
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4143
|
+
await this.#completeIssue(record);
|
|
3891
4144
|
return;
|
|
3892
4145
|
}
|
|
3893
4146
|
const tracked = exiting;
|
|
@@ -3895,14 +4148,59 @@ export class FactoryLoop {
|
|
|
3895
4148
|
return;
|
|
3896
4149
|
}
|
|
3897
4150
|
try {
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
4151
|
+
const hasCompletionPr = tracked.spec.role === 'implementer'
|
|
4152
|
+
? await this.#issueHasCompletionPr(record, {
|
|
4153
|
+
openOnly: this.#config.babysitter.enabled,
|
|
4154
|
+
}, tracked)
|
|
4155
|
+
: false;
|
|
4156
|
+
if (tracingReconciledExit) {
|
|
4157
|
+
this.#logger.info?.('[factory] reconciled agent exit completion PR lookup completed', {
|
|
4158
|
+
issue: record.issue.key,
|
|
4159
|
+
name,
|
|
4160
|
+
hasCompletionPr,
|
|
4161
|
+
});
|
|
4162
|
+
}
|
|
4163
|
+
if (hasCompletionPr) {
|
|
4164
|
+
let reconciledPr;
|
|
4165
|
+
if (tracingReconciledExit && tracked.spec.role === 'implementer') {
|
|
4166
|
+
// A restart can discover that the implementer is gone after its PR
|
|
4167
|
+
// reached GitHub but before the durable receipt was saved. Persist the
|
|
4168
|
+
// exact existing-branch receipt before handing off to a babysitter;
|
|
4169
|
+
// otherwise the lifecycle remains `running` forever and consumes a
|
|
4170
|
+
// batch slot even though useful implementation work has finished.
|
|
4171
|
+
if (!await this.#saveDispatchLifecycle(record, 'publishing'))
|
|
4172
|
+
return;
|
|
4173
|
+
try {
|
|
4174
|
+
reconciledPr = await this.#publishImplementerPullRequest(record, tracked, {
|
|
4175
|
+
reconcileExisting: true,
|
|
4176
|
+
});
|
|
4177
|
+
if (reconciledPr && !await this.#saveDispatchLifecycle(record, 'published', reconciledPr))
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
catch (error) {
|
|
4181
|
+
this.#increment('githubPullRequestPublishFailures');
|
|
4182
|
+
this.#error(error, record.issue);
|
|
4183
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
4184
|
+
return;
|
|
4185
|
+
}
|
|
4186
|
+
}
|
|
3901
4187
|
if (this.#config.babysitter.enabled) {
|
|
3902
|
-
|
|
4188
|
+
if (reconciledPr) {
|
|
4189
|
+
await this.#ensureBabysitter(record, {
|
|
4190
|
+
repo: reconciledPr.repo,
|
|
4191
|
+
prNumber: reconciledPr.number,
|
|
4192
|
+
url: reconciledPr.url,
|
|
4193
|
+
headRef: reconciledPr.headRef,
|
|
4194
|
+
authoritative: true,
|
|
4195
|
+
});
|
|
4196
|
+
}
|
|
4197
|
+
else {
|
|
4198
|
+
await this.#ensureBabysitterForIssue(record);
|
|
4199
|
+
}
|
|
3903
4200
|
return;
|
|
3904
4201
|
}
|
|
3905
|
-
await this.#
|
|
4202
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4203
|
+
await this.#completeIssue(record);
|
|
3906
4204
|
return;
|
|
3907
4205
|
}
|
|
3908
4206
|
// The implementer's turn ended without a PR of record. Agents reliably
|
|
@@ -3915,6 +4213,8 @@ export class FactoryLoop {
|
|
|
3915
4213
|
// ahead of base, clone gone) it returns undefined and we fall through.
|
|
3916
4214
|
if (tracked.spec.role === 'implementer') {
|
|
3917
4215
|
await this.#saveDispatchLifecycle(record, 'publishing');
|
|
4216
|
+
if (tracingReconciledExit)
|
|
4217
|
+
this.#logger.info?.('[factory] reconciled agent exit PR publication started', { issue: record.issue.key, name });
|
|
3918
4218
|
const publishedPr = await this.#tryPublishImplementerPr(record, tracked);
|
|
3919
4219
|
if (publishedPr) {
|
|
3920
4220
|
await this.#saveDispatchLifecycle(record, 'published', publishedPr);
|
|
@@ -3923,10 +4223,12 @@ export class FactoryLoop {
|
|
|
3923
4223
|
repo: publishedPr.repo,
|
|
3924
4224
|
prNumber: publishedPr.number,
|
|
3925
4225
|
url: publishedPr.url,
|
|
4226
|
+
authoritative: true,
|
|
3926
4227
|
});
|
|
3927
4228
|
}
|
|
3928
4229
|
else {
|
|
3929
|
-
await this.#
|
|
4230
|
+
if (await this.#allImplementersHaveCompletionPr(record))
|
|
4231
|
+
await this.#completeIssue(record);
|
|
3930
4232
|
}
|
|
3931
4233
|
return;
|
|
3932
4234
|
}
|
|
@@ -4045,6 +4347,10 @@ export class FactoryLoop {
|
|
|
4045
4347
|
return undefined;
|
|
4046
4348
|
}
|
|
4047
4349
|
try {
|
|
4350
|
+
// A missed exit can be reconciled after the worker checkout was pruned.
|
|
4351
|
+
// Re-create its deterministic worktree from the retained local branch so
|
|
4352
|
+
// publication can still push/read the completed commit.
|
|
4353
|
+
await this.#prepareAgentWorktree(record, implementer.spec);
|
|
4048
4354
|
const published = await this.#publishImplementerPullRequest(record, implementer);
|
|
4049
4355
|
if (published) {
|
|
4050
4356
|
this.#increment('implementerPrsPublishedOnExit');
|
|
@@ -4069,8 +4375,6 @@ export class FactoryLoop {
|
|
|
4069
4375
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
4070
4376
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
4071
4377
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
4072
|
-
if (durable?.pullRequest)
|
|
4073
|
-
return durable.pullRequest;
|
|
4074
4378
|
const cached = this.#publishedPullRequests.get(key);
|
|
4075
4379
|
if (cached)
|
|
4076
4380
|
return cached;
|
|
@@ -4095,7 +4399,11 @@ export class FactoryLoop {
|
|
|
4095
4399
|
? sourceRepoParts.owner
|
|
4096
4400
|
: undefined;
|
|
4097
4401
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
4402
|
+
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
4098
4403
|
const expectedHeadRef = implementer.spec.branch ?? remoteBranch;
|
|
4404
|
+
if (durableReceipt &&
|
|
4405
|
+
(!opts.reconcileExisting || !expectedHeadRef || durableReceipt.headRef === expectedHeadRef))
|
|
4406
|
+
return durableReceipt;
|
|
4099
4407
|
if (opts.reconcileExisting && expectedHeadRef) {
|
|
4100
4408
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
4101
4409
|
if (existing) {
|
|
@@ -4139,6 +4447,44 @@ export class FactoryLoop {
|
|
|
4139
4447
|
return result;
|
|
4140
4448
|
}
|
|
4141
4449
|
async #openPullRequestByHead(repo, expectedHeadRef) {
|
|
4450
|
+
if (this.#hasProbePrGhRunner) {
|
|
4451
|
+
try {
|
|
4452
|
+
const result = await this.#probePrGhRunner([
|
|
4453
|
+
'pr',
|
|
4454
|
+
'list',
|
|
4455
|
+
'--repo',
|
|
4456
|
+
repo,
|
|
4457
|
+
'--head',
|
|
4458
|
+
expectedHeadRef,
|
|
4459
|
+
'--state',
|
|
4460
|
+
'open',
|
|
4461
|
+
'--json',
|
|
4462
|
+
'number,url,headRefName,isDraft',
|
|
4463
|
+
'--limit',
|
|
4464
|
+
'10',
|
|
4465
|
+
]);
|
|
4466
|
+
const payload = parseJsonContent(result.stdout);
|
|
4467
|
+
if (Array.isArray(payload)) {
|
|
4468
|
+
const candidates = payload.flatMap((entry) => {
|
|
4469
|
+
const candidate = asRecord(entry);
|
|
4470
|
+
const number = numberValue(candidate?.number);
|
|
4471
|
+
const url = stringValue(candidate?.url);
|
|
4472
|
+
const headRef = stringValue(candidate?.headRefName);
|
|
4473
|
+
if (!number || !url || headRef !== expectedHeadRef || candidate?.isDraft !== false)
|
|
4474
|
+
return [];
|
|
4475
|
+
return [{ repo, number, url, headRef }];
|
|
4476
|
+
});
|
|
4477
|
+
return candidates.sort((a, b) => b.number - a.number)[0];
|
|
4478
|
+
}
|
|
4479
|
+
}
|
|
4480
|
+
catch (error) {
|
|
4481
|
+
this.#logger.warn?.('[factory] exact-head gh PR lookup failed; falling back to mounted metadata', {
|
|
4482
|
+
repo,
|
|
4483
|
+
headRef: expectedHeadRef,
|
|
4484
|
+
error: describeError(error).errorMessage,
|
|
4485
|
+
});
|
|
4486
|
+
}
|
|
4487
|
+
}
|
|
4142
4488
|
const parts = githubRepoParts(repo);
|
|
4143
4489
|
if (!parts)
|
|
4144
4490
|
return undefined;
|
|
@@ -4450,12 +4796,22 @@ export class FactoryLoop {
|
|
|
4450
4796
|
timer.unref?.();
|
|
4451
4797
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
4452
4798
|
}
|
|
4453
|
-
async #issueHasCompletionPr(record, opts = {}) {
|
|
4799
|
+
async #issueHasCompletionPr(record, opts = {}, implementer) {
|
|
4454
4800
|
try {
|
|
4455
4801
|
const issue = await this.#readIssue(record.issue.path);
|
|
4456
4802
|
if (!issue) {
|
|
4457
4803
|
return false;
|
|
4458
4804
|
}
|
|
4805
|
+
if (implementer?.spec.branch && record.decision.implementers.length > 1) {
|
|
4806
|
+
const sourceOwner = record.issue.path
|
|
4807
|
+
? githubIssuePathParts(record.issue.path)?.owner
|
|
4808
|
+
: undefined;
|
|
4809
|
+
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
4810
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
4811
|
+
if (publishedPullRequests(lifecycle).some((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase()))
|
|
4812
|
+
return true;
|
|
4813
|
+
return Boolean(await this.#openPullRequestByHead(repo, implementer.spec.branch));
|
|
4814
|
+
}
|
|
4459
4815
|
// Only a NON-DRAFT (ready) PR counts as completion. A draft PR means the
|
|
4460
4816
|
// work isn't review-ready, so an implementer exiting with only a draft PR
|
|
4461
4817
|
// must NOT mark the issue done / release agents — mirror the
|
|
@@ -4474,11 +4830,30 @@ export class FactoryLoop {
|
|
|
4474
4830
|
return false;
|
|
4475
4831
|
}
|
|
4476
4832
|
}
|
|
4833
|
+
async #allImplementersHaveCompletionPr(record, opts = {}) {
|
|
4834
|
+
const implementers = [...record.agents.values()].filter((agent) => agent.spec.role === 'implementer');
|
|
4835
|
+
if (implementers.length === 0)
|
|
4836
|
+
return false;
|
|
4837
|
+
if (implementers.length === 1)
|
|
4838
|
+
return true;
|
|
4839
|
+
const completed = await Promise.all(implementers.map(async (implementer) => this.#issueHasCompletionPr(record, opts, implementer)));
|
|
4840
|
+
return completed.every(Boolean);
|
|
4841
|
+
}
|
|
4477
4842
|
async #resumeTrackedAgent(record, name, tracked) {
|
|
4478
4843
|
if (!tracked.sessionRef) {
|
|
4479
4844
|
return;
|
|
4480
4845
|
}
|
|
4846
|
+
this.#logger.debug?.('[factory] tracked agent resume preparation started', {
|
|
4847
|
+
issue: record.issue.key,
|
|
4848
|
+
name,
|
|
4849
|
+
role: tracked.spec.role,
|
|
4850
|
+
});
|
|
4481
4851
|
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
4852
|
+
this.#logger.debug?.('[factory] tracked agent resume spawn started', {
|
|
4853
|
+
issue: record.issue.key,
|
|
4854
|
+
name,
|
|
4855
|
+
role: tracked.spec.role,
|
|
4856
|
+
});
|
|
4482
4857
|
const result = await this.#fleet.resume({
|
|
4483
4858
|
name,
|
|
4484
4859
|
sessionRef: tracked.sessionRef,
|
|
@@ -4487,6 +4862,12 @@ export class FactoryLoop {
|
|
|
4487
4862
|
repo: tracked.spec.repo,
|
|
4488
4863
|
clonePath: tracked.spec.clonePath,
|
|
4489
4864
|
});
|
|
4865
|
+
this.#logger.debug?.('[factory] tracked agent resume spawn completed', {
|
|
4866
|
+
issue: record.issue.key,
|
|
4867
|
+
name,
|
|
4868
|
+
resumedName: result.name,
|
|
4869
|
+
role: tracked.spec.role,
|
|
4870
|
+
});
|
|
4490
4871
|
tracked.result = {
|
|
4491
4872
|
...result,
|
|
4492
4873
|
node: result.node ?? tracked.result?.node,
|
|
@@ -4495,34 +4876,156 @@ export class FactoryLoop {
|
|
|
4495
4876
|
tracked.sessionRef = result.sessionRef ?? tracked.sessionRef;
|
|
4496
4877
|
record.agents.delete(name);
|
|
4497
4878
|
record.agents.set(result.name, tracked);
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4879
|
+
await this.#retargetBabysitterAgent(record, name, tracked);
|
|
4880
|
+
this.#logger.debug?.('[factory] tracked agent resume ownership retargeted', {
|
|
4881
|
+
issue: record.issue.key,
|
|
4882
|
+
name,
|
|
4883
|
+
resumedName: result.name,
|
|
4884
|
+
role: tracked.spec.role,
|
|
4885
|
+
});
|
|
4886
|
+
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4887
|
+
}
|
|
4888
|
+
async #retargetBabysitterAgent(record, previousName, tracked) {
|
|
4889
|
+
if (tracked.spec.role !== 'babysitter')
|
|
4890
|
+
return;
|
|
4891
|
+
const currentName = tracked.result?.name ?? tracked.spec.name;
|
|
4892
|
+
this.#babysitterCriticalAgents.delete(previousName);
|
|
4893
|
+
const ownership = [...this.#babysitterPr.entries()]
|
|
4894
|
+
.find(([, candidate]) => candidate.agentName === previousName);
|
|
4895
|
+
const [ownershipKey, ref] = ownership ?? [];
|
|
4896
|
+
if (!ref)
|
|
4897
|
+
return;
|
|
4898
|
+
ref.agentName = currentName;
|
|
4899
|
+
// Resuming a session commonly preserves its Relaycast name. Iterate a
|
|
4900
|
+
// snapshot because deleting and re-inserting that same key while walking
|
|
4901
|
+
// the live Map would append it to the iterator again indefinitely.
|
|
4902
|
+
for (const [wakeKey, state] of [...this.#babysitterWakeStates]) {
|
|
4903
|
+
if (state.agentName !== previousName)
|
|
4904
|
+
continue;
|
|
4905
|
+
if (state.timer)
|
|
4906
|
+
clearTimeout(state.timer);
|
|
4907
|
+
this.#babysitterWakeStates.delete(wakeKey);
|
|
4908
|
+
state.timer = undefined;
|
|
4909
|
+
state.agentName = currentName;
|
|
4910
|
+
state.tracked = tracked;
|
|
4911
|
+
if (state.deferredSubmitTargets) {
|
|
4912
|
+
state.deferredSubmitTargets = undefined;
|
|
4913
|
+
state.deliveringKinds = undefined;
|
|
4914
|
+
state.kinds.add('pull-request-state');
|
|
4915
|
+
await this.#recordPendingBabysitterWake(state);
|
|
4916
|
+
}
|
|
4917
|
+
this.#babysitterWakeStates.set(babysitterWakeKey(record.issue, ref), state);
|
|
4918
|
+
if (state.kinds.size > 0)
|
|
4919
|
+
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
4920
|
+
}
|
|
4921
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked, ownershipKey);
|
|
4922
|
+
}
|
|
4923
|
+
async #recoverUnreachableBabysitter(state) {
|
|
4924
|
+
const batch = await this.#batch();
|
|
4925
|
+
const record = batch.getIssueByAgent(state.agentName);
|
|
4926
|
+
const tracked = record?.agents.get(state.agentName);
|
|
4927
|
+
if (!record || !tracked || tracked.spec.role !== 'babysitter')
|
|
4928
|
+
return false;
|
|
4929
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
4930
|
+
return false;
|
|
4931
|
+
const previousName = state.agentName;
|
|
4932
|
+
this.#fleet.markAgentTerminal?.(previousName, 'babysitter-unreachable');
|
|
4933
|
+
try {
|
|
4934
|
+
await this.#fleet.release(previousName, 'babysitter-unreachable');
|
|
4935
|
+
this.#logger.debug?.('[factory] unreachable babysitter release completed', {
|
|
4936
|
+
issue: record.issue.key,
|
|
4937
|
+
babysitter: previousName,
|
|
4938
|
+
});
|
|
4939
|
+
}
|
|
4940
|
+
catch (error) {
|
|
4941
|
+
// An unresolvable Relaycast identity is frequently already absent from
|
|
4942
|
+
// placement too. Release is best-effort; the fresh spawn below is the
|
|
4943
|
+
// recovery operation that matters.
|
|
4944
|
+
this.#increment('babysitterUnreachableReleaseFailures');
|
|
4945
|
+
this.#logger.warn?.('[factory] unreachable babysitter release failed; attempting session recovery', {
|
|
4946
|
+
issue: record.issue.key,
|
|
4947
|
+
babysitter: previousName,
|
|
4948
|
+
error: describeError(error).errorMessage,
|
|
4949
|
+
});
|
|
4950
|
+
}
|
|
4951
|
+
try {
|
|
4952
|
+
const configuredCapability = this.#config.agentCapabilities.babysitter;
|
|
4953
|
+
const capabilityChanged = tracked.spec.capability !== configuredCapability;
|
|
4954
|
+
if (tracked.sessionRef && !capabilityChanged) {
|
|
4955
|
+
await this.#resumeTrackedAgent(record, previousName, tracked);
|
|
4956
|
+
}
|
|
4957
|
+
else {
|
|
4958
|
+
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:unreachable:${this.#clock.now()}`;
|
|
4959
|
+
const { sessionRef: _staleSessionRef, ...persistedSpec } = tracked.spec;
|
|
4960
|
+
const replacementSpec = capabilityChanged
|
|
4961
|
+
? {
|
|
4962
|
+
...persistedSpec,
|
|
4963
|
+
capability: configuredCapability,
|
|
4964
|
+
model: this.#config.models.babysitter,
|
|
4517
4965
|
}
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4966
|
+
: persistedSpec;
|
|
4967
|
+
await this.#prepareAgentWorktree(record, replacementSpec);
|
|
4968
|
+
const result = await this.#fleet.spawn({
|
|
4969
|
+
name: replacementSpec.name,
|
|
4970
|
+
capability: replacementSpec.capability,
|
|
4971
|
+
node: tracked.result?.node ?? replacementSpec.node ?? 'self',
|
|
4972
|
+
repo: replacementSpec.repo,
|
|
4973
|
+
task: replacementSpec.task,
|
|
4974
|
+
model: replacementSpec.model,
|
|
4975
|
+
cwd: replacementSpec.clonePath,
|
|
4976
|
+
invocationId,
|
|
4977
|
+
restartPolicy: defaultRestartPolicy(replacementSpec),
|
|
4978
|
+
channel: replacementSpec.channel,
|
|
4979
|
+
});
|
|
4980
|
+
batch.recordSpawn(record, replacementSpec, invocationId, result);
|
|
4981
|
+
const restarted = record.agents.get(result.name);
|
|
4982
|
+
if (!restarted)
|
|
4983
|
+
throw new Error(`Recovered babysitter ${result.name} was not tracked`);
|
|
4984
|
+
await this.#retargetBabysitterAgent(record, previousName, restarted);
|
|
4985
|
+
await this.#reportAgent(record, restarted, 'agent.resumed');
|
|
4986
|
+
if (capabilityChanged) {
|
|
4987
|
+
this.#increment('babysitterCapabilityMigrations');
|
|
4988
|
+
this.#logger.info?.('[factory] cold-started unreachable babysitter on configured capability', {
|
|
4989
|
+
issue: record.issue.key,
|
|
4990
|
+
babysitter: result.name,
|
|
4991
|
+
previousCapability: tracked.spec.capability,
|
|
4992
|
+
capability: configuredCapability,
|
|
4993
|
+
});
|
|
4521
4994
|
}
|
|
4522
|
-
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
4523
4995
|
}
|
|
4996
|
+
this.#logger.debug?.('[factory] unreachable babysitter replacement started', {
|
|
4997
|
+
issue: record.issue.key,
|
|
4998
|
+
previousBabysitter: previousName,
|
|
4999
|
+
babysitter: state.agentName,
|
|
5000
|
+
});
|
|
5001
|
+
await this.#writeInFlightRegistry();
|
|
5002
|
+
this.#logger.debug?.('[factory] unreachable babysitter registry refreshed', {
|
|
5003
|
+
issue: record.issue.key,
|
|
5004
|
+
babysitter: state.agentName,
|
|
5005
|
+
});
|
|
5006
|
+
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
5007
|
+
return false;
|
|
5008
|
+
this.#increment('babysitterEventWakeUnreachableRecoveries');
|
|
5009
|
+
this.#logger.info?.('[factory] restarted unreachable babysitter session', {
|
|
5010
|
+
issue: record.issue.key,
|
|
5011
|
+
repo: state.repo,
|
|
5012
|
+
prNumber: state.prNumber,
|
|
5013
|
+
previousBabysitter: previousName,
|
|
5014
|
+
babysitter: state.agentName,
|
|
5015
|
+
});
|
|
5016
|
+
return true;
|
|
5017
|
+
}
|
|
5018
|
+
catch (error) {
|
|
5019
|
+
this.#increment('babysitterEventWakeUnreachableRecoveryFailures');
|
|
5020
|
+
this.#logger.warn?.('[factory] failed to restart unreachable babysitter session', {
|
|
5021
|
+
issue: record.issue.key,
|
|
5022
|
+
repo: state.repo,
|
|
5023
|
+
prNumber: state.prNumber,
|
|
5024
|
+
babysitter: previousName,
|
|
5025
|
+
error: describeError(error).errorMessage,
|
|
5026
|
+
});
|
|
5027
|
+
return false;
|
|
4524
5028
|
}
|
|
4525
|
-
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4526
5029
|
}
|
|
4527
5030
|
async #handleDeliveryFailed(info) {
|
|
4528
5031
|
const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
|
|
@@ -4586,7 +5089,7 @@ export class FactoryLoop {
|
|
|
4586
5089
|
return;
|
|
4587
5090
|
}
|
|
4588
5091
|
this.#increment('agentLifecycleReadySignals');
|
|
4589
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
5092
|
+
await this.#maybeAdvanceToHumanReview(record, signal.name);
|
|
4590
5093
|
return;
|
|
4591
5094
|
}
|
|
4592
5095
|
if (tracked.spec.role === 'babysitter') {
|
|
@@ -4686,7 +5189,7 @@ export class FactoryLoop {
|
|
|
4686
5189
|
this.#increment('prReadySignalsIgnoredIssueMismatch');
|
|
4687
5190
|
return;
|
|
4688
5191
|
}
|
|
4689
|
-
await this.#maybeAdvanceToHumanReview(record);
|
|
5192
|
+
await this.#maybeAdvanceToHumanReview(record, ready.agentName);
|
|
4690
5193
|
return;
|
|
4691
5194
|
}
|
|
4692
5195
|
}
|
|
@@ -5504,13 +6007,25 @@ export class FactoryLoop {
|
|
|
5504
6007
|
}
|
|
5505
6008
|
}
|
|
5506
6009
|
async #replayGithubIssueComments(key) {
|
|
6010
|
+
const active = this.#githubIssueCommentReplays.get(key);
|
|
6011
|
+
if (active)
|
|
6012
|
+
return await active;
|
|
6013
|
+
const replay = this.#runGithubIssueCommentReplay(key).finally(() => {
|
|
6014
|
+
if (this.#githubIssueCommentReplays.get(key) === replay) {
|
|
6015
|
+
this.#githubIssueCommentReplays.delete(key);
|
|
6016
|
+
}
|
|
6017
|
+
});
|
|
6018
|
+
this.#githubIssueCommentReplays.set(key, replay);
|
|
6019
|
+
return await replay;
|
|
6020
|
+
}
|
|
6021
|
+
async #runGithubIssueCommentReplay(key) {
|
|
5507
6022
|
const watch = this.#githubIssueCommentWatchStates.get(key);
|
|
5508
6023
|
if (!watch)
|
|
5509
6024
|
return;
|
|
5510
6025
|
const comments = [];
|
|
5511
6026
|
const sinceCommentId = githubCommentNumericId(watch.sinceCommentId ?? watch.lastSeenCommentId);
|
|
5512
6027
|
const processedCommentIds = new Set(watch.processedCommentIds ?? []);
|
|
5513
|
-
for (const path of await this.#githubIssueCommentPaths(watch.source)) {
|
|
6028
|
+
for (const path of await this.#githubIssueCommentPaths(watch.source, watch.issue.path)) {
|
|
5514
6029
|
const parts = githubIssueCommentPathParts(path);
|
|
5515
6030
|
const id = parts ? githubCommentNumericId(parts.commentId) : undefined;
|
|
5516
6031
|
if (id !== undefined && id > sinceCommentId && !processedCommentIds.has(String(id))) {
|
|
@@ -5537,14 +6052,25 @@ export class FactoryLoop {
|
|
|
5537
6052
|
}
|
|
5538
6053
|
}
|
|
5539
6054
|
}
|
|
5540
|
-
async #githubIssueCommentPaths(source) {
|
|
6055
|
+
async #githubIssueCommentPaths(source, issuePath) {
|
|
5541
6056
|
const paths = new Set();
|
|
5542
6057
|
const owner = encodeURIComponent(source.owner);
|
|
5543
6058
|
const repo = encodeURIComponent(source.repo);
|
|
5544
|
-
|
|
5545
|
-
|
|
5546
|
-
|
|
5547
|
-
|
|
6059
|
+
const issueParts = issuePath ? githubIssuePathParts(issuePath) : undefined;
|
|
6060
|
+
const canonicalIssueRoot = issuePath
|
|
6061
|
+
&& issueParts?.owner.toLowerCase() === source.owner.toLowerCase()
|
|
6062
|
+
&& issueParts.repo.toLowerCase() === source.repo.toLowerCase()
|
|
6063
|
+
&& issueParts.number === source.number
|
|
6064
|
+
&& /\/(?:meta|metadata)\.json$/u.test(issuePath)
|
|
6065
|
+
? dirname(issuePath)
|
|
6066
|
+
: undefined;
|
|
6067
|
+
const prefixes = canonicalIssueRoot
|
|
6068
|
+
? [canonicalIssueRoot]
|
|
6069
|
+
: [
|
|
6070
|
+
`${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues`,
|
|
6071
|
+
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
6072
|
+
];
|
|
6073
|
+
for (const prefix of prefixes) {
|
|
5548
6074
|
try {
|
|
5549
6075
|
for (const path of await this.#mount.listTree(prefix)) {
|
|
5550
6076
|
const parts = githubIssueCommentPathParts(path);
|
|
@@ -5941,7 +6467,8 @@ export class FactoryLoop {
|
|
|
5941
6467
|
async #restoreBabysitterOwnership() {
|
|
5942
6468
|
const batch = await this.#batch();
|
|
5943
6469
|
for (const [persistedKey, session] of await this.#state.listBabysitterSessions(this.#workspaceId)) {
|
|
5944
|
-
|
|
6470
|
+
const ownershipKey = babysitterOwnershipKey(session.issue, session);
|
|
6471
|
+
if ((persistedKey !== issueKey(session.issue) && persistedKey !== ownershipKey) ||
|
|
5945
6472
|
!validGithubRepo(session.repo) ||
|
|
5946
6473
|
!validPrNumber(session.prNumber) ||
|
|
5947
6474
|
!session.agentName) {
|
|
@@ -5970,20 +6497,30 @@ export class FactoryLoop {
|
|
|
5970
6497
|
continue;
|
|
5971
6498
|
}
|
|
5972
6499
|
const record = batch.getIssue(session.issue);
|
|
5973
|
-
const
|
|
5974
|
-
|
|
6500
|
+
const trackedEntry = record?.agents.has(session.agentName)
|
|
6501
|
+
? [session.agentName, record.agents.get(session.agentName)]
|
|
6502
|
+
: [...(record?.agents.entries() ?? [])].find(([, agent]) => agent.spec.role === 'babysitter' &&
|
|
6503
|
+
githubPrIdentity(agent.spec.ownedPullRequest?.repo ?? '', agent.spec.ownedPullRequest?.number ?? 0) ===
|
|
6504
|
+
githubPrIdentity(session.repo, session.prNumber));
|
|
6505
|
+
const tracked = trackedEntry?.[1]
|
|
5975
6506
|
?? durableBabysitterTrackedAgent(session, this.#config.agentCapabilities.babysitter);
|
|
6507
|
+
if (record && !trackedEntry)
|
|
6508
|
+
record.agents.set(session.agentName, tracked);
|
|
5976
6509
|
const ref = {
|
|
5977
6510
|
repo: session.repo,
|
|
5978
6511
|
prNumber: session.prNumber,
|
|
5979
6512
|
path: session.path,
|
|
5980
6513
|
agentName: session.agentName,
|
|
5981
6514
|
};
|
|
5982
|
-
this.#babysitterPr.set(
|
|
5983
|
-
this.#babysitterIssueRefs.set(
|
|
5984
|
-
this.#babysitterSpawned.add(
|
|
6515
|
+
this.#babysitterPr.set(ownershipKey, ref);
|
|
6516
|
+
this.#babysitterIssueRefs.set(ownershipKey, { ...session.issue });
|
|
6517
|
+
this.#babysitterSpawned.add(ownershipKey);
|
|
5985
6518
|
if (session.critical)
|
|
5986
6519
|
this.#babysitterCriticalAgents.add(session.agentName);
|
|
6520
|
+
if (persistedKey !== ownershipKey) {
|
|
6521
|
+
await this.#state.setBabysitterSession(this.#workspaceId, ownershipKey, session);
|
|
6522
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, persistedKey);
|
|
6523
|
+
}
|
|
5987
6524
|
this.#increment('babysitterOwnershipRestored');
|
|
5988
6525
|
const pendingKinds = session.pendingKinds.filter(isBabysitterWakeKind);
|
|
5989
6526
|
if (pendingKinds.length > 0) {
|
|
@@ -5992,6 +6529,74 @@ export class FactoryLoop {
|
|
|
5992
6529
|
}
|
|
5993
6530
|
}
|
|
5994
6531
|
}
|
|
6532
|
+
async #reconcileRestoredBabysitterReceipts(onlyRecord) {
|
|
6533
|
+
const records = onlyRecord ? [onlyRecord] : (await this.#batch()).inFlight;
|
|
6534
|
+
for (const record of records) {
|
|
6535
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
6536
|
+
continue;
|
|
6537
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
6538
|
+
// Babysitter sessions are independently durable and restore only after
|
|
6539
|
+
// their mounted PR metadata passes the open/draft/issue-identity guard.
|
|
6540
|
+
// Fold those validated receipts back into the lifecycle on takeover.
|
|
6541
|
+
// This closes the crash gap where the babysitter session persisted but
|
|
6542
|
+
// the lifecycle PR receipt did not, and lets exact ownership retire any
|
|
6543
|
+
// earlier weak-match babysitter for the same repository. The lifecycle's
|
|
6544
|
+
// own exact receipts are authoritative independently of the session
|
|
6545
|
+
// index: session restoration can legitimately be delayed or skipped
|
|
6546
|
+
// while mounted PR metadata converges, but that must never preserve a
|
|
6547
|
+
// superseded weak-match babysitter already disproved by publication.
|
|
6548
|
+
const authoritative = new Map();
|
|
6549
|
+
for (const receipt of lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : [])) {
|
|
6550
|
+
if (!receipt.repo || !validPrNumber(receipt.number) || !receipt.url || !receipt.headRef)
|
|
6551
|
+
continue;
|
|
6552
|
+
const identity = githubPrIdentity(receipt.repo, receipt.number);
|
|
6553
|
+
if (identity)
|
|
6554
|
+
authoritative.set(identity, { ...receipt });
|
|
6555
|
+
}
|
|
6556
|
+
const restored = [...this.#babysitterPr.entries()]
|
|
6557
|
+
.filter(([ownershipKey, ref]) => ref.agentName && issueKey(this.#babysitterIssueRefs.get(ownershipKey) ?? record.issue) === issueKey(record.issue))
|
|
6558
|
+
.map(([, ref]) => ref);
|
|
6559
|
+
for (const receipt of restored) {
|
|
6560
|
+
if (!receipt.repo || !validPrNumber(receipt.prNumber))
|
|
6561
|
+
continue;
|
|
6562
|
+
const snapshot = await this.#readPrSnapshot(receipt);
|
|
6563
|
+
const headRef = snapshot?.headRef ?? record.decision.implementers
|
|
6564
|
+
.find((implementer) => implementer.repo.toLowerCase() === receipt.repo.toLowerCase())?.branch;
|
|
6565
|
+
if (!headRef)
|
|
6566
|
+
continue;
|
|
6567
|
+
const identity = githubPrIdentity(receipt.repo, receipt.prNumber);
|
|
6568
|
+
if (!identity)
|
|
6569
|
+
continue;
|
|
6570
|
+
authoritative.set(identity, {
|
|
6571
|
+
repo: receipt.repo,
|
|
6572
|
+
number: receipt.prNumber,
|
|
6573
|
+
url: snapshot?.url ?? `https://github.com/${receipt.repo}/pull/${receipt.prNumber}`,
|
|
6574
|
+
headRef,
|
|
6575
|
+
...(receipt.path ? { path: receipt.path } : {}),
|
|
6576
|
+
});
|
|
6577
|
+
}
|
|
6578
|
+
if (authoritative.size > 0) {
|
|
6579
|
+
this.#logger.debug?.('[factory] reconciling authoritative babysitter receipts', {
|
|
6580
|
+
issue: record.issue.key,
|
|
6581
|
+
lifecycleReceipts: lifecycle?.pullRequests?.map((receipt) => receipt.number) ?? [],
|
|
6582
|
+
restoredReceipts: restored.map((receipt) => receipt.prNumber),
|
|
6583
|
+
authoritativeReceipts: [...authoritative.values()].map((receipt) => receipt.number),
|
|
6584
|
+
});
|
|
6585
|
+
await this.#retireBabysittersOutsideCurrentRoutes(record);
|
|
6586
|
+
}
|
|
6587
|
+
for (const published of authoritative.values()) {
|
|
6588
|
+
if (!await this.#saveDispatchLifecycle(record, 'running', published))
|
|
6589
|
+
return;
|
|
6590
|
+
await this.#ensureBabysitter(record, {
|
|
6591
|
+
repo: published.repo,
|
|
6592
|
+
prNumber: published.number,
|
|
6593
|
+
url: published.url,
|
|
6594
|
+
path: published.path,
|
|
6595
|
+
authoritative: true,
|
|
6596
|
+
});
|
|
6597
|
+
}
|
|
6598
|
+
}
|
|
6599
|
+
}
|
|
5995
6600
|
async #drainBabysitterWakesForStop() {
|
|
5996
6601
|
for (const state of this.#babysitterWakeStates.values()) {
|
|
5997
6602
|
state.cancelled = true;
|
|
@@ -6006,12 +6611,13 @@ export class FactoryLoop {
|
|
|
6006
6611
|
}
|
|
6007
6612
|
this.#babysitterWakeStates.clear();
|
|
6008
6613
|
}
|
|
6009
|
-
async #cancelBabysitterWake(
|
|
6010
|
-
const issue = this.#babysitterIssueRefs.get(
|
|
6614
|
+
async #cancelBabysitterWake(ownershipKey) {
|
|
6615
|
+
const issue = this.#babysitterIssueRefs.get(ownershipKey);
|
|
6616
|
+
const ref = this.#babysitterPr.get(ownershipKey);
|
|
6011
6617
|
const mayClearDurable = !this.#usesDurableDispatchLifecycle()
|
|
6012
6618
|
|| Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue));
|
|
6013
6619
|
for (const [key, state] of this.#babysitterWakeStates) {
|
|
6014
|
-
if (
|
|
6620
|
+
if (!ref || babysitterOwnershipKey(state.issue, state) !== ownershipKey)
|
|
6015
6621
|
continue;
|
|
6016
6622
|
state.cancelled = true;
|
|
6017
6623
|
delete state.tracked.spec.pendingPullRequestWake;
|
|
@@ -6020,11 +6626,19 @@ export class FactoryLoop {
|
|
|
6020
6626
|
this.#babysitterWakeStates.delete(key);
|
|
6021
6627
|
this.#babysitterCriticalAgents.delete(state.agentName);
|
|
6022
6628
|
}
|
|
6023
|
-
this.#babysitterPr.delete(
|
|
6024
|
-
this.#babysitterIssueRefs.delete(
|
|
6025
|
-
this.#babysitterSpawned.delete(
|
|
6629
|
+
this.#babysitterPr.delete(ownershipKey);
|
|
6630
|
+
this.#babysitterIssueRefs.delete(ownershipKey);
|
|
6631
|
+
this.#babysitterSpawned.delete(ownershipKey);
|
|
6632
|
+
this.#babysitterReady.delete(ownershipKey);
|
|
6026
6633
|
if (mayClearDurable)
|
|
6027
|
-
await this.#state.clearBabysitterSession(this.#workspaceId,
|
|
6634
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, ownershipKey);
|
|
6635
|
+
}
|
|
6636
|
+
async #cancelBabysittersForIssue(issue) {
|
|
6637
|
+
const wanted = issueKey(issue);
|
|
6638
|
+
const keys = [...this.#babysitterIssueRefs.entries()]
|
|
6639
|
+
.filter(([, candidate]) => issueKey(candidate) === wanted)
|
|
6640
|
+
.map(([key]) => key);
|
|
6641
|
+
await Promise.all(keys.map(async (key) => this.#cancelBabysitterWake(key)));
|
|
6028
6642
|
}
|
|
6029
6643
|
async #routeBabysitterEvent(path, extraKinds = []) {
|
|
6030
6644
|
const event = githubBabysitterEventPathParts(path);
|
|
@@ -6086,7 +6700,7 @@ export class FactoryLoop {
|
|
|
6086
6700
|
const tracked = record?.agents.get(ref.agentName)
|
|
6087
6701
|
?? [...(record?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
6088
6702
|
?? durableBabysitterTrackedAgent({ issue, repo: ref.repo, prNumber: ref.prNumber, path: ref.path, agentName: ref.agentName, critical: false, pendingKinds: [] }, this.#config.agentCapabilities.babysitter);
|
|
6089
|
-
return { issue, record, ref, tracked };
|
|
6703
|
+
return { key, issue, record, ref, tracked };
|
|
6090
6704
|
}
|
|
6091
6705
|
}
|
|
6092
6706
|
return undefined;
|
|
@@ -6125,13 +6739,16 @@ export class FactoryLoop {
|
|
|
6125
6739
|
// Owner lookup and queueing straddle async mount/state reads. Revalidate
|
|
6126
6740
|
// the exact composite owner so a concurrent close/merge cancellation can
|
|
6127
6741
|
// never recreate durable state from a stale child event.
|
|
6128
|
-
const
|
|
6742
|
+
const ownershipKey = babysitterOwnershipKey(issue, ref);
|
|
6743
|
+
const current = this.#babysitterPr.get(ownershipKey);
|
|
6129
6744
|
if (!current ||
|
|
6130
6745
|
current.agentName !== ref.agentName ||
|
|
6131
6746
|
githubPrIdentity(current.repo, current.prNumber) !== githubPrIdentity(ref.repo, ref.prNumber)) {
|
|
6132
6747
|
this.#increment('babysitterEventsIgnoredStaleOwner');
|
|
6133
6748
|
return;
|
|
6134
6749
|
}
|
|
6750
|
+
// Any new event invalidates a prior readiness assertion for this exact PR.
|
|
6751
|
+
this.#babysitterReady.delete(ownershipKey);
|
|
6135
6752
|
const key = babysitterWakeKey(issue, ref);
|
|
6136
6753
|
let state = this.#babysitterWakeStates.get(key);
|
|
6137
6754
|
if (!state) {
|
|
@@ -6177,18 +6794,18 @@ export class FactoryLoop {
|
|
|
6177
6794
|
kinds: [...kinds].sort(compareBabysitterWakeKinds),
|
|
6178
6795
|
};
|
|
6179
6796
|
}
|
|
6180
|
-
await this.#persistBabysitterSession(state.issue, this.#babysitterPr.get(
|
|
6797
|
+
await this.#persistBabysitterSession(state.issue, this.#babysitterPr.get(babysitterOwnershipKey(state.issue, state)) ?? {
|
|
6181
6798
|
repo: state.repo,
|
|
6182
6799
|
prNumber: state.prNumber,
|
|
6183
6800
|
agentName: state.agentName,
|
|
6184
6801
|
}, state.tracked);
|
|
6185
6802
|
}
|
|
6186
|
-
async #persistBabysitterSession(issue, ref, tracked) {
|
|
6803
|
+
async #persistBabysitterSession(issue, ref, tracked, ownershipKey = babysitterOwnershipKey(issue, ref)) {
|
|
6187
6804
|
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
6188
6805
|
throw new Error(`Babysitter lifecycle ownership lost for ${issue.key}`);
|
|
6189
6806
|
}
|
|
6190
6807
|
const pending = tracked?.spec.pendingPullRequestWake;
|
|
6191
|
-
await this.#state.setBabysitterSession(this.#workspaceId,
|
|
6808
|
+
await this.#state.setBabysitterSession(this.#workspaceId, ownershipKey, {
|
|
6192
6809
|
issue: { ...issue },
|
|
6193
6810
|
repo: ref.repo,
|
|
6194
6811
|
prNumber: ref.prNumber,
|
|
@@ -6273,6 +6890,7 @@ export class FactoryLoop {
|
|
|
6273
6890
|
await this.#fleet.sendMessage(input);
|
|
6274
6891
|
state.unreachableSinceMs = undefined;
|
|
6275
6892
|
state.unreachableEscalated = false;
|
|
6893
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6276
6894
|
if (this.#stopping || state.cancelled) {
|
|
6277
6895
|
state.deliveringKinds = undefined;
|
|
6278
6896
|
return;
|
|
@@ -6288,6 +6906,7 @@ export class FactoryLoop {
|
|
|
6288
6906
|
// registration-lag backoff state accumulated by prior failures.
|
|
6289
6907
|
state.unreachableSinceMs = undefined;
|
|
6290
6908
|
state.unreachableEscalated = false;
|
|
6909
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6291
6910
|
targets = ack.targets.length > 0 ? [...new Set(ack.targets)] : [input.to];
|
|
6292
6911
|
}
|
|
6293
6912
|
if (this.#stopping || state.cancelled)
|
|
@@ -6343,21 +6962,23 @@ export class FactoryLoop {
|
|
|
6343
6962
|
// unreachable window so a later genuine registration lag starts fresh.
|
|
6344
6963
|
state.unreachableSinceMs = undefined;
|
|
6345
6964
|
state.unreachableEscalated = false;
|
|
6965
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6346
6966
|
}
|
|
6347
6967
|
const unreachableMs = state.unreachableSinceMs !== undefined
|
|
6348
6968
|
? this.#clock.now() - state.unreachableSinceMs
|
|
6349
6969
|
: 0;
|
|
6350
6970
|
if (registrationLag && unreachableMs >= this.#babysitterWakeUnreachableEscalateMs) {
|
|
6351
6971
|
// The agent is up but its relay identity never became resolvable. Stop
|
|
6352
|
-
// the tight 1s loop
|
|
6353
|
-
//
|
|
6354
|
-
//
|
|
6355
|
-
// failure spinning silently forever.
|
|
6972
|
+
// the tight 1s loop, reconcile once, and restart the session. A recovery
|
|
6973
|
+
// cooldown prevents a still-converging Relaycast registration from
|
|
6974
|
+
// turning that restart into another tight loop.
|
|
6356
6975
|
state.nextDelayMs = this.#babysitterWakeUnreachableRetryMs;
|
|
6357
6976
|
if (!state.unreachableEscalated) {
|
|
6358
6977
|
state.unreachableEscalated = true;
|
|
6978
|
+
await this.#fleet.reconcileTrackedAgents?.();
|
|
6979
|
+
this.#increment('babysitterEventWakeUnreachableReconciliations');
|
|
6359
6980
|
this.#increment('babysitterEventWakeUnreachableEscalations');
|
|
6360
|
-
this.#logger.warn?.('[factory] babysitter unreachable past grace window;
|
|
6981
|
+
this.#logger.warn?.('[factory] babysitter unreachable past grace window; reconciling and restarting its session', {
|
|
6361
6982
|
issue: state.issue.key,
|
|
6362
6983
|
repo: state.repo,
|
|
6363
6984
|
prNumber: state.prNumber,
|
|
@@ -6367,6 +6988,19 @@ export class FactoryLoop {
|
|
|
6367
6988
|
error: describeError(error).errorMessage,
|
|
6368
6989
|
});
|
|
6369
6990
|
}
|
|
6991
|
+
if (state.unreachableRecoveryAfterMs === undefined ||
|
|
6992
|
+
this.#clock.now() >= state.unreachableRecoveryAfterMs) {
|
|
6993
|
+
state.unreachableRecoveryAfterMs = this.#clock.now() + this.#babysitterWakeUnreachableRetryMs;
|
|
6994
|
+
if (await this.#recoverUnreachableBabysitter(state)) {
|
|
6995
|
+
// Probe the newly registered identity promptly. If Relaycast still
|
|
6996
|
+
// cannot resolve it, the recovery cooldown above prevents another
|
|
6997
|
+
// teardown loop and the next attempt uses the slow cadence.
|
|
6998
|
+
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
6999
|
+
}
|
|
7000
|
+
}
|
|
7001
|
+
else {
|
|
7002
|
+
state.nextDelayMs = Math.max(BABYSITTER_EVENT_RETRY_MS, state.unreachableRecoveryAfterMs - this.#clock.now());
|
|
7003
|
+
}
|
|
6370
7004
|
}
|
|
6371
7005
|
else {
|
|
6372
7006
|
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
@@ -6454,18 +7088,17 @@ export class FactoryLoop {
|
|
|
6454
7088
|
// only and can never redirect a live babysitter.
|
|
6455
7089
|
const owned = await this.#babysitterOwnerFor(repo, snapshot.number);
|
|
6456
7090
|
if (owned) {
|
|
6457
|
-
const ownedKey = issueKey(owned.issue);
|
|
6458
7091
|
if (prMetaShowsMerged(snapshot)) {
|
|
6459
7092
|
if (owned.record)
|
|
6460
7093
|
await this.#advanceMergedPrToDone(snapshot, owned.record);
|
|
6461
7094
|
else
|
|
6462
|
-
await this.#cancelBabysitterWake(
|
|
7095
|
+
await this.#cancelBabysitterWake(owned.key);
|
|
6463
7096
|
return;
|
|
6464
7097
|
}
|
|
6465
7098
|
if (!this.#config.babysitter.enabled)
|
|
6466
7099
|
return;
|
|
6467
7100
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
6468
|
-
await this.#cancelBabysitterWake(
|
|
7101
|
+
await this.#cancelBabysitterWake(owned.key);
|
|
6469
7102
|
return;
|
|
6470
7103
|
}
|
|
6471
7104
|
if (snapshot.draft)
|
|
@@ -6474,8 +7107,23 @@ export class FactoryLoop {
|
|
|
6474
7107
|
return;
|
|
6475
7108
|
}
|
|
6476
7109
|
const record = this.#inFlightIssueForPrSnapshot(snapshot, await this.#batch(), repo);
|
|
6477
|
-
const babysitterKey = record ?
|
|
7110
|
+
const babysitterKey = record ? babysitterOwnershipKey(record.issue, { repo, prNumber: snapshot.number }) : undefined;
|
|
6478
7111
|
const existing = babysitterKey ? this.#babysitterPr.get(babysitterKey) : undefined;
|
|
7112
|
+
const sameRepoOwner = record
|
|
7113
|
+
? [...this.#babysitterPr.entries()].find(([key, candidate]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue) &&
|
|
7114
|
+
candidate.repo.toLowerCase() === repo.toLowerCase())?.[1]
|
|
7115
|
+
: undefined;
|
|
7116
|
+
if (sameRepoOwner && githubPrIdentity(sameRepoOwner.repo, sameRepoOwner.prNumber) !== githubPrIdentity(repo, snapshot.number)) {
|
|
7117
|
+
this.#increment('babysitterEventsIgnoredOwnershipMismatch');
|
|
7118
|
+
this.#logger.warn?.('[factory] ignored PR event that conflicts with established babysitter ownership', {
|
|
7119
|
+
issue: record?.issue.key,
|
|
7120
|
+
ownedRepo: sameRepoOwner.repo,
|
|
7121
|
+
ownedPrNumber: sameRepoOwner.prNumber,
|
|
7122
|
+
eventRepo: repo,
|
|
7123
|
+
eventPrNumber: snapshot.number,
|
|
7124
|
+
});
|
|
7125
|
+
return;
|
|
7126
|
+
}
|
|
6479
7127
|
if (existing && githubPrIdentity(existing.repo, existing.prNumber) !== githubPrIdentity(repo, snapshot.number)) {
|
|
6480
7128
|
this.#increment('babysitterEventsIgnoredOwnershipMismatch');
|
|
6481
7129
|
this.#logger.warn?.('[factory] ignored PR event that conflicts with established babysitter ownership', {
|
|
@@ -6641,8 +7289,21 @@ export class FactoryLoop {
|
|
|
6641
7289
|
// probe resolver and spawn the babysitter. Triggered by an implementer exiting
|
|
6642
7290
|
// after opening its PR (an event, not a poll).
|
|
6643
7291
|
async #ensureBabysitterForIssue(record) {
|
|
6644
|
-
if (this.#
|
|
6645
|
-
|
|
7292
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
7293
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7294
|
+
const receipts = lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : []);
|
|
7295
|
+
if (receipts.length > 0) {
|
|
7296
|
+
for (const receipt of receipts) {
|
|
7297
|
+
await this.#ensureBabysitter(record, {
|
|
7298
|
+
repo: receipt.repo,
|
|
7299
|
+
prNumber: receipt.number,
|
|
7300
|
+
url: receipt.url,
|
|
7301
|
+
headRef: receipt.headRef,
|
|
7302
|
+
authoritative: true,
|
|
7303
|
+
});
|
|
7304
|
+
}
|
|
7305
|
+
return;
|
|
7306
|
+
}
|
|
6646
7307
|
}
|
|
6647
7308
|
const issue = await this.#readIssue(record.issue.path);
|
|
6648
7309
|
if (!issue) {
|
|
@@ -6655,11 +7316,14 @@ export class FactoryLoop {
|
|
|
6655
7316
|
await this.#ensureBabysitter(record, { repo: pr.repo, prNumber: pr.prNumber });
|
|
6656
7317
|
}
|
|
6657
7318
|
async #ensureBabysitter(record, prRef) {
|
|
6658
|
-
const babysitterKey =
|
|
7319
|
+
const babysitterKey = babysitterOwnershipKey(record.issue, prRef);
|
|
6659
7320
|
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
6660
7321
|
this.#increment('babysitterLifecycleOwnershipRejected');
|
|
6661
7322
|
return;
|
|
6662
7323
|
}
|
|
7324
|
+
const replacedSuperseded = prRef.authoritative
|
|
7325
|
+
? await this.#retireSupersededBabysitters(record, prRef)
|
|
7326
|
+
: false;
|
|
6663
7327
|
this.#babysitterIssueRefs.set(babysitterKey, { ...record.issue });
|
|
6664
7328
|
const existing = this.#babysitterPr.get(babysitterKey);
|
|
6665
7329
|
if (existing && githubPrIdentity(existing.repo, existing.prNumber) !== githubPrIdentity(prRef.repo, prRef.prNumber)) {
|
|
@@ -6684,7 +7348,9 @@ export class FactoryLoop {
|
|
|
6684
7348
|
settled.path = prRef.path;
|
|
6685
7349
|
return;
|
|
6686
7350
|
}
|
|
6687
|
-
const
|
|
7351
|
+
const wantedPr = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
7352
|
+
const trackedBabysitter = [...record.agents.entries()].find(([, agent]) => agent.spec.role === 'babysitter' &&
|
|
7353
|
+
githubPrIdentity(agent.spec.ownedPullRequest?.repo ?? '', agent.spec.ownedPullRequest?.number ?? 0) === wantedPr);
|
|
6688
7354
|
if (trackedBabysitter) {
|
|
6689
7355
|
const [trackedName, tracked] = trackedBabysitter;
|
|
6690
7356
|
const owned = tracked.spec.ownedPullRequest;
|
|
@@ -6718,6 +7384,12 @@ export class FactoryLoop {
|
|
|
6718
7384
|
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
6719
7385
|
?? record.decision.routes[0];
|
|
6720
7386
|
const initialSpec = babysitterSpec(issue, this.#config, route);
|
|
7387
|
+
if (replacedSuperseded || [...this.#babysitterIssueRefs.entries()].some(([key, candidate]) => key !== babysitterKey && issueKey(candidate) === issueKey(record.issue))) {
|
|
7388
|
+
initialSpec.name = agentNameForRole(issue, 'babysit', {
|
|
7389
|
+
repo: prRef.repo,
|
|
7390
|
+
discriminator: `${sanitizeAgentSlug(prRef.repo)}-${prRef.prNumber}`,
|
|
7391
|
+
});
|
|
7392
|
+
}
|
|
6721
7393
|
const sharedCheckout = [...record.agents.values()]
|
|
6722
7394
|
.map((agent) => agent.spec)
|
|
6723
7395
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath)
|
|
@@ -6814,24 +7486,129 @@ export class FactoryLoop {
|
|
|
6814
7486
|
}
|
|
6815
7487
|
}
|
|
6816
7488
|
}
|
|
7489
|
+
async #retireSupersededBabysitters(record, prRef) {
|
|
7490
|
+
const wanted = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
7491
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7492
|
+
const trackedAgents = new Map([
|
|
7493
|
+
...(lifecycle?.agents ?? [])
|
|
7494
|
+
.filter((agent) => agent.releasedAtMs === undefined)
|
|
7495
|
+
.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)]),
|
|
7496
|
+
...record.agents,
|
|
7497
|
+
]);
|
|
7498
|
+
const superseded = [...trackedAgents.entries()].filter(([, tracked]) => {
|
|
7499
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7500
|
+
return tracked.spec.role === 'babysitter' &&
|
|
7501
|
+
owned?.repo.toLowerCase() === prRef.repo.toLowerCase() &&
|
|
7502
|
+
githubPrIdentity(owned.repo, owned.number) !== wanted;
|
|
7503
|
+
});
|
|
7504
|
+
for (const [agentName, tracked] of superseded) {
|
|
7505
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7506
|
+
const failed = await this.#releaseAndTerminateAgents([[agentName, tracked]], 'superseded-pr-receipt', 'completion');
|
|
7507
|
+
if (failed.length > 0) {
|
|
7508
|
+
this.#increment('supersededBabysitterReleaseFailures');
|
|
7509
|
+
throw new Error(`Failed to release superseded babysitter ${agentName}`);
|
|
7510
|
+
}
|
|
7511
|
+
if (owned) {
|
|
7512
|
+
const staleKey = babysitterOwnershipKey(record.issue, {
|
|
7513
|
+
repo: owned.repo,
|
|
7514
|
+
prNumber: owned.number,
|
|
7515
|
+
});
|
|
7516
|
+
await this.#cancelBabysitterWake(staleKey);
|
|
7517
|
+
if (await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
7518
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, staleKey);
|
|
7519
|
+
}
|
|
7520
|
+
}
|
|
7521
|
+
record.agents.delete(agentName);
|
|
7522
|
+
this.#babysitterCriticalAgents.delete(agentName);
|
|
7523
|
+
this.#increment('supersededBabysittersReleased');
|
|
7524
|
+
this.#logger.info?.('[factory] released babysitter superseded by exact PR receipt', {
|
|
7525
|
+
issue: record.issue.key,
|
|
7526
|
+
babysitter: agentName,
|
|
7527
|
+
previousRepo: owned?.repo,
|
|
7528
|
+
previousPrNumber: owned?.number,
|
|
7529
|
+
repo: prRef.repo,
|
|
7530
|
+
prNumber: prRef.prNumber,
|
|
7531
|
+
});
|
|
7532
|
+
}
|
|
7533
|
+
if (superseded.length > 0) {
|
|
7534
|
+
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7535
|
+
if (latest && !await this.#saveDispatchLifecycle(record, latest.phase)) {
|
|
7536
|
+
throw new Error(`Failed to persist superseded babysitter cleanup for ${record.issue.key}`);
|
|
7537
|
+
}
|
|
7538
|
+
}
|
|
7539
|
+
return superseded.length > 0;
|
|
7540
|
+
}
|
|
7541
|
+
async #retireBabysittersOutsideCurrentRoutes(record) {
|
|
7542
|
+
const wantedRepos = new Set(record.decision.implementers.map((implementer) => implementer.repo.toLowerCase()));
|
|
7543
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7544
|
+
const trackedAgents = new Map([
|
|
7545
|
+
...(lifecycle?.agents ?? [])
|
|
7546
|
+
.filter((agent) => agent.releasedAtMs === undefined)
|
|
7547
|
+
.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)]),
|
|
7548
|
+
...record.agents,
|
|
7549
|
+
]);
|
|
7550
|
+
const unrouted = [...trackedAgents.entries()].filter(([, tracked]) => {
|
|
7551
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7552
|
+
return tracked.spec.role === 'babysitter' &&
|
|
7553
|
+
Boolean(owned) &&
|
|
7554
|
+
!wantedRepos.has(owned.repo.toLowerCase());
|
|
7555
|
+
});
|
|
7556
|
+
for (const [agentName, tracked] of unrouted) {
|
|
7557
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7558
|
+
const failed = await this.#releaseAndTerminateAgents([[agentName, tracked]], 'superseded-pr-route', 'completion');
|
|
7559
|
+
if (failed.length > 0) {
|
|
7560
|
+
this.#increment('supersededBabysitterReleaseFailures');
|
|
7561
|
+
throw new Error(`Failed to release unrouted babysitter ${agentName}`);
|
|
7562
|
+
}
|
|
7563
|
+
if (owned) {
|
|
7564
|
+
const staleKey = babysitterOwnershipKey(record.issue, {
|
|
7565
|
+
repo: owned.repo,
|
|
7566
|
+
prNumber: owned.number,
|
|
7567
|
+
});
|
|
7568
|
+
await this.#cancelBabysitterWake(staleKey);
|
|
7569
|
+
if (await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
7570
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, staleKey);
|
|
7571
|
+
}
|
|
7572
|
+
}
|
|
7573
|
+
record.agents.delete(agentName);
|
|
7574
|
+
this.#babysitterCriticalAgents.delete(agentName);
|
|
7575
|
+
this.#increment('supersededBabysittersReleased');
|
|
7576
|
+
this.#logger.info?.('[factory] released babysitter outside the current dispatch routes', {
|
|
7577
|
+
issue: record.issue.key,
|
|
7578
|
+
babysitter: agentName,
|
|
7579
|
+
previousRepo: owned?.repo,
|
|
7580
|
+
previousPrNumber: owned?.number,
|
|
7581
|
+
currentRepos: [...wantedRepos],
|
|
7582
|
+
});
|
|
7583
|
+
}
|
|
7584
|
+
if (unrouted.length > 0) {
|
|
7585
|
+
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7586
|
+
if (latest && !await this.#saveDispatchLifecycle(record, latest.phase)) {
|
|
7587
|
+
throw new Error(`Failed to persist unrouted babysitter cleanup for ${record.issue.key}`);
|
|
7588
|
+
}
|
|
7589
|
+
}
|
|
7590
|
+
}
|
|
6817
7591
|
// The babysitter owns the readiness verdict (CI green + conflicts resolved +
|
|
6818
7592
|
// review comments addressed) — it sees the per-event PR webhook data in its
|
|
6819
7593
|
// sandbox, exactly like AgentWorkforce/agents review. It signals readiness by
|
|
6820
7594
|
// invoking the lifecycle action with `kind: ready`. The orchestrator trusts that signal and only
|
|
6821
7595
|
// guards on the PR's OWN webhook-fed meta (still open, not a draft, not already
|
|
6822
7596
|
// merged) before flipping the issue to Human Review. No `gh` call.
|
|
6823
|
-
async #maybeAdvanceToHumanReview(record) {
|
|
7597
|
+
async #maybeAdvanceToHumanReview(record, agentName) {
|
|
6824
7598
|
if (this.#completionInFlight.has(issueKey(record.issue))) {
|
|
6825
7599
|
return;
|
|
6826
7600
|
}
|
|
6827
|
-
|
|
7601
|
+
const ownership = [...this.#babysitterPr.entries()].find(([key, ref]) => ref.agentName === agentName && issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue));
|
|
7602
|
+
if (!ownership) {
|
|
6828
7603
|
this.#increment('babysitterReadinessGuardBlocked');
|
|
6829
7604
|
this.#logger.info?.('[factory] babysitter ready signal ignored; PR ownership is no longer active', {
|
|
6830
7605
|
issue: record.issue.key,
|
|
7606
|
+
babysitter: agentName,
|
|
6831
7607
|
});
|
|
6832
7608
|
return;
|
|
6833
7609
|
}
|
|
6834
|
-
const
|
|
7610
|
+
const [ownershipKey, ref] = ownership;
|
|
7611
|
+
const snapshot = await this.#readPrSnapshot(ref);
|
|
6835
7612
|
if (!snapshot) {
|
|
6836
7613
|
this.#increment('babysitterReadinessGuardBlocked');
|
|
6837
7614
|
this.#logger.info?.('[factory] babysitter ready signal ignored; authoritative PR meta is unavailable', {
|
|
@@ -6848,6 +7625,25 @@ export class FactoryLoop {
|
|
|
6848
7625
|
});
|
|
6849
7626
|
return;
|
|
6850
7627
|
}
|
|
7628
|
+
this.#babysitterReady.add(ownershipKey);
|
|
7629
|
+
await this.#ensureBabysitterForIssue(record);
|
|
7630
|
+
const owners = [...this.#babysitterIssueRefs.entries()]
|
|
7631
|
+
.filter(([, issue]) => issueKey(issue) === issueKey(record.issue))
|
|
7632
|
+
.map(([key]) => key);
|
|
7633
|
+
const expectedPrOwners = new Set(record.decision.implementers.map((implementer) => implementer.repo)).size;
|
|
7634
|
+
if (owners.length < expectedPrOwners) {
|
|
7635
|
+
this.#increment('babysitterReadinessWaitingForPeers');
|
|
7636
|
+
this.#logger.info?.('[factory] babysitter ready; waiting for remaining repository PRs', {
|
|
7637
|
+
issue: record.issue.key,
|
|
7638
|
+
repo: ref.repo,
|
|
7639
|
+
prNumber: ref.prNumber,
|
|
7640
|
+
});
|
|
7641
|
+
return;
|
|
7642
|
+
}
|
|
7643
|
+
if (owners.length === 0 || owners.some((key) => !this.#babysitterReady.has(key))) {
|
|
7644
|
+
this.#increment('babysitterReadinessWaitingForPeers');
|
|
7645
|
+
return;
|
|
7646
|
+
}
|
|
6851
7647
|
this.#increment('babysitterReadinessReady');
|
|
6852
7648
|
this.#logger.info?.('[factory] babysitter signalled PR ready; advancing to human review', {
|
|
6853
7649
|
issue: record.issue.key,
|
|
@@ -6859,7 +7655,8 @@ export class FactoryLoop {
|
|
|
6859
7655
|
// exact path captured when the babysitter was spawned; otherwise scans the
|
|
6860
7656
|
// repo's pulls subtree for the PR number across known layout shapes.
|
|
6861
7657
|
async #readBabysatPrSnapshot(record) {
|
|
6862
|
-
const ref = this.#babysitterPr.
|
|
7658
|
+
const ref = [...this.#babysitterPr.entries()]
|
|
7659
|
+
.find(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))?.[1];
|
|
6863
7660
|
if (!ref) {
|
|
6864
7661
|
return undefined;
|
|
6865
7662
|
}
|
|
@@ -6909,7 +7706,9 @@ export class FactoryLoop {
|
|
|
6909
7706
|
if (babysatSnapshot && prMetaShowsMerged(babysatSnapshot)) {
|
|
6910
7707
|
return true;
|
|
6911
7708
|
}
|
|
6912
|
-
const pr = this.#babysitterPr.
|
|
7709
|
+
const pr = [...this.#babysitterPr.entries()]
|
|
7710
|
+
.find(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))?.[1]
|
|
7711
|
+
?? await this.#completionPrForIssue(issue);
|
|
6913
7712
|
if (!pr) {
|
|
6914
7713
|
return false;
|
|
6915
7714
|
}
|
|
@@ -7045,9 +7844,7 @@ export class FactoryLoop {
|
|
|
7045
7844
|
const stateKey = issueStateKey(record.issue);
|
|
7046
7845
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
7047
7846
|
this.#probePrResolvedCache.delete(stateKey);
|
|
7048
|
-
this.#
|
|
7049
|
-
this.#babysitterPr.delete(completionKey);
|
|
7050
|
-
await this.#cancelBabysitterWake(completionKey);
|
|
7847
|
+
await this.#cancelBabysittersForIssue(record.issue);
|
|
7051
7848
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
7052
7849
|
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
7053
7850
|
for (const publishedKey of this.#publishedPullRequests.keys()) {
|
|
@@ -8944,6 +9741,19 @@ function dispatchSpecs(decision) {
|
|
|
8944
9741
|
}
|
|
8945
9742
|
return [...decision.implementers, decision.reviewer];
|
|
8946
9743
|
}
|
|
9744
|
+
function authoritativeRoutedDecision(triaged, routed) {
|
|
9745
|
+
if (triaged.confidence !== 'low' || triaged.routes.length > 0 || routed.routes.length === 0) {
|
|
9746
|
+
return routed;
|
|
9747
|
+
}
|
|
9748
|
+
return {
|
|
9749
|
+
...routed,
|
|
9750
|
+
confidence: 'high',
|
|
9751
|
+
rationale: [
|
|
9752
|
+
routed.routes.map((route) => route.rationale).filter(Boolean).join(' '),
|
|
9753
|
+
'Repository identity was resolved authoritatively from the live issue labels or GitHub source repository.',
|
|
9754
|
+
].filter(Boolean).join(' '),
|
|
9755
|
+
};
|
|
9756
|
+
}
|
|
8947
9757
|
function labelDerivedDispatchDecision(liveIssue, decision, config) {
|
|
8948
9758
|
const routesByLabel = labelRoutesForIssue(liveIssue, config);
|
|
8949
9759
|
if (routesByLabel.labels.length === 0) {
|
|
@@ -10072,6 +10882,7 @@ const decodeGithubPathSegment = (value) => {
|
|
|
10072
10882
|
const validGithubRepo = (repo) => /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})\/[A-Za-z0-9_.-]{1,100}$/u.test(repo);
|
|
10073
10883
|
const validPrNumber = (value) => Number.isInteger(value) && value > 0;
|
|
10074
10884
|
const githubPrIdentity = (repo, prNumber) => validGithubRepo(repo) && validPrNumber(prNumber) ? `${repo.toLowerCase()}#${prNumber}` : undefined;
|
|
10885
|
+
const babysitterOwnershipKey = (issue, ref) => `${issueKey(issue)}:${githubPrIdentity(ref.repo, ref.prNumber) ?? 'invalid'}`;
|
|
10075
10886
|
const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
10076
10887
|
if (!validGithubRepo(eventRepo))
|
|
10077
10888
|
return false;
|
|
@@ -10088,7 +10899,7 @@ const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
|
10088
10899
|
}
|
|
10089
10900
|
});
|
|
10090
10901
|
};
|
|
10091
|
-
const babysitterWakeKey = (issue, ref) => `${
|
|
10902
|
+
const babysitterWakeKey = (issue, ref) => `${babysitterOwnershipKey(issue, ref)}:${ref.agentName}`;
|
|
10092
10903
|
const BABYSITTER_WAKE_KIND_ORDER = [
|
|
10093
10904
|
'changes-requested',
|
|
10094
10905
|
'review-comment',
|
|
@@ -10612,7 +11423,32 @@ const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') =>
|
|
|
10612
11423
|
result: { name: session.agentName },
|
|
10613
11424
|
});
|
|
10614
11425
|
const isTerminalDispatchLifecycle = (lifecycle) => lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned';
|
|
10615
|
-
const
|
|
11426
|
+
const publishedPullRequests = (lifecycle) => {
|
|
11427
|
+
const receipts = [
|
|
11428
|
+
...(lifecycle?.pullRequests ?? []).filter(Boolean),
|
|
11429
|
+
...(lifecycle?.pullRequest ? [lifecycle.pullRequest] : []),
|
|
11430
|
+
];
|
|
11431
|
+
return [...new Map(receipts.map((receipt) => [
|
|
11432
|
+
`${receipt.repo.toLowerCase()}#${receipt.number}`,
|
|
11433
|
+
{ ...receipt },
|
|
11434
|
+
])).values()];
|
|
11435
|
+
};
|
|
11436
|
+
const mergePublishedPullRequests = (lifecycle, receipt) => {
|
|
11437
|
+
const receipts = publishedPullRequests(lifecycle);
|
|
11438
|
+
if (receipt)
|
|
11439
|
+
receipts.push(receipt);
|
|
11440
|
+
return [...new Map(receipts.map((candidate) => [
|
|
11441
|
+
candidate.repo.toLowerCase(),
|
|
11442
|
+
{ ...candidate },
|
|
11443
|
+
])).values()];
|
|
11444
|
+
};
|
|
11445
|
+
const primaryPublishedPullRequest = (previous, receipt, receipts) => {
|
|
11446
|
+
if (receipt &&
|
|
11447
|
+
(!previous?.pullRequest || previous.pullRequest.repo.toLowerCase() === receipt.repo.toLowerCase()))
|
|
11448
|
+
return receipt;
|
|
11449
|
+
return previous?.pullRequest ?? receipt ?? receipts[0];
|
|
11450
|
+
};
|
|
11451
|
+
const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequest, pullRequests = [], releaseReason) => ({
|
|
10616
11452
|
runId,
|
|
10617
11453
|
issue: { ...record.issue },
|
|
10618
11454
|
decision: structuredClone(record.decision),
|
|
@@ -10621,6 +11457,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
10621
11457
|
agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
|
|
10622
11458
|
invocationIds: [...record.invocationIds],
|
|
10623
11459
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
11460
|
+
...(pullRequests.length > 0 ? { pullRequests: pullRequests.map((receipt) => ({ ...receipt })) } : {}),
|
|
10624
11461
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
10625
11462
|
...(releaseReason ? { releaseReason } : {}),
|
|
10626
11463
|
updatedAtMs,
|