@agent-relay/factory 0.1.37 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -6
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +39 -4
- package/dist/cli/fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +13 -0
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +1 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +4 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/github/repo-identity.d.ts +8 -0
- package/dist/github/repo-identity.d.ts.map +1 -0
- package/dist/github/repo-identity.js +28 -0
- package/dist/github/repo-identity.js.map +1 -0
- package/dist/hosted/index.d.ts +5 -0
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +3 -0
- package/dist/hosted/index.js.map +1 -0
- package/dist/hosted/orchestrator.d.ts +12 -0
- package/dist/hosted/orchestrator.d.ts.map +1 -0
- package/dist/hosted/orchestrator.js +513 -0
- package/dist/hosted/orchestrator.js.map +1 -0
- package/dist/hosted/state-store.d.ts +48 -0
- package/dist/hosted/state-store.d.ts.map +1 -0
- package/dist/hosted/state-store.js +210 -0
- package/dist/hosted/state-store.js.map +1 -0
- package/dist/hosted/types.d.ts +173 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/hosted/types.js +2 -0
- package/dist/hosted/types.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +20 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +209 -22
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +17 -1
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +588 -87
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +13 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +13 -1
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/writeback/index.d.ts +2 -0
- package/dist/writeback/index.d.ts.map +1 -1
- package/dist/writeback/index.js +1 -0
- package/dist/writeback/index.js.map +1 -1
- package/dist/writeback/mount-health.d.ts +15 -0
- package/dist/writeback/mount-health.d.ts.map +1 -0
- package/dist/writeback/mount-health.js +15 -0
- package/dist/writeback/mount-health.js.map +1 -0
- package/package.json +6 -2
|
@@ -85,6 +85,7 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
85
85
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
86
86
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
87
87
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
88
|
+
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
88
89
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
89
90
|
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
90
91
|
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
@@ -99,6 +100,8 @@ const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
|
99
100
|
const GITHUB_MIRROR_SOURCE_PREFIX = 'Source: ';
|
|
100
101
|
export const DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH = '/tmp/factory-run/factory-loop-heartbeat.json';
|
|
101
102
|
export const DEFAULT_FACTORY_LOOP_REGISTRY_PATH = '/tmp/factory-run/factory-loop-registry.json';
|
|
103
|
+
class DispatchLifecycleCapacityError extends Error {
|
|
104
|
+
}
|
|
102
105
|
const realClock = {
|
|
103
106
|
now: () => Date.now(),
|
|
104
107
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -130,6 +133,7 @@ export class FactoryLoop {
|
|
|
130
133
|
#terminationGraceMs;
|
|
131
134
|
#babysitterWakeUnreachableEscalateMs;
|
|
132
135
|
#babysitterWakeUnreachableRetryMs;
|
|
136
|
+
#startupAgentExitDrainTimeoutMs;
|
|
133
137
|
#state;
|
|
134
138
|
#workspaceId;
|
|
135
139
|
#relayflows;
|
|
@@ -176,6 +180,7 @@ export class FactoryLoop {
|
|
|
176
180
|
#dispatchTerminalWaiters = new Map();
|
|
177
181
|
#dispatchLifecycleRetryTimers = new Map();
|
|
178
182
|
#dispatchLifecycleDrives = new Set();
|
|
183
|
+
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
179
184
|
#localReleaseCheckpoints = new Map();
|
|
180
185
|
#dispatchLifecycleRenewTimer;
|
|
181
186
|
#clarificationSweepTimer;
|
|
@@ -286,6 +291,7 @@ export class FactoryLoop {
|
|
|
286
291
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
287
292
|
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
288
293
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
294
|
+
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
289
295
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
290
296
|
this.#relayflows = ports.relayflows;
|
|
291
297
|
this.#worktrees = ports.worktrees;
|
|
@@ -480,7 +486,12 @@ export class FactoryLoop {
|
|
|
480
486
|
this.#wireFleetEvents();
|
|
481
487
|
await this.#adoptInFlightAgents(legacyRegistry);
|
|
482
488
|
this.#startupAgentAdoptionActive = false;
|
|
483
|
-
|
|
489
|
+
if (this.#config.babysitter.enabled) {
|
|
490
|
+
// Re-run the idempotent receipt fold after adoption returns. This
|
|
491
|
+
// catches records restored by lifecycle work that completed while the
|
|
492
|
+
// startup roster drain was in progress.
|
|
493
|
+
await this.#reconcileRestoredBabysitterReceipts();
|
|
494
|
+
}
|
|
484
495
|
}
|
|
485
496
|
catch (error) {
|
|
486
497
|
this.#startupAgentAdoptionActive = false;
|
|
@@ -559,6 +570,12 @@ export class FactoryLoop {
|
|
|
559
570
|
this.#completionSweepTimer = undefined;
|
|
560
571
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
561
572
|
try {
|
|
573
|
+
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
574
|
+
// drives. A slow Relayfile scan must not consume the shutdown deadline
|
|
575
|
+
// while every issue remains fenced to a publisher that is already
|
|
576
|
+
// stopping. The owner/epoch fence makes any late completion from those
|
|
577
|
+
// drives harmless; a second sweep below catches claims racing this one.
|
|
578
|
+
await this.#releaseOwnedDispatchLifecycleLeases();
|
|
562
579
|
await Promise.allSettled([...this.#dispatchLifecycleDrives]);
|
|
563
580
|
// Fence every source of new clarification work before touching the fleet.
|
|
564
581
|
// A wake already past the fence is allowed to unwind, and is awaited
|
|
@@ -581,10 +598,7 @@ export class FactoryLoop {
|
|
|
581
598
|
// non-durable (local/internal) records; terminal completion performs the
|
|
582
599
|
// normal remote release before clearing the lifecycle.
|
|
583
600
|
await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
|
|
584
|
-
|
|
585
|
-
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
586
|
-
}
|
|
587
|
-
this.#dispatchLifecycleEpochs.clear();
|
|
601
|
+
await this.#releaseOwnedDispatchLifecycleLeases();
|
|
588
602
|
if (this.#livePollTimer)
|
|
589
603
|
clearTimeout(this.#livePollTimer);
|
|
590
604
|
this.#livePollTimer = undefined;
|
|
@@ -622,6 +636,14 @@ export class FactoryLoop {
|
|
|
622
636
|
this.#stoppingHeartbeatRefreshActive = false;
|
|
623
637
|
}
|
|
624
638
|
}
|
|
639
|
+
async #releaseOwnedDispatchLifecycleLeases() {
|
|
640
|
+
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
641
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
642
|
+
if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
|
|
643
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
625
647
|
async #drainClarificationWakesForStop() {
|
|
626
648
|
// A wake may add its promise just as the sweep that discovered it settles.
|
|
627
649
|
// Re-snapshot until the map is empty rather than assuming one await is a
|
|
@@ -1709,6 +1731,7 @@ export class FactoryLoop {
|
|
|
1709
1731
|
url: pr.url ?? `https://github.com/${pr.repo}/pull/${pr.prNumber}`,
|
|
1710
1732
|
path: pr.path,
|
|
1711
1733
|
headRef,
|
|
1734
|
+
authoritative: true,
|
|
1712
1735
|
});
|
|
1713
1736
|
const babysitter = [...record.agents.values()].find((tracked) => tracked.spec.role === 'babysitter');
|
|
1714
1737
|
if (!babysitter) {
|
|
@@ -2279,6 +2302,14 @@ export class FactoryLoop {
|
|
|
2279
2302
|
if (agents.length > 0 && this.#fleet.hydrateTracked) {
|
|
2280
2303
|
this.#fleet.hydrateTracked(agents);
|
|
2281
2304
|
}
|
|
2305
|
+
if (this.#config.babysitter.enabled) {
|
|
2306
|
+
// Restore and reconcile exact PR ownership before asking the fleet to
|
|
2307
|
+
// report missing agents. Otherwise a stale weak-match babysitter from
|
|
2308
|
+
// the lifecycle can be resumed before its independently durable,
|
|
2309
|
+
// metadata-validated replacement session becomes authoritative.
|
|
2310
|
+
await this.#restoreBabysitterOwnership();
|
|
2311
|
+
await this.#reconcileRestoredBabysitterReceipts();
|
|
2312
|
+
}
|
|
2282
2313
|
this.#scheduleDispatchLifecycleRenewal();
|
|
2283
2314
|
if (this.#fleet.hydrateTracked) {
|
|
2284
2315
|
this.#startupAgentAdoptionActive = false;
|
|
@@ -2293,23 +2324,50 @@ export class FactoryLoop {
|
|
|
2293
2324
|
// but recovery work is asynchronous (issue reads, worktree restore,
|
|
2294
2325
|
// PR publication). Finish exits discovered by the startup reconcile
|
|
2295
2326
|
// before the full ready-issue backfill can monopolize mount I/O.
|
|
2296
|
-
|
|
2297
|
-
this.#
|
|
2327
|
+
const exitNames = new Set(agents.map((agent) => agent.name));
|
|
2328
|
+
const drained = await this.#drainAgentExitsInFlight(exitNames, this.#startMode === 'live' ? this.#startupAgentExitDrainTimeoutMs : undefined);
|
|
2329
|
+
if (drained) {
|
|
2330
|
+
this.#logger.info?.('[factory] durable startup reconciled exits drained');
|
|
2331
|
+
}
|
|
2332
|
+
else {
|
|
2333
|
+
this.#increment('startupAgentExitDrainTimeouts');
|
|
2334
|
+
this.#logger.warn?.('[factory] startup agent exit reconciliation is still running; continuing ready-issue discovery', {
|
|
2335
|
+
timeoutMs: this.#startupAgentExitDrainTimeoutMs,
|
|
2336
|
+
pendingExits: [...this.#agentExitsInFlight.keys()].filter((name) => exitNames.has(name)),
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2298
2339
|
}
|
|
2299
2340
|
}
|
|
2300
2341
|
catch (error) {
|
|
2301
2342
|
this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
|
|
2302
2343
|
}
|
|
2303
2344
|
}
|
|
2304
|
-
async #drainAgentExitsInFlight(names) {
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2345
|
+
async #drainAgentExitsInFlight(names, timeoutMs) {
|
|
2346
|
+
const drain = async () => {
|
|
2347
|
+
for (;;) {
|
|
2348
|
+
const pending = [...this.#agentExitsInFlight]
|
|
2349
|
+
.filter(([name]) => !names || names.has(name))
|
|
2350
|
+
.map(([, handling]) => handling);
|
|
2351
|
+
if (pending.length === 0)
|
|
2352
|
+
return;
|
|
2353
|
+
await Promise.allSettled(pending);
|
|
2354
|
+
}
|
|
2355
|
+
};
|
|
2356
|
+
if (timeoutMs === undefined) {
|
|
2357
|
+
await drain();
|
|
2358
|
+
return true;
|
|
2312
2359
|
}
|
|
2360
|
+
let timer;
|
|
2361
|
+
const completed = await Promise.race([
|
|
2362
|
+
drain().then(() => true),
|
|
2363
|
+
new Promise((resolve) => {
|
|
2364
|
+
timer = setTimeout(() => resolve(false), Math.max(0, timeoutMs));
|
|
2365
|
+
timer.unref?.();
|
|
2366
|
+
}),
|
|
2367
|
+
]);
|
|
2368
|
+
if (timer)
|
|
2369
|
+
clearTimeout(timer);
|
|
2370
|
+
return completed;
|
|
2313
2371
|
}
|
|
2314
2372
|
#scheduleDispatchLifecycleRenewal() {
|
|
2315
2373
|
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
@@ -2457,7 +2515,7 @@ export class FactoryLoop {
|
|
|
2457
2515
|
}
|
|
2458
2516
|
const previous = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2459
2517
|
const pullRequests = mergePublishedPullRequests(previous, pullRequest);
|
|
2460
|
-
const primaryPullRequest = previous
|
|
2518
|
+
const primaryPullRequest = primaryPublishedPullRequest(previous, pullRequest, pullRequests);
|
|
2461
2519
|
const lifecycle = lifecycleFromInFlightRecord(record, previous?.runId ?? randomUUID(), phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason);
|
|
2462
2520
|
for (const agent of lifecycle.agents) {
|
|
2463
2521
|
const previouslyReleasedAtMs = previous?.agents.find((candidate) => candidate.name === agent.name)?.releasedAtMs;
|
|
@@ -2502,11 +2560,27 @@ export class FactoryLoop {
|
|
|
2502
2560
|
const timer = setTimeout(() => {
|
|
2503
2561
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2504
2562
|
const drive = this.#driveDispatchLifecycle(key)
|
|
2563
|
+
.then(() => {
|
|
2564
|
+
this.#dispatchLifecycleCapacityWaitLogged.delete(key);
|
|
2565
|
+
})
|
|
2505
2566
|
.catch((error) => {
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2567
|
+
if (error instanceof DispatchLifecycleCapacityError) {
|
|
2568
|
+
if (!this.#dispatchLifecycleCapacityWaitLogged.has(key)) {
|
|
2569
|
+
this.#dispatchLifecycleCapacityWaitLogged.add(key);
|
|
2570
|
+
this.#increment('dispatchLifecycleCapacityWaits');
|
|
2571
|
+
this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
|
|
2572
|
+
issue: record.issue.key,
|
|
2573
|
+
retryMs: DISPATCH_LIFECYCLE_RETRY_MS,
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
else {
|
|
2578
|
+
this.#dispatchLifecycleCapacityWaitLogged.delete(key);
|
|
2579
|
+
this.#logger.warn?.('[factory] durable dispatch lifecycle retry failed', {
|
|
2580
|
+
issue: record.issue.key,
|
|
2581
|
+
error: describeError(error).errorMessage,
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2510
2584
|
this.#scheduleDispatchLifecycleRetry(record);
|
|
2511
2585
|
})
|
|
2512
2586
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
@@ -2565,7 +2639,7 @@ export class FactoryLoop {
|
|
|
2565
2639
|
if (lifecycle.phase === 'queued') {
|
|
2566
2640
|
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
2567
2641
|
if (epoch === undefined || !await this.#state.promoteDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now())) {
|
|
2568
|
-
throw new
|
|
2642
|
+
throw new DispatchLifecycleCapacityError(`durable dispatch ${lifecycle.issue.key} is waiting for batch capacity`);
|
|
2569
2643
|
}
|
|
2570
2644
|
const promoted = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2571
2645
|
if (!promoted || promoted.phase !== 'dispatching') {
|
|
@@ -2601,6 +2675,8 @@ export class FactoryLoop {
|
|
|
2601
2675
|
})));
|
|
2602
2676
|
await this.#fleet.reconcileTrackedAgents?.();
|
|
2603
2677
|
}
|
|
2678
|
+
if (this.#config.babysitter.enabled)
|
|
2679
|
+
await this.#reconcileRestoredBabysitterReceipts(record);
|
|
2604
2680
|
return;
|
|
2605
2681
|
}
|
|
2606
2682
|
if (lifecycle.phase === 'parking') {
|
|
@@ -2637,6 +2713,7 @@ export class FactoryLoop {
|
|
|
2637
2713
|
prNumber: receipt.number,
|
|
2638
2714
|
url: receipt.url,
|
|
2639
2715
|
headRef: receipt.headRef,
|
|
2716
|
+
authoritative: true,
|
|
2640
2717
|
});
|
|
2641
2718
|
}
|
|
2642
2719
|
return;
|
|
@@ -2651,6 +2728,7 @@ export class FactoryLoop {
|
|
|
2651
2728
|
prNumber: receipt.number,
|
|
2652
2729
|
url: receipt.url,
|
|
2653
2730
|
headRef: receipt.headRef,
|
|
2731
|
+
authoritative: true,
|
|
2654
2732
|
});
|
|
2655
2733
|
}
|
|
2656
2734
|
return;
|
|
@@ -3040,40 +3118,55 @@ export class FactoryLoop {
|
|
|
3040
3118
|
async #githubIssuePaths() {
|
|
3041
3119
|
try {
|
|
3042
3120
|
const issuePaths = new Map();
|
|
3043
|
-
for (const
|
|
3044
|
-
const
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3121
|
+
for (const { owner, repo } of configuredGithubRepoParts(this.#config)) {
|
|
3122
|
+
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
3123
|
+
const roots = githubIssueRepoRoots(owner, repo);
|
|
3124
|
+
// Keep the fallback roots as separate batches. Flattening a very large
|
|
3125
|
+
// provider result is synchronous work and can starve the durable loop
|
|
3126
|
+
// heartbeat before the bounded scan below gets a chance to yield.
|
|
3127
|
+
const pathBatches = indexedPaths
|
|
3128
|
+
? [indexedPaths]
|
|
3129
|
+
: await Promise.all(roots.map(async (root) => await this.#listRelayfileTree(root, 'GitHub issue ingestion')));
|
|
3130
|
+
if (indexedPaths) {
|
|
3131
|
+
this.#increment('githubIssueIndexReposUsed');
|
|
3132
|
+
}
|
|
3133
|
+
else {
|
|
3134
|
+
this.#increment('githubIssueIndexFallbacks');
|
|
3135
|
+
}
|
|
3136
|
+
for (const paths of pathBatches) {
|
|
3137
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
3138
|
+
const path = paths[index];
|
|
3139
|
+
const parts = githubIssuePathParts(path);
|
|
3140
|
+
if (parts) {
|
|
3141
|
+
const identity = githubIssueIdentity(parts.owner, parts.repo, parts.number);
|
|
3142
|
+
const existing = issuePaths.get(identity);
|
|
3143
|
+
if (!existing || githubIssuePathPreference(path) < githubIssuePathPreference(existing)) {
|
|
3144
|
+
if (existing)
|
|
3145
|
+
this.#increment('githubIssueAliasPathsSuppressed');
|
|
3146
|
+
issuePaths.set(identity, path);
|
|
3147
|
+
}
|
|
3148
|
+
else {
|
|
3053
3149
|
this.#increment('githubIssueAliasPathsSuppressed');
|
|
3054
|
-
|
|
3150
|
+
}
|
|
3055
3151
|
}
|
|
3056
|
-
else {
|
|
3057
|
-
|
|
3152
|
+
else if (githubIssueDirectoryPathParts(path) !== undefined) {
|
|
3153
|
+
// listTree returns the issue directory entry alongside its
|
|
3154
|
+
// meta.json file; githubIssuePathParts() already collected the
|
|
3155
|
+
// file, so skip the directory to avoid reading the same issue
|
|
3156
|
+
// twice in one backfill pass. Directory paths are only meaningful
|
|
3157
|
+
// for live change events, not the tree scan.
|
|
3158
|
+
continue;
|
|
3159
|
+
}
|
|
3160
|
+
else if (isGithubIssueTreePath(path)) {
|
|
3161
|
+
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
3162
|
+
}
|
|
3163
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
3164
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
3165
|
+
await liveEventYield();
|
|
3058
3166
|
}
|
|
3059
3167
|
}
|
|
3060
|
-
|
|
3061
|
-
// listTree returns the issue directory entry alongside its
|
|
3062
|
-
// meta.json file; githubIssuePathParts() already collected the
|
|
3063
|
-
// file, so skip the directory to avoid reading the same issue
|
|
3064
|
-
// twice in one backfill pass. Directory paths are only meaningful
|
|
3065
|
-
// for live change events, not the tree scan.
|
|
3066
|
-
continue;
|
|
3067
|
-
}
|
|
3068
|
-
else if (isGithubIssueTreePath(path)) {
|
|
3069
|
-
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
3070
|
-
}
|
|
3071
|
-
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
3072
|
-
await this.#refreshLiveHeartbeatIfDue();
|
|
3073
|
-
await liveEventYield();
|
|
3074
|
-
}
|
|
3168
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
3075
3169
|
}
|
|
3076
|
-
await this.#refreshLiveHeartbeatIfDue();
|
|
3077
3170
|
}
|
|
3078
3171
|
for (const [identity, path] of issuePaths) {
|
|
3079
3172
|
this.#githubIssuePreferredPaths.set(identity, path);
|
|
@@ -3088,6 +3181,41 @@ export class FactoryLoop {
|
|
|
3088
3181
|
return [];
|
|
3089
3182
|
}
|
|
3090
3183
|
}
|
|
3184
|
+
async #githubIssuePathsFromIndex(owner, repo) {
|
|
3185
|
+
const indexPath = `${GITHUB_ISSUE_ROOT}/${owner}/${repo}/issues/_index.json`;
|
|
3186
|
+
let parsed;
|
|
3187
|
+
try {
|
|
3188
|
+
const { content } = await this.#readRelayfileFile(indexPath, 'GitHub issue index discovery');
|
|
3189
|
+
parsed = parseJsonContent(content);
|
|
3190
|
+
}
|
|
3191
|
+
catch {
|
|
3192
|
+
return undefined;
|
|
3193
|
+
}
|
|
3194
|
+
if (!Array.isArray(parsed))
|
|
3195
|
+
return undefined;
|
|
3196
|
+
const requiredLabel = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
3197
|
+
if (!requiredLabel)
|
|
3198
|
+
return undefined;
|
|
3199
|
+
const paths = [];
|
|
3200
|
+
for (const entry of parsed) {
|
|
3201
|
+
const row = asRecord(entry);
|
|
3202
|
+
const number = row?.number;
|
|
3203
|
+
const state = typeof row?.state === 'string' ? row.state.trim().toLowerCase() : undefined;
|
|
3204
|
+
const labels = row?.labels;
|
|
3205
|
+
// Labels were added to the public GitHub issue index contract after the
|
|
3206
|
+
// first index version. Fall back for the entire repository if any row is
|
|
3207
|
+
// legacy or malformed so an eligible issue can never be filtered out.
|
|
3208
|
+
if (!Number.isSafeInteger(number) || Number(number) <= 0 || !state || !Array.isArray(labels) ||
|
|
3209
|
+
!labels.every((label) => typeof label === 'string')) {
|
|
3210
|
+
return undefined;
|
|
3211
|
+
}
|
|
3212
|
+
if (state !== 'open' || !labels.some((label) => label.trim().toLowerCase() === requiredLabel)) {
|
|
3213
|
+
continue;
|
|
3214
|
+
}
|
|
3215
|
+
paths.push(`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues/by-id/${number}.json`);
|
|
3216
|
+
}
|
|
3217
|
+
return paths;
|
|
3218
|
+
}
|
|
3091
3219
|
async #handleGithubIssueChange(path, opts = {}) {
|
|
3092
3220
|
if (this.#githubIngestionEnabled === false || !isGithubIssueFilePath(path)) {
|
|
3093
3221
|
return;
|
|
@@ -4003,6 +4131,7 @@ export class FactoryLoop {
|
|
|
4003
4131
|
repo: publishedPr.repo,
|
|
4004
4132
|
prNumber: publishedPr.number,
|
|
4005
4133
|
url: publishedPr.url,
|
|
4134
|
+
authoritative: true,
|
|
4006
4135
|
});
|
|
4007
4136
|
}
|
|
4008
4137
|
else {
|
|
@@ -4032,8 +4161,42 @@ export class FactoryLoop {
|
|
|
4032
4161
|
});
|
|
4033
4162
|
}
|
|
4034
4163
|
if (hasCompletionPr) {
|
|
4164
|
+
let reconciledPr;
|
|
4165
|
+
if (tracingReconciledExit && tracked.spec.role === 'implementer') {
|
|
4166
|
+
// A restart can discover that the implementer is gone after its PR
|
|
4167
|
+
// reached GitHub but before the durable receipt was saved. Persist the
|
|
4168
|
+
// exact existing-branch receipt before handing off to a babysitter;
|
|
4169
|
+
// otherwise the lifecycle remains `running` forever and consumes a
|
|
4170
|
+
// batch slot even though useful implementation work has finished.
|
|
4171
|
+
if (!await this.#saveDispatchLifecycle(record, 'publishing'))
|
|
4172
|
+
return;
|
|
4173
|
+
try {
|
|
4174
|
+
reconciledPr = await this.#publishImplementerPullRequest(record, tracked, {
|
|
4175
|
+
reconcileExisting: true,
|
|
4176
|
+
});
|
|
4177
|
+
if (reconciledPr && !await this.#saveDispatchLifecycle(record, 'published', reconciledPr))
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
catch (error) {
|
|
4181
|
+
this.#increment('githubPullRequestPublishFailures');
|
|
4182
|
+
this.#error(error, record.issue);
|
|
4183
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
4184
|
+
return;
|
|
4185
|
+
}
|
|
4186
|
+
}
|
|
4035
4187
|
if (this.#config.babysitter.enabled) {
|
|
4036
|
-
|
|
4188
|
+
if (reconciledPr) {
|
|
4189
|
+
await this.#ensureBabysitter(record, {
|
|
4190
|
+
repo: reconciledPr.repo,
|
|
4191
|
+
prNumber: reconciledPr.number,
|
|
4192
|
+
url: reconciledPr.url,
|
|
4193
|
+
headRef: reconciledPr.headRef,
|
|
4194
|
+
authoritative: true,
|
|
4195
|
+
});
|
|
4196
|
+
}
|
|
4197
|
+
else {
|
|
4198
|
+
await this.#ensureBabysitterForIssue(record);
|
|
4199
|
+
}
|
|
4037
4200
|
return;
|
|
4038
4201
|
}
|
|
4039
4202
|
if (await this.#allImplementersHaveCompletionPr(record))
|
|
@@ -4060,6 +4223,7 @@ export class FactoryLoop {
|
|
|
4060
4223
|
repo: publishedPr.repo,
|
|
4061
4224
|
prNumber: publishedPr.number,
|
|
4062
4225
|
url: publishedPr.url,
|
|
4226
|
+
authoritative: true,
|
|
4063
4227
|
});
|
|
4064
4228
|
}
|
|
4065
4229
|
else {
|
|
@@ -4236,9 +4400,10 @@ export class FactoryLoop {
|
|
|
4236
4400
|
: undefined;
|
|
4237
4401
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
4238
4402
|
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
4239
|
-
if (durableReceipt)
|
|
4240
|
-
return durableReceipt;
|
|
4241
4403
|
const expectedHeadRef = implementer.spec.branch ?? remoteBranch;
|
|
4404
|
+
if (durableReceipt &&
|
|
4405
|
+
(!opts.reconcileExisting || !expectedHeadRef || durableReceipt.headRef === expectedHeadRef))
|
|
4406
|
+
return durableReceipt;
|
|
4242
4407
|
if (opts.reconcileExisting && expectedHeadRef) {
|
|
4243
4408
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
4244
4409
|
if (existing) {
|
|
@@ -4678,7 +4843,17 @@ export class FactoryLoop {
|
|
|
4678
4843
|
if (!tracked.sessionRef) {
|
|
4679
4844
|
return;
|
|
4680
4845
|
}
|
|
4846
|
+
this.#logger.debug?.('[factory] tracked agent resume preparation started', {
|
|
4847
|
+
issue: record.issue.key,
|
|
4848
|
+
name,
|
|
4849
|
+
role: tracked.spec.role,
|
|
4850
|
+
});
|
|
4681
4851
|
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
4852
|
+
this.#logger.debug?.('[factory] tracked agent resume spawn started', {
|
|
4853
|
+
issue: record.issue.key,
|
|
4854
|
+
name,
|
|
4855
|
+
role: tracked.spec.role,
|
|
4856
|
+
});
|
|
4682
4857
|
const result = await this.#fleet.resume({
|
|
4683
4858
|
name,
|
|
4684
4859
|
sessionRef: tracked.sessionRef,
|
|
@@ -4687,6 +4862,12 @@ export class FactoryLoop {
|
|
|
4687
4862
|
repo: tracked.spec.repo,
|
|
4688
4863
|
clonePath: tracked.spec.clonePath,
|
|
4689
4864
|
});
|
|
4865
|
+
this.#logger.debug?.('[factory] tracked agent resume spawn completed', {
|
|
4866
|
+
issue: record.issue.key,
|
|
4867
|
+
name,
|
|
4868
|
+
resumedName: result.name,
|
|
4869
|
+
role: tracked.spec.role,
|
|
4870
|
+
});
|
|
4690
4871
|
tracked.result = {
|
|
4691
4872
|
...result,
|
|
4692
4873
|
node: result.node ?? tracked.result?.node,
|
|
@@ -4695,35 +4876,156 @@ export class FactoryLoop {
|
|
|
4695
4876
|
tracked.sessionRef = result.sessionRef ?? tracked.sessionRef;
|
|
4696
4877
|
record.agents.delete(name);
|
|
4697
4878
|
record.agents.set(result.name, tracked);
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4879
|
+
await this.#retargetBabysitterAgent(record, name, tracked);
|
|
4880
|
+
this.#logger.debug?.('[factory] tracked agent resume ownership retargeted', {
|
|
4881
|
+
issue: record.issue.key,
|
|
4882
|
+
name,
|
|
4883
|
+
resumedName: result.name,
|
|
4884
|
+
role: tracked.spec.role,
|
|
4885
|
+
});
|
|
4886
|
+
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4887
|
+
}
|
|
4888
|
+
async #retargetBabysitterAgent(record, previousName, tracked) {
|
|
4889
|
+
if (tracked.spec.role !== 'babysitter')
|
|
4890
|
+
return;
|
|
4891
|
+
const currentName = tracked.result?.name ?? tracked.spec.name;
|
|
4892
|
+
this.#babysitterCriticalAgents.delete(previousName);
|
|
4893
|
+
const ownership = [...this.#babysitterPr.entries()]
|
|
4894
|
+
.find(([, candidate]) => candidate.agentName === previousName);
|
|
4895
|
+
const [ownershipKey, ref] = ownership ?? [];
|
|
4896
|
+
if (!ref)
|
|
4897
|
+
return;
|
|
4898
|
+
ref.agentName = currentName;
|
|
4899
|
+
// Resuming a session commonly preserves its Relaycast name. Iterate a
|
|
4900
|
+
// snapshot because deleting and re-inserting that same key while walking
|
|
4901
|
+
// the live Map would append it to the iterator again indefinitely.
|
|
4902
|
+
for (const [wakeKey, state] of [...this.#babysitterWakeStates]) {
|
|
4903
|
+
if (state.agentName !== previousName)
|
|
4904
|
+
continue;
|
|
4905
|
+
if (state.timer)
|
|
4906
|
+
clearTimeout(state.timer);
|
|
4907
|
+
this.#babysitterWakeStates.delete(wakeKey);
|
|
4908
|
+
state.timer = undefined;
|
|
4909
|
+
state.agentName = currentName;
|
|
4910
|
+
state.tracked = tracked;
|
|
4911
|
+
if (state.deferredSubmitTargets) {
|
|
4912
|
+
state.deferredSubmitTargets = undefined;
|
|
4913
|
+
state.deliveringKinds = undefined;
|
|
4914
|
+
state.kinds.add('pull-request-state');
|
|
4915
|
+
await this.#recordPendingBabysitterWake(state);
|
|
4916
|
+
}
|
|
4917
|
+
this.#babysitterWakeStates.set(babysitterWakeKey(record.issue, ref), state);
|
|
4918
|
+
if (state.kinds.size > 0)
|
|
4919
|
+
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
4920
|
+
}
|
|
4921
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked, ownershipKey);
|
|
4922
|
+
}
|
|
4923
|
+
async #recoverUnreachableBabysitter(state) {
|
|
4924
|
+
const batch = await this.#batch();
|
|
4925
|
+
const record = batch.getIssueByAgent(state.agentName);
|
|
4926
|
+
const tracked = record?.agents.get(state.agentName);
|
|
4927
|
+
if (!record || !tracked || tracked.spec.role !== 'babysitter')
|
|
4928
|
+
return false;
|
|
4929
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
4930
|
+
return false;
|
|
4931
|
+
const previousName = state.agentName;
|
|
4932
|
+
this.#fleet.markAgentTerminal?.(previousName, 'babysitter-unreachable');
|
|
4933
|
+
try {
|
|
4934
|
+
await this.#fleet.release(previousName, 'babysitter-unreachable');
|
|
4935
|
+
this.#logger.debug?.('[factory] unreachable babysitter release completed', {
|
|
4936
|
+
issue: record.issue.key,
|
|
4937
|
+
babysitter: previousName,
|
|
4938
|
+
});
|
|
4939
|
+
}
|
|
4940
|
+
catch (error) {
|
|
4941
|
+
// An unresolvable Relaycast identity is frequently already absent from
|
|
4942
|
+
// placement too. Release is best-effort; the fresh spawn below is the
|
|
4943
|
+
// recovery operation that matters.
|
|
4944
|
+
this.#increment('babysitterUnreachableReleaseFailures');
|
|
4945
|
+
this.#logger.warn?.('[factory] unreachable babysitter release failed; attempting session recovery', {
|
|
4946
|
+
issue: record.issue.key,
|
|
4947
|
+
babysitter: previousName,
|
|
4948
|
+
error: describeError(error).errorMessage,
|
|
4949
|
+
});
|
|
4950
|
+
}
|
|
4951
|
+
try {
|
|
4952
|
+
const configuredCapability = this.#config.agentCapabilities.babysitter;
|
|
4953
|
+
const capabilityChanged = tracked.spec.capability !== configuredCapability;
|
|
4954
|
+
if (tracked.sessionRef && !capabilityChanged) {
|
|
4955
|
+
await this.#resumeTrackedAgent(record, previousName, tracked);
|
|
4956
|
+
}
|
|
4957
|
+
else {
|
|
4958
|
+
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:unreachable:${this.#clock.now()}`;
|
|
4959
|
+
const { sessionRef: _staleSessionRef, ...persistedSpec } = tracked.spec;
|
|
4960
|
+
const replacementSpec = capabilityChanged
|
|
4961
|
+
? {
|
|
4962
|
+
...persistedSpec,
|
|
4963
|
+
capability: configuredCapability,
|
|
4964
|
+
model: this.#config.models.babysitter,
|
|
4718
4965
|
}
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4966
|
+
: persistedSpec;
|
|
4967
|
+
await this.#prepareAgentWorktree(record, replacementSpec);
|
|
4968
|
+
const result = await this.#fleet.spawn({
|
|
4969
|
+
name: replacementSpec.name,
|
|
4970
|
+
capability: replacementSpec.capability,
|
|
4971
|
+
node: tracked.result?.node ?? replacementSpec.node ?? 'self',
|
|
4972
|
+
repo: replacementSpec.repo,
|
|
4973
|
+
task: replacementSpec.task,
|
|
4974
|
+
model: replacementSpec.model,
|
|
4975
|
+
cwd: replacementSpec.clonePath,
|
|
4976
|
+
invocationId,
|
|
4977
|
+
restartPolicy: defaultRestartPolicy(replacementSpec),
|
|
4978
|
+
channel: replacementSpec.channel,
|
|
4979
|
+
});
|
|
4980
|
+
batch.recordSpawn(record, replacementSpec, invocationId, result);
|
|
4981
|
+
const restarted = record.agents.get(result.name);
|
|
4982
|
+
if (!restarted)
|
|
4983
|
+
throw new Error(`Recovered babysitter ${result.name} was not tracked`);
|
|
4984
|
+
await this.#retargetBabysitterAgent(record, previousName, restarted);
|
|
4985
|
+
await this.#reportAgent(record, restarted, 'agent.resumed');
|
|
4986
|
+
if (capabilityChanged) {
|
|
4987
|
+
this.#increment('babysitterCapabilityMigrations');
|
|
4988
|
+
this.#logger.info?.('[factory] cold-started unreachable babysitter on configured capability', {
|
|
4989
|
+
issue: record.issue.key,
|
|
4990
|
+
babysitter: result.name,
|
|
4991
|
+
previousCapability: tracked.spec.capability,
|
|
4992
|
+
capability: configuredCapability,
|
|
4993
|
+
});
|
|
4722
4994
|
}
|
|
4723
|
-
await this.#persistBabysitterSession(record.issue, ref, tracked, ownershipKey);
|
|
4724
4995
|
}
|
|
4996
|
+
this.#logger.debug?.('[factory] unreachable babysitter replacement started', {
|
|
4997
|
+
issue: record.issue.key,
|
|
4998
|
+
previousBabysitter: previousName,
|
|
4999
|
+
babysitter: state.agentName,
|
|
5000
|
+
});
|
|
5001
|
+
await this.#writeInFlightRegistry();
|
|
5002
|
+
this.#logger.debug?.('[factory] unreachable babysitter registry refreshed', {
|
|
5003
|
+
issue: record.issue.key,
|
|
5004
|
+
babysitter: state.agentName,
|
|
5005
|
+
});
|
|
5006
|
+
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
5007
|
+
return false;
|
|
5008
|
+
this.#increment('babysitterEventWakeUnreachableRecoveries');
|
|
5009
|
+
this.#logger.info?.('[factory] restarted unreachable babysitter session', {
|
|
5010
|
+
issue: record.issue.key,
|
|
5011
|
+
repo: state.repo,
|
|
5012
|
+
prNumber: state.prNumber,
|
|
5013
|
+
previousBabysitter: previousName,
|
|
5014
|
+
babysitter: state.agentName,
|
|
5015
|
+
});
|
|
5016
|
+
return true;
|
|
5017
|
+
}
|
|
5018
|
+
catch (error) {
|
|
5019
|
+
this.#increment('babysitterEventWakeUnreachableRecoveryFailures');
|
|
5020
|
+
this.#logger.warn?.('[factory] failed to restart unreachable babysitter session', {
|
|
5021
|
+
issue: record.issue.key,
|
|
5022
|
+
repo: state.repo,
|
|
5023
|
+
prNumber: state.prNumber,
|
|
5024
|
+
babysitter: previousName,
|
|
5025
|
+
error: describeError(error).errorMessage,
|
|
5026
|
+
});
|
|
5027
|
+
return false;
|
|
4725
5028
|
}
|
|
4726
|
-
await this.#reportAgent(record, tracked, 'agent.resumed');
|
|
4727
5029
|
}
|
|
4728
5030
|
async #handleDeliveryFailed(info) {
|
|
4729
5031
|
const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
|
|
@@ -6195,11 +6497,15 @@ export class FactoryLoop {
|
|
|
6195
6497
|
continue;
|
|
6196
6498
|
}
|
|
6197
6499
|
const record = batch.getIssue(session.issue);
|
|
6198
|
-
const
|
|
6199
|
-
|
|
6500
|
+
const trackedEntry = record?.agents.has(session.agentName)
|
|
6501
|
+
? [session.agentName, record.agents.get(session.agentName)]
|
|
6502
|
+
: [...(record?.agents.entries() ?? [])].find(([, agent]) => agent.spec.role === 'babysitter' &&
|
|
6200
6503
|
githubPrIdentity(agent.spec.ownedPullRequest?.repo ?? '', agent.spec.ownedPullRequest?.number ?? 0) ===
|
|
6201
|
-
githubPrIdentity(session.repo, session.prNumber))
|
|
6504
|
+
githubPrIdentity(session.repo, session.prNumber));
|
|
6505
|
+
const tracked = trackedEntry?.[1]
|
|
6202
6506
|
?? durableBabysitterTrackedAgent(session, this.#config.agentCapabilities.babysitter);
|
|
6507
|
+
if (record && !trackedEntry)
|
|
6508
|
+
record.agents.set(session.agentName, tracked);
|
|
6203
6509
|
const ref = {
|
|
6204
6510
|
repo: session.repo,
|
|
6205
6511
|
prNumber: session.prNumber,
|
|
@@ -6223,6 +6529,74 @@ export class FactoryLoop {
|
|
|
6223
6529
|
}
|
|
6224
6530
|
}
|
|
6225
6531
|
}
|
|
6532
|
+
async #reconcileRestoredBabysitterReceipts(onlyRecord) {
|
|
6533
|
+
const records = onlyRecord ? [onlyRecord] : (await this.#batch()).inFlight;
|
|
6534
|
+
for (const record of records) {
|
|
6535
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
6536
|
+
continue;
|
|
6537
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
6538
|
+
// Babysitter sessions are independently durable and restore only after
|
|
6539
|
+
// their mounted PR metadata passes the open/draft/issue-identity guard.
|
|
6540
|
+
// Fold those validated receipts back into the lifecycle on takeover.
|
|
6541
|
+
// This closes the crash gap where the babysitter session persisted but
|
|
6542
|
+
// the lifecycle PR receipt did not, and lets exact ownership retire any
|
|
6543
|
+
// earlier weak-match babysitter for the same repository. The lifecycle's
|
|
6544
|
+
// own exact receipts are authoritative independently of the session
|
|
6545
|
+
// index: session restoration can legitimately be delayed or skipped
|
|
6546
|
+
// while mounted PR metadata converges, but that must never preserve a
|
|
6547
|
+
// superseded weak-match babysitter already disproved by publication.
|
|
6548
|
+
const authoritative = new Map();
|
|
6549
|
+
for (const receipt of lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : [])) {
|
|
6550
|
+
if (!receipt.repo || !validPrNumber(receipt.number) || !receipt.url || !receipt.headRef)
|
|
6551
|
+
continue;
|
|
6552
|
+
const identity = githubPrIdentity(receipt.repo, receipt.number);
|
|
6553
|
+
if (identity)
|
|
6554
|
+
authoritative.set(identity, { ...receipt });
|
|
6555
|
+
}
|
|
6556
|
+
const restored = [...this.#babysitterPr.entries()]
|
|
6557
|
+
.filter(([ownershipKey, ref]) => ref.agentName && issueKey(this.#babysitterIssueRefs.get(ownershipKey) ?? record.issue) === issueKey(record.issue))
|
|
6558
|
+
.map(([, ref]) => ref);
|
|
6559
|
+
for (const receipt of restored) {
|
|
6560
|
+
if (!receipt.repo || !validPrNumber(receipt.prNumber))
|
|
6561
|
+
continue;
|
|
6562
|
+
const snapshot = await this.#readPrSnapshot(receipt);
|
|
6563
|
+
const headRef = snapshot?.headRef ?? record.decision.implementers
|
|
6564
|
+
.find((implementer) => implementer.repo.toLowerCase() === receipt.repo.toLowerCase())?.branch;
|
|
6565
|
+
if (!headRef)
|
|
6566
|
+
continue;
|
|
6567
|
+
const identity = githubPrIdentity(receipt.repo, receipt.prNumber);
|
|
6568
|
+
if (!identity)
|
|
6569
|
+
continue;
|
|
6570
|
+
authoritative.set(identity, {
|
|
6571
|
+
repo: receipt.repo,
|
|
6572
|
+
number: receipt.prNumber,
|
|
6573
|
+
url: snapshot?.url ?? `https://github.com/${receipt.repo}/pull/${receipt.prNumber}`,
|
|
6574
|
+
headRef,
|
|
6575
|
+
...(receipt.path ? { path: receipt.path } : {}),
|
|
6576
|
+
});
|
|
6577
|
+
}
|
|
6578
|
+
if (authoritative.size > 0) {
|
|
6579
|
+
this.#logger.debug?.('[factory] reconciling authoritative babysitter receipts', {
|
|
6580
|
+
issue: record.issue.key,
|
|
6581
|
+
lifecycleReceipts: lifecycle?.pullRequests?.map((receipt) => receipt.number) ?? [],
|
|
6582
|
+
restoredReceipts: restored.map((receipt) => receipt.prNumber),
|
|
6583
|
+
authoritativeReceipts: [...authoritative.values()].map((receipt) => receipt.number),
|
|
6584
|
+
});
|
|
6585
|
+
await this.#retireBabysittersOutsideCurrentRoutes(record);
|
|
6586
|
+
}
|
|
6587
|
+
for (const published of authoritative.values()) {
|
|
6588
|
+
if (!await this.#saveDispatchLifecycle(record, 'running', published))
|
|
6589
|
+
return;
|
|
6590
|
+
await this.#ensureBabysitter(record, {
|
|
6591
|
+
repo: published.repo,
|
|
6592
|
+
prNumber: published.number,
|
|
6593
|
+
url: published.url,
|
|
6594
|
+
path: published.path,
|
|
6595
|
+
authoritative: true,
|
|
6596
|
+
});
|
|
6597
|
+
}
|
|
6598
|
+
}
|
|
6599
|
+
}
|
|
6226
6600
|
async #drainBabysitterWakesForStop() {
|
|
6227
6601
|
for (const state of this.#babysitterWakeStates.values()) {
|
|
6228
6602
|
state.cancelled = true;
|
|
@@ -6516,6 +6890,7 @@ export class FactoryLoop {
|
|
|
6516
6890
|
await this.#fleet.sendMessage(input);
|
|
6517
6891
|
state.unreachableSinceMs = undefined;
|
|
6518
6892
|
state.unreachableEscalated = false;
|
|
6893
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6519
6894
|
if (this.#stopping || state.cancelled) {
|
|
6520
6895
|
state.deliveringKinds = undefined;
|
|
6521
6896
|
return;
|
|
@@ -6531,6 +6906,7 @@ export class FactoryLoop {
|
|
|
6531
6906
|
// registration-lag backoff state accumulated by prior failures.
|
|
6532
6907
|
state.unreachableSinceMs = undefined;
|
|
6533
6908
|
state.unreachableEscalated = false;
|
|
6909
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6534
6910
|
targets = ack.targets.length > 0 ? [...new Set(ack.targets)] : [input.to];
|
|
6535
6911
|
}
|
|
6536
6912
|
if (this.#stopping || state.cancelled)
|
|
@@ -6586,23 +6962,23 @@ export class FactoryLoop {
|
|
|
6586
6962
|
// unreachable window so a later genuine registration lag starts fresh.
|
|
6587
6963
|
state.unreachableSinceMs = undefined;
|
|
6588
6964
|
state.unreachableEscalated = false;
|
|
6965
|
+
state.unreachableRecoveryAfterMs = undefined;
|
|
6589
6966
|
}
|
|
6590
6967
|
const unreachableMs = state.unreachableSinceMs !== undefined
|
|
6591
6968
|
? this.#clock.now() - state.unreachableSinceMs
|
|
6592
6969
|
: 0;
|
|
6593
6970
|
if (registrationLag && unreachableMs >= this.#babysitterWakeUnreachableEscalateMs) {
|
|
6594
6971
|
// The agent is up but its relay identity never became resolvable. Stop
|
|
6595
|
-
// the tight 1s loop
|
|
6596
|
-
//
|
|
6597
|
-
//
|
|
6598
|
-
// failure spinning silently forever.
|
|
6972
|
+
// the tight 1s loop, reconcile once, and restart the session. A recovery
|
|
6973
|
+
// cooldown prevents a still-converging Relaycast registration from
|
|
6974
|
+
// turning that restart into another tight loop.
|
|
6599
6975
|
state.nextDelayMs = this.#babysitterWakeUnreachableRetryMs;
|
|
6600
6976
|
if (!state.unreachableEscalated) {
|
|
6601
6977
|
state.unreachableEscalated = true;
|
|
6602
6978
|
await this.#fleet.reconcileTrackedAgents?.();
|
|
6603
6979
|
this.#increment('babysitterEventWakeUnreachableReconciliations');
|
|
6604
6980
|
this.#increment('babysitterEventWakeUnreachableEscalations');
|
|
6605
|
-
this.#logger.warn?.('[factory] babysitter unreachable past grace window;
|
|
6981
|
+
this.#logger.warn?.('[factory] babysitter unreachable past grace window; reconciling and restarting its session', {
|
|
6606
6982
|
issue: state.issue.key,
|
|
6607
6983
|
repo: state.repo,
|
|
6608
6984
|
prNumber: state.prNumber,
|
|
@@ -6612,6 +6988,19 @@ export class FactoryLoop {
|
|
|
6612
6988
|
error: describeError(error).errorMessage,
|
|
6613
6989
|
});
|
|
6614
6990
|
}
|
|
6991
|
+
if (state.unreachableRecoveryAfterMs === undefined ||
|
|
6992
|
+
this.#clock.now() >= state.unreachableRecoveryAfterMs) {
|
|
6993
|
+
state.unreachableRecoveryAfterMs = this.#clock.now() + this.#babysitterWakeUnreachableRetryMs;
|
|
6994
|
+
if (await this.#recoverUnreachableBabysitter(state)) {
|
|
6995
|
+
// Probe the newly registered identity promptly. If Relaycast still
|
|
6996
|
+
// cannot resolve it, the recovery cooldown above prevents another
|
|
6997
|
+
// teardown loop and the next attempt uses the slow cadence.
|
|
6998
|
+
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
6999
|
+
}
|
|
7000
|
+
}
|
|
7001
|
+
else {
|
|
7002
|
+
state.nextDelayMs = Math.max(BABYSITTER_EVENT_RETRY_MS, state.unreachableRecoveryAfterMs - this.#clock.now());
|
|
7003
|
+
}
|
|
6615
7004
|
}
|
|
6616
7005
|
else {
|
|
6617
7006
|
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
@@ -6910,6 +7299,7 @@ export class FactoryLoop {
|
|
|
6910
7299
|
prNumber: receipt.number,
|
|
6911
7300
|
url: receipt.url,
|
|
6912
7301
|
headRef: receipt.headRef,
|
|
7302
|
+
authoritative: true,
|
|
6913
7303
|
});
|
|
6914
7304
|
}
|
|
6915
7305
|
return;
|
|
@@ -6931,6 +7321,9 @@ export class FactoryLoop {
|
|
|
6931
7321
|
this.#increment('babysitterLifecycleOwnershipRejected');
|
|
6932
7322
|
return;
|
|
6933
7323
|
}
|
|
7324
|
+
const replacedSuperseded = prRef.authoritative
|
|
7325
|
+
? await this.#retireSupersededBabysitters(record, prRef)
|
|
7326
|
+
: false;
|
|
6934
7327
|
this.#babysitterIssueRefs.set(babysitterKey, { ...record.issue });
|
|
6935
7328
|
const existing = this.#babysitterPr.get(babysitterKey);
|
|
6936
7329
|
if (existing && githubPrIdentity(existing.repo, existing.prNumber) !== githubPrIdentity(prRef.repo, prRef.prNumber)) {
|
|
@@ -6991,7 +7384,7 @@ export class FactoryLoop {
|
|
|
6991
7384
|
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
6992
7385
|
?? record.decision.routes[0];
|
|
6993
7386
|
const initialSpec = babysitterSpec(issue, this.#config, route);
|
|
6994
|
-
if ([...this.#babysitterIssueRefs.entries()].some(([key, candidate]) => key !== babysitterKey && issueKey(candidate) === issueKey(record.issue))) {
|
|
7387
|
+
if (replacedSuperseded || [...this.#babysitterIssueRefs.entries()].some(([key, candidate]) => key !== babysitterKey && issueKey(candidate) === issueKey(record.issue))) {
|
|
6995
7388
|
initialSpec.name = agentNameForRole(issue, 'babysit', {
|
|
6996
7389
|
repo: prRef.repo,
|
|
6997
7390
|
discriminator: `${sanitizeAgentSlug(prRef.repo)}-${prRef.prNumber}`,
|
|
@@ -7093,6 +7486,108 @@ export class FactoryLoop {
|
|
|
7093
7486
|
}
|
|
7094
7487
|
}
|
|
7095
7488
|
}
|
|
7489
|
+
async #retireSupersededBabysitters(record, prRef) {
|
|
7490
|
+
const wanted = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
7491
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7492
|
+
const trackedAgents = new Map([
|
|
7493
|
+
...(lifecycle?.agents ?? [])
|
|
7494
|
+
.filter((agent) => agent.releasedAtMs === undefined)
|
|
7495
|
+
.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)]),
|
|
7496
|
+
...record.agents,
|
|
7497
|
+
]);
|
|
7498
|
+
const superseded = [...trackedAgents.entries()].filter(([, tracked]) => {
|
|
7499
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7500
|
+
return tracked.spec.role === 'babysitter' &&
|
|
7501
|
+
owned?.repo.toLowerCase() === prRef.repo.toLowerCase() &&
|
|
7502
|
+
githubPrIdentity(owned.repo, owned.number) !== wanted;
|
|
7503
|
+
});
|
|
7504
|
+
for (const [agentName, tracked] of superseded) {
|
|
7505
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7506
|
+
const failed = await this.#releaseAndTerminateAgents([[agentName, tracked]], 'superseded-pr-receipt', 'completion');
|
|
7507
|
+
if (failed.length > 0) {
|
|
7508
|
+
this.#increment('supersededBabysitterReleaseFailures');
|
|
7509
|
+
throw new Error(`Failed to release superseded babysitter ${agentName}`);
|
|
7510
|
+
}
|
|
7511
|
+
if (owned) {
|
|
7512
|
+
const staleKey = babysitterOwnershipKey(record.issue, {
|
|
7513
|
+
repo: owned.repo,
|
|
7514
|
+
prNumber: owned.number,
|
|
7515
|
+
});
|
|
7516
|
+
await this.#cancelBabysitterWake(staleKey);
|
|
7517
|
+
if (await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
7518
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, staleKey);
|
|
7519
|
+
}
|
|
7520
|
+
}
|
|
7521
|
+
record.agents.delete(agentName);
|
|
7522
|
+
this.#babysitterCriticalAgents.delete(agentName);
|
|
7523
|
+
this.#increment('supersededBabysittersReleased');
|
|
7524
|
+
this.#logger.info?.('[factory] released babysitter superseded by exact PR receipt', {
|
|
7525
|
+
issue: record.issue.key,
|
|
7526
|
+
babysitter: agentName,
|
|
7527
|
+
previousRepo: owned?.repo,
|
|
7528
|
+
previousPrNumber: owned?.number,
|
|
7529
|
+
repo: prRef.repo,
|
|
7530
|
+
prNumber: prRef.prNumber,
|
|
7531
|
+
});
|
|
7532
|
+
}
|
|
7533
|
+
if (superseded.length > 0) {
|
|
7534
|
+
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7535
|
+
if (latest && !await this.#saveDispatchLifecycle(record, latest.phase)) {
|
|
7536
|
+
throw new Error(`Failed to persist superseded babysitter cleanup for ${record.issue.key}`);
|
|
7537
|
+
}
|
|
7538
|
+
}
|
|
7539
|
+
return superseded.length > 0;
|
|
7540
|
+
}
|
|
7541
|
+
async #retireBabysittersOutsideCurrentRoutes(record) {
|
|
7542
|
+
const wantedRepos = new Set(record.decision.implementers.map((implementer) => implementer.repo.toLowerCase()));
|
|
7543
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7544
|
+
const trackedAgents = new Map([
|
|
7545
|
+
...(lifecycle?.agents ?? [])
|
|
7546
|
+
.filter((agent) => agent.releasedAtMs === undefined)
|
|
7547
|
+
.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)]),
|
|
7548
|
+
...record.agents,
|
|
7549
|
+
]);
|
|
7550
|
+
const unrouted = [...trackedAgents.entries()].filter(([, tracked]) => {
|
|
7551
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7552
|
+
return tracked.spec.role === 'babysitter' &&
|
|
7553
|
+
Boolean(owned) &&
|
|
7554
|
+
!wantedRepos.has(owned.repo.toLowerCase());
|
|
7555
|
+
});
|
|
7556
|
+
for (const [agentName, tracked] of unrouted) {
|
|
7557
|
+
const owned = tracked.spec.ownedPullRequest;
|
|
7558
|
+
const failed = await this.#releaseAndTerminateAgents([[agentName, tracked]], 'superseded-pr-route', 'completion');
|
|
7559
|
+
if (failed.length > 0) {
|
|
7560
|
+
this.#increment('supersededBabysitterReleaseFailures');
|
|
7561
|
+
throw new Error(`Failed to release unrouted babysitter ${agentName}`);
|
|
7562
|
+
}
|
|
7563
|
+
if (owned) {
|
|
7564
|
+
const staleKey = babysitterOwnershipKey(record.issue, {
|
|
7565
|
+
repo: owned.repo,
|
|
7566
|
+
prNumber: owned.number,
|
|
7567
|
+
});
|
|
7568
|
+
await this.#cancelBabysitterWake(staleKey);
|
|
7569
|
+
if (await this.#assertIssueDispatchLifecycleOwner(record.issue)) {
|
|
7570
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, staleKey);
|
|
7571
|
+
}
|
|
7572
|
+
}
|
|
7573
|
+
record.agents.delete(agentName);
|
|
7574
|
+
this.#babysitterCriticalAgents.delete(agentName);
|
|
7575
|
+
this.#increment('supersededBabysittersReleased');
|
|
7576
|
+
this.#logger.info?.('[factory] released babysitter outside the current dispatch routes', {
|
|
7577
|
+
issue: record.issue.key,
|
|
7578
|
+
babysitter: agentName,
|
|
7579
|
+
previousRepo: owned?.repo,
|
|
7580
|
+
previousPrNumber: owned?.number,
|
|
7581
|
+
currentRepos: [...wantedRepos],
|
|
7582
|
+
});
|
|
7583
|
+
}
|
|
7584
|
+
if (unrouted.length > 0) {
|
|
7585
|
+
const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
7586
|
+
if (latest && !await this.#saveDispatchLifecycle(record, latest.phase)) {
|
|
7587
|
+
throw new Error(`Failed to persist unrouted babysitter cleanup for ${record.issue.key}`);
|
|
7588
|
+
}
|
|
7589
|
+
}
|
|
7590
|
+
}
|
|
7096
7591
|
// The babysitter owns the readiness verdict (CI green + conflicts resolved +
|
|
7097
7592
|
// review comments addressed) — it sees the per-event PR webhook data in its
|
|
7098
7593
|
// sandbox, exactly like AgentWorkforce/agents review. It signals readiness by
|
|
@@ -10947,6 +11442,12 @@ const mergePublishedPullRequests = (lifecycle, receipt) => {
|
|
|
10947
11442
|
{ ...candidate },
|
|
10948
11443
|
])).values()];
|
|
10949
11444
|
};
|
|
11445
|
+
const primaryPublishedPullRequest = (previous, receipt, receipts) => {
|
|
11446
|
+
if (receipt &&
|
|
11447
|
+
(!previous?.pullRequest || previous.pullRequest.repo.toLowerCase() === receipt.repo.toLowerCase()))
|
|
11448
|
+
return receipt;
|
|
11449
|
+
return previous?.pullRequest ?? receipt ?? receipts[0];
|
|
11450
|
+
};
|
|
10950
11451
|
const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequest, pullRequests = [], releaseReason) => ({
|
|
10951
11452
|
runId,
|
|
10952
11453
|
issue: { ...record.issue },
|