@agent-relay/factory 0.1.49 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -11
- package/dist/cli/fleet.d.ts +4 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +87 -42
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +1671 -10
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +83 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/dispatch/templates.d.ts +6 -0
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +16 -0
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/environments/connection-registry.d.ts +617 -0
- package/dist/environments/connection-registry.d.ts.map +1 -0
- package/dist/environments/connection-registry.js +174 -0
- package/dist/environments/connection-registry.js.map +1 -0
- package/dist/environments/index.d.ts +21 -0
- package/dist/environments/index.d.ts.map +1 -0
- package/dist/environments/index.js +11 -0
- package/dist/environments/index.js.map +1 -0
- package/dist/environments/k6-job.d.ts +64 -0
- package/dist/environments/k6-job.d.ts.map +1 -0
- package/dist/environments/k6-job.js +376 -0
- package/dist/environments/k6-job.js.map +1 -0
- package/dist/environments/kubernetes-client.d.ts +72 -0
- package/dist/environments/kubernetes-client.d.ts.map +1 -0
- package/dist/environments/kubernetes-client.js +337 -0
- package/dist/environments/kubernetes-client.js.map +1 -0
- package/dist/environments/kubernetes-command.d.ts +30 -0
- package/dist/environments/kubernetes-command.d.ts.map +1 -0
- package/dist/environments/kubernetes-command.js +108 -0
- package/dist/environments/kubernetes-command.js.map +1 -0
- package/dist/environments/kubernetes-environment.d.ts +38 -0
- package/dist/environments/kubernetes-environment.d.ts.map +1 -0
- package/dist/environments/kubernetes-environment.js +333 -0
- package/dist/environments/kubernetes-environment.js.map +1 -0
- package/dist/environments/kubernetes-provider.d.ts +83 -0
- package/dist/environments/kubernetes-provider.d.ts.map +1 -0
- package/dist/environments/kubernetes-provider.js +956 -0
- package/dist/environments/kubernetes-provider.js.map +1 -0
- package/dist/environments/load-harness.d.ts +199 -0
- package/dist/environments/load-harness.d.ts.map +1 -0
- package/dist/environments/load-harness.js +259 -0
- package/dist/environments/load-harness.js.map +1 -0
- package/dist/environments/load-profile.d.ts +294 -0
- package/dist/environments/load-profile.d.ts.map +1 -0
- package/dist/environments/load-profile.js +125 -0
- package/dist/environments/load-profile.js.map +1 -0
- package/dist/environments/stack-deployer.d.ts +22 -0
- package/dist/environments/stack-deployer.d.ts.map +1 -0
- package/dist/environments/stack-deployer.js +27 -0
- package/dist/environments/stack-deployer.js.map +1 -0
- package/dist/environments/stack-descriptor.d.ts +243 -0
- package/dist/environments/stack-descriptor.d.ts.map +1 -0
- package/dist/environments/stack-descriptor.js +88 -0
- package/dist/environments/stack-descriptor.js.map +1 -0
- package/dist/environments/verification-pipeline.d.ts +117 -0
- package/dist/environments/verification-pipeline.d.ts.map +1 -0
- package/dist/environments/verification-pipeline.js +643 -0
- package/dist/environments/verification-pipeline.js.map +1 -0
- package/dist/environments/verification-stack-deployer.d.ts +73 -0
- package/dist/environments/verification-stack-deployer.d.ts.map +1 -0
- package/dist/environments/verification-stack-deployer.js +484 -0
- package/dist/environments/verification-stack-deployer.js.map +1 -0
- package/dist/environments/verification-stack-descriptor.d.ts +1134 -0
- package/dist/environments/verification-stack-descriptor.d.ts.map +1 -0
- package/dist/environments/verification-stack-descriptor.js +275 -0
- package/dist/environments/verification-stack-descriptor.js.map +1 -0
- package/dist/environments/verification-stack.d.ts +29 -0
- package/dist/environments/verification-stack.d.ts.map +1 -0
- package/dist/environments/verification-stack.js +43 -0
- package/dist/environments/verification-stack.js.map +1 -0
- package/dist/featuremap/generate.d.ts +72 -0
- package/dist/featuremap/generate.d.ts.map +1 -0
- package/dist/featuremap/generate.js +854 -0
- package/dist/featuremap/generate.js.map +1 -0
- package/dist/featuremap/index.d.ts +2 -0
- package/dist/featuremap/index.d.ts.map +1 -1
- package/dist/featuremap/index.js +1 -0
- package/dist/featuremap/index.js.map +1 -1
- package/dist/featuremap/validate.d.ts +6 -0
- package/dist/featuremap/validate.d.ts.map +1 -1
- package/dist/featuremap/validate.js +95 -13
- package/dist/featuremap/validate.js.map +1 -1
- package/dist/fleet/create-fleet.d.ts +2 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -1
- package/dist/fleet/create-fleet.js +1 -0
- package/dist/fleet/create-fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +8 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +31 -6
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +4 -1
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +142 -1
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/git/agent-worktree.d.ts +4 -1
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +151 -8
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/index.d.ts +32 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -3
- package/dist/index.js.map +1 -1
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +6 -2
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +8 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +11 -1
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/node/factory-node.d.ts +2 -0
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +107 -2
- package/dist/node/factory-node.js.map +1 -1
- package/dist/node/preview-process.d.ts +65 -0
- package/dist/node/preview-process.d.ts.map +1 -0
- package/dist/node/preview-process.js +431 -0
- package/dist/node/preview-process.js.map +1 -0
- package/dist/node/tailscale-preview.d.ts +45 -0
- package/dist/node/tailscale-preview.d.ts.map +1 -0
- package/dist/node/tailscale-preview.js +748 -0
- package/dist/node/tailscale-preview.js.map +1 -0
- package/dist/observability/events.d.ts +1838 -28
- package/dist/observability/events.d.ts.map +1 -1
- package/dist/observability/events.js +52 -0
- package/dist/observability/events.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1416 -78
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +2 -1
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/reaper.d.ts +24 -0
- package/dist/orchestrator/reaper.d.ts.map +1 -1
- package/dist/orchestrator/reaper.js +117 -0
- package/dist/orchestrator/reaper.js.map +1 -1
- package/dist/ports/environment.d.ts +83 -0
- package/dist/ports/environment.d.ts.map +1 -0
- package/dist/ports/environment.js +2 -0
- package/dist/ports/environment.js.map +1 -0
- package/dist/ports/fleet.d.ts +68 -1
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +3 -2
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +9 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/ports/worktree.d.ts +14 -0
- package/dist/ports/worktree.d.ts.map +1 -1
- package/dist/state/file-state-store.js +28 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/subscriptions/index.d.ts +2 -0
- package/dist/subscriptions/index.d.ts.map +1 -1
- package/dist/subscriptions/index.js +1 -0
- package/dist/subscriptions/index.js.map +1 -1
- package/dist/subscriptions/resource-subscriptions.d.ts +77 -0
- package/dist/subscriptions/resource-subscriptions.d.ts.map +1 -0
- package/dist/subscriptions/resource-subscriptions.js +91 -0
- package/dist/subscriptions/resource-subscriptions.js.map +1 -0
- package/dist/testing/fakes.d.ts +9 -1
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +40 -0
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +32 -32
- package/dist/types.d.ts +4 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +18 -4
- package/schemas/kubernetes-environment-stack.schema.json +86 -0
- package/schemas/verification-stack.schema.json +292 -0
|
@@ -5,7 +5,8 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
-
import {
|
|
8
|
+
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
|
+
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
9
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
10
11
|
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
@@ -16,6 +17,7 @@ import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/temp
|
|
|
16
17
|
import { resolveTestGuidance } from '../dispatch/test-guidance.js';
|
|
17
18
|
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
|
|
18
19
|
import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
|
|
20
|
+
import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
|
|
19
21
|
import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
20
22
|
import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
|
|
21
23
|
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
|
|
@@ -44,6 +46,7 @@ const LIVE_DEDUPE_LIMIT = 5_000;
|
|
|
44
46
|
const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
45
47
|
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
46
48
|
const COMPLETION_SWEEP_BATCH_SIZE = 2;
|
|
49
|
+
const PREVIEW_SWEEP_INTERVAL_MS = 60_000;
|
|
47
50
|
const PROBE_PR_GH_BACKOFF_MS = 60_000;
|
|
48
51
|
const PROBE_PR_GH_CANDIDATE_LIMIT = 200;
|
|
49
52
|
const PUBLISHED_PR_CONFIRM_ATTEMPTS = 20;
|
|
@@ -66,6 +69,24 @@ const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
|
66
69
|
const INJECTION_MAX_ATTEMPTS = 6;
|
|
67
70
|
const BABYSITTER_EVENT_COALESCE_MS = 750;
|
|
68
71
|
const BABYSITTER_EVENT_RETRY_MS = 1_000;
|
|
72
|
+
const BABYSITTER_SUBSCRIPTION_TTL_SECONDS = 60 * 60;
|
|
73
|
+
// Relayfile receives provider-native GitHub events, not the materialized file
|
|
74
|
+
// changes that the legacy local router consumed. `closed` is separately
|
|
75
|
+
// indexed as a terminal event below, so terminal delivery does not need a
|
|
76
|
+
// broad normal-event subscription.
|
|
77
|
+
const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
78
|
+
'pull_request.opened',
|
|
79
|
+
'pull_request.reopened',
|
|
80
|
+
'pull_request.synchronize',
|
|
81
|
+
'pull_request.ready_for_review',
|
|
82
|
+
'pull_request_review.submitted',
|
|
83
|
+
'pull_request_review_comment.created',
|
|
84
|
+
'issue_comment.created',
|
|
85
|
+
'check_run.completed',
|
|
86
|
+
];
|
|
87
|
+
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
88
|
+
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
89
|
+
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
69
90
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
70
91
|
// recipient unavailable) both when a freshly spawned agent has not finished
|
|
71
92
|
// enrolling AND when an agent is up but its relay identity never becomes
|
|
@@ -90,6 +111,7 @@ const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
|
90
111
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
91
112
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
92
113
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
114
|
+
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
93
115
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
94
116
|
const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
|
|
95
117
|
const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
|
|
@@ -137,6 +159,7 @@ export class FactoryLoop {
|
|
|
137
159
|
#githubWritebackProvided;
|
|
138
160
|
#slack;
|
|
139
161
|
#mergeGate;
|
|
162
|
+
#verificationGate;
|
|
140
163
|
#probeCloser;
|
|
141
164
|
#probePrResolver;
|
|
142
165
|
#customProbePrResolver;
|
|
@@ -207,6 +230,7 @@ export class FactoryLoop {
|
|
|
207
230
|
#dispatchTerminalWaiters = new Map();
|
|
208
231
|
#dispatchLifecycleRetryTimers = new Map();
|
|
209
232
|
#dispatchLifecycleDrives = new Set();
|
|
233
|
+
#abandonedDispatchReasons = new Map();
|
|
210
234
|
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
211
235
|
#dispatchLifecycleOwnershipWaitLogged = new Set();
|
|
212
236
|
#localReleaseCheckpoints = new Map();
|
|
@@ -244,8 +268,12 @@ export class FactoryLoop {
|
|
|
244
268
|
#deferLiveEventDrain = false;
|
|
245
269
|
#completionSweepTimer;
|
|
246
270
|
#completionSweepActive = false;
|
|
271
|
+
#previewSweepTimer;
|
|
272
|
+
#previewSweepInFlight;
|
|
247
273
|
#completionInFlight = new Set();
|
|
248
274
|
#agentExitsInFlight = new Map();
|
|
275
|
+
#reconciledAgentExitsActive = 0;
|
|
276
|
+
#reconciledAgentExitWaiters = [];
|
|
249
277
|
#agentLifecycleSignalsInFlight = new Map();
|
|
250
278
|
#startupAgentAdoptionActive = false;
|
|
251
279
|
#startupRosterExitSignals;
|
|
@@ -258,6 +286,14 @@ export class FactoryLoop {
|
|
|
258
286
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
259
287
|
#babysitterPr = new Map();
|
|
260
288
|
#babysitterIssueRefs = new Map();
|
|
289
|
+
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
290
|
+
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
291
|
+
// owning babysitter.
|
|
292
|
+
#babysitterSubscriptionOwners = new Map();
|
|
293
|
+
#babysitterResourceSubscriptionFault = false;
|
|
294
|
+
#babysitterResourceSubscriptionUnavailable = false;
|
|
295
|
+
#babysitterResourceDeliveryRetryTimer;
|
|
296
|
+
#babysitterResourceSubscriptionRenewTimer;
|
|
261
297
|
#babysitterReady = new Set();
|
|
262
298
|
#babysitterWakeStates = new Map();
|
|
263
299
|
// A babysitter announces this fence before invoking destructive git tooling
|
|
@@ -265,6 +301,8 @@ export class FactoryLoop {
|
|
|
265
301
|
// is active, but the PTY submit must never land in that critical window.
|
|
266
302
|
#babysitterCriticalAgents = new Set();
|
|
267
303
|
#publishedPullRequests = new Map();
|
|
304
|
+
#previewReferences = new Map();
|
|
305
|
+
#removedPreviewIds = new Set();
|
|
268
306
|
#probePrGhBackoffUntilMs = new Map();
|
|
269
307
|
#probePrResolvedCache = new Map();
|
|
270
308
|
// GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion
|
|
@@ -304,6 +342,17 @@ export class FactoryLoop {
|
|
|
304
342
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
305
343
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
306
344
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
345
|
+
this.#verificationGate = ports.verificationGate ?? (config.verification.enabled
|
|
346
|
+
? new VerificationPipeline({
|
|
347
|
+
descriptorPath: config.verification.descriptorPath,
|
|
348
|
+
reporter: ports.reporter,
|
|
349
|
+
logger: ports.logger,
|
|
350
|
+
maxConcurrentEnvironments: config.verification.maxConcurrentEnvironments,
|
|
351
|
+
maxRunTimeoutMs: config.verification.maxRunTimeoutMs,
|
|
352
|
+
maxEnvironmentTtlMs: config.verification.maxEnvironmentTtlMs,
|
|
353
|
+
maxTeardownTimeoutMs: config.verification.maxTeardownTimeoutMs,
|
|
354
|
+
})
|
|
355
|
+
: undefined);
|
|
307
356
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
308
357
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
309
358
|
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
@@ -529,12 +578,15 @@ export class FactoryLoop {
|
|
|
529
578
|
this.#wireFleetEvents();
|
|
530
579
|
await this.#adoptInFlightAgents(legacyRegistry);
|
|
531
580
|
this.#startupAgentAdoptionActive = false;
|
|
581
|
+
if (opts.mode !== 'dispatch-owner')
|
|
582
|
+
await this.#reapOrphanedWorktreesOnStartup(legacyRegistry);
|
|
532
583
|
if (this.#config.babysitter.enabled) {
|
|
533
584
|
// Re-run the idempotent receipt fold after adoption returns. This
|
|
534
585
|
// catches records restored by lifecycle work that completed while the
|
|
535
586
|
// startup roster drain was in progress.
|
|
536
587
|
await this.#reconcileRestoredBabysitterReceipts();
|
|
537
588
|
}
|
|
589
|
+
await this.#reapPreviewOrphans();
|
|
538
590
|
}
|
|
539
591
|
catch (error) {
|
|
540
592
|
this.#startupAgentAdoptionActive = false;
|
|
@@ -544,6 +596,7 @@ export class FactoryLoop {
|
|
|
544
596
|
}
|
|
545
597
|
if (opts.mode === 'dispatch-owner') {
|
|
546
598
|
this.#started = true;
|
|
599
|
+
this.#schedulePreviewSweep();
|
|
547
600
|
this.#scheduleDispatchLifecycleRenewal();
|
|
548
601
|
// A replacement one-shot owner must also recover a team parked for
|
|
549
602
|
// human input; it intentionally does not subscribe to the full issue
|
|
@@ -555,6 +608,7 @@ export class FactoryLoop {
|
|
|
555
608
|
}
|
|
556
609
|
if (live) {
|
|
557
610
|
this.#started = true;
|
|
611
|
+
this.#schedulePreviewSweep();
|
|
558
612
|
try {
|
|
559
613
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
560
614
|
await this.#rearmSlackReplyWatchers();
|
|
@@ -594,6 +648,7 @@ export class FactoryLoop {
|
|
|
594
648
|
void this.#handleChange(path);
|
|
595
649
|
});
|
|
596
650
|
this.#started = true;
|
|
651
|
+
this.#schedulePreviewSweep();
|
|
597
652
|
await this.#rearmSlackReplyWatchers();
|
|
598
653
|
await this.#drainReadyClarificationWake();
|
|
599
654
|
await this.#rearmGithubIssueCommentWatchers();
|
|
@@ -602,16 +657,27 @@ export class FactoryLoop {
|
|
|
602
657
|
async stop() {
|
|
603
658
|
this.#started = false;
|
|
604
659
|
this.#stopping = true;
|
|
660
|
+
if (this.#babysitterResourceDeliveryRetryTimer)
|
|
661
|
+
clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
|
|
662
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
663
|
+
if (this.#babysitterResourceSubscriptionRenewTimer)
|
|
664
|
+
clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
|
|
665
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
605
666
|
if (this.#dispatchLifecycleRenewTimer)
|
|
606
667
|
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
607
668
|
this.#dispatchLifecycleRenewTimer = undefined;
|
|
608
669
|
for (const timer of this.#dispatchLifecycleRetryTimers.values())
|
|
609
670
|
clearTimeout(timer);
|
|
610
671
|
this.#dispatchLifecycleRetryTimers.clear();
|
|
672
|
+
this.#abandonedDispatchReasons.clear();
|
|
611
673
|
this.#dispatchLifecycleOwnershipWaitLogged.clear();
|
|
612
674
|
if (this.#completionSweepTimer)
|
|
613
675
|
clearTimeout(this.#completionSweepTimer);
|
|
614
676
|
this.#completionSweepTimer = undefined;
|
|
677
|
+
if (this.#previewSweepTimer)
|
|
678
|
+
clearTimeout(this.#previewSweepTimer);
|
|
679
|
+
this.#previewSweepTimer = undefined;
|
|
680
|
+
await this.#previewSweepInFlight;
|
|
615
681
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
616
682
|
try {
|
|
617
683
|
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
@@ -652,6 +718,7 @@ export class FactoryLoop {
|
|
|
652
718
|
this.#babysitterSpawned.clear();
|
|
653
719
|
this.#babysitterPr.clear();
|
|
654
720
|
this.#babysitterIssueRefs.clear();
|
|
721
|
+
this.#babysitterSubscriptionOwners.clear();
|
|
655
722
|
this.#babysitterReady.clear();
|
|
656
723
|
this.#babysitterCriticalAgents.clear();
|
|
657
724
|
const subscription = this.#subscription;
|
|
@@ -2090,9 +2157,6 @@ export class FactoryLoop {
|
|
|
2090
2157
|
};
|
|
2091
2158
|
}
|
|
2092
2159
|
this.#clearDependencyPark(batch, dispatchDecision.issue);
|
|
2093
|
-
// Full task rendering is part of the durable spawn specification. It must
|
|
2094
|
-
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2095
|
-
// recover a persisted minimal triage task after a crash in this gap.
|
|
2096
2160
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
2097
2161
|
// Local dispatches need the same deterministic branch identity as remote
|
|
2098
2162
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -2104,10 +2168,17 @@ export class FactoryLoop {
|
|
|
2104
2168
|
isolateLocalWorktree,
|
|
2105
2169
|
});
|
|
2106
2170
|
}
|
|
2171
|
+
// Full task rendering is part of the durable spawn specification. It must
|
|
2172
|
+
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2173
|
+
// recover a persisted minimal triage task after a crash in this gap. The
|
|
2174
|
+
// task is rendered again below after preview provisioning adds its URL.
|
|
2107
2175
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2176
|
+
let claimedLifecycle;
|
|
2177
|
+
let recoveredRecord;
|
|
2108
2178
|
if (durableDispatch) {
|
|
2109
2179
|
const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
|
|
2110
2180
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2181
|
+
claimedLifecycle = lifecycleClaim.lifecycle;
|
|
2111
2182
|
dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
|
|
2112
2183
|
if (lifecycleClaim.lifecycle.phase === 'waiting-for-human') {
|
|
2113
2184
|
return lifecycleClaim.lifecycle.result ?? { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
@@ -2125,16 +2196,70 @@ export class FactoryLoop {
|
|
|
2125
2196
|
};
|
|
2126
2197
|
}
|
|
2127
2198
|
if (!lifecycleClaim.created) {
|
|
2128
|
-
|
|
2129
|
-
if (
|
|
2130
|
-
return
|
|
2199
|
+
recoveredRecord = inFlightRecordFromLifecycle(lifecycleClaim.lifecycle);
|
|
2200
|
+
if (recoveredRecord.result)
|
|
2201
|
+
return recoveredRecord.result;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
// External preview creation must happen only after the durable lease is
|
|
2205
|
+
// acquired and capacity admission has promoted the lifecycle. Persist the
|
|
2206
|
+
// fully rendered, preview-bearing decision before any worker can spawn so
|
|
2207
|
+
// takeover never recovers a minimal triage task.
|
|
2208
|
+
const previouslyPersistedPreviewIds = new Set(dispatchSpecs(dispatchDecision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
2209
|
+
try {
|
|
2210
|
+
if (!dryRun) {
|
|
2211
|
+
dispatchDecision = await this.#withPreviewReferences(dispatchDecision);
|
|
2212
|
+
}
|
|
2213
|
+
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2214
|
+
if (durableDispatch && claimedLifecycle) {
|
|
2215
|
+
const stagedRecord = inFlightRecordFromLifecycle({
|
|
2216
|
+
...claimedLifecycle,
|
|
2217
|
+
decision: structuredClone(dispatchDecision),
|
|
2218
|
+
});
|
|
2219
|
+
if (!await this.#saveDispatchLifecycle(stagedRecord, 'dispatching')) {
|
|
2220
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${dispatchDecision.issue.key}`);
|
|
2221
|
+
}
|
|
2222
|
+
if (recoveredRecord) {
|
|
2223
|
+
recoveredRecord.decision = structuredClone(dispatchDecision);
|
|
2224
|
+
recoveredRecord = batch.restore(recoveredRecord);
|
|
2225
|
+
}
|
|
2131
2226
|
}
|
|
2132
2227
|
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
const newlyCreated = uniquePreviewReferences([
|
|
2230
|
+
...dispatchSpecs(dispatchDecision).map((spec) => spec.preview),
|
|
2231
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue)) ?? []),
|
|
2232
|
+
]).filter((preview) => !previouslyPersistedPreviewIds.has(preview.id));
|
|
2233
|
+
// Once the durable fence is lost, the successor may already have
|
|
2234
|
+
// adopted this deterministic issue preview. Leave cleanup to the
|
|
2235
|
+
// identity-aware sweep instead of letting a stale owner tear down the
|
|
2236
|
+
// successor's route.
|
|
2237
|
+
const mayRollback = !claimedLifecycle ||
|
|
2238
|
+
await this.#assertIssueDispatchLifecycleOwner(dispatchDecision.issue);
|
|
2239
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
2240
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
2241
|
+
this.#logger.warn?.('[factory] failed to roll back preview provisioning', {
|
|
2242
|
+
issue: dispatchDecision.issue.key,
|
|
2243
|
+
error: describeError(cleanupError).errorMessage,
|
|
2244
|
+
});
|
|
2245
|
+
});
|
|
2246
|
+
}
|
|
2247
|
+
if (mayRollback)
|
|
2248
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2249
|
+
if (claimedLifecycle) {
|
|
2250
|
+
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(claimedLifecycle));
|
|
2251
|
+
}
|
|
2252
|
+
throw error;
|
|
2253
|
+
}
|
|
2133
2254
|
if (!durableDispatch)
|
|
2134
2255
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2135
2256
|
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
2136
|
-
const record = batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2257
|
+
const record = recoveredRecord ?? batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2137
2258
|
if (!record) {
|
|
2259
|
+
if (!dryRun) {
|
|
2260
|
+
await this.#teardownPreviewReferences(dispatchSpecs(dispatchDecision).map((spec) => spec.preview));
|
|
2261
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2262
|
+
}
|
|
2138
2263
|
await this.#clearDispatchInFlight(dispatchDecision.issue);
|
|
2139
2264
|
this.#increment('queued');
|
|
2140
2265
|
this.#emit('issue-queued', { issue: dispatchDecision.issue });
|
|
@@ -2143,7 +2268,9 @@ export class FactoryLoop {
|
|
|
2143
2268
|
if (record.result) {
|
|
2144
2269
|
return record.result;
|
|
2145
2270
|
}
|
|
2146
|
-
await this.#saveDispatchLifecycle(record, 'dispatching')
|
|
2271
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
2272
|
+
throw new Error(`Dispatch lifecycle ownership lost immediately before spawning ${dispatchDecision.issue.key}`);
|
|
2273
|
+
}
|
|
2147
2274
|
if (!dryRun)
|
|
2148
2275
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
2149
2276
|
const spawnedForReaperHandoff = [];
|
|
@@ -2198,6 +2325,9 @@ export class FactoryLoop {
|
|
|
2198
2325
|
agents,
|
|
2199
2326
|
comments: [comment],
|
|
2200
2327
|
stateId: implementingStateId,
|
|
2328
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue))?.length
|
|
2329
|
+
? { previews: this.#previewReferences.get(issueKey(dispatchDecision.issue)) }
|
|
2330
|
+
: {}),
|
|
2201
2331
|
dryRun,
|
|
2202
2332
|
};
|
|
2203
2333
|
record.result = result;
|
|
@@ -2218,6 +2348,14 @@ export class FactoryLoop {
|
|
|
2218
2348
|
const liveStateChanged = error instanceof LiveDispatchStateChangedError;
|
|
2219
2349
|
const cancellationReason = factoryCloudDispatchCancellationReason(error);
|
|
2220
2350
|
const cleanupReason = liveStateChanged ? 'live dispatch state changed' : 'dispatch failed';
|
|
2351
|
+
let failedState;
|
|
2352
|
+
if (!liveStateChanged) {
|
|
2353
|
+
await this.#recordDispatchFailure(decision.issue);
|
|
2354
|
+
failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
2355
|
+
}
|
|
2356
|
+
const terminalFailure = liveStateChanged || Boolean(failedState?.terminal);
|
|
2357
|
+
if (terminalFailure && !await this.#saveDispatchLifecycle(record, 'abandoning', undefined, cleanupReason, new Set(), { cancellationReason }))
|
|
2358
|
+
throw error;
|
|
2221
2359
|
let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs, cleanupReason);
|
|
2222
2360
|
if (liveStateChanged && !failureHandoffs.some((handoff) => handoff.worktree)) {
|
|
2223
2361
|
const failed = await this.#releaseAndTerminateAgents(failureHandoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion');
|
|
@@ -2228,16 +2366,43 @@ export class FactoryLoop {
|
|
|
2228
2366
|
worktreesTornDown = failureHandoffs.length > 0;
|
|
2229
2367
|
}
|
|
2230
2368
|
}
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2369
|
+
if (terminalFailure) {
|
|
2370
|
+
try {
|
|
2371
|
+
await this.#teardownPreviews(record);
|
|
2372
|
+
}
|
|
2373
|
+
catch (previewError) {
|
|
2374
|
+
this.#logger.warn?.('[factory] failed to tear down previews after terminal dispatch failure', {
|
|
2375
|
+
issue: record.issue.key,
|
|
2376
|
+
error: describeError(previewError).errorMessage,
|
|
2377
|
+
});
|
|
2378
|
+
// Do not commit a terminal lifecycle while an externally reachable
|
|
2379
|
+
// preview remains. The abandonment driver retries the identity-
|
|
2380
|
+
// checked teardown and commits terminal state only after it succeeds.
|
|
2381
|
+
this.#scheduleAbandonedDispatchRetry(record, cleanupReason);
|
|
2382
|
+
if (!liveStateChanged)
|
|
2383
|
+
this.#error(error, decision.issue);
|
|
2384
|
+
if (worktreesTornDown) {
|
|
2385
|
+
await this.#writeInFlightRegistry().catch((registryError) => {
|
|
2386
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after dispatch worktree teardown', {
|
|
2387
|
+
issue: record.issue,
|
|
2388
|
+
error: describeError(registryError).errorMessage,
|
|
2389
|
+
});
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, undefined, new Set(), { cancellationReason }))
|
|
2395
|
+
throw error;
|
|
2396
|
+
if (liveStateChanged)
|
|
2397
|
+
await this.#clearDispatchInFlight(decision.issue);
|
|
2398
|
+
else
|
|
2399
|
+
await this.#recordDispatchTerminal(decision.issue);
|
|
2400
|
+
if (liveStateChanged)
|
|
2401
|
+
this.#increment('dispatchLiveStateRaces');
|
|
2236
2402
|
}
|
|
2237
2403
|
else {
|
|
2238
|
-
await this.#
|
|
2239
|
-
|
|
2240
|
-
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable', undefined, undefined, new Set(), { cancellationReason: failedState?.terminal ? cancellationReason : undefined });
|
|
2404
|
+
if (!await this.#saveDispatchLifecycle(record, 'retryable'))
|
|
2405
|
+
throw error;
|
|
2241
2406
|
}
|
|
2242
2407
|
batch.abandon(decision.issue);
|
|
2243
2408
|
if (!liveStateChanged && !failedState?.terminal)
|
|
@@ -2341,7 +2506,13 @@ export class FactoryLoop {
|
|
|
2341
2506
|
const previous = this.#agentExitsInFlight.get(name) ?? Promise.resolve();
|
|
2342
2507
|
const handling = previous
|
|
2343
2508
|
.catch(() => undefined)
|
|
2344
|
-
.then(async () =>
|
|
2509
|
+
.then(async () => {
|
|
2510
|
+
if (reason === 'reconciled-missing') {
|
|
2511
|
+
await this.#withReconciledAgentExitSlot(async () => this.#handleAgentExit(name, reason));
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
2514
|
+
await this.#handleAgentExit(name, reason);
|
|
2515
|
+
})
|
|
2345
2516
|
.catch((error) => this.#error(error))
|
|
2346
2517
|
.finally(() => {
|
|
2347
2518
|
if (this.#agentExitsInFlight.get(name) === handling) {
|
|
@@ -2367,6 +2538,12 @@ export class FactoryLoop {
|
|
|
2367
2538
|
for (const [key, lifecycle] of durableLifecycles) {
|
|
2368
2539
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
2369
2540
|
continue;
|
|
2541
|
+
const previews = uniquePreviewReferences([
|
|
2542
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
2543
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
2544
|
+
]);
|
|
2545
|
+
if (previews.length > 0)
|
|
2546
|
+
this.#previewReferences.set(issueKey(lifecycle.issue), previews);
|
|
2370
2547
|
hasNonterminalDurableLifecycle = true;
|
|
2371
2548
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2372
2549
|
if (!claim.acquired || !claim.lease) {
|
|
@@ -2380,6 +2557,33 @@ export class FactoryLoop {
|
|
|
2380
2557
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2381
2558
|
continue;
|
|
2382
2559
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
2560
|
+
if (!durableRecord.dryRun &&
|
|
2561
|
+
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2562
|
+
claim.lifecycle.phase !== 'releasing') {
|
|
2563
|
+
const liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2564
|
+
// A babysat Linear issue already at Done may have merged while this
|
|
2565
|
+
// process was down. Let authoritative PR restoration drive the
|
|
2566
|
+
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
2567
|
+
const deferDoneToBabysitterRecovery = Boolean(liveIssue &&
|
|
2568
|
+
!isGithubIssue(liveIssue) &&
|
|
2569
|
+
this.#states.roleOf(liveIssue.stateId) === 'done' &&
|
|
2570
|
+
this.#config.babysitter.enabled &&
|
|
2571
|
+
await this.#hasRestorableMergedBabysitterSession(durableRecord.issue));
|
|
2572
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue) && !deferDoneToBabysitterRecovery) {
|
|
2573
|
+
const restored = batch.restore(durableRecord);
|
|
2574
|
+
try {
|
|
2575
|
+
await this.#abandonDurableResume(restored, 'source issue is already terminal during startup recovery');
|
|
2576
|
+
}
|
|
2577
|
+
catch (error) {
|
|
2578
|
+
this.#logger.warn?.('[factory] terminal source preview cleanup will retry after startup', {
|
|
2579
|
+
issue: durableRecord.issue.key,
|
|
2580
|
+
error: describeError(error).errorMessage,
|
|
2581
|
+
});
|
|
2582
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2583
|
+
}
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2383
2587
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2384
2588
|
? durableRecord
|
|
2385
2589
|
: batch.restore(durableRecord);
|
|
@@ -2448,7 +2652,12 @@ export class FactoryLoop {
|
|
|
2448
2652
|
// the hydrated durable session is no longer usable. Classify it as a
|
|
2449
2653
|
// reconciled miss so remote implementers recover a PR or restart
|
|
2450
2654
|
// instead of waiting forever for branch replication after a crash.
|
|
2451
|
-
|
|
2655
|
+
const rolePriority = (name) => {
|
|
2656
|
+
const role = batch.getIssueByAgent(name)?.agents.get(name)?.spec.role;
|
|
2657
|
+
return role === 'implementer' ? 0 : role === 'babysitter' ? 1 : 2;
|
|
2658
|
+
};
|
|
2659
|
+
const orderedSignals = [...signalled].sort((left, right) => rolePriority(left) - rolePriority(right) || left.localeCompare(right));
|
|
2660
|
+
for (const name of orderedSignals)
|
|
2452
2661
|
this.#queueAgentExit(name, 'reconciled-missing');
|
|
2453
2662
|
if (synthesized.length > 0) {
|
|
2454
2663
|
this.#increment('startupRosterMissingExitsSynthesized', synthesized.length);
|
|
@@ -2566,6 +2775,20 @@ export class FactoryLoop {
|
|
|
2566
2775
|
clearTimeout(timer);
|
|
2567
2776
|
return completed;
|
|
2568
2777
|
}
|
|
2778
|
+
async #withReconciledAgentExitSlot(run) {
|
|
2779
|
+
if (this.#reconciledAgentExitsActive >= RECONCILED_AGENT_EXIT_CONCURRENCY) {
|
|
2780
|
+
this.#increment('reconciledAgentExitBackpressure');
|
|
2781
|
+
await new Promise((resolve) => this.#reconciledAgentExitWaiters.push(resolve));
|
|
2782
|
+
}
|
|
2783
|
+
this.#reconciledAgentExitsActive += 1;
|
|
2784
|
+
try {
|
|
2785
|
+
await run();
|
|
2786
|
+
}
|
|
2787
|
+
finally {
|
|
2788
|
+
this.#reconciledAgentExitsActive -= 1;
|
|
2789
|
+
this.#reconciledAgentExitWaiters.shift()?.();
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2569
2792
|
#scheduleDispatchLifecycleRenewal() {
|
|
2570
2793
|
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
2571
2794
|
return;
|
|
@@ -2893,6 +3116,18 @@ export class FactoryLoop {
|
|
|
2893
3116
|
return;
|
|
2894
3117
|
if (acquiredNow && this.#config.babysitter.enabled)
|
|
2895
3118
|
await this.#restoreBabysitterOwnership();
|
|
3119
|
+
const abandonedReason = this.#abandonedDispatchReasons.get(key);
|
|
3120
|
+
if (abandonedReason !== undefined) {
|
|
3121
|
+
await this.#abandonStuckDispatch(record, abandonedReason);
|
|
3122
|
+
return;
|
|
3123
|
+
}
|
|
3124
|
+
if (lifecycle.phase === 'running' && !record.dryRun) {
|
|
3125
|
+
const liveIssue = await this.#readIssue(record.issue.path);
|
|
3126
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3127
|
+
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3128
|
+
return;
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
2896
3131
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
2897
3132
|
if (this.#fleet.hydrateTracked) {
|
|
2898
3133
|
this.#fleet.hydrateTracked(lifecycle.agents.map((agent) => ({
|
|
@@ -2914,6 +3149,10 @@ export class FactoryLoop {
|
|
|
2914
3149
|
await this.#finishClarificationPark(waiting, true);
|
|
2915
3150
|
return;
|
|
2916
3151
|
}
|
|
3152
|
+
if (lifecycle.phase === 'abandoning') {
|
|
3153
|
+
await this.#abandonStuckDispatch(record, lifecycle.releaseReason ?? 'dispatch failed');
|
|
3154
|
+
return;
|
|
3155
|
+
}
|
|
2917
3156
|
if (lifecycle.phase === 'dispatching' || lifecycle.phase === 'retryable') {
|
|
2918
3157
|
await this.#resumeDurableDispatch(record);
|
|
2919
3158
|
return;
|
|
@@ -2972,15 +3211,39 @@ export class FactoryLoop {
|
|
|
2972
3211
|
}
|
|
2973
3212
|
}
|
|
2974
3213
|
async #resumeDurableDispatch(record) {
|
|
3214
|
+
let liveIssue;
|
|
2975
3215
|
if (!record.dryRun) {
|
|
2976
|
-
|
|
2977
|
-
if (!
|
|
3216
|
+
liveIssue = await this.#readIssue(record.issue.path);
|
|
3217
|
+
if (!liveIssue) {
|
|
2978
3218
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
|
|
2979
3219
|
}
|
|
2980
|
-
if (
|
|
2981
|
-
await this.#abandonDurableResume(record, 'live
|
|
3220
|
+
if (this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3221
|
+
await this.#abandonDurableResume(record, 'live source issue is already terminal');
|
|
2982
3222
|
return;
|
|
2983
3223
|
}
|
|
3224
|
+
const persistedPreviewIds = new Set(dispatchSpecs(record.decision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
3225
|
+
try {
|
|
3226
|
+
record.decision = await this.#withPreviewReferences(record.decision);
|
|
3227
|
+
record.decision = await this.#withRenderedDispatchTasks(record.decision, liveIssue);
|
|
3228
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching'))
|
|
3229
|
+
return;
|
|
3230
|
+
}
|
|
3231
|
+
catch (error) {
|
|
3232
|
+
const newlyCreated = uniquePreviewReferences([
|
|
3233
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3234
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
3235
|
+
]).filter((preview) => !persistedPreviewIds.has(preview.id));
|
|
3236
|
+
const mayRollback = await this.#assertDispatchLifecycleOwner(record);
|
|
3237
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
3238
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
3239
|
+
this.#logger.warn?.('[factory] failed to roll back recovered preview provisioning', {
|
|
3240
|
+
issue: record.issue.key,
|
|
3241
|
+
error: describeError(cleanupError).errorMessage,
|
|
3242
|
+
});
|
|
3243
|
+
});
|
|
3244
|
+
}
|
|
3245
|
+
throw error;
|
|
3246
|
+
}
|
|
2984
3247
|
}
|
|
2985
3248
|
const agents = [];
|
|
2986
3249
|
const specs = dispatchSpecs(record.decision);
|
|
@@ -2997,7 +3260,7 @@ export class FactoryLoop {
|
|
|
2997
3260
|
}
|
|
2998
3261
|
await this.#writeInFlightRegistry();
|
|
2999
3262
|
if (!record.dryRun) {
|
|
3000
|
-
const issue = await this.#readIssue(record.issue.path);
|
|
3263
|
+
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3001
3264
|
if (!issue)
|
|
3002
3265
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3003
3266
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
@@ -3008,15 +3271,24 @@ export class FactoryLoop {
|
|
|
3008
3271
|
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3009
3272
|
}
|
|
3010
3273
|
}
|
|
3274
|
+
const recoveredPreviews = uniquePreviewReferences([
|
|
3275
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3276
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
3277
|
+
]);
|
|
3011
3278
|
record.result ??= {
|
|
3012
3279
|
issue: record.issue,
|
|
3013
3280
|
agents,
|
|
3014
3281
|
comments: [dispatchComment(record.decision, agents)],
|
|
3282
|
+
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3015
3283
|
dryRun: record.dryRun,
|
|
3016
3284
|
};
|
|
3285
|
+
if (recoveredPreviews.length > 0 && !record.result.previews?.length) {
|
|
3286
|
+
record.result = { ...record.result, previews: recoveredPreviews };
|
|
3287
|
+
}
|
|
3017
3288
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3018
3289
|
return;
|
|
3019
3290
|
if (!record.dryRun) {
|
|
3291
|
+
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3020
3292
|
for (const tracked of record.agents.values()) {
|
|
3021
3293
|
const owned = tracked.spec.ownedPullRequest;
|
|
3022
3294
|
if (tracked.spec.role !== 'babysitter' || !owned)
|
|
@@ -3041,6 +3313,31 @@ export class FactoryLoop {
|
|
|
3041
3313
|
labels.has('factory:in-progress') &&
|
|
3042
3314
|
!labels.has('factory:human-review');
|
|
3043
3315
|
}
|
|
3316
|
+
#isIssueExternallyTerminal(issue) {
|
|
3317
|
+
if (isGithubIssue(issue)) {
|
|
3318
|
+
if (githubFactoryIssueIsClosed(issue))
|
|
3319
|
+
return true;
|
|
3320
|
+
return issue.labels.some((label) => label.trim().toLowerCase() === 'factory:human-review');
|
|
3321
|
+
}
|
|
3322
|
+
const role = this.#states.roleOf(issue.stateId);
|
|
3323
|
+
return role === 'humanReview' || role === 'done';
|
|
3324
|
+
}
|
|
3325
|
+
async #hasRestorableMergedBabysitterSession(issue) {
|
|
3326
|
+
const wanted = issueKey(issue);
|
|
3327
|
+
for (const [, session] of await this.#state.listBabysitterSessions(this.#workspaceId)) {
|
|
3328
|
+
if (issueKey(session.issue) !== wanted ||
|
|
3329
|
+
!validGithubRepo(session.repo) ||
|
|
3330
|
+
!validPrNumber(session.prNumber) ||
|
|
3331
|
+
!session.agentName)
|
|
3332
|
+
continue;
|
|
3333
|
+
const snapshot = await this.#readPrSnapshot(session);
|
|
3334
|
+
if (snapshot &&
|
|
3335
|
+
prMetaShowsMerged(snapshot) &&
|
|
3336
|
+
prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30)
|
|
3337
|
+
return true;
|
|
3338
|
+
}
|
|
3339
|
+
return false;
|
|
3340
|
+
}
|
|
3044
3341
|
async #abandonDurableResume(record, reason) {
|
|
3045
3342
|
const handoffs = this.#dispatchFailureHandoffs(record, [...record.agents].map(([name, tracked]) => ({
|
|
3046
3343
|
issue: record.issue,
|
|
@@ -3049,25 +3346,32 @@ export class FactoryLoop {
|
|
|
3049
3346
|
persistedAtMs: this.#clock.now(),
|
|
3050
3347
|
})));
|
|
3051
3348
|
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
batch.abandon(record.issue);
|
|
3057
|
-
for (const [name] of record.agents) {
|
|
3058
|
-
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3059
|
-
}
|
|
3349
|
+
// Keep the lifecycle nonterminal until every externally reachable route
|
|
3350
|
+
// is confirmed gone. A restart can then retry cleanup instead of treating
|
|
3351
|
+
// an abandoned row as finished and leaking its issue preview forever.
|
|
3352
|
+
await this.#teardownPreviews(record);
|
|
3060
3353
|
if (handoffs.some((handoff) => handoff.worktree)) {
|
|
3061
|
-
await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')
|
|
3354
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')) {
|
|
3355
|
+
throw new Error(`Unable to finish stale dispatch worktree teardown for ${record.issue.key}`);
|
|
3356
|
+
}
|
|
3062
3357
|
}
|
|
3063
3358
|
else if (handoffs.length > 0) {
|
|
3064
3359
|
const failed = new Set(await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion'));
|
|
3360
|
+
if (failed.size > 0) {
|
|
3361
|
+
throw new Error(`Unable to release stale dispatch agents for ${record.issue.key}: ${[...failed].join(', ')}`);
|
|
3362
|
+
}
|
|
3065
3363
|
for (const handoff of handoffs) {
|
|
3066
|
-
if (failed.has(handoff.name))
|
|
3067
|
-
continue;
|
|
3068
3364
|
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
3069
3365
|
}
|
|
3070
3366
|
}
|
|
3367
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'source_state_changed' }))
|
|
3368
|
+
return;
|
|
3369
|
+
await this.#recordDispatchTerminal(record.issue);
|
|
3370
|
+
const batch = await this.#batch();
|
|
3371
|
+
batch.abandon(record.issue);
|
|
3372
|
+
for (const [name] of record.agents) {
|
|
3373
|
+
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3374
|
+
}
|
|
3071
3375
|
await this.#stopSlackWatcher(record.issue);
|
|
3072
3376
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
3073
3377
|
await this.#writeInFlightRegistry();
|
|
@@ -3082,6 +3386,16 @@ export class FactoryLoop {
|
|
|
3082
3386
|
const batch = await this.#batch();
|
|
3083
3387
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
3084
3388
|
const releaseKey = issueKey(record.issue);
|
|
3389
|
+
// Terminal writeback has already been acknowledged before this method is
|
|
3390
|
+
// entered. Remove externally reachable routes first so a stuck agent
|
|
3391
|
+
// release cannot leave a preview live after Human Review or Done.
|
|
3392
|
+
try {
|
|
3393
|
+
await this.#teardownPreviews(record);
|
|
3394
|
+
}
|
|
3395
|
+
catch {
|
|
3396
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
3397
|
+
return false;
|
|
3398
|
+
}
|
|
3085
3399
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, releaseKey);
|
|
3086
3400
|
const released = new Set(lifecycle?.agents
|
|
3087
3401
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
@@ -4567,6 +4881,7 @@ export class FactoryLoop {
|
|
|
4567
4881
|
if (isCompletionReason(reason)) {
|
|
4568
4882
|
if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
|
|
4569
4883
|
openOnly: this.#config.babysitter.enabled,
|
|
4884
|
+
preferExactBranch: tracingReconciledExit,
|
|
4570
4885
|
}, exiting)) {
|
|
4571
4886
|
if (this.#config.babysitter.enabled)
|
|
4572
4887
|
await this.#ensureBabysitterForIssue(record);
|
|
@@ -4624,6 +4939,7 @@ export class FactoryLoop {
|
|
|
4624
4939
|
const hasCompletionPr = tracked.spec.role === 'implementer'
|
|
4625
4940
|
? await this.#issueHasCompletionPr(record, {
|
|
4626
4941
|
openOnly: this.#config.babysitter.enabled,
|
|
4942
|
+
preferExactBranch: tracingReconciledExit,
|
|
4627
4943
|
}, tracked)
|
|
4628
4944
|
: false;
|
|
4629
4945
|
if (tracingReconciledExit) {
|
|
@@ -4729,30 +5045,43 @@ export class FactoryLoop {
|
|
|
4729
5045
|
return;
|
|
4730
5046
|
}
|
|
4731
5047
|
}
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
5048
|
+
// Bound recovery by the durable logical agent, not by the session ref.
|
|
5049
|
+
// Harnesses may return a fresh session ref from every successful resume,
|
|
5050
|
+
// and no-sessionRef workers use respawn instead. Keying only resumptions
|
|
5051
|
+
// by the changing ref (and not recording respawns at all) let an agent
|
|
5052
|
+
// that exited immediately consume a batch slot forever while Factory
|
|
5053
|
+
// continuously recreated it. One successful recovery is enough; a
|
|
5054
|
+
// subsequent no-PR implementer exit is terminal and frees the slot.
|
|
5055
|
+
const recoveryLifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
5056
|
+
const recoveryRunIdentity = recoveryLifecycle?.runId
|
|
5057
|
+
?? tracked.spec.branch
|
|
5058
|
+
?? batch.invocationIdFor(record.issue, tracked.spec);
|
|
5059
|
+
const recoveryKey = `${recoveryRunIdentity}:${tracked.spec.name}`;
|
|
5060
|
+
if (await this.#state.isResumed(this.#workspaceId, recoveryKey)) {
|
|
5061
|
+
if (tracked.spec.role === 'implementer') {
|
|
5062
|
+
await this.#concludeTerminalImplementer(record, name, 'stalled-no-pr');
|
|
4745
5063
|
}
|
|
4746
|
-
|
|
4747
|
-
|
|
5064
|
+
return;
|
|
5065
|
+
}
|
|
5066
|
+
const existing = this.#resumeInFlight.get(recoveryKey);
|
|
5067
|
+
if (existing) {
|
|
5068
|
+
try {
|
|
4748
5069
|
await existing;
|
|
4749
|
-
return;
|
|
4750
5070
|
}
|
|
5071
|
+
catch {
|
|
5072
|
+
// The initiating recovery handler owns classification and logging.
|
|
5073
|
+
// Followers only coalesce onto its lifetime and must not re-report
|
|
5074
|
+
// the same failure.
|
|
5075
|
+
}
|
|
5076
|
+
return;
|
|
5077
|
+
}
|
|
5078
|
+
let recovered = false;
|
|
5079
|
+
if (tracked.sessionRef) {
|
|
4751
5080
|
const resume = this.#resumeTrackedAgent(record, name, tracked);
|
|
4752
|
-
this.#resumeInFlight.set(
|
|
5081
|
+
this.#resumeInFlight.set(recoveryKey, resume);
|
|
4753
5082
|
try {
|
|
4754
5083
|
await resume;
|
|
4755
|
-
await this.#state.markResumed(this.#workspaceId,
|
|
5084
|
+
await this.#state.markResumed(this.#workspaceId, recoveryKey);
|
|
4756
5085
|
recovered = true;
|
|
4757
5086
|
}
|
|
4758
5087
|
catch (error) {
|
|
@@ -4760,11 +5089,11 @@ export class FactoryLoop {
|
|
|
4760
5089
|
// The broker never released this agent's name on exit
|
|
4761
5090
|
// (relay#1116-family), so re-registering collides with the stuck
|
|
4762
5091
|
// name. Retrying just re-collides forever. Treat it as terminal for
|
|
4763
|
-
// this name: record the
|
|
5092
|
+
// this name: record the recovery key so subsequent exit events
|
|
4764
5093
|
// short-circuit, count it, and warn once. The external reaper / a
|
|
4765
5094
|
// broker restart reclaims the leaked name.
|
|
4766
5095
|
this.#fleet.markAgentTerminal?.(name, 'resume-already-exists');
|
|
4767
|
-
await this.#state.markResumed(this.#workspaceId,
|
|
5096
|
+
await this.#state.markResumed(this.#workspaceId, recoveryKey);
|
|
4768
5097
|
this.#increment('resumeNameCollisions');
|
|
4769
5098
|
this.#logger.warn?.('[factory] resume skipped: broker still holds agent name (relay#1116); not retrying', {
|
|
4770
5099
|
issue: record.issue.key,
|
|
@@ -4783,12 +5112,12 @@ export class FactoryLoop {
|
|
|
4783
5112
|
}
|
|
4784
5113
|
}
|
|
4785
5114
|
finally {
|
|
4786
|
-
this.#resumeInFlight.delete(
|
|
5115
|
+
this.#resumeInFlight.delete(recoveryKey);
|
|
4787
5116
|
}
|
|
4788
5117
|
}
|
|
4789
5118
|
else {
|
|
4790
5119
|
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:restart:${this.#clock.now()}`;
|
|
4791
|
-
|
|
5120
|
+
const respawn = (async () => {
|
|
4792
5121
|
await this.#prepareAgentWorktree(record, tracked.spec);
|
|
4793
5122
|
const result = await this.#fleet.spawn({
|
|
4794
5123
|
name: tracked.spec.name,
|
|
@@ -4804,6 +5133,11 @@ export class FactoryLoop {
|
|
|
4804
5133
|
channel: tracked.spec.channel,
|
|
4805
5134
|
});
|
|
4806
5135
|
batch.recordSpawn(record, tracked.spec, invocationId, result);
|
|
5136
|
+
})();
|
|
5137
|
+
this.#resumeInFlight.set(recoveryKey, respawn);
|
|
5138
|
+
try {
|
|
5139
|
+
await respawn;
|
|
5140
|
+
await this.#state.markResumed(this.#workspaceId, recoveryKey);
|
|
4807
5141
|
recovered = true;
|
|
4808
5142
|
}
|
|
4809
5143
|
catch (error) {
|
|
@@ -4817,6 +5151,7 @@ export class FactoryLoop {
|
|
|
4817
5151
|
// the dispatch so a reviewer blocked on the implementer's DM does not
|
|
4818
5152
|
// hang the owned-broker dispose-wait (#67).
|
|
4819
5153
|
this.#fleet.markAgentTerminal?.(name, 'respawn-already-exists');
|
|
5154
|
+
await this.#state.markResumed(this.#workspaceId, recoveryKey);
|
|
4820
5155
|
this.#increment('resumeNameCollisions');
|
|
4821
5156
|
this.#logger.warn?.('[factory] respawn skipped: broker still holds agent name (relay#1116); not retrying', {
|
|
4822
5157
|
issue: record.issue.key,
|
|
@@ -4826,6 +5161,9 @@ export class FactoryLoop {
|
|
|
4826
5161
|
await this.#concludeTerminalImplementer(record, name, 'respawn-already-exists');
|
|
4827
5162
|
}
|
|
4828
5163
|
}
|
|
5164
|
+
finally {
|
|
5165
|
+
this.#resumeInFlight.delete(recoveryKey);
|
|
5166
|
+
}
|
|
4829
5167
|
}
|
|
4830
5168
|
if (recovered && tracked.spec.role === 'implementer') {
|
|
4831
5169
|
await this.#writeInFlightRegistry();
|
|
@@ -4954,7 +5292,7 @@ export class FactoryLoop {
|
|
|
4954
5292
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
4955
5293
|
baseRef,
|
|
4956
5294
|
title: `${issue.key}: ${issue.title}`,
|
|
4957
|
-
body: githubPullRequestBody(issue),
|
|
5295
|
+
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
4958
5296
|
});
|
|
4959
5297
|
const published = result.author
|
|
4960
5298
|
? result
|
|
@@ -5137,9 +5475,42 @@ export class FactoryLoop {
|
|
|
5137
5475
|
if (worktree)
|
|
5138
5476
|
unique.set(worktree.worktreePath, worktree);
|
|
5139
5477
|
}
|
|
5478
|
+
const issueSlug = factoryWorktreeIssueSlug(record.issue.key);
|
|
5140
5479
|
const failures = [];
|
|
5480
|
+
for (const repository of this.#worktreeRepositories(record)) {
|
|
5481
|
+
try {
|
|
5482
|
+
const candidates = await this.#worktrees.listWorktrees(repository);
|
|
5483
|
+
for (const candidate of candidates) {
|
|
5484
|
+
if (factoryWorktreeIssueSlug(candidate.issueKey) === issueSlug) {
|
|
5485
|
+
unique.set(candidate.worktreePath, candidate);
|
|
5486
|
+
}
|
|
5487
|
+
}
|
|
5488
|
+
}
|
|
5489
|
+
catch (error) {
|
|
5490
|
+
const message = `${repository.baseClonePath}: ${describeError(error).errorMessage}`;
|
|
5491
|
+
failures.push(message);
|
|
5492
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5493
|
+
this.#logger.warn?.('[factory] failed to enumerate completed issue worktrees', {
|
|
5494
|
+
issue: record.issue.key,
|
|
5495
|
+
repo: repository.repo,
|
|
5496
|
+
baseClonePath: repository.baseClonePath,
|
|
5497
|
+
error: describeError(error).errorMessage,
|
|
5498
|
+
});
|
|
5499
|
+
}
|
|
5500
|
+
}
|
|
5141
5501
|
for (const worktree of unique.values()) {
|
|
5142
5502
|
try {
|
|
5503
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5504
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5505
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5506
|
+
this.#logger.warn?.('[factory] retained completed issue worktree with local state', {
|
|
5507
|
+
issue: record.issue.key,
|
|
5508
|
+
repo: worktree.repo,
|
|
5509
|
+
worktreePath: worktree.worktreePath,
|
|
5510
|
+
retentionReasons: inspection.retentionReasons,
|
|
5511
|
+
});
|
|
5512
|
+
continue;
|
|
5513
|
+
}
|
|
5143
5514
|
await this.#worktrees.cleanup(worktree);
|
|
5144
5515
|
this.#increment('agentWorktreesCleaned');
|
|
5145
5516
|
}
|
|
@@ -5158,6 +5529,132 @@ export class FactoryLoop {
|
|
|
5158
5529
|
throw new Error(`Factory worktree cleanup incomplete for ${record.issue.key}: ${failures.join('; ')}`);
|
|
5159
5530
|
}
|
|
5160
5531
|
}
|
|
5532
|
+
#worktreeRepositories(record) {
|
|
5533
|
+
const repositories = new Map();
|
|
5534
|
+
const add = (repo, baseClonePath) => {
|
|
5535
|
+
if (!baseClonePath)
|
|
5536
|
+
return;
|
|
5537
|
+
const key = `${repo}\u0000${resolve(baseClonePath)}`;
|
|
5538
|
+
repositories.set(key, { repo, baseClonePath });
|
|
5539
|
+
};
|
|
5540
|
+
if (record) {
|
|
5541
|
+
for (const route of record.decision.routes) {
|
|
5542
|
+
add(route.repo, this.#config.repos.clonePaths[route.repo] ?? route.clonePath);
|
|
5543
|
+
}
|
|
5544
|
+
for (const tracked of record.agents.values()) {
|
|
5545
|
+
add(tracked.spec.repo, tracked.spec.baseClonePath);
|
|
5546
|
+
}
|
|
5547
|
+
for (const spec of record.decision.implementers) {
|
|
5548
|
+
add(spec.repo, spec.baseClonePath);
|
|
5549
|
+
}
|
|
5550
|
+
}
|
|
5551
|
+
else {
|
|
5552
|
+
for (const [repo, baseClonePath] of Object.entries(this.#config.repos.clonePaths)) {
|
|
5553
|
+
add(repo, baseClonePath);
|
|
5554
|
+
}
|
|
5555
|
+
}
|
|
5556
|
+
return [...repositories.values()];
|
|
5557
|
+
}
|
|
5558
|
+
async #reapOrphanedWorktreesOnStartup(legacyRegistry) {
|
|
5559
|
+
if (!this.#worktrees)
|
|
5560
|
+
return;
|
|
5561
|
+
const legacyAgents = (legacyRegistry?.agents ?? []).filter((agent) => agent.issue);
|
|
5562
|
+
let durableLifecycles;
|
|
5563
|
+
let waitingClarifications;
|
|
5564
|
+
let onlineAgentNames;
|
|
5565
|
+
try {
|
|
5566
|
+
const [lifecycles, clarifications, roster] = await Promise.all([
|
|
5567
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
5568
|
+
this.#state.listWaitingClarifications(this.#workspaceId),
|
|
5569
|
+
legacyAgents.length > 0 ? this.#fleet.roster() : undefined,
|
|
5570
|
+
]);
|
|
5571
|
+
durableLifecycles = lifecycles;
|
|
5572
|
+
waitingClarifications = clarifications;
|
|
5573
|
+
onlineAgentNames = new Set((roster?.agents ?? []).map((agent) => agent.name));
|
|
5574
|
+
}
|
|
5575
|
+
catch (error) {
|
|
5576
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5577
|
+
this.#logger.warn?.('[factory] startup worktree reaper skipped because active lifecycle or roster state could not be loaded', {
|
|
5578
|
+
error: describeError(error).errorMessage,
|
|
5579
|
+
});
|
|
5580
|
+
return;
|
|
5581
|
+
}
|
|
5582
|
+
const activeIssueSlugs = new Set((await this.#batch()).inFlight.map((record) => factoryWorktreeIssueSlug(record.issue.key)));
|
|
5583
|
+
for (const [, lifecycle] of durableLifecycles) {
|
|
5584
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
5585
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(lifecycle.issue.key));
|
|
5586
|
+
}
|
|
5587
|
+
}
|
|
5588
|
+
for (const [, waiting] of waitingClarifications) {
|
|
5589
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(waiting.issue.key));
|
|
5590
|
+
}
|
|
5591
|
+
for (const agent of legacyAgents) {
|
|
5592
|
+
if (agent.issue && onlineAgentNames.has(agent.name)) {
|
|
5593
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(agent.issue.key));
|
|
5594
|
+
}
|
|
5595
|
+
}
|
|
5596
|
+
const candidates = new Map();
|
|
5597
|
+
let reaped = 0;
|
|
5598
|
+
let reclaimedBytes = 0;
|
|
5599
|
+
let retained = 0;
|
|
5600
|
+
let failures = 0;
|
|
5601
|
+
for (const repository of this.#worktreeRepositories()) {
|
|
5602
|
+
try {
|
|
5603
|
+
for (const candidate of await this.#worktrees.listWorktrees(repository)) {
|
|
5604
|
+
candidates.set(candidate.worktreePath, candidate);
|
|
5605
|
+
}
|
|
5606
|
+
}
|
|
5607
|
+
catch (error) {
|
|
5608
|
+
failures += 1;
|
|
5609
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5610
|
+
this.#logger.warn?.('[factory] startup worktree reaper failed to enumerate repository', {
|
|
5611
|
+
repo: repository.repo,
|
|
5612
|
+
baseClonePath: repository.baseClonePath,
|
|
5613
|
+
error: describeError(error).errorMessage,
|
|
5614
|
+
});
|
|
5615
|
+
}
|
|
5616
|
+
}
|
|
5617
|
+
for (const worktree of candidates.values()) {
|
|
5618
|
+
if (activeIssueSlugs.has(factoryWorktreeIssueSlug(worktree.issueKey)))
|
|
5619
|
+
continue;
|
|
5620
|
+
try {
|
|
5621
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5622
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5623
|
+
retained += 1;
|
|
5624
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5625
|
+
this.#logger.warn?.('[factory] retained startup orphan worktree with local state', {
|
|
5626
|
+
issue: worktree.issueKey,
|
|
5627
|
+
repo: worktree.repo,
|
|
5628
|
+
worktreePath: worktree.worktreePath,
|
|
5629
|
+
retentionReasons: inspection.retentionReasons,
|
|
5630
|
+
});
|
|
5631
|
+
continue;
|
|
5632
|
+
}
|
|
5633
|
+
await this.#worktrees.cleanup(worktree);
|
|
5634
|
+
reaped += 1;
|
|
5635
|
+
reclaimedBytes += inspection.bytes;
|
|
5636
|
+
this.#increment('agentWorktreesCleaned');
|
|
5637
|
+
this.#increment('agentWorktreesReapedOnStartup');
|
|
5638
|
+
}
|
|
5639
|
+
catch (error) {
|
|
5640
|
+
failures += 1;
|
|
5641
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5642
|
+
this.#logger.warn?.('[factory] startup worktree reaper retained checkout after cleanup failure', {
|
|
5643
|
+
issue: worktree.issueKey,
|
|
5644
|
+
repo: worktree.repo,
|
|
5645
|
+
worktreePath: worktree.worktreePath,
|
|
5646
|
+
error: describeError(error).errorMessage,
|
|
5647
|
+
});
|
|
5648
|
+
}
|
|
5649
|
+
}
|
|
5650
|
+
this.#logger.info?.('[factory] startup worktree reaper completed', {
|
|
5651
|
+
reaped,
|
|
5652
|
+
reclaimedBytes,
|
|
5653
|
+
reclaimed: formatByteCount(reclaimedBytes),
|
|
5654
|
+
retained,
|
|
5655
|
+
failures,
|
|
5656
|
+
});
|
|
5657
|
+
}
|
|
5161
5658
|
async #confirmPublishedRemotePullRequest(repo, result, expectedHeadRef) {
|
|
5162
5659
|
const parts = githubRepoParts(repo);
|
|
5163
5660
|
if (!parts)
|
|
@@ -5302,6 +5799,30 @@ export class FactoryLoop {
|
|
|
5302
5799
|
// the release-driven exit event so it cannot re-trigger a resume before the
|
|
5303
5800
|
// record leaves the batch.
|
|
5304
5801
|
async #abandonStuckDispatch(record, reason) {
|
|
5802
|
+
const key = issueKey(record.issue);
|
|
5803
|
+
this.#abandonedDispatchReasons.set(key, reason);
|
|
5804
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5805
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5806
|
+
// The generic durable retry can recover the in-memory reason in this
|
|
5807
|
+
// process; after restart the persisted `abandoning` phase is the fence.
|
|
5808
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5809
|
+
return;
|
|
5810
|
+
}
|
|
5811
|
+
try {
|
|
5812
|
+
// Remove externally reachable routes before releasing the agents that
|
|
5813
|
+
// could still be serving the upstream. A failed provider teardown keeps
|
|
5814
|
+
// the durable lifecycle retryable instead of terminalizing a leak.
|
|
5815
|
+
await this.#teardownPreviews(record);
|
|
5816
|
+
}
|
|
5817
|
+
catch (error) {
|
|
5818
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5819
|
+
this.#logger.warn?.('[factory] abandoned dispatch preview teardown failed; retrying', {
|
|
5820
|
+
issue: record.issue.key,
|
|
5821
|
+
error: describeError(error).errorMessage,
|
|
5822
|
+
});
|
|
5823
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5824
|
+
return;
|
|
5825
|
+
}
|
|
5305
5826
|
const agents = [...record.agents];
|
|
5306
5827
|
for (const [agentName, tracked] of agents) {
|
|
5307
5828
|
if (tracked.spec.role === 'implementer')
|
|
@@ -5334,6 +5855,19 @@ export class FactoryLoop {
|
|
|
5334
5855
|
await this.#writeInFlightRegistry();
|
|
5335
5856
|
return;
|
|
5336
5857
|
}
|
|
5858
|
+
// Batch completion alone only frees the process-local slot. Durable
|
|
5859
|
+
// capacity is computed from lifecycle phases, so commit the terminal phase
|
|
5860
|
+
// only after the preview, agents, and worktrees have all been cleaned up.
|
|
5861
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5862
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5863
|
+
// #saveDispatchLifecycle already schedules the generic durable retry. The
|
|
5864
|
+
// pending reason makes an in-process retry return here, while the durable
|
|
5865
|
+
// `abandoning` phase provides the same recovery guarantee after restart.
|
|
5866
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5867
|
+
await this.#writeInFlightRegistry();
|
|
5868
|
+
return;
|
|
5869
|
+
}
|
|
5870
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
5337
5871
|
await this.#recordDispatchTerminal(record.issue);
|
|
5338
5872
|
const next = (await this.#batch()).complete(record.issue);
|
|
5339
5873
|
await this.#drainReadyClarificationWake();
|
|
@@ -5367,7 +5901,12 @@ export class FactoryLoop {
|
|
|
5367
5901
|
if (!issue) {
|
|
5368
5902
|
return false;
|
|
5369
5903
|
}
|
|
5370
|
-
|
|
5904
|
+
// During startup recovery, the implementer's deterministic branch is the
|
|
5905
|
+
// strongest and cheapest lookup. Avoid scanning mounted PR metadata across
|
|
5906
|
+
// every configured repository. Normal event-driven exits keep using the
|
|
5907
|
+
// webhook-fed mount so they do not introduce a GitHub API dependency.
|
|
5908
|
+
if (implementer?.spec.branch &&
|
|
5909
|
+
(opts.preferExactBranch || record.decision.implementers.length > 1)) {
|
|
5371
5910
|
const sourceOwner = record.issue.path
|
|
5372
5911
|
? githubIssuePathParts(record.issue.path)?.owner
|
|
5373
5912
|
: undefined;
|
|
@@ -6952,11 +7491,12 @@ export class FactoryLoop {
|
|
|
6952
7491
|
const integrationInstructions = await this.#resolveIntegrationInstructions();
|
|
6953
7492
|
const render = async (spec) => {
|
|
6954
7493
|
const route = routeForSpec(decision, spec);
|
|
7494
|
+
const previewUrl = previewUrlFromSpec(spec);
|
|
6955
7495
|
const testGuidance = await resolveTestGuidance({
|
|
6956
7496
|
repoPath: route.clonePath,
|
|
6957
7497
|
issue: templateIssue,
|
|
6958
7498
|
route,
|
|
6959
|
-
previewUrl
|
|
7499
|
+
previewUrl,
|
|
6960
7500
|
});
|
|
6961
7501
|
return {
|
|
6962
7502
|
...spec,
|
|
@@ -6973,6 +7513,11 @@ export class FactoryLoop {
|
|
|
6973
7513
|
branchName: spec.branch ?? decision.implementers.find((candidate) => candidate.repo === spec.repo)?.branch,
|
|
6974
7514
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
6975
7515
|
agentName: spec.name,
|
|
7516
|
+
...(previewUrl ? {
|
|
7517
|
+
previewUrl,
|
|
7518
|
+
previewTargetPort: spec.preview?.targetPort,
|
|
7519
|
+
previewStartCommand: spec.preview?.startCommand,
|
|
7520
|
+
} : {}),
|
|
6976
7521
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
6977
7522
|
}),
|
|
6978
7523
|
};
|
|
@@ -6984,6 +7529,193 @@ export class FactoryLoop {
|
|
|
6984
7529
|
reviewer: rendered[decision.implementers.length],
|
|
6985
7530
|
};
|
|
6986
7531
|
}
|
|
7532
|
+
async #withPreviewReferences(decision) {
|
|
7533
|
+
if (!this.#config.preview || decision.scope === 'workflow')
|
|
7534
|
+
return decision;
|
|
7535
|
+
if (!this.#fleet.createPreview) {
|
|
7536
|
+
throw new Error('Preview services are configured but the selected fleet backend cannot create previews');
|
|
7537
|
+
}
|
|
7538
|
+
const owner = issueKey(decision.issue);
|
|
7539
|
+
const byRepo = new Map();
|
|
7540
|
+
for (const implementer of decision.implementers) {
|
|
7541
|
+
const service = previewServiceForRepo(this.#config, implementer.repo);
|
|
7542
|
+
if (byRepo.has(implementer.repo))
|
|
7543
|
+
continue;
|
|
7544
|
+
const persisted = implementer.preview;
|
|
7545
|
+
if (!service) {
|
|
7546
|
+
if (persisted)
|
|
7547
|
+
byRepo.set(implementer.repo, persisted);
|
|
7548
|
+
continue;
|
|
7549
|
+
}
|
|
7550
|
+
if (!implementer.clonePath) {
|
|
7551
|
+
throw new Error(`Preview service ${service.name} requires a configured checkout for ${implementer.repo}`);
|
|
7552
|
+
}
|
|
7553
|
+
await this.#preparePreviewCheckout(decision, implementer);
|
|
7554
|
+
const preview = await this.#fleet.createPreview({
|
|
7555
|
+
namespace: this.#workspaceId,
|
|
7556
|
+
owner,
|
|
7557
|
+
issueKey: decision.issue.key,
|
|
7558
|
+
service: service.name,
|
|
7559
|
+
repo: implementer.repo,
|
|
7560
|
+
targetPort: service.config.port,
|
|
7561
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7562
|
+
startCommand: service.config.startCommand,
|
|
7563
|
+
checkoutPath: implementer.clonePath,
|
|
7564
|
+
node: persisted?.node ?? implementer.node,
|
|
7565
|
+
});
|
|
7566
|
+
this.#removedPreviewIds.delete(preview.id);
|
|
7567
|
+
byRepo.set(implementer.repo, preview);
|
|
7568
|
+
// Make the provider identity visible immediately. If validation or the
|
|
7569
|
+
// following durable save fails, the caller can roll it back while it
|
|
7570
|
+
// still owns the lifecycle; after fence loss the startup sweep owns it.
|
|
7571
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7572
|
+
assertPublishablePreview(preview, {
|
|
7573
|
+
namespace: this.#workspaceId,
|
|
7574
|
+
owner,
|
|
7575
|
+
service: service.name,
|
|
7576
|
+
repo: implementer.repo,
|
|
7577
|
+
targetPort: service.config.port,
|
|
7578
|
+
portSpan: service.config.portSpan ?? 100,
|
|
7579
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7580
|
+
startCommand: service.config.startCommand,
|
|
7581
|
+
checkoutPath: implementer.clonePath,
|
|
7582
|
+
requireNode: this.#fleet.placementLocality === 'remote',
|
|
7583
|
+
});
|
|
7584
|
+
}
|
|
7585
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7586
|
+
return {
|
|
7587
|
+
...decision,
|
|
7588
|
+
implementers: decision.implementers.map((spec) => specWithPreview(spec, byRepo.get(spec.repo))),
|
|
7589
|
+
reviewer: specWithPreview(decision.reviewer, byRepo.get(decision.reviewer.repo)),
|
|
7590
|
+
};
|
|
7591
|
+
}
|
|
7592
|
+
async #preparePreviewCheckout(decision, spec) {
|
|
7593
|
+
if (!this.#worktrees ||
|
|
7594
|
+
!spec.baseClonePath ||
|
|
7595
|
+
!spec.clonePath ||
|
|
7596
|
+
spec.baseClonePath === spec.clonePath ||
|
|
7597
|
+
!spec.branch)
|
|
7598
|
+
return;
|
|
7599
|
+
try {
|
|
7600
|
+
await this.#worktrees.prepare({
|
|
7601
|
+
repo: spec.repo,
|
|
7602
|
+
issueKey: decision.issue.key,
|
|
7603
|
+
baseClonePath: spec.baseClonePath,
|
|
7604
|
+
worktreePath: spec.clonePath,
|
|
7605
|
+
branch: spec.branch,
|
|
7606
|
+
...(spec.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
7607
|
+
});
|
|
7608
|
+
this.#increment('agentWorktreesPrepared');
|
|
7609
|
+
}
|
|
7610
|
+
catch (error) {
|
|
7611
|
+
throw contextualError(`Unable to prepare preview checkout for ${decision.issue.key}/${spec.repo} at ${spec.clonePath}`, error);
|
|
7612
|
+
}
|
|
7613
|
+
}
|
|
7614
|
+
async #teardownPreviews(record) {
|
|
7615
|
+
const previews = uniquePreviewReferences([
|
|
7616
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7617
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7618
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
7619
|
+
]);
|
|
7620
|
+
if (previews.length === 0)
|
|
7621
|
+
return;
|
|
7622
|
+
await this.#teardownPreviewReferences(previews);
|
|
7623
|
+
this.#previewReferences.delete(issueKey(record.issue));
|
|
7624
|
+
}
|
|
7625
|
+
async #teardownPreviewReferences(references) {
|
|
7626
|
+
const previews = uniquePreviewReferences(references)
|
|
7627
|
+
.filter((preview) => !this.#removedPreviewIds.has(preview.id));
|
|
7628
|
+
if (previews.length === 0)
|
|
7629
|
+
return;
|
|
7630
|
+
if (!this.#fleet.removePreview) {
|
|
7631
|
+
throw new Error('Fleet backend cannot remove its configured previews');
|
|
7632
|
+
}
|
|
7633
|
+
const results = await Promise.allSettled(previews.map(async (preview) => await this.#fleet.removePreview(preview)));
|
|
7634
|
+
const failures = results.flatMap((result, index) => {
|
|
7635
|
+
const preview = previews[index];
|
|
7636
|
+
if (result.status === 'rejected')
|
|
7637
|
+
return [{ preview, reason: result.reason }];
|
|
7638
|
+
if (!result.value) {
|
|
7639
|
+
return [{
|
|
7640
|
+
preview,
|
|
7641
|
+
reason: new Error(`Preview provider could not confirm removal of ${preview.id}`),
|
|
7642
|
+
}];
|
|
7643
|
+
}
|
|
7644
|
+
this.#removedPreviewIds.add(preview.id);
|
|
7645
|
+
return [];
|
|
7646
|
+
});
|
|
7647
|
+
if (failures.length > 0) {
|
|
7648
|
+
this.#logger.warn?.('[factory] preview teardown failed', {
|
|
7649
|
+
owners: [...new Set(failures.map(({ preview }) => preview.owner))],
|
|
7650
|
+
previews: failures.map(({ preview }) => preview.id),
|
|
7651
|
+
});
|
|
7652
|
+
throw new AggregateError(failures.map(({ reason }) => reason), 'Unable to tear down every issue preview');
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7655
|
+
async #reapPreviewOrphans() {
|
|
7656
|
+
if (!this.#config.preview || !this.#fleet.reapPreviews)
|
|
7657
|
+
return;
|
|
7658
|
+
const [lifecycles, batch] = await Promise.all([
|
|
7659
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
7660
|
+
this.#batch(),
|
|
7661
|
+
]);
|
|
7662
|
+
const activePreviewIds = new Set();
|
|
7663
|
+
const activeOwners = new Set(lifecycles
|
|
7664
|
+
.map(([, lifecycle]) => lifecycle)
|
|
7665
|
+
.filter((lifecycle) => !isTerminalDispatchLifecycle(lifecycle))
|
|
7666
|
+
.map((lifecycle) => {
|
|
7667
|
+
for (const preview of uniquePreviewReferences([
|
|
7668
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
7669
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
7670
|
+
]))
|
|
7671
|
+
activePreviewIds.add(preview.id);
|
|
7672
|
+
return issueKey(lifecycle.issue);
|
|
7673
|
+
}));
|
|
7674
|
+
for (const record of batch.inFlight) {
|
|
7675
|
+
activeOwners.add(issueKey(record.issue));
|
|
7676
|
+
for (const preview of uniquePreviewReferences([
|
|
7677
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7678
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7679
|
+
]))
|
|
7680
|
+
activePreviewIds.add(preview.id);
|
|
7681
|
+
}
|
|
7682
|
+
const report = await this.#fleet.reapPreviews({
|
|
7683
|
+
namespace: this.#workspaceId,
|
|
7684
|
+
activeOwners: [...activeOwners],
|
|
7685
|
+
activePreviewIds: [...activePreviewIds],
|
|
7686
|
+
});
|
|
7687
|
+
if (report.reaped.length > 0 || report.skipped.length > 0) {
|
|
7688
|
+
this.#logger.info?.('[factory] preview orphan sweep completed', {
|
|
7689
|
+
reaped: report.reaped.map((preview) => preview.id),
|
|
7690
|
+
skipped: report.skipped,
|
|
7691
|
+
});
|
|
7692
|
+
}
|
|
7693
|
+
}
|
|
7694
|
+
#schedulePreviewSweep(delayMs = PREVIEW_SWEEP_INTERVAL_MS) {
|
|
7695
|
+
if (this.#stopping ||
|
|
7696
|
+
!this.#started ||
|
|
7697
|
+
!this.#config.preview ||
|
|
7698
|
+
!this.#fleet.reapPreviews ||
|
|
7699
|
+
this.#previewSweepTimer ||
|
|
7700
|
+
this.#previewSweepInFlight)
|
|
7701
|
+
return;
|
|
7702
|
+
this.#previewSweepTimer = setTimeout(() => {
|
|
7703
|
+
this.#previewSweepTimer = undefined;
|
|
7704
|
+
if (this.#stopping || !this.#started)
|
|
7705
|
+
return;
|
|
7706
|
+
this.#previewSweepInFlight = this.#reapPreviewOrphans()
|
|
7707
|
+
.catch((error) => {
|
|
7708
|
+
this.#logger.warn?.('[factory] periodic preview orphan sweep failed', {
|
|
7709
|
+
error: describeError(error).errorMessage,
|
|
7710
|
+
});
|
|
7711
|
+
})
|
|
7712
|
+
.finally(() => {
|
|
7713
|
+
this.#previewSweepInFlight = undefined;
|
|
7714
|
+
this.#schedulePreviewSweep();
|
|
7715
|
+
});
|
|
7716
|
+
}, delayMs);
|
|
7717
|
+
this.#previewSweepTimer.unref?.();
|
|
7718
|
+
}
|
|
6987
7719
|
#consumePendingDispatchClarifications(issue) {
|
|
6988
7720
|
const key = issueKey(issue);
|
|
6989
7721
|
this.#pendingSlackClarifications.delete(key);
|
|
@@ -7100,8 +7832,13 @@ export class FactoryLoop {
|
|
|
7100
7832
|
prNumber: session.prNumber,
|
|
7101
7833
|
path: session.path,
|
|
7102
7834
|
agentName: session.agentName,
|
|
7835
|
+
resourceSubscription: session.resourceSubscription,
|
|
7836
|
+
pendingDeliveryClaims: session.pendingDeliveryClaims,
|
|
7103
7837
|
};
|
|
7104
7838
|
this.#babysitterPr.set(ownershipKey, ref);
|
|
7839
|
+
if (ref.resourceSubscription) {
|
|
7840
|
+
this.#babysitterSubscriptionOwners.set(ref.resourceSubscription.subscriptionId, ownershipKey);
|
|
7841
|
+
}
|
|
7105
7842
|
this.#babysitterIssueRefs.set(ownershipKey, { ...session.issue });
|
|
7106
7843
|
this.#babysitterSpawned.add(ownershipKey);
|
|
7107
7844
|
if (session.critical)
|
|
@@ -7113,10 +7850,21 @@ export class FactoryLoop {
|
|
|
7113
7850
|
this.#increment('babysitterOwnershipRestored');
|
|
7114
7851
|
const pendingKinds = session.pendingKinds.filter(isBabysitterWakeKind);
|
|
7115
7852
|
if (pendingKinds.length > 0) {
|
|
7116
|
-
|
|
7853
|
+
// A terminal marker can coexist with the one wake persisted before
|
|
7854
|
+
// acknowledgement. Rehydrate that durable hand-off once, while later
|
|
7855
|
+
// raw events remain quarantined by #queueBabysitterWake.
|
|
7856
|
+
await this.#queueBabysitterWake(session.issue, ref, pendingKinds, tracked, { allowTerminal: true });
|
|
7117
7857
|
this.#increment('babysitterPendingWakesRestored');
|
|
7118
7858
|
}
|
|
7859
|
+
await this.#ensureBabysitterResourceSubscription(session.issue, ref, tracked);
|
|
7119
7860
|
}
|
|
7861
|
+
// A crash after the local queue write but before (or just after) the
|
|
7862
|
+
// remote acceptance leaves an ID in state. Accept is idempotent once the
|
|
7863
|
+
// lease was accepted, so settle those durable hand-offs before claiming
|
|
7864
|
+
// new work; lease expiry is retried below when a server has not released
|
|
7865
|
+
// the original claim yet.
|
|
7866
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
7867
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
7120
7868
|
}
|
|
7121
7869
|
async #reconcileRestoredBabysitterReceipts(onlyRecord) {
|
|
7122
7870
|
const records = onlyRecord ? [onlyRecord] : (await this.#batch()).inFlight;
|
|
@@ -7215,6 +7963,29 @@ export class FactoryLoop {
|
|
|
7215
7963
|
this.#babysitterWakeStates.delete(key);
|
|
7216
7964
|
this.#babysitterCriticalAgents.delete(state.agentName);
|
|
7217
7965
|
}
|
|
7966
|
+
if (mayClearDurable && ref?.resourceSubscription) {
|
|
7967
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
7968
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
7969
|
+
if (subscriptions) {
|
|
7970
|
+
try {
|
|
7971
|
+
await subscriptions.cancel(this.#workspaceId, {
|
|
7972
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
7973
|
+
});
|
|
7974
|
+
this.#increment('babysitterResourceSubscriptionsCancelled');
|
|
7975
|
+
}
|
|
7976
|
+
catch (error) {
|
|
7977
|
+
// Cancellation is deliberately idempotent. A terminal acceptance may
|
|
7978
|
+
// have retired this record already; an outage leaves the bounded TTL
|
|
7979
|
+
// as the leak backstop and must not prevent local session cleanup.
|
|
7980
|
+
this.#increment('babysitterResourceSubscriptionCancelFailures');
|
|
7981
|
+
this.#logger.warn?.('[factory] could not cancel durable babysitter resource subscription', {
|
|
7982
|
+
issue: issue?.key,
|
|
7983
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
7984
|
+
error: describeError(error).errorMessage,
|
|
7985
|
+
});
|
|
7986
|
+
}
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7218
7989
|
this.#babysitterPr.delete(ownershipKey);
|
|
7219
7990
|
this.#babysitterIssueRefs.delete(ownershipKey);
|
|
7220
7991
|
this.#babysitterSpawned.delete(ownershipKey);
|
|
@@ -7229,7 +8000,323 @@ export class FactoryLoop {
|
|
|
7229
8000
|
.map(([key]) => key);
|
|
7230
8001
|
await Promise.all(keys.map(async (key) => this.#cancelBabysitterWake(key)));
|
|
7231
8002
|
}
|
|
8003
|
+
async #ensureBabysitterResourceSubscription(issue, ref, tracked) {
|
|
8004
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8005
|
+
if (!subscriptions || !ref.agentName || !await this.#assertIssueDispatchLifecycleOwner(issue))
|
|
8006
|
+
return;
|
|
8007
|
+
// A terminal claim is persisted before Relayfile acceptance so a crash in
|
|
8008
|
+
// that gap can never renew a retired record into a fresh generation.
|
|
8009
|
+
if (ref.resourceSubscription?.terminal)
|
|
8010
|
+
return;
|
|
8011
|
+
const resourceRef = babysitterResourceRef(ref.repo, ref.prNumber);
|
|
8012
|
+
const subscriberId = babysitterSubscriberId(issue);
|
|
8013
|
+
try {
|
|
8014
|
+
const subscription = await subscriptions.createOrRenew(this.#workspaceId, {
|
|
8015
|
+
provider: 'github',
|
|
8016
|
+
resourceRef,
|
|
8017
|
+
eventTypes: [...BABYSITTER_SUBSCRIPTION_EVENT_TYPES],
|
|
8018
|
+
terminalEventTypes: [...BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES],
|
|
8019
|
+
subscriberId,
|
|
8020
|
+
ttlSeconds: BABYSITTER_SUBSCRIPTION_TTL_SECONDS,
|
|
8021
|
+
});
|
|
8022
|
+
if (!subscription.subscriptionId ||
|
|
8023
|
+
subscription.provider !== 'github' ||
|
|
8024
|
+
subscription.resourceRef !== resourceRef ||
|
|
8025
|
+
subscription.subscriberId !== subscriberId ||
|
|
8026
|
+
!subscription.ownerId ||
|
|
8027
|
+
!subscription.expiresAt ||
|
|
8028
|
+
!subscription.terminalEventTypes?.includes('pull_request.closed')) {
|
|
8029
|
+
throw new Error('Relayfile returned an invalid durable resource subscription');
|
|
8030
|
+
}
|
|
8031
|
+
if (ref.resourceSubscription?.subscriptionId && ref.resourceSubscription.subscriptionId !== subscription.subscriptionId) {
|
|
8032
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
8033
|
+
}
|
|
8034
|
+
ref.resourceSubscription = {
|
|
8035
|
+
subscriptionId: subscription.subscriptionId,
|
|
8036
|
+
provider: subscription.provider,
|
|
8037
|
+
resourceRef: subscription.resourceRef,
|
|
8038
|
+
subscriberId: subscription.subscriberId,
|
|
8039
|
+
ownerId: subscription.ownerId,
|
|
8040
|
+
expiresAt: subscription.expiresAt,
|
|
8041
|
+
};
|
|
8042
|
+
this.#babysitterSubscriptionOwners.set(subscription.subscriptionId, babysitterOwnershipKey(issue, ref));
|
|
8043
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8044
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8045
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8046
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8047
|
+
this.#increment('babysitterResourceSubscriptionsRenewed');
|
|
8048
|
+
}
|
|
8049
|
+
catch (error) {
|
|
8050
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8051
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8052
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8053
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8054
|
+
return;
|
|
8055
|
+
}
|
|
8056
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8057
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8058
|
+
this.#increment('babysitterResourceSubscriptionRenewFailures');
|
|
8059
|
+
this.#logger.warn?.('[factory] could not create or renew durable babysitter resource subscription', {
|
|
8060
|
+
issue: issue.key,
|
|
8061
|
+
repo: ref.repo,
|
|
8062
|
+
prNumber: ref.prNumber,
|
|
8063
|
+
error: describeError(error).errorMessage,
|
|
8064
|
+
});
|
|
8065
|
+
}
|
|
8066
|
+
}
|
|
8067
|
+
async #routeDurableBabysitterDeliveries() {
|
|
8068
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8069
|
+
if (!subscriptions || !this.#config.babysitter.enabled || this.#stopping)
|
|
8070
|
+
return false;
|
|
8071
|
+
// Do not bypass the proven local router until every active babysitter has
|
|
8072
|
+
// completed its own create/renew. This closes the rollout and transient
|
|
8073
|
+
// provisioning gap without making a successful API response for some
|
|
8074
|
+
// other subscription suppress an unregistered PR's wake.
|
|
8075
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.agentName && !ref.resourceSubscription)) {
|
|
8076
|
+
return false;
|
|
8077
|
+
}
|
|
8078
|
+
let claims;
|
|
8079
|
+
try {
|
|
8080
|
+
claims = await subscriptions.claimDeliveryClaims(this.#workspaceId);
|
|
8081
|
+
}
|
|
8082
|
+
catch (error) {
|
|
8083
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8084
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8085
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8086
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8087
|
+
}
|
|
8088
|
+
else {
|
|
8089
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8090
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8091
|
+
this.#increment('babysitterResourceDeliveryLookupFailures');
|
|
8092
|
+
this.#logger.warn?.('[factory] durable babysitter delivery-claim lookup failed; retaining durable delivery retry', {
|
|
8093
|
+
error: describeError(error).errorMessage,
|
|
8094
|
+
});
|
|
8095
|
+
}
|
|
8096
|
+
return !isResourceSubscriptionsUnavailable(error);
|
|
8097
|
+
}
|
|
8098
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8099
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8100
|
+
for (const claim of claims) {
|
|
8101
|
+
const issueIdentity = this.#babysitterSubscriptionOwners.get(claim.subscriptionId);
|
|
8102
|
+
const issue = issueIdentity ? this.#babysitterIssueRefs.get(issueIdentity) : undefined;
|
|
8103
|
+
const ref = issueIdentity ? this.#babysitterPr.get(issueIdentity) : undefined;
|
|
8104
|
+
const subscription = ref?.resourceSubscription;
|
|
8105
|
+
if (!issue ||
|
|
8106
|
+
!ref ||
|
|
8107
|
+
!subscription ||
|
|
8108
|
+
subscription.subscriptionId !== claim.subscriptionId ||
|
|
8109
|
+
subscription.provider !== claim.provider ||
|
|
8110
|
+
subscription.resourceRef !== claim.resourceRef ||
|
|
8111
|
+
subscription.subscriberId !== claim.subscriberId ||
|
|
8112
|
+
subscription.ownerId !== claim.ownerId) {
|
|
8113
|
+
// The service is owner-isolated, but Factory may have just retired a
|
|
8114
|
+
// local owner. Never route a stale or other-session claim by resource.
|
|
8115
|
+
this.#increment('babysitterResourceDeliveriesIgnoredUnowned');
|
|
8116
|
+
continue;
|
|
8117
|
+
}
|
|
8118
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
8119
|
+
this.#increment('babysitterResourceDeliveriesIgnoredNonOwner');
|
|
8120
|
+
continue;
|
|
8121
|
+
}
|
|
8122
|
+
// A terminal delivery may be reclaimed after a process crash before its
|
|
8123
|
+
// remote acceptance. Only that already-persisted delivery may finish;
|
|
8124
|
+
// no later claim is allowed to wake or re-open the retired session.
|
|
8125
|
+
if (subscription.terminal && !ref.pendingDeliveryClaims?.some((pending) => pending.deliveryId === claim.deliveryId)) {
|
|
8126
|
+
this.#increment('babysitterResourceDeliveriesIgnoredTerminal');
|
|
8127
|
+
continue;
|
|
8128
|
+
}
|
|
8129
|
+
const batch = await this.#batch();
|
|
8130
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8131
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8132
|
+
?? durableBabysitterTrackedAgent({
|
|
8133
|
+
issue,
|
|
8134
|
+
repo: ref.repo,
|
|
8135
|
+
prNumber: ref.prNumber,
|
|
8136
|
+
path: ref.path,
|
|
8137
|
+
agentName: ref.agentName,
|
|
8138
|
+
critical: false,
|
|
8139
|
+
pendingKinds: [],
|
|
8140
|
+
resourceSubscription: subscription,
|
|
8141
|
+
pendingDeliveryClaims: ref.pendingDeliveryClaims,
|
|
8142
|
+
});
|
|
8143
|
+
const pendingClaim = ref.pendingDeliveryClaims?.find((pending) => pending.deliveryId === claim.deliveryId);
|
|
8144
|
+
const alreadyQueued = Boolean(pendingClaim);
|
|
8145
|
+
if (!alreadyQueued) {
|
|
8146
|
+
const queued = await this.#queueBabysitterWake(issue, ref, ['pull-request-state'], tracked);
|
|
8147
|
+
if (!queued)
|
|
8148
|
+
continue;
|
|
8149
|
+
}
|
|
8150
|
+
if (!pendingClaim || pendingClaim.claimToken !== claim.claimToken) {
|
|
8151
|
+
ref.pendingDeliveryClaims = [
|
|
8152
|
+
...(ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId),
|
|
8153
|
+
{ deliveryId: claim.deliveryId, claimToken: claim.claimToken },
|
|
8154
|
+
];
|
|
8155
|
+
// The claim lease joins Factory's durable pending-wake state before
|
|
8156
|
+
// the external acceptance. A crash after this point can retry the
|
|
8157
|
+
// exact hand-off without delivering the same wake a second time.
|
|
8158
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8159
|
+
}
|
|
8160
|
+
try {
|
|
8161
|
+
if (claim.terminal && !subscription.terminal) {
|
|
8162
|
+
subscription.terminal = true;
|
|
8163
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8164
|
+
}
|
|
8165
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, {
|
|
8166
|
+
deliveryId: claim.deliveryId,
|
|
8167
|
+
claimToken: claim.claimToken,
|
|
8168
|
+
});
|
|
8169
|
+
if (accepted.deliveryId !== claim.deliveryId || accepted.subscriptionId !== claim.subscriptionId) {
|
|
8170
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8171
|
+
}
|
|
8172
|
+
if (accepted.terminal || claim.terminal) {
|
|
8173
|
+
// Keep the terminal marker and subscription identity locally until
|
|
8174
|
+
// normal PR/session teardown. That quarantines the babysitter from
|
|
8175
|
+
// both legacy fallback and a restart-time create-or-renew.
|
|
8176
|
+
subscription.terminal = true;
|
|
8177
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8178
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8179
|
+
this.#increment('babysitterResourceSubscriptionsRetiredTerminal');
|
|
8180
|
+
}
|
|
8181
|
+
else {
|
|
8182
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8183
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8184
|
+
}
|
|
8185
|
+
this.#increment('babysitterResourceDeliveriesAccepted');
|
|
8186
|
+
}
|
|
8187
|
+
catch (error) {
|
|
8188
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8189
|
+
this.#logger.warn?.('[factory] durable babysitter delivery claim remains pending after wake queue', {
|
|
8190
|
+
issue: issue.key,
|
|
8191
|
+
subscriptionId: claim.subscriptionId,
|
|
8192
|
+
deliveryId: claim.deliveryId,
|
|
8193
|
+
error: describeError(error).errorMessage,
|
|
8194
|
+
});
|
|
8195
|
+
}
|
|
8196
|
+
}
|
|
8197
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8198
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8199
|
+
}
|
|
8200
|
+
return true;
|
|
8201
|
+
}
|
|
8202
|
+
async #retryPendingBabysitterDeliveryAcceptances() {
|
|
8203
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8204
|
+
if (!subscriptions)
|
|
8205
|
+
return;
|
|
8206
|
+
const retrySubscriptionRenewal = this.#babysitterResourceSubscriptionFault;
|
|
8207
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8208
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8209
|
+
if (issue && ref.agentName && !ref.resourceSubscription?.terminal && (!ref.resourceSubscription || retrySubscriptionRenewal)) {
|
|
8210
|
+
const batch = await this.#batch();
|
|
8211
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8212
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8213
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8214
|
+
}
|
|
8215
|
+
const subscription = ref.resourceSubscription;
|
|
8216
|
+
const pendingDeliveryClaims = [...(ref.pendingDeliveryClaims ?? [])];
|
|
8217
|
+
if (!subscription || !issue || pendingDeliveryClaims.length === 0)
|
|
8218
|
+
continue;
|
|
8219
|
+
const batch = await this.#batch();
|
|
8220
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8221
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8222
|
+
?? durableBabysitterTrackedAgent({
|
|
8223
|
+
issue,
|
|
8224
|
+
repo: ref.repo,
|
|
8225
|
+
prNumber: ref.prNumber,
|
|
8226
|
+
path: ref.path,
|
|
8227
|
+
agentName: ref.agentName,
|
|
8228
|
+
critical: false,
|
|
8229
|
+
pendingKinds: [],
|
|
8230
|
+
resourceSubscription: subscription,
|
|
8231
|
+
pendingDeliveryClaims,
|
|
8232
|
+
});
|
|
8233
|
+
for (const { deliveryId, claimToken } of pendingDeliveryClaims) {
|
|
8234
|
+
try {
|
|
8235
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, { deliveryId, claimToken });
|
|
8236
|
+
if (accepted.deliveryId !== deliveryId || accepted.subscriptionId !== subscription.subscriptionId) {
|
|
8237
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8238
|
+
}
|
|
8239
|
+
if (accepted.terminal)
|
|
8240
|
+
subscription.terminal = true;
|
|
8241
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== deliveryId);
|
|
8242
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8243
|
+
this.#increment('babysitterResourceDeliveriesAcceptedAfterRestore');
|
|
8244
|
+
}
|
|
8245
|
+
catch (error) {
|
|
8246
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8247
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8248
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8249
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8250
|
+
}
|
|
8251
|
+
else {
|
|
8252
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8253
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8254
|
+
this.#logger.warn?.('[factory] durable babysitter delivery acceptance remains pending after restore', {
|
|
8255
|
+
issue: issue.key,
|
|
8256
|
+
subscriptionId: subscription.subscriptionId,
|
|
8257
|
+
deliveryId,
|
|
8258
|
+
error: describeError(error).errorMessage,
|
|
8259
|
+
});
|
|
8260
|
+
}
|
|
8261
|
+
}
|
|
8262
|
+
}
|
|
8263
|
+
}
|
|
8264
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8265
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8266
|
+
}
|
|
8267
|
+
}
|
|
8268
|
+
#scheduleBabysitterResourceSubscriptionRenewal() {
|
|
8269
|
+
if (this.#babysitterResourceSubscriptionRenewTimer ||
|
|
8270
|
+
this.#stopping ||
|
|
8271
|
+
!this.#mount.resourceSubscriptions ||
|
|
8272
|
+
![...this.#babysitterPr.values()].some((ref) => ref.resourceSubscription && !ref.resourceSubscription.terminal))
|
|
8273
|
+
return;
|
|
8274
|
+
this.#babysitterResourceSubscriptionRenewTimer = setTimeout(() => {
|
|
8275
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
8276
|
+
void (async () => {
|
|
8277
|
+
const batch = await this.#batch();
|
|
8278
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8279
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8280
|
+
if (!issue || !ref.resourceSubscription || ref.resourceSubscription.terminal)
|
|
8281
|
+
continue;
|
|
8282
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8283
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8284
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8285
|
+
}
|
|
8286
|
+
})().catch((error) => {
|
|
8287
|
+
this.#logger.warn?.('[factory] durable babysitter subscription renewal rejected', {
|
|
8288
|
+
error: describeError(error).errorMessage,
|
|
8289
|
+
});
|
|
8290
|
+
}).finally(() => {
|
|
8291
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8292
|
+
});
|
|
8293
|
+
}, BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS);
|
|
8294
|
+
this.#babysitterResourceSubscriptionRenewTimer.unref?.();
|
|
8295
|
+
}
|
|
8296
|
+
#scheduleDurableBabysitterDeliveryRetry() {
|
|
8297
|
+
if (this.#babysitterResourceDeliveryRetryTimer || this.#stopping || !this.#mount.resourceSubscriptions)
|
|
8298
|
+
return;
|
|
8299
|
+
this.#babysitterResourceDeliveryRetryTimer = setTimeout(() => {
|
|
8300
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
8301
|
+
void (async () => {
|
|
8302
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
8303
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8304
|
+
})().catch((error) => {
|
|
8305
|
+
this.#logger.warn?.('[factory] durable babysitter delivery retry rejected', {
|
|
8306
|
+
error: describeError(error).errorMessage,
|
|
8307
|
+
});
|
|
8308
|
+
});
|
|
8309
|
+
}, BABYSITTER_RESOURCE_DELIVERY_RETRY_MS);
|
|
8310
|
+
this.#babysitterResourceDeliveryRetryTimer.unref?.();
|
|
8311
|
+
}
|
|
7232
8312
|
async #routeBabysitterEvent(path, extraKinds = []) {
|
|
8313
|
+
// A successful Relayfile claim lookup is the new exact demux. While some
|
|
8314
|
+
// owners are still unregistered, the legacy path router remains available
|
|
8315
|
+
// only to those owners. Registered owners retain and retry service claims,
|
|
8316
|
+
// so a mixed rollout or transient create failure neither double-delivers a
|
|
8317
|
+
// registered PR nor drops an event for an unregistered PR.
|
|
8318
|
+
if (await this.#routeDurableBabysitterDeliveries())
|
|
8319
|
+
return;
|
|
7233
8320
|
const event = githubBabysitterEventPathParts(path);
|
|
7234
8321
|
if (!event || !this.#config.babysitter.enabled || this.#stopping)
|
|
7235
8322
|
return;
|
|
@@ -7262,6 +8349,12 @@ export class FactoryLoop {
|
|
|
7262
8349
|
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', { ...event, prNumber: target.prNumber });
|
|
7263
8350
|
continue;
|
|
7264
8351
|
}
|
|
8352
|
+
if (this.#mount.resourceSubscriptions &&
|
|
8353
|
+
!this.#babysitterResourceSubscriptionUnavailable &&
|
|
8354
|
+
owner.ref.resourceSubscription) {
|
|
8355
|
+
this.#increment('babysitterEventsDeferredToDurableSubscription');
|
|
8356
|
+
continue;
|
|
8357
|
+
}
|
|
7265
8358
|
if (!await this.#assertIssueDispatchLifecycleOwner(owner.issue)) {
|
|
7266
8359
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7267
8360
|
continue;
|
|
@@ -7320,10 +8413,14 @@ export class FactoryLoop {
|
|
|
7320
8413
|
return;
|
|
7321
8414
|
}
|
|
7322
8415
|
}
|
|
7323
|
-
async #queueBabysitterWake(issue, ref, kinds, tracked) {
|
|
8416
|
+
async #queueBabysitterWake(issue, ref, kinds, tracked, options = {}) {
|
|
8417
|
+
if (ref.resourceSubscription?.terminal && !options.allowTerminal) {
|
|
8418
|
+
this.#increment('babysitterEventsIgnoredTerminal');
|
|
8419
|
+
return false;
|
|
8420
|
+
}
|
|
7324
8421
|
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
7325
8422
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7326
|
-
return;
|
|
8423
|
+
return false;
|
|
7327
8424
|
}
|
|
7328
8425
|
// Owner lookup and queueing straddle async mount/state reads. Revalidate
|
|
7329
8426
|
// the exact composite owner so a concurrent close/merge cancellation can
|
|
@@ -7334,7 +8431,7 @@ export class FactoryLoop {
|
|
|
7334
8431
|
current.agentName !== ref.agentName ||
|
|
7335
8432
|
githubPrIdentity(current.repo, current.prNumber) !== githubPrIdentity(ref.repo, ref.prNumber)) {
|
|
7336
8433
|
this.#increment('babysitterEventsIgnoredStaleOwner');
|
|
7337
|
-
return;
|
|
8434
|
+
return false;
|
|
7338
8435
|
}
|
|
7339
8436
|
// Any new event invalidates a prior readiness assertion for this exact PR.
|
|
7340
8437
|
this.#babysitterReady.delete(ownershipKey);
|
|
@@ -7364,9 +8461,10 @@ export class FactoryLoop {
|
|
|
7364
8461
|
});
|
|
7365
8462
|
if (state.deferredSubmitTargets || state.inFlight || this.#babysitterCriticalAgents.has(state.agentName)) {
|
|
7366
8463
|
this.#increment('babysitterEventWakesDeferred');
|
|
7367
|
-
return;
|
|
8464
|
+
return true;
|
|
7368
8465
|
}
|
|
7369
8466
|
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
8467
|
+
return true;
|
|
7370
8468
|
}
|
|
7371
8469
|
async #recordPendingBabysitterWake(state) {
|
|
7372
8470
|
const kinds = new Set([
|
|
@@ -7402,6 +8500,8 @@ export class FactoryLoop {
|
|
|
7402
8500
|
path: ref.path,
|
|
7403
8501
|
critical: this.#babysitterCriticalAgents.has(ref.agentName),
|
|
7404
8502
|
pendingKinds: pending?.kinds.filter(isBabysitterWakeKind).sort(compareBabysitterWakeKinds) ?? [],
|
|
8503
|
+
...(ref.resourceSubscription ? { resourceSubscription: { ...ref.resourceSubscription } } : {}),
|
|
8504
|
+
...(ref.pendingDeliveryClaims?.length ? { pendingDeliveryClaims: structuredClone(ref.pendingDeliveryClaims) } : {}),
|
|
7405
8505
|
});
|
|
7406
8506
|
}
|
|
7407
8507
|
#scheduleBabysitterWake(state, delayMs) {
|
|
@@ -7687,11 +8787,21 @@ export class FactoryLoop {
|
|
|
7687
8787
|
if (!this.#config.babysitter.enabled)
|
|
7688
8788
|
return;
|
|
7689
8789
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8790
|
+
// A provider close produces a separately indexed terminal claim. Keep
|
|
8791
|
+
// the durable owner until it has been accepted (or a transient retry
|
|
8792
|
+
// has claimed it); do not let closed-state cleanup erase that hand-off.
|
|
8793
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8794
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8795
|
+
return;
|
|
7690
8796
|
await this.#cancelBabysitterWake(owned.key);
|
|
7691
8797
|
return;
|
|
7692
8798
|
}
|
|
7693
8799
|
if (snapshot.draft)
|
|
7694
8800
|
this.#increment('babysitterDraftPrSkipped');
|
|
8801
|
+
// PR meta events are also the normal renewal heartbeat for the durable
|
|
8802
|
+
// record. The store's identity makes this a create-or-renew, never a
|
|
8803
|
+
// second subscription for the same babysitter.
|
|
8804
|
+
await this.#ensureBabysitterResourceSubscription(owned.issue, owned.ref, owned.tracked);
|
|
7695
8805
|
await this.#routeBabysitterEvent(path, babysitterWakeKindsFromSnapshot(snapshot));
|
|
7696
8806
|
return;
|
|
7697
8807
|
}
|
|
@@ -7739,6 +8849,13 @@ export class FactoryLoop {
|
|
|
7739
8849
|
return;
|
|
7740
8850
|
}
|
|
7741
8851
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8852
|
+
// `pull_request.closed` is a separately indexed Relayfile terminal
|
|
8853
|
+
// event. Claim and accept its durable hand-off before the local closed
|
|
8854
|
+
// PR cleanup drops the subscription owner. On a transient service fault,
|
|
8855
|
+
// retain the owner so the retry loop can claim it without local fallback.
|
|
8856
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8857
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8858
|
+
return;
|
|
7742
8859
|
if (babysitterKey && existing)
|
|
7743
8860
|
await this.#cancelBabysitterWake(babysitterKey);
|
|
7744
8861
|
return;
|
|
@@ -7936,6 +9053,11 @@ export class FactoryLoop {
|
|
|
7936
9053
|
const settled = this.#babysitterPr.get(babysitterKey);
|
|
7937
9054
|
if (settled && prRef.path)
|
|
7938
9055
|
settled.path = prRef.path;
|
|
9056
|
+
if (settled) {
|
|
9057
|
+
const tracked = record.agents.get(settled.agentName)
|
|
9058
|
+
?? [...record.agents.values()].find((agent) => agent.spec.role === 'babysitter');
|
|
9059
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, settled, tracked);
|
|
9060
|
+
}
|
|
7939
9061
|
return;
|
|
7940
9062
|
}
|
|
7941
9063
|
const wantedPr = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
@@ -7956,7 +9078,9 @@ export class FactoryLoop {
|
|
|
7956
9078
|
agentName: tracked.result?.name ?? trackedName,
|
|
7957
9079
|
});
|
|
7958
9080
|
this.#babysitterSpawned.add(babysitterKey);
|
|
7959
|
-
|
|
9081
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9082
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9083
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
7960
9084
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
7961
9085
|
return;
|
|
7962
9086
|
}
|
|
@@ -7986,9 +9110,13 @@ export class FactoryLoop {
|
|
|
7986
9110
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath)
|
|
7987
9111
|
?? record.decision.implementers
|
|
7988
9112
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath);
|
|
9113
|
+
const preview = [...record.agents.values()]
|
|
9114
|
+
.map((agent) => agent.spec)
|
|
9115
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.preview)?.preview
|
|
9116
|
+
?? record.decision.implementers.find((candidate) => candidate.repo === initialSpec.repo)?.preview;
|
|
7989
9117
|
const implementerBranch = prRef.headRef ?? record.decision.implementers
|
|
7990
9118
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.branch)?.branch;
|
|
7991
|
-
const
|
|
9119
|
+
const checkoutSpec = sharedCheckout
|
|
7992
9120
|
? {
|
|
7993
9121
|
...initialSpec,
|
|
7994
9122
|
baseClonePath: sharedCheckout.baseClonePath,
|
|
@@ -7997,6 +9125,7 @@ export class FactoryLoop {
|
|
|
7997
9125
|
...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
7998
9126
|
}
|
|
7999
9127
|
: initialSpec;
|
|
9128
|
+
const spec = specWithPreview(checkoutSpec, preview);
|
|
8000
9129
|
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
8001
9130
|
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name
|
|
8002
9131
|
?? agentNameForRole(issue, 'review', { repo: route?.repo ?? prRef.repo });
|
|
@@ -8029,6 +9158,11 @@ export class FactoryLoop {
|
|
|
8029
9158
|
branchName: spec.branch,
|
|
8030
9159
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
8031
9160
|
agentName: spec.name,
|
|
9161
|
+
...(spec.preview ? {
|
|
9162
|
+
previewUrl: spec.preview.url,
|
|
9163
|
+
previewTargetPort: spec.preview.targetPort,
|
|
9164
|
+
previewStartCommand: spec.preview.startCommand,
|
|
9165
|
+
} : {}),
|
|
8032
9166
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
8033
9167
|
});
|
|
8034
9168
|
const spawned = await this.#spawnAgent(record, {
|
|
@@ -8043,7 +9177,9 @@ export class FactoryLoop {
|
|
|
8043
9177
|
path: prRef.path,
|
|
8044
9178
|
agentName: tracked?.result?.name ?? spawned.name,
|
|
8045
9179
|
});
|
|
8046
|
-
|
|
9180
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9181
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9182
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
8047
9183
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
8048
9184
|
await this.#writeInFlightRegistry();
|
|
8049
9185
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
@@ -8346,6 +9482,7 @@ export class FactoryLoop {
|
|
|
8346
9482
|
// state.
|
|
8347
9483
|
const issueTeam = issue?.team;
|
|
8348
9484
|
const githubIssue = issue ? isGithubIssue(issue) : false;
|
|
9485
|
+
const syntheticProbe = issue ? this.#isSyntheticProbeIssue(issue) : false;
|
|
8349
9486
|
const configuredHumanReview = opts.targetState !== 'done' &&
|
|
8350
9487
|
this.#config.terminalState === 'human-review' &&
|
|
8351
9488
|
(githubIssue || this.#states.hasHumanReview(issueTeam));
|
|
@@ -8353,7 +9490,7 @@ export class FactoryLoop {
|
|
|
8353
9490
|
if (issue && githubIssue && !configuredHumanReview && !githubMerged) {
|
|
8354
9491
|
githubMerged = await this.#githubPrObservedMerged(record, issue);
|
|
8355
9492
|
if (!githubMerged && opts.runMergeGate !== false) {
|
|
8356
|
-
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue);
|
|
9493
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
8357
9494
|
// A successful merge command can mean queued/auto-merge rather than
|
|
8358
9495
|
// merged. Only mounted PR state or a merged webhook may prove merge.
|
|
8359
9496
|
if (mergeCommandAccepted) {
|
|
@@ -8361,6 +9498,21 @@ export class FactoryLoop {
|
|
|
8361
9498
|
}
|
|
8362
9499
|
}
|
|
8363
9500
|
}
|
|
9501
|
+
// Linear-backed issues historically wrote Done before invoking the
|
|
9502
|
+
// guarded merge. With live verification in that guard, a red verdict
|
|
9503
|
+
// would therefore release the team and make the failure terminal even
|
|
9504
|
+
// though the PR remained open. Require the entire merge gate (including
|
|
9505
|
+
// verification) to accept the merge before applying terminal writeback.
|
|
9506
|
+
if (issue &&
|
|
9507
|
+
!githubIssue &&
|
|
9508
|
+
!syntheticProbe &&
|
|
9509
|
+
!configuredHumanReview &&
|
|
9510
|
+
opts.runMergeGate !== false &&
|
|
9511
|
+
this.#config.mergePolicy === 'on-green-with-review') {
|
|
9512
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
9513
|
+
if (!mergeCommandAccepted)
|
|
9514
|
+
return;
|
|
9515
|
+
}
|
|
8364
9516
|
// A GitHub issue only closes after its PR merges. If a configured done
|
|
8365
9517
|
// path cannot merge (including mergePolicy: never), park it for a human
|
|
8366
9518
|
// instead of closing the issue while its PR remains open.
|
|
@@ -8415,11 +9567,11 @@ export class FactoryLoop {
|
|
|
8415
9567
|
this.#markSlackWritebackFailure('completion-thread', error);
|
|
8416
9568
|
}
|
|
8417
9569
|
}
|
|
8418
|
-
//
|
|
8419
|
-
//
|
|
8420
|
-
//
|
|
8421
|
-
if (issue &&
|
|
8422
|
-
await this.#runCompletionMergeGate(issue);
|
|
9570
|
+
// Synthetic canaries are cleanup probes, not merge candidates. Preserve
|
|
9571
|
+
// their close-before-release path under every merge policy without
|
|
9572
|
+
// subjecting them to the required feature verification gate.
|
|
9573
|
+
if (issue && syntheticProbe && !githubIssue && opts.runMergeGate !== false) {
|
|
9574
|
+
await this.#runCompletionMergeGate(issue, record);
|
|
8423
9575
|
}
|
|
8424
9576
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
8425
9577
|
releaseReasonForRetry = releaseReason;
|
|
@@ -8449,6 +9601,8 @@ export class FactoryLoop {
|
|
|
8449
9601
|
const stateKey = issueStateKey(record.issue);
|
|
8450
9602
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
8451
9603
|
this.#probePrResolvedCache.delete(stateKey);
|
|
9604
|
+
// Cancellation must see the subscription identity so it can issue the
|
|
9605
|
+
// idempotent Relayfile DELETE before clearing the local owner maps.
|
|
8452
9606
|
await this.#cancelBabysittersForIssue(record.issue);
|
|
8453
9607
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
8454
9608
|
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
@@ -8787,6 +9941,10 @@ export class FactoryLoop {
|
|
|
8787
9941
|
await this.#rearmSlackWatcher(record, existingThread, {
|
|
8788
9942
|
replayConversationReplies: Boolean(durableConversation),
|
|
8789
9943
|
});
|
|
9944
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
9945
|
+
if (previews.length > 0) {
|
|
9946
|
+
await this.#slack.reply(existingThread, previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join('\n'));
|
|
9947
|
+
}
|
|
8790
9948
|
}
|
|
8791
9949
|
return;
|
|
8792
9950
|
}
|
|
@@ -8807,12 +9965,15 @@ export class FactoryLoop {
|
|
|
8807
9965
|
if (!this.#slack || !this.#config.slack) {
|
|
8808
9966
|
return;
|
|
8809
9967
|
}
|
|
9968
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
8810
9969
|
const root = await this.#slack.postThread({
|
|
8811
9970
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
8812
9971
|
text: [
|
|
8813
9972
|
`${record.issue.key}: factory agents dispatched.`,
|
|
8814
|
-
`State: ${result.stateId ?? 'dispatching'}`,
|
|
8815
|
-
|
|
9973
|
+
`State: ${result.stateId ?? 'dispatching'} · Agents: ${result.agents.map((agent) => agent.name).join(', ') || 'none'}`,
|
|
9974
|
+
...(previews.length > 0
|
|
9975
|
+
? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
|
|
9976
|
+
: []),
|
|
8816
9977
|
].join('\n'),
|
|
8817
9978
|
});
|
|
8818
9979
|
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
@@ -10284,7 +11445,7 @@ export class FactoryLoop {
|
|
|
10284
11445
|
? { channel, threadId, mountRoot: this.#integrationsMountRoot() }
|
|
10285
11446
|
: undefined;
|
|
10286
11447
|
}
|
|
10287
|
-
async #runCompletionMergeGate(issue) {
|
|
11448
|
+
async #runCompletionMergeGate(issue, record) {
|
|
10288
11449
|
if (this.#isSyntheticProbeIssue(issue)) {
|
|
10289
11450
|
await this.#closeSyntheticProbeIfPresent(issue);
|
|
10290
11451
|
return false;
|
|
@@ -10309,6 +11470,48 @@ export class FactoryLoop {
|
|
|
10309
11470
|
this.#increment('mergeGateNotMerged');
|
|
10310
11471
|
return false;
|
|
10311
11472
|
}
|
|
11473
|
+
if (this.#verificationGate) {
|
|
11474
|
+
const repositoryPath = this.#verificationRepositoryPath(record, pr.repo);
|
|
11475
|
+
if (!repositoryPath) {
|
|
11476
|
+
this.#logger.warn?.('[factory] verification gate has no feature checkout for merge candidate', {
|
|
11477
|
+
issue: issue.key,
|
|
11478
|
+
repo: pr.repo,
|
|
11479
|
+
prNumber: pr.prNumber,
|
|
11480
|
+
});
|
|
11481
|
+
this.#increment('verificationGateMissingRepository');
|
|
11482
|
+
return false;
|
|
11483
|
+
}
|
|
11484
|
+
try {
|
|
11485
|
+
const verification = await this.#verificationGate.verify({
|
|
11486
|
+
repository: pr.repo,
|
|
11487
|
+
repositoryPath,
|
|
11488
|
+
issueKey: issue.key,
|
|
11489
|
+
expectedHeadSha: headSha,
|
|
11490
|
+
});
|
|
11491
|
+
if (!verification.passed) {
|
|
11492
|
+
this.#logger.warn?.('[factory] verification gate blocked merge', {
|
|
11493
|
+
issue: issue.key,
|
|
11494
|
+
repo: pr.repo,
|
|
11495
|
+
prNumber: pr.prNumber,
|
|
11496
|
+
environmentId: verification.evidence.environmentId,
|
|
11497
|
+
reason: verification.reason,
|
|
11498
|
+
});
|
|
11499
|
+
this.#increment('verificationGateFailed');
|
|
11500
|
+
return false;
|
|
11501
|
+
}
|
|
11502
|
+
this.#increment('verificationGatePassed');
|
|
11503
|
+
}
|
|
11504
|
+
catch (error) {
|
|
11505
|
+
this.#logger.warn?.('[factory] verification gate failed closed', {
|
|
11506
|
+
issue: issue.key,
|
|
11507
|
+
repo: pr.repo,
|
|
11508
|
+
prNumber: pr.prNumber,
|
|
11509
|
+
error: describeError(error).errorMessage,
|
|
11510
|
+
});
|
|
11511
|
+
this.#increment('verificationGateFailed');
|
|
11512
|
+
return false;
|
|
11513
|
+
}
|
|
11514
|
+
}
|
|
10312
11515
|
const result = await this.#mergeGate.merge({
|
|
10313
11516
|
repo: pr.repo,
|
|
10314
11517
|
number: pr.prNumber,
|
|
@@ -10334,6 +11537,14 @@ export class FactoryLoop {
|
|
|
10334
11537
|
this.#increment('mergeGateMerged');
|
|
10335
11538
|
return true;
|
|
10336
11539
|
}
|
|
11540
|
+
#verificationRepositoryPath(record, repo) {
|
|
11541
|
+
const normalized = repo.toLowerCase();
|
|
11542
|
+
const active = [...record.agents.values()]
|
|
11543
|
+
.map((tracked) => tracked.spec)
|
|
11544
|
+
.find((spec) => spec.role === 'implementer' && spec.repo.toLowerCase() === normalized && spec.clonePath)
|
|
11545
|
+
?? record.decision.implementers.find((spec) => spec.repo.toLowerCase() === normalized && spec.clonePath);
|
|
11546
|
+
return active?.clonePath ?? this.#config.repos.clonePaths[repo];
|
|
11547
|
+
}
|
|
10337
11548
|
async #closeSyntheticProbeIfPresent(issue) {
|
|
10338
11549
|
const probe = this.#customProbePrResolver
|
|
10339
11550
|
? await this.#probePrResolver(issue)
|
|
@@ -10688,6 +11899,101 @@ function dispatchSpecs(decision) {
|
|
|
10688
11899
|
}
|
|
10689
11900
|
return [...decision.implementers, decision.reviewer];
|
|
10690
11901
|
}
|
|
11902
|
+
function previewServiceForRepo(config, repo) {
|
|
11903
|
+
const services = config.preview?.services;
|
|
11904
|
+
if (!services)
|
|
11905
|
+
return undefined;
|
|
11906
|
+
const normalizedRepo = repo.replace(/^\/+|\/+$/gu, '').toLowerCase();
|
|
11907
|
+
const basename = normalizedRepo.slice(normalizedRepo.lastIndexOf('/') + 1);
|
|
11908
|
+
const entries = Object.entries(services).map(([name, service]) => ({
|
|
11909
|
+
name,
|
|
11910
|
+
service,
|
|
11911
|
+
normalized: name.replace(/^\/+|\/+$/gu, '').toLowerCase(),
|
|
11912
|
+
}));
|
|
11913
|
+
const exact = entries.find((entry) => entry.normalized === normalizedRepo);
|
|
11914
|
+
if (exact)
|
|
11915
|
+
return { name: exact.name, config: exact.service };
|
|
11916
|
+
const basenameMatches = entries.filter((entry) => entry.normalized.slice(entry.normalized.lastIndexOf('/') + 1) === basename);
|
|
11917
|
+
return basenameMatches.length === 1
|
|
11918
|
+
? { name: basenameMatches[0].name, config: basenameMatches[0].service }
|
|
11919
|
+
: undefined;
|
|
11920
|
+
}
|
|
11921
|
+
function uniquePreviewReferences(previews) {
|
|
11922
|
+
const unique = new Map();
|
|
11923
|
+
for (const preview of previews) {
|
|
11924
|
+
if (preview)
|
|
11925
|
+
unique.set(preview.id, preview);
|
|
11926
|
+
}
|
|
11927
|
+
return [...unique.values()];
|
|
11928
|
+
}
|
|
11929
|
+
function assertPublishablePreview(preview, expected) {
|
|
11930
|
+
const refuse = (reason) => {
|
|
11931
|
+
throw new Error(`Refusing insecure preview for ${expected.repo}: ${reason}`);
|
|
11932
|
+
};
|
|
11933
|
+
if (preview.provider !== 'tailscale-serve')
|
|
11934
|
+
refuse('unexpected provider');
|
|
11935
|
+
if (preview.access !== 'tailnet')
|
|
11936
|
+
refuse('provider did not guarantee tailnet access');
|
|
11937
|
+
if (preview.lifetime !== 'issue')
|
|
11938
|
+
refuse('provider did not guarantee issue-scoped lifetime');
|
|
11939
|
+
if (preview.namespace !== expected.namespace ||
|
|
11940
|
+
preview.owner !== expected.owner ||
|
|
11941
|
+
preview.service !== expected.service ||
|
|
11942
|
+
preview.repo !== expected.repo ||
|
|
11943
|
+
preview.startCommand !== expected.startCommand ||
|
|
11944
|
+
(preview.configuredTargetPort ?? preview.targetPort) !== expected.targetPort ||
|
|
11945
|
+
preview.targetPort < expected.targetPort ||
|
|
11946
|
+
preview.targetPort >= expected.targetPort + expected.portSpan) {
|
|
11947
|
+
refuse('provider returned a reference for a different dispatch identity');
|
|
11948
|
+
}
|
|
11949
|
+
if (expected.preferredHttpsPort !== undefined && preview.httpsPort !== expected.preferredHttpsPort) {
|
|
11950
|
+
refuse('provider ignored the configured HTTPS port');
|
|
11951
|
+
}
|
|
11952
|
+
if (expected.requireNode && (!preview.node || preview.node === 'self')) {
|
|
11953
|
+
refuse('remote provider did not identify the placement node');
|
|
11954
|
+
}
|
|
11955
|
+
const managedProcess = preview.process;
|
|
11956
|
+
if (!managedProcess ||
|
|
11957
|
+
!Number.isInteger(managedProcess.pid) ||
|
|
11958
|
+
managedProcess.pid <= 0 ||
|
|
11959
|
+
!managedProcess.startTime ||
|
|
11960
|
+
!managedProcess.cmdline ||
|
|
11961
|
+
!managedProcess.cwd ||
|
|
11962
|
+
!managedProcess.marker) {
|
|
11963
|
+
refuse('provider did not return an identity-checked managed process');
|
|
11964
|
+
}
|
|
11965
|
+
if (!expected.requireNode && managedProcess.cwd !== expected.checkoutPath) {
|
|
11966
|
+
refuse('provider started the managed process in a different checkout');
|
|
11967
|
+
}
|
|
11968
|
+
const url = (() => {
|
|
11969
|
+
try {
|
|
11970
|
+
return new URL(preview.url);
|
|
11971
|
+
}
|
|
11972
|
+
catch {
|
|
11973
|
+
return refuse('provider returned an invalid URL');
|
|
11974
|
+
}
|
|
11975
|
+
})();
|
|
11976
|
+
if (url.protocol !== 'https:' || url.username || url.password || !url.hostname) {
|
|
11977
|
+
refuse('provider URL is not credential-free HTTPS');
|
|
11978
|
+
}
|
|
11979
|
+
if (!url.hostname.endsWith('.ts.net')) {
|
|
11980
|
+
refuse('provider URL is not a Tailscale HTTPS name');
|
|
11981
|
+
}
|
|
11982
|
+
const urlPort = url.port ? Number(url.port) : 443;
|
|
11983
|
+
if (urlPort !== preview.httpsPort)
|
|
11984
|
+
refuse('provider URL does not match its guarded HTTPS route');
|
|
11985
|
+
}
|
|
11986
|
+
function specWithPreview(spec, preview) {
|
|
11987
|
+
if (!preview)
|
|
11988
|
+
return { ...spec };
|
|
11989
|
+
return {
|
|
11990
|
+
...spec,
|
|
11991
|
+
preview,
|
|
11992
|
+
// The provider route forwards to loopback on its placement node. Pin every
|
|
11993
|
+
// agent using that checkout to the same node so its dev server is reachable.
|
|
11994
|
+
...(preview.node ? { node: preview.node } : {}),
|
|
11995
|
+
};
|
|
11996
|
+
}
|
|
10691
11997
|
function authoritativeRoutedDecision(triaged, routed) {
|
|
10692
11998
|
if (triaged.confidence !== 'low' || triaged.routes.length > 0 || routed.routes.length === 0) {
|
|
10693
11999
|
return routed;
|
|
@@ -11059,6 +12365,8 @@ const routeForSpec = (decision, spec) => {
|
|
|
11059
12365
|
};
|
|
11060
12366
|
};
|
|
11061
12367
|
const previewUrlFromSpec = (spec) => {
|
|
12368
|
+
if (spec.preview?.url.trim())
|
|
12369
|
+
return spec.preview.url.trim();
|
|
11062
12370
|
const previewUrl = spec.previewUrl;
|
|
11063
12371
|
return typeof previewUrl === 'string' && previewUrl.trim() ? previewUrl.trim() : undefined;
|
|
11064
12372
|
};
|
|
@@ -11851,6 +13159,17 @@ const decodeGithubPathSegment = (value) => {
|
|
|
11851
13159
|
const validGithubRepo = (repo) => /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})\/[A-Za-z0-9_.-]{1,100}$/u.test(repo);
|
|
11852
13160
|
const validPrNumber = (value) => Number.isInteger(value) && value > 0;
|
|
11853
13161
|
const githubPrIdentity = (repo, prNumber) => validGithubRepo(repo) && validPrNumber(prNumber) ? `${repo.toLowerCase()}#${prNumber}` : undefined;
|
|
13162
|
+
// This is the public Relayfile stable identity for a GitHub pull request. It
|
|
13163
|
+
// deliberately comes from the PR's repo/number ownership record, never by
|
|
13164
|
+
// transforming an incoming canonical path (whose title slug can be renamed).
|
|
13165
|
+
const babysitterResourceRef = (repo, prNumber) => {
|
|
13166
|
+
if (!validGithubRepo(repo) || !validPrNumber(prNumber)) {
|
|
13167
|
+
throw new Error('Cannot create a durable babysitter subscription for an invalid GitHub PR identity');
|
|
13168
|
+
}
|
|
13169
|
+
const [owner, name] = repo.split('/');
|
|
13170
|
+
return `/github/repos/${owner}__${name}/pulls/by-id/${prNumber}.json`;
|
|
13171
|
+
};
|
|
13172
|
+
const babysitterSubscriberId = (issue) => `factory-babysitter:${issue.uuid}`;
|
|
11854
13173
|
const babysitterOwnershipKey = (issue, ref) => `${issueKey(issue)}:${githubPrIdentity(ref.repo, ref.prNumber) ?? 'invalid'}`;
|
|
11855
13174
|
const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
11856
13175
|
if (!validGithubRepo(eventRepo))
|
|
@@ -12184,12 +13503,17 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
12184
13503
|
}
|
|
12185
13504
|
return `${owner}/${repo}`;
|
|
12186
13505
|
};
|
|
12187
|
-
const githubPullRequestBody = (issue) => [
|
|
13506
|
+
const githubPullRequestBody = (issue, preview) => [
|
|
12188
13507
|
issue.description,
|
|
12189
13508
|
'',
|
|
12190
13509
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
12191
13510
|
? `Fixes #${issue.key}`
|
|
12192
13511
|
: `Factory issue ${issue.key}`,
|
|
13512
|
+
...(preview ? [
|
|
13513
|
+
'',
|
|
13514
|
+
`Live preview: ${preview.url}`,
|
|
13515
|
+
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
13516
|
+
] : []),
|
|
12193
13517
|
].join('\n').trim();
|
|
12194
13518
|
// The broker rejects re-registering a name it never released on exit
|
|
12195
13519
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
@@ -12647,6 +13971,20 @@ const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
|
|
|
12647
13971
|
`Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
|
|
12648
13972
|
].filter((line) => Boolean(line)).join('\n');
|
|
12649
13973
|
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
13974
|
+
const formatByteCount = (bytes) => {
|
|
13975
|
+
if (bytes < 1024)
|
|
13976
|
+
return `${bytes} B`;
|
|
13977
|
+
const units = ['KiB', 'MiB', 'GiB', 'TiB'];
|
|
13978
|
+
let value = bytes;
|
|
13979
|
+
let unit = 'B';
|
|
13980
|
+
for (const candidate of units) {
|
|
13981
|
+
value /= 1024;
|
|
13982
|
+
unit = candidate;
|
|
13983
|
+
if (value < 1024)
|
|
13984
|
+
break;
|
|
13985
|
+
}
|
|
13986
|
+
return `${value.toFixed(value >= 10 ? 1 : 2)} ${unit}`;
|
|
13987
|
+
};
|
|
12650
13988
|
const unrefDelay = (ms) => new Promise((resolve) => {
|
|
12651
13989
|
const timer = setTimeout(resolve, ms);
|
|
12652
13990
|
timer.unref?.();
|