@agent-relay/factory 0.1.50 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -11
- package/dist/cli/fleet.d.ts +4 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +87 -42
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +1671 -10
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +83 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/dispatch/templates.d.ts +6 -0
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +16 -0
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/environments/connection-registry.d.ts +617 -0
- package/dist/environments/connection-registry.d.ts.map +1 -0
- package/dist/environments/connection-registry.js +174 -0
- package/dist/environments/connection-registry.js.map +1 -0
- package/dist/environments/index.d.ts +21 -0
- package/dist/environments/index.d.ts.map +1 -0
- package/dist/environments/index.js +11 -0
- package/dist/environments/index.js.map +1 -0
- package/dist/environments/k6-job.d.ts +64 -0
- package/dist/environments/k6-job.d.ts.map +1 -0
- package/dist/environments/k6-job.js +376 -0
- package/dist/environments/k6-job.js.map +1 -0
- package/dist/environments/kubernetes-client.d.ts +72 -0
- package/dist/environments/kubernetes-client.d.ts.map +1 -0
- package/dist/environments/kubernetes-client.js +337 -0
- package/dist/environments/kubernetes-client.js.map +1 -0
- package/dist/environments/kubernetes-command.d.ts +30 -0
- package/dist/environments/kubernetes-command.d.ts.map +1 -0
- package/dist/environments/kubernetes-command.js +108 -0
- package/dist/environments/kubernetes-command.js.map +1 -0
- package/dist/environments/kubernetes-environment.d.ts +38 -0
- package/dist/environments/kubernetes-environment.d.ts.map +1 -0
- package/dist/environments/kubernetes-environment.js +333 -0
- package/dist/environments/kubernetes-environment.js.map +1 -0
- package/dist/environments/kubernetes-provider.d.ts +83 -0
- package/dist/environments/kubernetes-provider.d.ts.map +1 -0
- package/dist/environments/kubernetes-provider.js +956 -0
- package/dist/environments/kubernetes-provider.js.map +1 -0
- package/dist/environments/load-harness.d.ts +199 -0
- package/dist/environments/load-harness.d.ts.map +1 -0
- package/dist/environments/load-harness.js +259 -0
- package/dist/environments/load-harness.js.map +1 -0
- package/dist/environments/load-profile.d.ts +294 -0
- package/dist/environments/load-profile.d.ts.map +1 -0
- package/dist/environments/load-profile.js +125 -0
- package/dist/environments/load-profile.js.map +1 -0
- package/dist/environments/stack-deployer.d.ts +22 -0
- package/dist/environments/stack-deployer.d.ts.map +1 -0
- package/dist/environments/stack-deployer.js +27 -0
- package/dist/environments/stack-deployer.js.map +1 -0
- package/dist/environments/stack-descriptor.d.ts +243 -0
- package/dist/environments/stack-descriptor.d.ts.map +1 -0
- package/dist/environments/stack-descriptor.js +88 -0
- package/dist/environments/stack-descriptor.js.map +1 -0
- package/dist/environments/verification-pipeline.d.ts +117 -0
- package/dist/environments/verification-pipeline.d.ts.map +1 -0
- package/dist/environments/verification-pipeline.js +643 -0
- package/dist/environments/verification-pipeline.js.map +1 -0
- package/dist/environments/verification-stack-deployer.d.ts +73 -0
- package/dist/environments/verification-stack-deployer.d.ts.map +1 -0
- package/dist/environments/verification-stack-deployer.js +484 -0
- package/dist/environments/verification-stack-deployer.js.map +1 -0
- package/dist/environments/verification-stack-descriptor.d.ts +1134 -0
- package/dist/environments/verification-stack-descriptor.d.ts.map +1 -0
- package/dist/environments/verification-stack-descriptor.js +275 -0
- package/dist/environments/verification-stack-descriptor.js.map +1 -0
- package/dist/environments/verification-stack.d.ts +29 -0
- package/dist/environments/verification-stack.d.ts.map +1 -0
- package/dist/environments/verification-stack.js +43 -0
- package/dist/environments/verification-stack.js.map +1 -0
- package/dist/featuremap/generate.d.ts +72 -0
- package/dist/featuremap/generate.d.ts.map +1 -0
- package/dist/featuremap/generate.js +854 -0
- package/dist/featuremap/generate.js.map +1 -0
- package/dist/featuremap/index.d.ts +2 -0
- package/dist/featuremap/index.d.ts.map +1 -1
- package/dist/featuremap/index.js +1 -0
- package/dist/featuremap/index.js.map +1 -1
- package/dist/featuremap/validate.d.ts +6 -0
- package/dist/featuremap/validate.d.ts.map +1 -1
- package/dist/featuremap/validate.js +95 -13
- package/dist/featuremap/validate.js.map +1 -1
- package/dist/fleet/create-fleet.d.ts +2 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -1
- package/dist/fleet/create-fleet.js +1 -0
- package/dist/fleet/create-fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +8 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +31 -6
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +4 -1
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +142 -1
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/git/agent-worktree.d.ts +4 -1
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +151 -8
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/index.d.ts +32 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -3
- package/dist/index.js.map +1 -1
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +6 -2
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +8 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +11 -1
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/node/factory-node.d.ts +2 -0
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +107 -2
- package/dist/node/factory-node.js.map +1 -1
- package/dist/node/preview-process.d.ts +65 -0
- package/dist/node/preview-process.d.ts.map +1 -0
- package/dist/node/preview-process.js +431 -0
- package/dist/node/preview-process.js.map +1 -0
- package/dist/node/tailscale-preview.d.ts +45 -0
- package/dist/node/tailscale-preview.d.ts.map +1 -0
- package/dist/node/tailscale-preview.js +748 -0
- package/dist/node/tailscale-preview.js.map +1 -0
- package/dist/observability/events.d.ts +1838 -28
- package/dist/observability/events.d.ts.map +1 -1
- package/dist/observability/events.js +52 -0
- package/dist/observability/events.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1317 -61
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +2 -1
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/reaper.d.ts +24 -0
- package/dist/orchestrator/reaper.d.ts.map +1 -1
- package/dist/orchestrator/reaper.js +117 -0
- package/dist/orchestrator/reaper.js.map +1 -1
- package/dist/ports/environment.d.ts +83 -0
- package/dist/ports/environment.d.ts.map +1 -0
- package/dist/ports/environment.js +2 -0
- package/dist/ports/environment.js.map +1 -0
- package/dist/ports/fleet.d.ts +68 -1
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +3 -2
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +9 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/ports/worktree.d.ts +14 -0
- package/dist/ports/worktree.d.ts.map +1 -1
- package/dist/state/file-state-store.js +28 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/subscriptions/index.d.ts +2 -0
- package/dist/subscriptions/index.d.ts.map +1 -1
- package/dist/subscriptions/index.js +1 -0
- package/dist/subscriptions/index.js.map +1 -1
- package/dist/subscriptions/resource-subscriptions.d.ts +77 -0
- package/dist/subscriptions/resource-subscriptions.d.ts.map +1 -0
- package/dist/subscriptions/resource-subscriptions.js +91 -0
- package/dist/subscriptions/resource-subscriptions.js.map +1 -0
- package/dist/testing/fakes.d.ts +9 -1
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +40 -0
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +32 -32
- package/dist/types.d.ts +4 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +18 -4
- package/schemas/kubernetes-environment-stack.schema.json +86 -0
- package/schemas/verification-stack.schema.json +292 -0
|
@@ -5,7 +5,8 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
-
import {
|
|
8
|
+
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
|
+
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
9
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
10
11
|
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
@@ -16,6 +17,7 @@ import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/temp
|
|
|
16
17
|
import { resolveTestGuidance } from '../dispatch/test-guidance.js';
|
|
17
18
|
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
|
|
18
19
|
import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
|
|
20
|
+
import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
|
|
19
21
|
import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
20
22
|
import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
|
|
21
23
|
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
|
|
@@ -44,6 +46,7 @@ const LIVE_DEDUPE_LIMIT = 5_000;
|
|
|
44
46
|
const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
45
47
|
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
46
48
|
const COMPLETION_SWEEP_BATCH_SIZE = 2;
|
|
49
|
+
const PREVIEW_SWEEP_INTERVAL_MS = 60_000;
|
|
47
50
|
const PROBE_PR_GH_BACKOFF_MS = 60_000;
|
|
48
51
|
const PROBE_PR_GH_CANDIDATE_LIMIT = 200;
|
|
49
52
|
const PUBLISHED_PR_CONFIRM_ATTEMPTS = 20;
|
|
@@ -66,6 +69,24 @@ const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
|
66
69
|
const INJECTION_MAX_ATTEMPTS = 6;
|
|
67
70
|
const BABYSITTER_EVENT_COALESCE_MS = 750;
|
|
68
71
|
const BABYSITTER_EVENT_RETRY_MS = 1_000;
|
|
72
|
+
const BABYSITTER_SUBSCRIPTION_TTL_SECONDS = 60 * 60;
|
|
73
|
+
// Relayfile receives provider-native GitHub events, not the materialized file
|
|
74
|
+
// changes that the legacy local router consumed. `closed` is separately
|
|
75
|
+
// indexed as a terminal event below, so terminal delivery does not need a
|
|
76
|
+
// broad normal-event subscription.
|
|
77
|
+
const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
78
|
+
'pull_request.opened',
|
|
79
|
+
'pull_request.reopened',
|
|
80
|
+
'pull_request.synchronize',
|
|
81
|
+
'pull_request.ready_for_review',
|
|
82
|
+
'pull_request_review.submitted',
|
|
83
|
+
'pull_request_review_comment.created',
|
|
84
|
+
'issue_comment.created',
|
|
85
|
+
'check_run.completed',
|
|
86
|
+
];
|
|
87
|
+
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
88
|
+
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
89
|
+
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
69
90
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
70
91
|
// recipient unavailable) both when a freshly spawned agent has not finished
|
|
71
92
|
// enrolling AND when an agent is up but its relay identity never becomes
|
|
@@ -138,6 +159,7 @@ export class FactoryLoop {
|
|
|
138
159
|
#githubWritebackProvided;
|
|
139
160
|
#slack;
|
|
140
161
|
#mergeGate;
|
|
162
|
+
#verificationGate;
|
|
141
163
|
#probeCloser;
|
|
142
164
|
#probePrResolver;
|
|
143
165
|
#customProbePrResolver;
|
|
@@ -246,6 +268,8 @@ export class FactoryLoop {
|
|
|
246
268
|
#deferLiveEventDrain = false;
|
|
247
269
|
#completionSweepTimer;
|
|
248
270
|
#completionSweepActive = false;
|
|
271
|
+
#previewSweepTimer;
|
|
272
|
+
#previewSweepInFlight;
|
|
249
273
|
#completionInFlight = new Set();
|
|
250
274
|
#agentExitsInFlight = new Map();
|
|
251
275
|
#reconciledAgentExitsActive = 0;
|
|
@@ -262,6 +286,14 @@ export class FactoryLoop {
|
|
|
262
286
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
263
287
|
#babysitterPr = new Map();
|
|
264
288
|
#babysitterIssueRefs = new Map();
|
|
289
|
+
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
290
|
+
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
291
|
+
// owning babysitter.
|
|
292
|
+
#babysitterSubscriptionOwners = new Map();
|
|
293
|
+
#babysitterResourceSubscriptionFault = false;
|
|
294
|
+
#babysitterResourceSubscriptionUnavailable = false;
|
|
295
|
+
#babysitterResourceDeliveryRetryTimer;
|
|
296
|
+
#babysitterResourceSubscriptionRenewTimer;
|
|
265
297
|
#babysitterReady = new Set();
|
|
266
298
|
#babysitterWakeStates = new Map();
|
|
267
299
|
// A babysitter announces this fence before invoking destructive git tooling
|
|
@@ -269,6 +301,8 @@ export class FactoryLoop {
|
|
|
269
301
|
// is active, but the PTY submit must never land in that critical window.
|
|
270
302
|
#babysitterCriticalAgents = new Set();
|
|
271
303
|
#publishedPullRequests = new Map();
|
|
304
|
+
#previewReferences = new Map();
|
|
305
|
+
#removedPreviewIds = new Set();
|
|
272
306
|
#probePrGhBackoffUntilMs = new Map();
|
|
273
307
|
#probePrResolvedCache = new Map();
|
|
274
308
|
// GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion
|
|
@@ -308,6 +342,17 @@ export class FactoryLoop {
|
|
|
308
342
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
309
343
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
310
344
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
345
|
+
this.#verificationGate = ports.verificationGate ?? (config.verification.enabled
|
|
346
|
+
? new VerificationPipeline({
|
|
347
|
+
descriptorPath: config.verification.descriptorPath,
|
|
348
|
+
reporter: ports.reporter,
|
|
349
|
+
logger: ports.logger,
|
|
350
|
+
maxConcurrentEnvironments: config.verification.maxConcurrentEnvironments,
|
|
351
|
+
maxRunTimeoutMs: config.verification.maxRunTimeoutMs,
|
|
352
|
+
maxEnvironmentTtlMs: config.verification.maxEnvironmentTtlMs,
|
|
353
|
+
maxTeardownTimeoutMs: config.verification.maxTeardownTimeoutMs,
|
|
354
|
+
})
|
|
355
|
+
: undefined);
|
|
311
356
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
312
357
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
313
358
|
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
@@ -533,12 +578,15 @@ export class FactoryLoop {
|
|
|
533
578
|
this.#wireFleetEvents();
|
|
534
579
|
await this.#adoptInFlightAgents(legacyRegistry);
|
|
535
580
|
this.#startupAgentAdoptionActive = false;
|
|
581
|
+
if (opts.mode !== 'dispatch-owner')
|
|
582
|
+
await this.#reapOrphanedWorktreesOnStartup(legacyRegistry);
|
|
536
583
|
if (this.#config.babysitter.enabled) {
|
|
537
584
|
// Re-run the idempotent receipt fold after adoption returns. This
|
|
538
585
|
// catches records restored by lifecycle work that completed while the
|
|
539
586
|
// startup roster drain was in progress.
|
|
540
587
|
await this.#reconcileRestoredBabysitterReceipts();
|
|
541
588
|
}
|
|
589
|
+
await this.#reapPreviewOrphans();
|
|
542
590
|
}
|
|
543
591
|
catch (error) {
|
|
544
592
|
this.#startupAgentAdoptionActive = false;
|
|
@@ -548,6 +596,7 @@ export class FactoryLoop {
|
|
|
548
596
|
}
|
|
549
597
|
if (opts.mode === 'dispatch-owner') {
|
|
550
598
|
this.#started = true;
|
|
599
|
+
this.#schedulePreviewSweep();
|
|
551
600
|
this.#scheduleDispatchLifecycleRenewal();
|
|
552
601
|
// A replacement one-shot owner must also recover a team parked for
|
|
553
602
|
// human input; it intentionally does not subscribe to the full issue
|
|
@@ -559,6 +608,7 @@ export class FactoryLoop {
|
|
|
559
608
|
}
|
|
560
609
|
if (live) {
|
|
561
610
|
this.#started = true;
|
|
611
|
+
this.#schedulePreviewSweep();
|
|
562
612
|
try {
|
|
563
613
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
564
614
|
await this.#rearmSlackReplyWatchers();
|
|
@@ -598,6 +648,7 @@ export class FactoryLoop {
|
|
|
598
648
|
void this.#handleChange(path);
|
|
599
649
|
});
|
|
600
650
|
this.#started = true;
|
|
651
|
+
this.#schedulePreviewSweep();
|
|
601
652
|
await this.#rearmSlackReplyWatchers();
|
|
602
653
|
await this.#drainReadyClarificationWake();
|
|
603
654
|
await this.#rearmGithubIssueCommentWatchers();
|
|
@@ -606,6 +657,12 @@ export class FactoryLoop {
|
|
|
606
657
|
async stop() {
|
|
607
658
|
this.#started = false;
|
|
608
659
|
this.#stopping = true;
|
|
660
|
+
if (this.#babysitterResourceDeliveryRetryTimer)
|
|
661
|
+
clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
|
|
662
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
663
|
+
if (this.#babysitterResourceSubscriptionRenewTimer)
|
|
664
|
+
clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
|
|
665
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
609
666
|
if (this.#dispatchLifecycleRenewTimer)
|
|
610
667
|
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
611
668
|
this.#dispatchLifecycleRenewTimer = undefined;
|
|
@@ -617,6 +674,10 @@ export class FactoryLoop {
|
|
|
617
674
|
if (this.#completionSweepTimer)
|
|
618
675
|
clearTimeout(this.#completionSweepTimer);
|
|
619
676
|
this.#completionSweepTimer = undefined;
|
|
677
|
+
if (this.#previewSweepTimer)
|
|
678
|
+
clearTimeout(this.#previewSweepTimer);
|
|
679
|
+
this.#previewSweepTimer = undefined;
|
|
680
|
+
await this.#previewSweepInFlight;
|
|
620
681
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
621
682
|
try {
|
|
622
683
|
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
@@ -657,6 +718,7 @@ export class FactoryLoop {
|
|
|
657
718
|
this.#babysitterSpawned.clear();
|
|
658
719
|
this.#babysitterPr.clear();
|
|
659
720
|
this.#babysitterIssueRefs.clear();
|
|
721
|
+
this.#babysitterSubscriptionOwners.clear();
|
|
660
722
|
this.#babysitterReady.clear();
|
|
661
723
|
this.#babysitterCriticalAgents.clear();
|
|
662
724
|
const subscription = this.#subscription;
|
|
@@ -2095,9 +2157,6 @@ export class FactoryLoop {
|
|
|
2095
2157
|
};
|
|
2096
2158
|
}
|
|
2097
2159
|
this.#clearDependencyPark(batch, dispatchDecision.issue);
|
|
2098
|
-
// Full task rendering is part of the durable spawn specification. It must
|
|
2099
|
-
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2100
|
-
// recover a persisted minimal triage task after a crash in this gap.
|
|
2101
2160
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
2102
2161
|
// Local dispatches need the same deterministic branch identity as remote
|
|
2103
2162
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -2109,10 +2168,17 @@ export class FactoryLoop {
|
|
|
2109
2168
|
isolateLocalWorktree,
|
|
2110
2169
|
});
|
|
2111
2170
|
}
|
|
2171
|
+
// Full task rendering is part of the durable spawn specification. It must
|
|
2172
|
+
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2173
|
+
// recover a persisted minimal triage task after a crash in this gap. The
|
|
2174
|
+
// task is rendered again below after preview provisioning adds its URL.
|
|
2112
2175
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2176
|
+
let claimedLifecycle;
|
|
2177
|
+
let recoveredRecord;
|
|
2113
2178
|
if (durableDispatch) {
|
|
2114
2179
|
const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
|
|
2115
2180
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2181
|
+
claimedLifecycle = lifecycleClaim.lifecycle;
|
|
2116
2182
|
dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
|
|
2117
2183
|
if (lifecycleClaim.lifecycle.phase === 'waiting-for-human') {
|
|
2118
2184
|
return lifecycleClaim.lifecycle.result ?? { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
@@ -2130,16 +2196,70 @@ export class FactoryLoop {
|
|
|
2130
2196
|
};
|
|
2131
2197
|
}
|
|
2132
2198
|
if (!lifecycleClaim.created) {
|
|
2133
|
-
|
|
2134
|
-
if (
|
|
2135
|
-
return
|
|
2199
|
+
recoveredRecord = inFlightRecordFromLifecycle(lifecycleClaim.lifecycle);
|
|
2200
|
+
if (recoveredRecord.result)
|
|
2201
|
+
return recoveredRecord.result;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
// External preview creation must happen only after the durable lease is
|
|
2205
|
+
// acquired and capacity admission has promoted the lifecycle. Persist the
|
|
2206
|
+
// fully rendered, preview-bearing decision before any worker can spawn so
|
|
2207
|
+
// takeover never recovers a minimal triage task.
|
|
2208
|
+
const previouslyPersistedPreviewIds = new Set(dispatchSpecs(dispatchDecision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
2209
|
+
try {
|
|
2210
|
+
if (!dryRun) {
|
|
2211
|
+
dispatchDecision = await this.#withPreviewReferences(dispatchDecision);
|
|
2212
|
+
}
|
|
2213
|
+
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2214
|
+
if (durableDispatch && claimedLifecycle) {
|
|
2215
|
+
const stagedRecord = inFlightRecordFromLifecycle({
|
|
2216
|
+
...claimedLifecycle,
|
|
2217
|
+
decision: structuredClone(dispatchDecision),
|
|
2218
|
+
});
|
|
2219
|
+
if (!await this.#saveDispatchLifecycle(stagedRecord, 'dispatching')) {
|
|
2220
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${dispatchDecision.issue.key}`);
|
|
2221
|
+
}
|
|
2222
|
+
if (recoveredRecord) {
|
|
2223
|
+
recoveredRecord.decision = structuredClone(dispatchDecision);
|
|
2224
|
+
recoveredRecord = batch.restore(recoveredRecord);
|
|
2225
|
+
}
|
|
2136
2226
|
}
|
|
2137
2227
|
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
const newlyCreated = uniquePreviewReferences([
|
|
2230
|
+
...dispatchSpecs(dispatchDecision).map((spec) => spec.preview),
|
|
2231
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue)) ?? []),
|
|
2232
|
+
]).filter((preview) => !previouslyPersistedPreviewIds.has(preview.id));
|
|
2233
|
+
// Once the durable fence is lost, the successor may already have
|
|
2234
|
+
// adopted this deterministic issue preview. Leave cleanup to the
|
|
2235
|
+
// identity-aware sweep instead of letting a stale owner tear down the
|
|
2236
|
+
// successor's route.
|
|
2237
|
+
const mayRollback = !claimedLifecycle ||
|
|
2238
|
+
await this.#assertIssueDispatchLifecycleOwner(dispatchDecision.issue);
|
|
2239
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
2240
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
2241
|
+
this.#logger.warn?.('[factory] failed to roll back preview provisioning', {
|
|
2242
|
+
issue: dispatchDecision.issue.key,
|
|
2243
|
+
error: describeError(cleanupError).errorMessage,
|
|
2244
|
+
});
|
|
2245
|
+
});
|
|
2246
|
+
}
|
|
2247
|
+
if (mayRollback)
|
|
2248
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2249
|
+
if (claimedLifecycle) {
|
|
2250
|
+
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(claimedLifecycle));
|
|
2251
|
+
}
|
|
2252
|
+
throw error;
|
|
2253
|
+
}
|
|
2138
2254
|
if (!durableDispatch)
|
|
2139
2255
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2140
2256
|
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
2141
|
-
const record = batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2257
|
+
const record = recoveredRecord ?? batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2142
2258
|
if (!record) {
|
|
2259
|
+
if (!dryRun) {
|
|
2260
|
+
await this.#teardownPreviewReferences(dispatchSpecs(dispatchDecision).map((spec) => spec.preview));
|
|
2261
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2262
|
+
}
|
|
2143
2263
|
await this.#clearDispatchInFlight(dispatchDecision.issue);
|
|
2144
2264
|
this.#increment('queued');
|
|
2145
2265
|
this.#emit('issue-queued', { issue: dispatchDecision.issue });
|
|
@@ -2148,7 +2268,9 @@ export class FactoryLoop {
|
|
|
2148
2268
|
if (record.result) {
|
|
2149
2269
|
return record.result;
|
|
2150
2270
|
}
|
|
2151
|
-
await this.#saveDispatchLifecycle(record, 'dispatching')
|
|
2271
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
2272
|
+
throw new Error(`Dispatch lifecycle ownership lost immediately before spawning ${dispatchDecision.issue.key}`);
|
|
2273
|
+
}
|
|
2152
2274
|
if (!dryRun)
|
|
2153
2275
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
2154
2276
|
const spawnedForReaperHandoff = [];
|
|
@@ -2203,6 +2325,9 @@ export class FactoryLoop {
|
|
|
2203
2325
|
agents,
|
|
2204
2326
|
comments: [comment],
|
|
2205
2327
|
stateId: implementingStateId,
|
|
2328
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue))?.length
|
|
2329
|
+
? { previews: this.#previewReferences.get(issueKey(dispatchDecision.issue)) }
|
|
2330
|
+
: {}),
|
|
2206
2331
|
dryRun,
|
|
2207
2332
|
};
|
|
2208
2333
|
record.result = result;
|
|
@@ -2223,6 +2348,14 @@ export class FactoryLoop {
|
|
|
2223
2348
|
const liveStateChanged = error instanceof LiveDispatchStateChangedError;
|
|
2224
2349
|
const cancellationReason = factoryCloudDispatchCancellationReason(error);
|
|
2225
2350
|
const cleanupReason = liveStateChanged ? 'live dispatch state changed' : 'dispatch failed';
|
|
2351
|
+
let failedState;
|
|
2352
|
+
if (!liveStateChanged) {
|
|
2353
|
+
await this.#recordDispatchFailure(decision.issue);
|
|
2354
|
+
failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
2355
|
+
}
|
|
2356
|
+
const terminalFailure = liveStateChanged || Boolean(failedState?.terminal);
|
|
2357
|
+
if (terminalFailure && !await this.#saveDispatchLifecycle(record, 'abandoning', undefined, cleanupReason, new Set(), { cancellationReason }))
|
|
2358
|
+
throw error;
|
|
2226
2359
|
let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs, cleanupReason);
|
|
2227
2360
|
if (liveStateChanged && !failureHandoffs.some((handoff) => handoff.worktree)) {
|
|
2228
2361
|
const failed = await this.#releaseAndTerminateAgents(failureHandoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion');
|
|
@@ -2233,16 +2366,43 @@ export class FactoryLoop {
|
|
|
2233
2366
|
worktreesTornDown = failureHandoffs.length > 0;
|
|
2234
2367
|
}
|
|
2235
2368
|
}
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2369
|
+
if (terminalFailure) {
|
|
2370
|
+
try {
|
|
2371
|
+
await this.#teardownPreviews(record);
|
|
2372
|
+
}
|
|
2373
|
+
catch (previewError) {
|
|
2374
|
+
this.#logger.warn?.('[factory] failed to tear down previews after terminal dispatch failure', {
|
|
2375
|
+
issue: record.issue.key,
|
|
2376
|
+
error: describeError(previewError).errorMessage,
|
|
2377
|
+
});
|
|
2378
|
+
// Do not commit a terminal lifecycle while an externally reachable
|
|
2379
|
+
// preview remains. The abandonment driver retries the identity-
|
|
2380
|
+
// checked teardown and commits terminal state only after it succeeds.
|
|
2381
|
+
this.#scheduleAbandonedDispatchRetry(record, cleanupReason);
|
|
2382
|
+
if (!liveStateChanged)
|
|
2383
|
+
this.#error(error, decision.issue);
|
|
2384
|
+
if (worktreesTornDown) {
|
|
2385
|
+
await this.#writeInFlightRegistry().catch((registryError) => {
|
|
2386
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after dispatch worktree teardown', {
|
|
2387
|
+
issue: record.issue,
|
|
2388
|
+
error: describeError(registryError).errorMessage,
|
|
2389
|
+
});
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, undefined, new Set(), { cancellationReason }))
|
|
2395
|
+
throw error;
|
|
2396
|
+
if (liveStateChanged)
|
|
2397
|
+
await this.#clearDispatchInFlight(decision.issue);
|
|
2398
|
+
else
|
|
2399
|
+
await this.#recordDispatchTerminal(decision.issue);
|
|
2400
|
+
if (liveStateChanged)
|
|
2401
|
+
this.#increment('dispatchLiveStateRaces');
|
|
2241
2402
|
}
|
|
2242
2403
|
else {
|
|
2243
|
-
await this.#
|
|
2244
|
-
|
|
2245
|
-
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable', undefined, undefined, new Set(), { cancellationReason: failedState?.terminal ? cancellationReason : undefined });
|
|
2404
|
+
if (!await this.#saveDispatchLifecycle(record, 'retryable'))
|
|
2405
|
+
throw error;
|
|
2246
2406
|
}
|
|
2247
2407
|
batch.abandon(decision.issue);
|
|
2248
2408
|
if (!liveStateChanged && !failedState?.terminal)
|
|
@@ -2378,6 +2538,12 @@ export class FactoryLoop {
|
|
|
2378
2538
|
for (const [key, lifecycle] of durableLifecycles) {
|
|
2379
2539
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
2380
2540
|
continue;
|
|
2541
|
+
const previews = uniquePreviewReferences([
|
|
2542
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
2543
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
2544
|
+
]);
|
|
2545
|
+
if (previews.length > 0)
|
|
2546
|
+
this.#previewReferences.set(issueKey(lifecycle.issue), previews);
|
|
2381
2547
|
hasNonterminalDurableLifecycle = true;
|
|
2382
2548
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2383
2549
|
if (!claim.acquired || !claim.lease) {
|
|
@@ -2391,6 +2557,33 @@ export class FactoryLoop {
|
|
|
2391
2557
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2392
2558
|
continue;
|
|
2393
2559
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
2560
|
+
if (!durableRecord.dryRun &&
|
|
2561
|
+
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2562
|
+
claim.lifecycle.phase !== 'releasing') {
|
|
2563
|
+
const liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2564
|
+
// A babysat Linear issue already at Done may have merged while this
|
|
2565
|
+
// process was down. Let authoritative PR restoration drive the
|
|
2566
|
+
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
2567
|
+
const deferDoneToBabysitterRecovery = Boolean(liveIssue &&
|
|
2568
|
+
!isGithubIssue(liveIssue) &&
|
|
2569
|
+
this.#states.roleOf(liveIssue.stateId) === 'done' &&
|
|
2570
|
+
this.#config.babysitter.enabled &&
|
|
2571
|
+
await this.#hasRestorableMergedBabysitterSession(durableRecord.issue));
|
|
2572
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue) && !deferDoneToBabysitterRecovery) {
|
|
2573
|
+
const restored = batch.restore(durableRecord);
|
|
2574
|
+
try {
|
|
2575
|
+
await this.#abandonDurableResume(restored, 'source issue is already terminal during startup recovery');
|
|
2576
|
+
}
|
|
2577
|
+
catch (error) {
|
|
2578
|
+
this.#logger.warn?.('[factory] terminal source preview cleanup will retry after startup', {
|
|
2579
|
+
issue: durableRecord.issue.key,
|
|
2580
|
+
error: describeError(error).errorMessage,
|
|
2581
|
+
});
|
|
2582
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2583
|
+
}
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2394
2587
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2395
2588
|
? durableRecord
|
|
2396
2589
|
: batch.restore(durableRecord);
|
|
@@ -2928,6 +3121,13 @@ export class FactoryLoop {
|
|
|
2928
3121
|
await this.#abandonStuckDispatch(record, abandonedReason);
|
|
2929
3122
|
return;
|
|
2930
3123
|
}
|
|
3124
|
+
if (lifecycle.phase === 'running' && !record.dryRun) {
|
|
3125
|
+
const liveIssue = await this.#readIssue(record.issue.path);
|
|
3126
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3127
|
+
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3128
|
+
return;
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
2931
3131
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
2932
3132
|
if (this.#fleet.hydrateTracked) {
|
|
2933
3133
|
this.#fleet.hydrateTracked(lifecycle.agents.map((agent) => ({
|
|
@@ -2949,6 +3149,10 @@ export class FactoryLoop {
|
|
|
2949
3149
|
await this.#finishClarificationPark(waiting, true);
|
|
2950
3150
|
return;
|
|
2951
3151
|
}
|
|
3152
|
+
if (lifecycle.phase === 'abandoning') {
|
|
3153
|
+
await this.#abandonStuckDispatch(record, lifecycle.releaseReason ?? 'dispatch failed');
|
|
3154
|
+
return;
|
|
3155
|
+
}
|
|
2952
3156
|
if (lifecycle.phase === 'dispatching' || lifecycle.phase === 'retryable') {
|
|
2953
3157
|
await this.#resumeDurableDispatch(record);
|
|
2954
3158
|
return;
|
|
@@ -3007,15 +3211,39 @@ export class FactoryLoop {
|
|
|
3007
3211
|
}
|
|
3008
3212
|
}
|
|
3009
3213
|
async #resumeDurableDispatch(record) {
|
|
3214
|
+
let liveIssue;
|
|
3010
3215
|
if (!record.dryRun) {
|
|
3011
|
-
|
|
3012
|
-
if (!
|
|
3216
|
+
liveIssue = await this.#readIssue(record.issue.path);
|
|
3217
|
+
if (!liveIssue) {
|
|
3013
3218
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
|
|
3014
3219
|
}
|
|
3015
|
-
if (
|
|
3016
|
-
await this.#abandonDurableResume(record, 'live
|
|
3220
|
+
if (this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3221
|
+
await this.#abandonDurableResume(record, 'live source issue is already terminal');
|
|
3017
3222
|
return;
|
|
3018
3223
|
}
|
|
3224
|
+
const persistedPreviewIds = new Set(dispatchSpecs(record.decision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
3225
|
+
try {
|
|
3226
|
+
record.decision = await this.#withPreviewReferences(record.decision);
|
|
3227
|
+
record.decision = await this.#withRenderedDispatchTasks(record.decision, liveIssue);
|
|
3228
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching'))
|
|
3229
|
+
return;
|
|
3230
|
+
}
|
|
3231
|
+
catch (error) {
|
|
3232
|
+
const newlyCreated = uniquePreviewReferences([
|
|
3233
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3234
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
3235
|
+
]).filter((preview) => !persistedPreviewIds.has(preview.id));
|
|
3236
|
+
const mayRollback = await this.#assertDispatchLifecycleOwner(record);
|
|
3237
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
3238
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
3239
|
+
this.#logger.warn?.('[factory] failed to roll back recovered preview provisioning', {
|
|
3240
|
+
issue: record.issue.key,
|
|
3241
|
+
error: describeError(cleanupError).errorMessage,
|
|
3242
|
+
});
|
|
3243
|
+
});
|
|
3244
|
+
}
|
|
3245
|
+
throw error;
|
|
3246
|
+
}
|
|
3019
3247
|
}
|
|
3020
3248
|
const agents = [];
|
|
3021
3249
|
const specs = dispatchSpecs(record.decision);
|
|
@@ -3032,7 +3260,7 @@ export class FactoryLoop {
|
|
|
3032
3260
|
}
|
|
3033
3261
|
await this.#writeInFlightRegistry();
|
|
3034
3262
|
if (!record.dryRun) {
|
|
3035
|
-
const issue = await this.#readIssue(record.issue.path);
|
|
3263
|
+
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3036
3264
|
if (!issue)
|
|
3037
3265
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3038
3266
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
@@ -3043,15 +3271,24 @@ export class FactoryLoop {
|
|
|
3043
3271
|
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3044
3272
|
}
|
|
3045
3273
|
}
|
|
3274
|
+
const recoveredPreviews = uniquePreviewReferences([
|
|
3275
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3276
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
3277
|
+
]);
|
|
3046
3278
|
record.result ??= {
|
|
3047
3279
|
issue: record.issue,
|
|
3048
3280
|
agents,
|
|
3049
3281
|
comments: [dispatchComment(record.decision, agents)],
|
|
3282
|
+
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3050
3283
|
dryRun: record.dryRun,
|
|
3051
3284
|
};
|
|
3285
|
+
if (recoveredPreviews.length > 0 && !record.result.previews?.length) {
|
|
3286
|
+
record.result = { ...record.result, previews: recoveredPreviews };
|
|
3287
|
+
}
|
|
3052
3288
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3053
3289
|
return;
|
|
3054
3290
|
if (!record.dryRun) {
|
|
3291
|
+
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3055
3292
|
for (const tracked of record.agents.values()) {
|
|
3056
3293
|
const owned = tracked.spec.ownedPullRequest;
|
|
3057
3294
|
if (tracked.spec.role !== 'babysitter' || !owned)
|
|
@@ -3076,6 +3313,31 @@ export class FactoryLoop {
|
|
|
3076
3313
|
labels.has('factory:in-progress') &&
|
|
3077
3314
|
!labels.has('factory:human-review');
|
|
3078
3315
|
}
|
|
3316
|
+
#isIssueExternallyTerminal(issue) {
|
|
3317
|
+
if (isGithubIssue(issue)) {
|
|
3318
|
+
if (githubFactoryIssueIsClosed(issue))
|
|
3319
|
+
return true;
|
|
3320
|
+
return issue.labels.some((label) => label.trim().toLowerCase() === 'factory:human-review');
|
|
3321
|
+
}
|
|
3322
|
+
const role = this.#states.roleOf(issue.stateId);
|
|
3323
|
+
return role === 'humanReview' || role === 'done';
|
|
3324
|
+
}
|
|
3325
|
+
async #hasRestorableMergedBabysitterSession(issue) {
|
|
3326
|
+
const wanted = issueKey(issue);
|
|
3327
|
+
for (const [, session] of await this.#state.listBabysitterSessions(this.#workspaceId)) {
|
|
3328
|
+
if (issueKey(session.issue) !== wanted ||
|
|
3329
|
+
!validGithubRepo(session.repo) ||
|
|
3330
|
+
!validPrNumber(session.prNumber) ||
|
|
3331
|
+
!session.agentName)
|
|
3332
|
+
continue;
|
|
3333
|
+
const snapshot = await this.#readPrSnapshot(session);
|
|
3334
|
+
if (snapshot &&
|
|
3335
|
+
prMetaShowsMerged(snapshot) &&
|
|
3336
|
+
prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30)
|
|
3337
|
+
return true;
|
|
3338
|
+
}
|
|
3339
|
+
return false;
|
|
3340
|
+
}
|
|
3079
3341
|
async #abandonDurableResume(record, reason) {
|
|
3080
3342
|
const handoffs = this.#dispatchFailureHandoffs(record, [...record.agents].map(([name, tracked]) => ({
|
|
3081
3343
|
issue: record.issue,
|
|
@@ -3084,25 +3346,32 @@ export class FactoryLoop {
|
|
|
3084
3346
|
persistedAtMs: this.#clock.now(),
|
|
3085
3347
|
})));
|
|
3086
3348
|
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
batch.abandon(record.issue);
|
|
3092
|
-
for (const [name] of record.agents) {
|
|
3093
|
-
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3094
|
-
}
|
|
3349
|
+
// Keep the lifecycle nonterminal until every externally reachable route
|
|
3350
|
+
// is confirmed gone. A restart can then retry cleanup instead of treating
|
|
3351
|
+
// an abandoned row as finished and leaking its issue preview forever.
|
|
3352
|
+
await this.#teardownPreviews(record);
|
|
3095
3353
|
if (handoffs.some((handoff) => handoff.worktree)) {
|
|
3096
|
-
await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')
|
|
3354
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')) {
|
|
3355
|
+
throw new Error(`Unable to finish stale dispatch worktree teardown for ${record.issue.key}`);
|
|
3356
|
+
}
|
|
3097
3357
|
}
|
|
3098
3358
|
else if (handoffs.length > 0) {
|
|
3099
3359
|
const failed = new Set(await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion'));
|
|
3360
|
+
if (failed.size > 0) {
|
|
3361
|
+
throw new Error(`Unable to release stale dispatch agents for ${record.issue.key}: ${[...failed].join(', ')}`);
|
|
3362
|
+
}
|
|
3100
3363
|
for (const handoff of handoffs) {
|
|
3101
|
-
if (failed.has(handoff.name))
|
|
3102
|
-
continue;
|
|
3103
3364
|
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
3104
3365
|
}
|
|
3105
3366
|
}
|
|
3367
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'source_state_changed' }))
|
|
3368
|
+
return;
|
|
3369
|
+
await this.#recordDispatchTerminal(record.issue);
|
|
3370
|
+
const batch = await this.#batch();
|
|
3371
|
+
batch.abandon(record.issue);
|
|
3372
|
+
for (const [name] of record.agents) {
|
|
3373
|
+
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3374
|
+
}
|
|
3106
3375
|
await this.#stopSlackWatcher(record.issue);
|
|
3107
3376
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
3108
3377
|
await this.#writeInFlightRegistry();
|
|
@@ -3117,6 +3386,16 @@ export class FactoryLoop {
|
|
|
3117
3386
|
const batch = await this.#batch();
|
|
3118
3387
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
3119
3388
|
const releaseKey = issueKey(record.issue);
|
|
3389
|
+
// Terminal writeback has already been acknowledged before this method is
|
|
3390
|
+
// entered. Remove externally reachable routes first so a stuck agent
|
|
3391
|
+
// release cannot leave a preview live after Human Review or Done.
|
|
3392
|
+
try {
|
|
3393
|
+
await this.#teardownPreviews(record);
|
|
3394
|
+
}
|
|
3395
|
+
catch {
|
|
3396
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
3397
|
+
return false;
|
|
3398
|
+
}
|
|
3120
3399
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, releaseKey);
|
|
3121
3400
|
const released = new Set(lifecycle?.agents
|
|
3122
3401
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
@@ -5013,7 +5292,7 @@ export class FactoryLoop {
|
|
|
5013
5292
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
5014
5293
|
baseRef,
|
|
5015
5294
|
title: `${issue.key}: ${issue.title}`,
|
|
5016
|
-
body: githubPullRequestBody(issue),
|
|
5295
|
+
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
5017
5296
|
});
|
|
5018
5297
|
const published = result.author
|
|
5019
5298
|
? result
|
|
@@ -5196,9 +5475,42 @@ export class FactoryLoop {
|
|
|
5196
5475
|
if (worktree)
|
|
5197
5476
|
unique.set(worktree.worktreePath, worktree);
|
|
5198
5477
|
}
|
|
5478
|
+
const issueSlug = factoryWorktreeIssueSlug(record.issue.key);
|
|
5199
5479
|
const failures = [];
|
|
5480
|
+
for (const repository of this.#worktreeRepositories(record)) {
|
|
5481
|
+
try {
|
|
5482
|
+
const candidates = await this.#worktrees.listWorktrees(repository);
|
|
5483
|
+
for (const candidate of candidates) {
|
|
5484
|
+
if (factoryWorktreeIssueSlug(candidate.issueKey) === issueSlug) {
|
|
5485
|
+
unique.set(candidate.worktreePath, candidate);
|
|
5486
|
+
}
|
|
5487
|
+
}
|
|
5488
|
+
}
|
|
5489
|
+
catch (error) {
|
|
5490
|
+
const message = `${repository.baseClonePath}: ${describeError(error).errorMessage}`;
|
|
5491
|
+
failures.push(message);
|
|
5492
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5493
|
+
this.#logger.warn?.('[factory] failed to enumerate completed issue worktrees', {
|
|
5494
|
+
issue: record.issue.key,
|
|
5495
|
+
repo: repository.repo,
|
|
5496
|
+
baseClonePath: repository.baseClonePath,
|
|
5497
|
+
error: describeError(error).errorMessage,
|
|
5498
|
+
});
|
|
5499
|
+
}
|
|
5500
|
+
}
|
|
5200
5501
|
for (const worktree of unique.values()) {
|
|
5201
5502
|
try {
|
|
5503
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5504
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5505
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5506
|
+
this.#logger.warn?.('[factory] retained completed issue worktree with local state', {
|
|
5507
|
+
issue: record.issue.key,
|
|
5508
|
+
repo: worktree.repo,
|
|
5509
|
+
worktreePath: worktree.worktreePath,
|
|
5510
|
+
retentionReasons: inspection.retentionReasons,
|
|
5511
|
+
});
|
|
5512
|
+
continue;
|
|
5513
|
+
}
|
|
5202
5514
|
await this.#worktrees.cleanup(worktree);
|
|
5203
5515
|
this.#increment('agentWorktreesCleaned');
|
|
5204
5516
|
}
|
|
@@ -5217,6 +5529,132 @@ export class FactoryLoop {
|
|
|
5217
5529
|
throw new Error(`Factory worktree cleanup incomplete for ${record.issue.key}: ${failures.join('; ')}`);
|
|
5218
5530
|
}
|
|
5219
5531
|
}
|
|
5532
|
+
#worktreeRepositories(record) {
|
|
5533
|
+
const repositories = new Map();
|
|
5534
|
+
const add = (repo, baseClonePath) => {
|
|
5535
|
+
if (!baseClonePath)
|
|
5536
|
+
return;
|
|
5537
|
+
const key = `${repo}\u0000${resolve(baseClonePath)}`;
|
|
5538
|
+
repositories.set(key, { repo, baseClonePath });
|
|
5539
|
+
};
|
|
5540
|
+
if (record) {
|
|
5541
|
+
for (const route of record.decision.routes) {
|
|
5542
|
+
add(route.repo, this.#config.repos.clonePaths[route.repo] ?? route.clonePath);
|
|
5543
|
+
}
|
|
5544
|
+
for (const tracked of record.agents.values()) {
|
|
5545
|
+
add(tracked.spec.repo, tracked.spec.baseClonePath);
|
|
5546
|
+
}
|
|
5547
|
+
for (const spec of record.decision.implementers) {
|
|
5548
|
+
add(spec.repo, spec.baseClonePath);
|
|
5549
|
+
}
|
|
5550
|
+
}
|
|
5551
|
+
else {
|
|
5552
|
+
for (const [repo, baseClonePath] of Object.entries(this.#config.repos.clonePaths)) {
|
|
5553
|
+
add(repo, baseClonePath);
|
|
5554
|
+
}
|
|
5555
|
+
}
|
|
5556
|
+
return [...repositories.values()];
|
|
5557
|
+
}
|
|
5558
|
+
async #reapOrphanedWorktreesOnStartup(legacyRegistry) {
|
|
5559
|
+
if (!this.#worktrees)
|
|
5560
|
+
return;
|
|
5561
|
+
const legacyAgents = (legacyRegistry?.agents ?? []).filter((agent) => agent.issue);
|
|
5562
|
+
let durableLifecycles;
|
|
5563
|
+
let waitingClarifications;
|
|
5564
|
+
let onlineAgentNames;
|
|
5565
|
+
try {
|
|
5566
|
+
const [lifecycles, clarifications, roster] = await Promise.all([
|
|
5567
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
5568
|
+
this.#state.listWaitingClarifications(this.#workspaceId),
|
|
5569
|
+
legacyAgents.length > 0 ? this.#fleet.roster() : undefined,
|
|
5570
|
+
]);
|
|
5571
|
+
durableLifecycles = lifecycles;
|
|
5572
|
+
waitingClarifications = clarifications;
|
|
5573
|
+
onlineAgentNames = new Set((roster?.agents ?? []).map((agent) => agent.name));
|
|
5574
|
+
}
|
|
5575
|
+
catch (error) {
|
|
5576
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5577
|
+
this.#logger.warn?.('[factory] startup worktree reaper skipped because active lifecycle or roster state could not be loaded', {
|
|
5578
|
+
error: describeError(error).errorMessage,
|
|
5579
|
+
});
|
|
5580
|
+
return;
|
|
5581
|
+
}
|
|
5582
|
+
const activeIssueSlugs = new Set((await this.#batch()).inFlight.map((record) => factoryWorktreeIssueSlug(record.issue.key)));
|
|
5583
|
+
for (const [, lifecycle] of durableLifecycles) {
|
|
5584
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
5585
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(lifecycle.issue.key));
|
|
5586
|
+
}
|
|
5587
|
+
}
|
|
5588
|
+
for (const [, waiting] of waitingClarifications) {
|
|
5589
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(waiting.issue.key));
|
|
5590
|
+
}
|
|
5591
|
+
for (const agent of legacyAgents) {
|
|
5592
|
+
if (agent.issue && onlineAgentNames.has(agent.name)) {
|
|
5593
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(agent.issue.key));
|
|
5594
|
+
}
|
|
5595
|
+
}
|
|
5596
|
+
const candidates = new Map();
|
|
5597
|
+
let reaped = 0;
|
|
5598
|
+
let reclaimedBytes = 0;
|
|
5599
|
+
let retained = 0;
|
|
5600
|
+
let failures = 0;
|
|
5601
|
+
for (const repository of this.#worktreeRepositories()) {
|
|
5602
|
+
try {
|
|
5603
|
+
for (const candidate of await this.#worktrees.listWorktrees(repository)) {
|
|
5604
|
+
candidates.set(candidate.worktreePath, candidate);
|
|
5605
|
+
}
|
|
5606
|
+
}
|
|
5607
|
+
catch (error) {
|
|
5608
|
+
failures += 1;
|
|
5609
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5610
|
+
this.#logger.warn?.('[factory] startup worktree reaper failed to enumerate repository', {
|
|
5611
|
+
repo: repository.repo,
|
|
5612
|
+
baseClonePath: repository.baseClonePath,
|
|
5613
|
+
error: describeError(error).errorMessage,
|
|
5614
|
+
});
|
|
5615
|
+
}
|
|
5616
|
+
}
|
|
5617
|
+
for (const worktree of candidates.values()) {
|
|
5618
|
+
if (activeIssueSlugs.has(factoryWorktreeIssueSlug(worktree.issueKey)))
|
|
5619
|
+
continue;
|
|
5620
|
+
try {
|
|
5621
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5622
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5623
|
+
retained += 1;
|
|
5624
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5625
|
+
this.#logger.warn?.('[factory] retained startup orphan worktree with local state', {
|
|
5626
|
+
issue: worktree.issueKey,
|
|
5627
|
+
repo: worktree.repo,
|
|
5628
|
+
worktreePath: worktree.worktreePath,
|
|
5629
|
+
retentionReasons: inspection.retentionReasons,
|
|
5630
|
+
});
|
|
5631
|
+
continue;
|
|
5632
|
+
}
|
|
5633
|
+
await this.#worktrees.cleanup(worktree);
|
|
5634
|
+
reaped += 1;
|
|
5635
|
+
reclaimedBytes += inspection.bytes;
|
|
5636
|
+
this.#increment('agentWorktreesCleaned');
|
|
5637
|
+
this.#increment('agentWorktreesReapedOnStartup');
|
|
5638
|
+
}
|
|
5639
|
+
catch (error) {
|
|
5640
|
+
failures += 1;
|
|
5641
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5642
|
+
this.#logger.warn?.('[factory] startup worktree reaper retained checkout after cleanup failure', {
|
|
5643
|
+
issue: worktree.issueKey,
|
|
5644
|
+
repo: worktree.repo,
|
|
5645
|
+
worktreePath: worktree.worktreePath,
|
|
5646
|
+
error: describeError(error).errorMessage,
|
|
5647
|
+
});
|
|
5648
|
+
}
|
|
5649
|
+
}
|
|
5650
|
+
this.#logger.info?.('[factory] startup worktree reaper completed', {
|
|
5651
|
+
reaped,
|
|
5652
|
+
reclaimedBytes,
|
|
5653
|
+
reclaimed: formatByteCount(reclaimedBytes),
|
|
5654
|
+
retained,
|
|
5655
|
+
failures,
|
|
5656
|
+
});
|
|
5657
|
+
}
|
|
5220
5658
|
async #confirmPublishedRemotePullRequest(repo, result, expectedHeadRef) {
|
|
5221
5659
|
const parts = githubRepoParts(repo);
|
|
5222
5660
|
if (!parts)
|
|
@@ -5363,6 +5801,28 @@ export class FactoryLoop {
|
|
|
5363
5801
|
async #abandonStuckDispatch(record, reason) {
|
|
5364
5802
|
const key = issueKey(record.issue);
|
|
5365
5803
|
this.#abandonedDispatchReasons.set(key, reason);
|
|
5804
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5805
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5806
|
+
// The generic durable retry can recover the in-memory reason in this
|
|
5807
|
+
// process; after restart the persisted `abandoning` phase is the fence.
|
|
5808
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5809
|
+
return;
|
|
5810
|
+
}
|
|
5811
|
+
try {
|
|
5812
|
+
// Remove externally reachable routes before releasing the agents that
|
|
5813
|
+
// could still be serving the upstream. A failed provider teardown keeps
|
|
5814
|
+
// the durable lifecycle retryable instead of terminalizing a leak.
|
|
5815
|
+
await this.#teardownPreviews(record);
|
|
5816
|
+
}
|
|
5817
|
+
catch (error) {
|
|
5818
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5819
|
+
this.#logger.warn?.('[factory] abandoned dispatch preview teardown failed; retrying', {
|
|
5820
|
+
issue: record.issue.key,
|
|
5821
|
+
error: describeError(error).errorMessage,
|
|
5822
|
+
});
|
|
5823
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5824
|
+
return;
|
|
5825
|
+
}
|
|
5366
5826
|
const agents = [...record.agents];
|
|
5367
5827
|
for (const [agentName, tracked] of agents) {
|
|
5368
5828
|
if (tracked.spec.role === 'implementer')
|
|
@@ -5396,16 +5856,13 @@ export class FactoryLoop {
|
|
|
5396
5856
|
return;
|
|
5397
5857
|
}
|
|
5398
5858
|
// Batch completion alone only frees the process-local slot. Durable
|
|
5399
|
-
// capacity is computed from lifecycle phases, so
|
|
5400
|
-
//
|
|
5401
|
-
|
|
5402
|
-
// before promoting the next issue.
|
|
5403
|
-
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason)) {
|
|
5859
|
+
// capacity is computed from lifecycle phases, so commit the terminal phase
|
|
5860
|
+
// only after the preview, agents, and worktrees have all been cleaned up.
|
|
5861
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5404
5862
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
5405
5863
|
// #saveDispatchLifecycle already schedules the generic durable retry. The
|
|
5406
|
-
// pending reason makes
|
|
5407
|
-
//
|
|
5408
|
-
// slot. The abandoned-specific scheduler remains the non-durable fallback.
|
|
5864
|
+
// pending reason makes an in-process retry return here, while the durable
|
|
5865
|
+
// `abandoning` phase provides the same recovery guarantee after restart.
|
|
5409
5866
|
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5410
5867
|
await this.#writeInFlightRegistry();
|
|
5411
5868
|
return;
|
|
@@ -7034,11 +7491,12 @@ export class FactoryLoop {
|
|
|
7034
7491
|
const integrationInstructions = await this.#resolveIntegrationInstructions();
|
|
7035
7492
|
const render = async (spec) => {
|
|
7036
7493
|
const route = routeForSpec(decision, spec);
|
|
7494
|
+
const previewUrl = previewUrlFromSpec(spec);
|
|
7037
7495
|
const testGuidance = await resolveTestGuidance({
|
|
7038
7496
|
repoPath: route.clonePath,
|
|
7039
7497
|
issue: templateIssue,
|
|
7040
7498
|
route,
|
|
7041
|
-
previewUrl
|
|
7499
|
+
previewUrl,
|
|
7042
7500
|
});
|
|
7043
7501
|
return {
|
|
7044
7502
|
...spec,
|
|
@@ -7055,6 +7513,11 @@ export class FactoryLoop {
|
|
|
7055
7513
|
branchName: spec.branch ?? decision.implementers.find((candidate) => candidate.repo === spec.repo)?.branch,
|
|
7056
7514
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
7057
7515
|
agentName: spec.name,
|
|
7516
|
+
...(previewUrl ? {
|
|
7517
|
+
previewUrl,
|
|
7518
|
+
previewTargetPort: spec.preview?.targetPort,
|
|
7519
|
+
previewStartCommand: spec.preview?.startCommand,
|
|
7520
|
+
} : {}),
|
|
7058
7521
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
7059
7522
|
}),
|
|
7060
7523
|
};
|
|
@@ -7066,6 +7529,193 @@ export class FactoryLoop {
|
|
|
7066
7529
|
reviewer: rendered[decision.implementers.length],
|
|
7067
7530
|
};
|
|
7068
7531
|
}
|
|
7532
|
+
async #withPreviewReferences(decision) {
|
|
7533
|
+
if (!this.#config.preview || decision.scope === 'workflow')
|
|
7534
|
+
return decision;
|
|
7535
|
+
if (!this.#fleet.createPreview) {
|
|
7536
|
+
throw new Error('Preview services are configured but the selected fleet backend cannot create previews');
|
|
7537
|
+
}
|
|
7538
|
+
const owner = issueKey(decision.issue);
|
|
7539
|
+
const byRepo = new Map();
|
|
7540
|
+
for (const implementer of decision.implementers) {
|
|
7541
|
+
const service = previewServiceForRepo(this.#config, implementer.repo);
|
|
7542
|
+
if (byRepo.has(implementer.repo))
|
|
7543
|
+
continue;
|
|
7544
|
+
const persisted = implementer.preview;
|
|
7545
|
+
if (!service) {
|
|
7546
|
+
if (persisted)
|
|
7547
|
+
byRepo.set(implementer.repo, persisted);
|
|
7548
|
+
continue;
|
|
7549
|
+
}
|
|
7550
|
+
if (!implementer.clonePath) {
|
|
7551
|
+
throw new Error(`Preview service ${service.name} requires a configured checkout for ${implementer.repo}`);
|
|
7552
|
+
}
|
|
7553
|
+
await this.#preparePreviewCheckout(decision, implementer);
|
|
7554
|
+
const preview = await this.#fleet.createPreview({
|
|
7555
|
+
namespace: this.#workspaceId,
|
|
7556
|
+
owner,
|
|
7557
|
+
issueKey: decision.issue.key,
|
|
7558
|
+
service: service.name,
|
|
7559
|
+
repo: implementer.repo,
|
|
7560
|
+
targetPort: service.config.port,
|
|
7561
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7562
|
+
startCommand: service.config.startCommand,
|
|
7563
|
+
checkoutPath: implementer.clonePath,
|
|
7564
|
+
node: persisted?.node ?? implementer.node,
|
|
7565
|
+
});
|
|
7566
|
+
this.#removedPreviewIds.delete(preview.id);
|
|
7567
|
+
byRepo.set(implementer.repo, preview);
|
|
7568
|
+
// Make the provider identity visible immediately. If validation or the
|
|
7569
|
+
// following durable save fails, the caller can roll it back while it
|
|
7570
|
+
// still owns the lifecycle; after fence loss the startup sweep owns it.
|
|
7571
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7572
|
+
assertPublishablePreview(preview, {
|
|
7573
|
+
namespace: this.#workspaceId,
|
|
7574
|
+
owner,
|
|
7575
|
+
service: service.name,
|
|
7576
|
+
repo: implementer.repo,
|
|
7577
|
+
targetPort: service.config.port,
|
|
7578
|
+
portSpan: service.config.portSpan ?? 100,
|
|
7579
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7580
|
+
startCommand: service.config.startCommand,
|
|
7581
|
+
checkoutPath: implementer.clonePath,
|
|
7582
|
+
requireNode: this.#fleet.placementLocality === 'remote',
|
|
7583
|
+
});
|
|
7584
|
+
}
|
|
7585
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7586
|
+
return {
|
|
7587
|
+
...decision,
|
|
7588
|
+
implementers: decision.implementers.map((spec) => specWithPreview(spec, byRepo.get(spec.repo))),
|
|
7589
|
+
reviewer: specWithPreview(decision.reviewer, byRepo.get(decision.reviewer.repo)),
|
|
7590
|
+
};
|
|
7591
|
+
}
|
|
7592
|
+
async #preparePreviewCheckout(decision, spec) {
|
|
7593
|
+
if (!this.#worktrees ||
|
|
7594
|
+
!spec.baseClonePath ||
|
|
7595
|
+
!spec.clonePath ||
|
|
7596
|
+
spec.baseClonePath === spec.clonePath ||
|
|
7597
|
+
!spec.branch)
|
|
7598
|
+
return;
|
|
7599
|
+
try {
|
|
7600
|
+
await this.#worktrees.prepare({
|
|
7601
|
+
repo: spec.repo,
|
|
7602
|
+
issueKey: decision.issue.key,
|
|
7603
|
+
baseClonePath: spec.baseClonePath,
|
|
7604
|
+
worktreePath: spec.clonePath,
|
|
7605
|
+
branch: spec.branch,
|
|
7606
|
+
...(spec.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
7607
|
+
});
|
|
7608
|
+
this.#increment('agentWorktreesPrepared');
|
|
7609
|
+
}
|
|
7610
|
+
catch (error) {
|
|
7611
|
+
throw contextualError(`Unable to prepare preview checkout for ${decision.issue.key}/${spec.repo} at ${spec.clonePath}`, error);
|
|
7612
|
+
}
|
|
7613
|
+
}
|
|
7614
|
+
async #teardownPreviews(record) {
|
|
7615
|
+
const previews = uniquePreviewReferences([
|
|
7616
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7617
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7618
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
7619
|
+
]);
|
|
7620
|
+
if (previews.length === 0)
|
|
7621
|
+
return;
|
|
7622
|
+
await this.#teardownPreviewReferences(previews);
|
|
7623
|
+
this.#previewReferences.delete(issueKey(record.issue));
|
|
7624
|
+
}
|
|
7625
|
+
async #teardownPreviewReferences(references) {
|
|
7626
|
+
const previews = uniquePreviewReferences(references)
|
|
7627
|
+
.filter((preview) => !this.#removedPreviewIds.has(preview.id));
|
|
7628
|
+
if (previews.length === 0)
|
|
7629
|
+
return;
|
|
7630
|
+
if (!this.#fleet.removePreview) {
|
|
7631
|
+
throw new Error('Fleet backend cannot remove its configured previews');
|
|
7632
|
+
}
|
|
7633
|
+
const results = await Promise.allSettled(previews.map(async (preview) => await this.#fleet.removePreview(preview)));
|
|
7634
|
+
const failures = results.flatMap((result, index) => {
|
|
7635
|
+
const preview = previews[index];
|
|
7636
|
+
if (result.status === 'rejected')
|
|
7637
|
+
return [{ preview, reason: result.reason }];
|
|
7638
|
+
if (!result.value) {
|
|
7639
|
+
return [{
|
|
7640
|
+
preview,
|
|
7641
|
+
reason: new Error(`Preview provider could not confirm removal of ${preview.id}`),
|
|
7642
|
+
}];
|
|
7643
|
+
}
|
|
7644
|
+
this.#removedPreviewIds.add(preview.id);
|
|
7645
|
+
return [];
|
|
7646
|
+
});
|
|
7647
|
+
if (failures.length > 0) {
|
|
7648
|
+
this.#logger.warn?.('[factory] preview teardown failed', {
|
|
7649
|
+
owners: [...new Set(failures.map(({ preview }) => preview.owner))],
|
|
7650
|
+
previews: failures.map(({ preview }) => preview.id),
|
|
7651
|
+
});
|
|
7652
|
+
throw new AggregateError(failures.map(({ reason }) => reason), 'Unable to tear down every issue preview');
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7655
|
+
async #reapPreviewOrphans() {
|
|
7656
|
+
if (!this.#config.preview || !this.#fleet.reapPreviews)
|
|
7657
|
+
return;
|
|
7658
|
+
const [lifecycles, batch] = await Promise.all([
|
|
7659
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
7660
|
+
this.#batch(),
|
|
7661
|
+
]);
|
|
7662
|
+
const activePreviewIds = new Set();
|
|
7663
|
+
const activeOwners = new Set(lifecycles
|
|
7664
|
+
.map(([, lifecycle]) => lifecycle)
|
|
7665
|
+
.filter((lifecycle) => !isTerminalDispatchLifecycle(lifecycle))
|
|
7666
|
+
.map((lifecycle) => {
|
|
7667
|
+
for (const preview of uniquePreviewReferences([
|
|
7668
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
7669
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
7670
|
+
]))
|
|
7671
|
+
activePreviewIds.add(preview.id);
|
|
7672
|
+
return issueKey(lifecycle.issue);
|
|
7673
|
+
}));
|
|
7674
|
+
for (const record of batch.inFlight) {
|
|
7675
|
+
activeOwners.add(issueKey(record.issue));
|
|
7676
|
+
for (const preview of uniquePreviewReferences([
|
|
7677
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7678
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7679
|
+
]))
|
|
7680
|
+
activePreviewIds.add(preview.id);
|
|
7681
|
+
}
|
|
7682
|
+
const report = await this.#fleet.reapPreviews({
|
|
7683
|
+
namespace: this.#workspaceId,
|
|
7684
|
+
activeOwners: [...activeOwners],
|
|
7685
|
+
activePreviewIds: [...activePreviewIds],
|
|
7686
|
+
});
|
|
7687
|
+
if (report.reaped.length > 0 || report.skipped.length > 0) {
|
|
7688
|
+
this.#logger.info?.('[factory] preview orphan sweep completed', {
|
|
7689
|
+
reaped: report.reaped.map((preview) => preview.id),
|
|
7690
|
+
skipped: report.skipped,
|
|
7691
|
+
});
|
|
7692
|
+
}
|
|
7693
|
+
}
|
|
7694
|
+
#schedulePreviewSweep(delayMs = PREVIEW_SWEEP_INTERVAL_MS) {
|
|
7695
|
+
if (this.#stopping ||
|
|
7696
|
+
!this.#started ||
|
|
7697
|
+
!this.#config.preview ||
|
|
7698
|
+
!this.#fleet.reapPreviews ||
|
|
7699
|
+
this.#previewSweepTimer ||
|
|
7700
|
+
this.#previewSweepInFlight)
|
|
7701
|
+
return;
|
|
7702
|
+
this.#previewSweepTimer = setTimeout(() => {
|
|
7703
|
+
this.#previewSweepTimer = undefined;
|
|
7704
|
+
if (this.#stopping || !this.#started)
|
|
7705
|
+
return;
|
|
7706
|
+
this.#previewSweepInFlight = this.#reapPreviewOrphans()
|
|
7707
|
+
.catch((error) => {
|
|
7708
|
+
this.#logger.warn?.('[factory] periodic preview orphan sweep failed', {
|
|
7709
|
+
error: describeError(error).errorMessage,
|
|
7710
|
+
});
|
|
7711
|
+
})
|
|
7712
|
+
.finally(() => {
|
|
7713
|
+
this.#previewSweepInFlight = undefined;
|
|
7714
|
+
this.#schedulePreviewSweep();
|
|
7715
|
+
});
|
|
7716
|
+
}, delayMs);
|
|
7717
|
+
this.#previewSweepTimer.unref?.();
|
|
7718
|
+
}
|
|
7069
7719
|
#consumePendingDispatchClarifications(issue) {
|
|
7070
7720
|
const key = issueKey(issue);
|
|
7071
7721
|
this.#pendingSlackClarifications.delete(key);
|
|
@@ -7182,8 +7832,13 @@ export class FactoryLoop {
|
|
|
7182
7832
|
prNumber: session.prNumber,
|
|
7183
7833
|
path: session.path,
|
|
7184
7834
|
agentName: session.agentName,
|
|
7835
|
+
resourceSubscription: session.resourceSubscription,
|
|
7836
|
+
pendingDeliveryClaims: session.pendingDeliveryClaims,
|
|
7185
7837
|
};
|
|
7186
7838
|
this.#babysitterPr.set(ownershipKey, ref);
|
|
7839
|
+
if (ref.resourceSubscription) {
|
|
7840
|
+
this.#babysitterSubscriptionOwners.set(ref.resourceSubscription.subscriptionId, ownershipKey);
|
|
7841
|
+
}
|
|
7187
7842
|
this.#babysitterIssueRefs.set(ownershipKey, { ...session.issue });
|
|
7188
7843
|
this.#babysitterSpawned.add(ownershipKey);
|
|
7189
7844
|
if (session.critical)
|
|
@@ -7195,10 +7850,21 @@ export class FactoryLoop {
|
|
|
7195
7850
|
this.#increment('babysitterOwnershipRestored');
|
|
7196
7851
|
const pendingKinds = session.pendingKinds.filter(isBabysitterWakeKind);
|
|
7197
7852
|
if (pendingKinds.length > 0) {
|
|
7198
|
-
|
|
7853
|
+
// A terminal marker can coexist with the one wake persisted before
|
|
7854
|
+
// acknowledgement. Rehydrate that durable hand-off once, while later
|
|
7855
|
+
// raw events remain quarantined by #queueBabysitterWake.
|
|
7856
|
+
await this.#queueBabysitterWake(session.issue, ref, pendingKinds, tracked, { allowTerminal: true });
|
|
7199
7857
|
this.#increment('babysitterPendingWakesRestored');
|
|
7200
7858
|
}
|
|
7859
|
+
await this.#ensureBabysitterResourceSubscription(session.issue, ref, tracked);
|
|
7201
7860
|
}
|
|
7861
|
+
// A crash after the local queue write but before (or just after) the
|
|
7862
|
+
// remote acceptance leaves an ID in state. Accept is idempotent once the
|
|
7863
|
+
// lease was accepted, so settle those durable hand-offs before claiming
|
|
7864
|
+
// new work; lease expiry is retried below when a server has not released
|
|
7865
|
+
// the original claim yet.
|
|
7866
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
7867
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
7202
7868
|
}
|
|
7203
7869
|
async #reconcileRestoredBabysitterReceipts(onlyRecord) {
|
|
7204
7870
|
const records = onlyRecord ? [onlyRecord] : (await this.#batch()).inFlight;
|
|
@@ -7297,6 +7963,29 @@ export class FactoryLoop {
|
|
|
7297
7963
|
this.#babysitterWakeStates.delete(key);
|
|
7298
7964
|
this.#babysitterCriticalAgents.delete(state.agentName);
|
|
7299
7965
|
}
|
|
7966
|
+
if (mayClearDurable && ref?.resourceSubscription) {
|
|
7967
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
7968
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
7969
|
+
if (subscriptions) {
|
|
7970
|
+
try {
|
|
7971
|
+
await subscriptions.cancel(this.#workspaceId, {
|
|
7972
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
7973
|
+
});
|
|
7974
|
+
this.#increment('babysitterResourceSubscriptionsCancelled');
|
|
7975
|
+
}
|
|
7976
|
+
catch (error) {
|
|
7977
|
+
// Cancellation is deliberately idempotent. A terminal acceptance may
|
|
7978
|
+
// have retired this record already; an outage leaves the bounded TTL
|
|
7979
|
+
// as the leak backstop and must not prevent local session cleanup.
|
|
7980
|
+
this.#increment('babysitterResourceSubscriptionCancelFailures');
|
|
7981
|
+
this.#logger.warn?.('[factory] could not cancel durable babysitter resource subscription', {
|
|
7982
|
+
issue: issue?.key,
|
|
7983
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
7984
|
+
error: describeError(error).errorMessage,
|
|
7985
|
+
});
|
|
7986
|
+
}
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7300
7989
|
this.#babysitterPr.delete(ownershipKey);
|
|
7301
7990
|
this.#babysitterIssueRefs.delete(ownershipKey);
|
|
7302
7991
|
this.#babysitterSpawned.delete(ownershipKey);
|
|
@@ -7311,7 +8000,323 @@ export class FactoryLoop {
|
|
|
7311
8000
|
.map(([key]) => key);
|
|
7312
8001
|
await Promise.all(keys.map(async (key) => this.#cancelBabysitterWake(key)));
|
|
7313
8002
|
}
|
|
8003
|
+
async #ensureBabysitterResourceSubscription(issue, ref, tracked) {
|
|
8004
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8005
|
+
if (!subscriptions || !ref.agentName || !await this.#assertIssueDispatchLifecycleOwner(issue))
|
|
8006
|
+
return;
|
|
8007
|
+
// A terminal claim is persisted before Relayfile acceptance so a crash in
|
|
8008
|
+
// that gap can never renew a retired record into a fresh generation.
|
|
8009
|
+
if (ref.resourceSubscription?.terminal)
|
|
8010
|
+
return;
|
|
8011
|
+
const resourceRef = babysitterResourceRef(ref.repo, ref.prNumber);
|
|
8012
|
+
const subscriberId = babysitterSubscriberId(issue);
|
|
8013
|
+
try {
|
|
8014
|
+
const subscription = await subscriptions.createOrRenew(this.#workspaceId, {
|
|
8015
|
+
provider: 'github',
|
|
8016
|
+
resourceRef,
|
|
8017
|
+
eventTypes: [...BABYSITTER_SUBSCRIPTION_EVENT_TYPES],
|
|
8018
|
+
terminalEventTypes: [...BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES],
|
|
8019
|
+
subscriberId,
|
|
8020
|
+
ttlSeconds: BABYSITTER_SUBSCRIPTION_TTL_SECONDS,
|
|
8021
|
+
});
|
|
8022
|
+
if (!subscription.subscriptionId ||
|
|
8023
|
+
subscription.provider !== 'github' ||
|
|
8024
|
+
subscription.resourceRef !== resourceRef ||
|
|
8025
|
+
subscription.subscriberId !== subscriberId ||
|
|
8026
|
+
!subscription.ownerId ||
|
|
8027
|
+
!subscription.expiresAt ||
|
|
8028
|
+
!subscription.terminalEventTypes?.includes('pull_request.closed')) {
|
|
8029
|
+
throw new Error('Relayfile returned an invalid durable resource subscription');
|
|
8030
|
+
}
|
|
8031
|
+
if (ref.resourceSubscription?.subscriptionId && ref.resourceSubscription.subscriptionId !== subscription.subscriptionId) {
|
|
8032
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
8033
|
+
}
|
|
8034
|
+
ref.resourceSubscription = {
|
|
8035
|
+
subscriptionId: subscription.subscriptionId,
|
|
8036
|
+
provider: subscription.provider,
|
|
8037
|
+
resourceRef: subscription.resourceRef,
|
|
8038
|
+
subscriberId: subscription.subscriberId,
|
|
8039
|
+
ownerId: subscription.ownerId,
|
|
8040
|
+
expiresAt: subscription.expiresAt,
|
|
8041
|
+
};
|
|
8042
|
+
this.#babysitterSubscriptionOwners.set(subscription.subscriptionId, babysitterOwnershipKey(issue, ref));
|
|
8043
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8044
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8045
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8046
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8047
|
+
this.#increment('babysitterResourceSubscriptionsRenewed');
|
|
8048
|
+
}
|
|
8049
|
+
catch (error) {
|
|
8050
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8051
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8052
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8053
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8054
|
+
return;
|
|
8055
|
+
}
|
|
8056
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8057
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8058
|
+
this.#increment('babysitterResourceSubscriptionRenewFailures');
|
|
8059
|
+
this.#logger.warn?.('[factory] could not create or renew durable babysitter resource subscription', {
|
|
8060
|
+
issue: issue.key,
|
|
8061
|
+
repo: ref.repo,
|
|
8062
|
+
prNumber: ref.prNumber,
|
|
8063
|
+
error: describeError(error).errorMessage,
|
|
8064
|
+
});
|
|
8065
|
+
}
|
|
8066
|
+
}
|
|
8067
|
+
async #routeDurableBabysitterDeliveries() {
|
|
8068
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8069
|
+
if (!subscriptions || !this.#config.babysitter.enabled || this.#stopping)
|
|
8070
|
+
return false;
|
|
8071
|
+
// Do not bypass the proven local router until every active babysitter has
|
|
8072
|
+
// completed its own create/renew. This closes the rollout and transient
|
|
8073
|
+
// provisioning gap without making a successful API response for some
|
|
8074
|
+
// other subscription suppress an unregistered PR's wake.
|
|
8075
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.agentName && !ref.resourceSubscription)) {
|
|
8076
|
+
return false;
|
|
8077
|
+
}
|
|
8078
|
+
let claims;
|
|
8079
|
+
try {
|
|
8080
|
+
claims = await subscriptions.claimDeliveryClaims(this.#workspaceId);
|
|
8081
|
+
}
|
|
8082
|
+
catch (error) {
|
|
8083
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8084
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8085
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8086
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8087
|
+
}
|
|
8088
|
+
else {
|
|
8089
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8090
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8091
|
+
this.#increment('babysitterResourceDeliveryLookupFailures');
|
|
8092
|
+
this.#logger.warn?.('[factory] durable babysitter delivery-claim lookup failed; retaining durable delivery retry', {
|
|
8093
|
+
error: describeError(error).errorMessage,
|
|
8094
|
+
});
|
|
8095
|
+
}
|
|
8096
|
+
return !isResourceSubscriptionsUnavailable(error);
|
|
8097
|
+
}
|
|
8098
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8099
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8100
|
+
for (const claim of claims) {
|
|
8101
|
+
const issueIdentity = this.#babysitterSubscriptionOwners.get(claim.subscriptionId);
|
|
8102
|
+
const issue = issueIdentity ? this.#babysitterIssueRefs.get(issueIdentity) : undefined;
|
|
8103
|
+
const ref = issueIdentity ? this.#babysitterPr.get(issueIdentity) : undefined;
|
|
8104
|
+
const subscription = ref?.resourceSubscription;
|
|
8105
|
+
if (!issue ||
|
|
8106
|
+
!ref ||
|
|
8107
|
+
!subscription ||
|
|
8108
|
+
subscription.subscriptionId !== claim.subscriptionId ||
|
|
8109
|
+
subscription.provider !== claim.provider ||
|
|
8110
|
+
subscription.resourceRef !== claim.resourceRef ||
|
|
8111
|
+
subscription.subscriberId !== claim.subscriberId ||
|
|
8112
|
+
subscription.ownerId !== claim.ownerId) {
|
|
8113
|
+
// The service is owner-isolated, but Factory may have just retired a
|
|
8114
|
+
// local owner. Never route a stale or other-session claim by resource.
|
|
8115
|
+
this.#increment('babysitterResourceDeliveriesIgnoredUnowned');
|
|
8116
|
+
continue;
|
|
8117
|
+
}
|
|
8118
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
8119
|
+
this.#increment('babysitterResourceDeliveriesIgnoredNonOwner');
|
|
8120
|
+
continue;
|
|
8121
|
+
}
|
|
8122
|
+
// A terminal delivery may be reclaimed after a process crash before its
|
|
8123
|
+
// remote acceptance. Only that already-persisted delivery may finish;
|
|
8124
|
+
// no later claim is allowed to wake or re-open the retired session.
|
|
8125
|
+
if (subscription.terminal && !ref.pendingDeliveryClaims?.some((pending) => pending.deliveryId === claim.deliveryId)) {
|
|
8126
|
+
this.#increment('babysitterResourceDeliveriesIgnoredTerminal');
|
|
8127
|
+
continue;
|
|
8128
|
+
}
|
|
8129
|
+
const batch = await this.#batch();
|
|
8130
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8131
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8132
|
+
?? durableBabysitterTrackedAgent({
|
|
8133
|
+
issue,
|
|
8134
|
+
repo: ref.repo,
|
|
8135
|
+
prNumber: ref.prNumber,
|
|
8136
|
+
path: ref.path,
|
|
8137
|
+
agentName: ref.agentName,
|
|
8138
|
+
critical: false,
|
|
8139
|
+
pendingKinds: [],
|
|
8140
|
+
resourceSubscription: subscription,
|
|
8141
|
+
pendingDeliveryClaims: ref.pendingDeliveryClaims,
|
|
8142
|
+
});
|
|
8143
|
+
const pendingClaim = ref.pendingDeliveryClaims?.find((pending) => pending.deliveryId === claim.deliveryId);
|
|
8144
|
+
const alreadyQueued = Boolean(pendingClaim);
|
|
8145
|
+
if (!alreadyQueued) {
|
|
8146
|
+
const queued = await this.#queueBabysitterWake(issue, ref, ['pull-request-state'], tracked);
|
|
8147
|
+
if (!queued)
|
|
8148
|
+
continue;
|
|
8149
|
+
}
|
|
8150
|
+
if (!pendingClaim || pendingClaim.claimToken !== claim.claimToken) {
|
|
8151
|
+
ref.pendingDeliveryClaims = [
|
|
8152
|
+
...(ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId),
|
|
8153
|
+
{ deliveryId: claim.deliveryId, claimToken: claim.claimToken },
|
|
8154
|
+
];
|
|
8155
|
+
// The claim lease joins Factory's durable pending-wake state before
|
|
8156
|
+
// the external acceptance. A crash after this point can retry the
|
|
8157
|
+
// exact hand-off without delivering the same wake a second time.
|
|
8158
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8159
|
+
}
|
|
8160
|
+
try {
|
|
8161
|
+
if (claim.terminal && !subscription.terminal) {
|
|
8162
|
+
subscription.terminal = true;
|
|
8163
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8164
|
+
}
|
|
8165
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, {
|
|
8166
|
+
deliveryId: claim.deliveryId,
|
|
8167
|
+
claimToken: claim.claimToken,
|
|
8168
|
+
});
|
|
8169
|
+
if (accepted.deliveryId !== claim.deliveryId || accepted.subscriptionId !== claim.subscriptionId) {
|
|
8170
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8171
|
+
}
|
|
8172
|
+
if (accepted.terminal || claim.terminal) {
|
|
8173
|
+
// Keep the terminal marker and subscription identity locally until
|
|
8174
|
+
// normal PR/session teardown. That quarantines the babysitter from
|
|
8175
|
+
// both legacy fallback and a restart-time create-or-renew.
|
|
8176
|
+
subscription.terminal = true;
|
|
8177
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8178
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8179
|
+
this.#increment('babysitterResourceSubscriptionsRetiredTerminal');
|
|
8180
|
+
}
|
|
8181
|
+
else {
|
|
8182
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8183
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8184
|
+
}
|
|
8185
|
+
this.#increment('babysitterResourceDeliveriesAccepted');
|
|
8186
|
+
}
|
|
8187
|
+
catch (error) {
|
|
8188
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8189
|
+
this.#logger.warn?.('[factory] durable babysitter delivery claim remains pending after wake queue', {
|
|
8190
|
+
issue: issue.key,
|
|
8191
|
+
subscriptionId: claim.subscriptionId,
|
|
8192
|
+
deliveryId: claim.deliveryId,
|
|
8193
|
+
error: describeError(error).errorMessage,
|
|
8194
|
+
});
|
|
8195
|
+
}
|
|
8196
|
+
}
|
|
8197
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8198
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8199
|
+
}
|
|
8200
|
+
return true;
|
|
8201
|
+
}
|
|
8202
|
+
async #retryPendingBabysitterDeliveryAcceptances() {
|
|
8203
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8204
|
+
if (!subscriptions)
|
|
8205
|
+
return;
|
|
8206
|
+
const retrySubscriptionRenewal = this.#babysitterResourceSubscriptionFault;
|
|
8207
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8208
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8209
|
+
if (issue && ref.agentName && !ref.resourceSubscription?.terminal && (!ref.resourceSubscription || retrySubscriptionRenewal)) {
|
|
8210
|
+
const batch = await this.#batch();
|
|
8211
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8212
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8213
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8214
|
+
}
|
|
8215
|
+
const subscription = ref.resourceSubscription;
|
|
8216
|
+
const pendingDeliveryClaims = [...(ref.pendingDeliveryClaims ?? [])];
|
|
8217
|
+
if (!subscription || !issue || pendingDeliveryClaims.length === 0)
|
|
8218
|
+
continue;
|
|
8219
|
+
const batch = await this.#batch();
|
|
8220
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8221
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8222
|
+
?? durableBabysitterTrackedAgent({
|
|
8223
|
+
issue,
|
|
8224
|
+
repo: ref.repo,
|
|
8225
|
+
prNumber: ref.prNumber,
|
|
8226
|
+
path: ref.path,
|
|
8227
|
+
agentName: ref.agentName,
|
|
8228
|
+
critical: false,
|
|
8229
|
+
pendingKinds: [],
|
|
8230
|
+
resourceSubscription: subscription,
|
|
8231
|
+
pendingDeliveryClaims,
|
|
8232
|
+
});
|
|
8233
|
+
for (const { deliveryId, claimToken } of pendingDeliveryClaims) {
|
|
8234
|
+
try {
|
|
8235
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, { deliveryId, claimToken });
|
|
8236
|
+
if (accepted.deliveryId !== deliveryId || accepted.subscriptionId !== subscription.subscriptionId) {
|
|
8237
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8238
|
+
}
|
|
8239
|
+
if (accepted.terminal)
|
|
8240
|
+
subscription.terminal = true;
|
|
8241
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== deliveryId);
|
|
8242
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8243
|
+
this.#increment('babysitterResourceDeliveriesAcceptedAfterRestore');
|
|
8244
|
+
}
|
|
8245
|
+
catch (error) {
|
|
8246
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8247
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8248
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8249
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8250
|
+
}
|
|
8251
|
+
else {
|
|
8252
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8253
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8254
|
+
this.#logger.warn?.('[factory] durable babysitter delivery acceptance remains pending after restore', {
|
|
8255
|
+
issue: issue.key,
|
|
8256
|
+
subscriptionId: subscription.subscriptionId,
|
|
8257
|
+
deliveryId,
|
|
8258
|
+
error: describeError(error).errorMessage,
|
|
8259
|
+
});
|
|
8260
|
+
}
|
|
8261
|
+
}
|
|
8262
|
+
}
|
|
8263
|
+
}
|
|
8264
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8265
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8266
|
+
}
|
|
8267
|
+
}
|
|
8268
|
+
#scheduleBabysitterResourceSubscriptionRenewal() {
|
|
8269
|
+
if (this.#babysitterResourceSubscriptionRenewTimer ||
|
|
8270
|
+
this.#stopping ||
|
|
8271
|
+
!this.#mount.resourceSubscriptions ||
|
|
8272
|
+
![...this.#babysitterPr.values()].some((ref) => ref.resourceSubscription && !ref.resourceSubscription.terminal))
|
|
8273
|
+
return;
|
|
8274
|
+
this.#babysitterResourceSubscriptionRenewTimer = setTimeout(() => {
|
|
8275
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
8276
|
+
void (async () => {
|
|
8277
|
+
const batch = await this.#batch();
|
|
8278
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8279
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8280
|
+
if (!issue || !ref.resourceSubscription || ref.resourceSubscription.terminal)
|
|
8281
|
+
continue;
|
|
8282
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8283
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8284
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8285
|
+
}
|
|
8286
|
+
})().catch((error) => {
|
|
8287
|
+
this.#logger.warn?.('[factory] durable babysitter subscription renewal rejected', {
|
|
8288
|
+
error: describeError(error).errorMessage,
|
|
8289
|
+
});
|
|
8290
|
+
}).finally(() => {
|
|
8291
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8292
|
+
});
|
|
8293
|
+
}, BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS);
|
|
8294
|
+
this.#babysitterResourceSubscriptionRenewTimer.unref?.();
|
|
8295
|
+
}
|
|
8296
|
+
#scheduleDurableBabysitterDeliveryRetry() {
|
|
8297
|
+
if (this.#babysitterResourceDeliveryRetryTimer || this.#stopping || !this.#mount.resourceSubscriptions)
|
|
8298
|
+
return;
|
|
8299
|
+
this.#babysitterResourceDeliveryRetryTimer = setTimeout(() => {
|
|
8300
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
8301
|
+
void (async () => {
|
|
8302
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
8303
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8304
|
+
})().catch((error) => {
|
|
8305
|
+
this.#logger.warn?.('[factory] durable babysitter delivery retry rejected', {
|
|
8306
|
+
error: describeError(error).errorMessage,
|
|
8307
|
+
});
|
|
8308
|
+
});
|
|
8309
|
+
}, BABYSITTER_RESOURCE_DELIVERY_RETRY_MS);
|
|
8310
|
+
this.#babysitterResourceDeliveryRetryTimer.unref?.();
|
|
8311
|
+
}
|
|
7314
8312
|
async #routeBabysitterEvent(path, extraKinds = []) {
|
|
8313
|
+
// A successful Relayfile claim lookup is the new exact demux. While some
|
|
8314
|
+
// owners are still unregistered, the legacy path router remains available
|
|
8315
|
+
// only to those owners. Registered owners retain and retry service claims,
|
|
8316
|
+
// so a mixed rollout or transient create failure neither double-delivers a
|
|
8317
|
+
// registered PR nor drops an event for an unregistered PR.
|
|
8318
|
+
if (await this.#routeDurableBabysitterDeliveries())
|
|
8319
|
+
return;
|
|
7315
8320
|
const event = githubBabysitterEventPathParts(path);
|
|
7316
8321
|
if (!event || !this.#config.babysitter.enabled || this.#stopping)
|
|
7317
8322
|
return;
|
|
@@ -7344,6 +8349,12 @@ export class FactoryLoop {
|
|
|
7344
8349
|
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', { ...event, prNumber: target.prNumber });
|
|
7345
8350
|
continue;
|
|
7346
8351
|
}
|
|
8352
|
+
if (this.#mount.resourceSubscriptions &&
|
|
8353
|
+
!this.#babysitterResourceSubscriptionUnavailable &&
|
|
8354
|
+
owner.ref.resourceSubscription) {
|
|
8355
|
+
this.#increment('babysitterEventsDeferredToDurableSubscription');
|
|
8356
|
+
continue;
|
|
8357
|
+
}
|
|
7347
8358
|
if (!await this.#assertIssueDispatchLifecycleOwner(owner.issue)) {
|
|
7348
8359
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7349
8360
|
continue;
|
|
@@ -7402,10 +8413,14 @@ export class FactoryLoop {
|
|
|
7402
8413
|
return;
|
|
7403
8414
|
}
|
|
7404
8415
|
}
|
|
7405
|
-
async #queueBabysitterWake(issue, ref, kinds, tracked) {
|
|
8416
|
+
async #queueBabysitterWake(issue, ref, kinds, tracked, options = {}) {
|
|
8417
|
+
if (ref.resourceSubscription?.terminal && !options.allowTerminal) {
|
|
8418
|
+
this.#increment('babysitterEventsIgnoredTerminal');
|
|
8419
|
+
return false;
|
|
8420
|
+
}
|
|
7406
8421
|
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
7407
8422
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7408
|
-
return;
|
|
8423
|
+
return false;
|
|
7409
8424
|
}
|
|
7410
8425
|
// Owner lookup and queueing straddle async mount/state reads. Revalidate
|
|
7411
8426
|
// the exact composite owner so a concurrent close/merge cancellation can
|
|
@@ -7416,7 +8431,7 @@ export class FactoryLoop {
|
|
|
7416
8431
|
current.agentName !== ref.agentName ||
|
|
7417
8432
|
githubPrIdentity(current.repo, current.prNumber) !== githubPrIdentity(ref.repo, ref.prNumber)) {
|
|
7418
8433
|
this.#increment('babysitterEventsIgnoredStaleOwner');
|
|
7419
|
-
return;
|
|
8434
|
+
return false;
|
|
7420
8435
|
}
|
|
7421
8436
|
// Any new event invalidates a prior readiness assertion for this exact PR.
|
|
7422
8437
|
this.#babysitterReady.delete(ownershipKey);
|
|
@@ -7446,9 +8461,10 @@ export class FactoryLoop {
|
|
|
7446
8461
|
});
|
|
7447
8462
|
if (state.deferredSubmitTargets || state.inFlight || this.#babysitterCriticalAgents.has(state.agentName)) {
|
|
7448
8463
|
this.#increment('babysitterEventWakesDeferred');
|
|
7449
|
-
return;
|
|
8464
|
+
return true;
|
|
7450
8465
|
}
|
|
7451
8466
|
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
8467
|
+
return true;
|
|
7452
8468
|
}
|
|
7453
8469
|
async #recordPendingBabysitterWake(state) {
|
|
7454
8470
|
const kinds = new Set([
|
|
@@ -7484,6 +8500,8 @@ export class FactoryLoop {
|
|
|
7484
8500
|
path: ref.path,
|
|
7485
8501
|
critical: this.#babysitterCriticalAgents.has(ref.agentName),
|
|
7486
8502
|
pendingKinds: pending?.kinds.filter(isBabysitterWakeKind).sort(compareBabysitterWakeKinds) ?? [],
|
|
8503
|
+
...(ref.resourceSubscription ? { resourceSubscription: { ...ref.resourceSubscription } } : {}),
|
|
8504
|
+
...(ref.pendingDeliveryClaims?.length ? { pendingDeliveryClaims: structuredClone(ref.pendingDeliveryClaims) } : {}),
|
|
7487
8505
|
});
|
|
7488
8506
|
}
|
|
7489
8507
|
#scheduleBabysitterWake(state, delayMs) {
|
|
@@ -7769,11 +8787,21 @@ export class FactoryLoop {
|
|
|
7769
8787
|
if (!this.#config.babysitter.enabled)
|
|
7770
8788
|
return;
|
|
7771
8789
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8790
|
+
// A provider close produces a separately indexed terminal claim. Keep
|
|
8791
|
+
// the durable owner until it has been accepted (or a transient retry
|
|
8792
|
+
// has claimed it); do not let closed-state cleanup erase that hand-off.
|
|
8793
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8794
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8795
|
+
return;
|
|
7772
8796
|
await this.#cancelBabysitterWake(owned.key);
|
|
7773
8797
|
return;
|
|
7774
8798
|
}
|
|
7775
8799
|
if (snapshot.draft)
|
|
7776
8800
|
this.#increment('babysitterDraftPrSkipped');
|
|
8801
|
+
// PR meta events are also the normal renewal heartbeat for the durable
|
|
8802
|
+
// record. The store's identity makes this a create-or-renew, never a
|
|
8803
|
+
// second subscription for the same babysitter.
|
|
8804
|
+
await this.#ensureBabysitterResourceSubscription(owned.issue, owned.ref, owned.tracked);
|
|
7777
8805
|
await this.#routeBabysitterEvent(path, babysitterWakeKindsFromSnapshot(snapshot));
|
|
7778
8806
|
return;
|
|
7779
8807
|
}
|
|
@@ -7821,6 +8849,13 @@ export class FactoryLoop {
|
|
|
7821
8849
|
return;
|
|
7822
8850
|
}
|
|
7823
8851
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8852
|
+
// `pull_request.closed` is a separately indexed Relayfile terminal
|
|
8853
|
+
// event. Claim and accept its durable hand-off before the local closed
|
|
8854
|
+
// PR cleanup drops the subscription owner. On a transient service fault,
|
|
8855
|
+
// retain the owner so the retry loop can claim it without local fallback.
|
|
8856
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8857
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8858
|
+
return;
|
|
7824
8859
|
if (babysitterKey && existing)
|
|
7825
8860
|
await this.#cancelBabysitterWake(babysitterKey);
|
|
7826
8861
|
return;
|
|
@@ -8018,6 +9053,11 @@ export class FactoryLoop {
|
|
|
8018
9053
|
const settled = this.#babysitterPr.get(babysitterKey);
|
|
8019
9054
|
if (settled && prRef.path)
|
|
8020
9055
|
settled.path = prRef.path;
|
|
9056
|
+
if (settled) {
|
|
9057
|
+
const tracked = record.agents.get(settled.agentName)
|
|
9058
|
+
?? [...record.agents.values()].find((agent) => agent.spec.role === 'babysitter');
|
|
9059
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, settled, tracked);
|
|
9060
|
+
}
|
|
8021
9061
|
return;
|
|
8022
9062
|
}
|
|
8023
9063
|
const wantedPr = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
@@ -8038,7 +9078,9 @@ export class FactoryLoop {
|
|
|
8038
9078
|
agentName: tracked.result?.name ?? trackedName,
|
|
8039
9079
|
});
|
|
8040
9080
|
this.#babysitterSpawned.add(babysitterKey);
|
|
8041
|
-
|
|
9081
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9082
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9083
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
8042
9084
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
8043
9085
|
return;
|
|
8044
9086
|
}
|
|
@@ -8068,9 +9110,13 @@ export class FactoryLoop {
|
|
|
8068
9110
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath)
|
|
8069
9111
|
?? record.decision.implementers
|
|
8070
9112
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath);
|
|
9113
|
+
const preview = [...record.agents.values()]
|
|
9114
|
+
.map((agent) => agent.spec)
|
|
9115
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.preview)?.preview
|
|
9116
|
+
?? record.decision.implementers.find((candidate) => candidate.repo === initialSpec.repo)?.preview;
|
|
8071
9117
|
const implementerBranch = prRef.headRef ?? record.decision.implementers
|
|
8072
9118
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.branch)?.branch;
|
|
8073
|
-
const
|
|
9119
|
+
const checkoutSpec = sharedCheckout
|
|
8074
9120
|
? {
|
|
8075
9121
|
...initialSpec,
|
|
8076
9122
|
baseClonePath: sharedCheckout.baseClonePath,
|
|
@@ -8079,6 +9125,7 @@ export class FactoryLoop {
|
|
|
8079
9125
|
...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
8080
9126
|
}
|
|
8081
9127
|
: initialSpec;
|
|
9128
|
+
const spec = specWithPreview(checkoutSpec, preview);
|
|
8082
9129
|
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
8083
9130
|
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name
|
|
8084
9131
|
?? agentNameForRole(issue, 'review', { repo: route?.repo ?? prRef.repo });
|
|
@@ -8111,6 +9158,11 @@ export class FactoryLoop {
|
|
|
8111
9158
|
branchName: spec.branch,
|
|
8112
9159
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
8113
9160
|
agentName: spec.name,
|
|
9161
|
+
...(spec.preview ? {
|
|
9162
|
+
previewUrl: spec.preview.url,
|
|
9163
|
+
previewTargetPort: spec.preview.targetPort,
|
|
9164
|
+
previewStartCommand: spec.preview.startCommand,
|
|
9165
|
+
} : {}),
|
|
8114
9166
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
8115
9167
|
});
|
|
8116
9168
|
const spawned = await this.#spawnAgent(record, {
|
|
@@ -8125,7 +9177,9 @@ export class FactoryLoop {
|
|
|
8125
9177
|
path: prRef.path,
|
|
8126
9178
|
agentName: tracked?.result?.name ?? spawned.name,
|
|
8127
9179
|
});
|
|
8128
|
-
|
|
9180
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9181
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9182
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
8129
9183
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
8130
9184
|
await this.#writeInFlightRegistry();
|
|
8131
9185
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
@@ -8428,6 +9482,7 @@ export class FactoryLoop {
|
|
|
8428
9482
|
// state.
|
|
8429
9483
|
const issueTeam = issue?.team;
|
|
8430
9484
|
const githubIssue = issue ? isGithubIssue(issue) : false;
|
|
9485
|
+
const syntheticProbe = issue ? this.#isSyntheticProbeIssue(issue) : false;
|
|
8431
9486
|
const configuredHumanReview = opts.targetState !== 'done' &&
|
|
8432
9487
|
this.#config.terminalState === 'human-review' &&
|
|
8433
9488
|
(githubIssue || this.#states.hasHumanReview(issueTeam));
|
|
@@ -8435,7 +9490,7 @@ export class FactoryLoop {
|
|
|
8435
9490
|
if (issue && githubIssue && !configuredHumanReview && !githubMerged) {
|
|
8436
9491
|
githubMerged = await this.#githubPrObservedMerged(record, issue);
|
|
8437
9492
|
if (!githubMerged && opts.runMergeGate !== false) {
|
|
8438
|
-
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue);
|
|
9493
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
8439
9494
|
// A successful merge command can mean queued/auto-merge rather than
|
|
8440
9495
|
// merged. Only mounted PR state or a merged webhook may prove merge.
|
|
8441
9496
|
if (mergeCommandAccepted) {
|
|
@@ -8443,6 +9498,21 @@ export class FactoryLoop {
|
|
|
8443
9498
|
}
|
|
8444
9499
|
}
|
|
8445
9500
|
}
|
|
9501
|
+
// Linear-backed issues historically wrote Done before invoking the
|
|
9502
|
+
// guarded merge. With live verification in that guard, a red verdict
|
|
9503
|
+
// would therefore release the team and make the failure terminal even
|
|
9504
|
+
// though the PR remained open. Require the entire merge gate (including
|
|
9505
|
+
// verification) to accept the merge before applying terminal writeback.
|
|
9506
|
+
if (issue &&
|
|
9507
|
+
!githubIssue &&
|
|
9508
|
+
!syntheticProbe &&
|
|
9509
|
+
!configuredHumanReview &&
|
|
9510
|
+
opts.runMergeGate !== false &&
|
|
9511
|
+
this.#config.mergePolicy === 'on-green-with-review') {
|
|
9512
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
9513
|
+
if (!mergeCommandAccepted)
|
|
9514
|
+
return;
|
|
9515
|
+
}
|
|
8446
9516
|
// A GitHub issue only closes after its PR merges. If a configured done
|
|
8447
9517
|
// path cannot merge (including mergePolicy: never), park it for a human
|
|
8448
9518
|
// instead of closing the issue while its PR remains open.
|
|
@@ -8497,11 +9567,11 @@ export class FactoryLoop {
|
|
|
8497
9567
|
this.#markSlackWritebackFailure('completion-thread', error);
|
|
8498
9568
|
}
|
|
8499
9569
|
}
|
|
8500
|
-
//
|
|
8501
|
-
//
|
|
8502
|
-
//
|
|
8503
|
-
if (issue &&
|
|
8504
|
-
await this.#runCompletionMergeGate(issue);
|
|
9570
|
+
// Synthetic canaries are cleanup probes, not merge candidates. Preserve
|
|
9571
|
+
// their close-before-release path under every merge policy without
|
|
9572
|
+
// subjecting them to the required feature verification gate.
|
|
9573
|
+
if (issue && syntheticProbe && !githubIssue && opts.runMergeGate !== false) {
|
|
9574
|
+
await this.#runCompletionMergeGate(issue, record);
|
|
8505
9575
|
}
|
|
8506
9576
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
8507
9577
|
releaseReasonForRetry = releaseReason;
|
|
@@ -8531,6 +9601,8 @@ export class FactoryLoop {
|
|
|
8531
9601
|
const stateKey = issueStateKey(record.issue);
|
|
8532
9602
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
8533
9603
|
this.#probePrResolvedCache.delete(stateKey);
|
|
9604
|
+
// Cancellation must see the subscription identity so it can issue the
|
|
9605
|
+
// idempotent Relayfile DELETE before clearing the local owner maps.
|
|
8534
9606
|
await this.#cancelBabysittersForIssue(record.issue);
|
|
8535
9607
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
8536
9608
|
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
@@ -8869,6 +9941,10 @@ export class FactoryLoop {
|
|
|
8869
9941
|
await this.#rearmSlackWatcher(record, existingThread, {
|
|
8870
9942
|
replayConversationReplies: Boolean(durableConversation),
|
|
8871
9943
|
});
|
|
9944
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
9945
|
+
if (previews.length > 0) {
|
|
9946
|
+
await this.#slack.reply(existingThread, previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join('\n'));
|
|
9947
|
+
}
|
|
8872
9948
|
}
|
|
8873
9949
|
return;
|
|
8874
9950
|
}
|
|
@@ -8889,12 +9965,15 @@ export class FactoryLoop {
|
|
|
8889
9965
|
if (!this.#slack || !this.#config.slack) {
|
|
8890
9966
|
return;
|
|
8891
9967
|
}
|
|
9968
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
8892
9969
|
const root = await this.#slack.postThread({
|
|
8893
9970
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
8894
9971
|
text: [
|
|
8895
9972
|
`${record.issue.key}: factory agents dispatched.`,
|
|
8896
|
-
`State: ${result.stateId ?? 'dispatching'}`,
|
|
8897
|
-
|
|
9973
|
+
`State: ${result.stateId ?? 'dispatching'} · Agents: ${result.agents.map((agent) => agent.name).join(', ') || 'none'}`,
|
|
9974
|
+
...(previews.length > 0
|
|
9975
|
+
? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
|
|
9976
|
+
: []),
|
|
8898
9977
|
].join('\n'),
|
|
8899
9978
|
});
|
|
8900
9979
|
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
@@ -10366,7 +11445,7 @@ export class FactoryLoop {
|
|
|
10366
11445
|
? { channel, threadId, mountRoot: this.#integrationsMountRoot() }
|
|
10367
11446
|
: undefined;
|
|
10368
11447
|
}
|
|
10369
|
-
async #runCompletionMergeGate(issue) {
|
|
11448
|
+
async #runCompletionMergeGate(issue, record) {
|
|
10370
11449
|
if (this.#isSyntheticProbeIssue(issue)) {
|
|
10371
11450
|
await this.#closeSyntheticProbeIfPresent(issue);
|
|
10372
11451
|
return false;
|
|
@@ -10391,6 +11470,48 @@ export class FactoryLoop {
|
|
|
10391
11470
|
this.#increment('mergeGateNotMerged');
|
|
10392
11471
|
return false;
|
|
10393
11472
|
}
|
|
11473
|
+
if (this.#verificationGate) {
|
|
11474
|
+
const repositoryPath = this.#verificationRepositoryPath(record, pr.repo);
|
|
11475
|
+
if (!repositoryPath) {
|
|
11476
|
+
this.#logger.warn?.('[factory] verification gate has no feature checkout for merge candidate', {
|
|
11477
|
+
issue: issue.key,
|
|
11478
|
+
repo: pr.repo,
|
|
11479
|
+
prNumber: pr.prNumber,
|
|
11480
|
+
});
|
|
11481
|
+
this.#increment('verificationGateMissingRepository');
|
|
11482
|
+
return false;
|
|
11483
|
+
}
|
|
11484
|
+
try {
|
|
11485
|
+
const verification = await this.#verificationGate.verify({
|
|
11486
|
+
repository: pr.repo,
|
|
11487
|
+
repositoryPath,
|
|
11488
|
+
issueKey: issue.key,
|
|
11489
|
+
expectedHeadSha: headSha,
|
|
11490
|
+
});
|
|
11491
|
+
if (!verification.passed) {
|
|
11492
|
+
this.#logger.warn?.('[factory] verification gate blocked merge', {
|
|
11493
|
+
issue: issue.key,
|
|
11494
|
+
repo: pr.repo,
|
|
11495
|
+
prNumber: pr.prNumber,
|
|
11496
|
+
environmentId: verification.evidence.environmentId,
|
|
11497
|
+
reason: verification.reason,
|
|
11498
|
+
});
|
|
11499
|
+
this.#increment('verificationGateFailed');
|
|
11500
|
+
return false;
|
|
11501
|
+
}
|
|
11502
|
+
this.#increment('verificationGatePassed');
|
|
11503
|
+
}
|
|
11504
|
+
catch (error) {
|
|
11505
|
+
this.#logger.warn?.('[factory] verification gate failed closed', {
|
|
11506
|
+
issue: issue.key,
|
|
11507
|
+
repo: pr.repo,
|
|
11508
|
+
prNumber: pr.prNumber,
|
|
11509
|
+
error: describeError(error).errorMessage,
|
|
11510
|
+
});
|
|
11511
|
+
this.#increment('verificationGateFailed');
|
|
11512
|
+
return false;
|
|
11513
|
+
}
|
|
11514
|
+
}
|
|
10394
11515
|
const result = await this.#mergeGate.merge({
|
|
10395
11516
|
repo: pr.repo,
|
|
10396
11517
|
number: pr.prNumber,
|
|
@@ -10416,6 +11537,14 @@ export class FactoryLoop {
|
|
|
10416
11537
|
this.#increment('mergeGateMerged');
|
|
10417
11538
|
return true;
|
|
10418
11539
|
}
|
|
11540
|
+
#verificationRepositoryPath(record, repo) {
|
|
11541
|
+
const normalized = repo.toLowerCase();
|
|
11542
|
+
const active = [...record.agents.values()]
|
|
11543
|
+
.map((tracked) => tracked.spec)
|
|
11544
|
+
.find((spec) => spec.role === 'implementer' && spec.repo.toLowerCase() === normalized && spec.clonePath)
|
|
11545
|
+
?? record.decision.implementers.find((spec) => spec.repo.toLowerCase() === normalized && spec.clonePath);
|
|
11546
|
+
return active?.clonePath ?? this.#config.repos.clonePaths[repo];
|
|
11547
|
+
}
|
|
10419
11548
|
async #closeSyntheticProbeIfPresent(issue) {
|
|
10420
11549
|
const probe = this.#customProbePrResolver
|
|
10421
11550
|
? await this.#probePrResolver(issue)
|
|
@@ -10770,6 +11899,101 @@ function dispatchSpecs(decision) {
|
|
|
10770
11899
|
}
|
|
10771
11900
|
return [...decision.implementers, decision.reviewer];
|
|
10772
11901
|
}
|
|
11902
|
+
function previewServiceForRepo(config, repo) {
|
|
11903
|
+
const services = config.preview?.services;
|
|
11904
|
+
if (!services)
|
|
11905
|
+
return undefined;
|
|
11906
|
+
const normalizedRepo = repo.replace(/^\/+|\/+$/gu, '').toLowerCase();
|
|
11907
|
+
const basename = normalizedRepo.slice(normalizedRepo.lastIndexOf('/') + 1);
|
|
11908
|
+
const entries = Object.entries(services).map(([name, service]) => ({
|
|
11909
|
+
name,
|
|
11910
|
+
service,
|
|
11911
|
+
normalized: name.replace(/^\/+|\/+$/gu, '').toLowerCase(),
|
|
11912
|
+
}));
|
|
11913
|
+
const exact = entries.find((entry) => entry.normalized === normalizedRepo);
|
|
11914
|
+
if (exact)
|
|
11915
|
+
return { name: exact.name, config: exact.service };
|
|
11916
|
+
const basenameMatches = entries.filter((entry) => entry.normalized.slice(entry.normalized.lastIndexOf('/') + 1) === basename);
|
|
11917
|
+
return basenameMatches.length === 1
|
|
11918
|
+
? { name: basenameMatches[0].name, config: basenameMatches[0].service }
|
|
11919
|
+
: undefined;
|
|
11920
|
+
}
|
|
11921
|
+
function uniquePreviewReferences(previews) {
|
|
11922
|
+
const unique = new Map();
|
|
11923
|
+
for (const preview of previews) {
|
|
11924
|
+
if (preview)
|
|
11925
|
+
unique.set(preview.id, preview);
|
|
11926
|
+
}
|
|
11927
|
+
return [...unique.values()];
|
|
11928
|
+
}
|
|
11929
|
+
function assertPublishablePreview(preview, expected) {
|
|
11930
|
+
const refuse = (reason) => {
|
|
11931
|
+
throw new Error(`Refusing insecure preview for ${expected.repo}: ${reason}`);
|
|
11932
|
+
};
|
|
11933
|
+
if (preview.provider !== 'tailscale-serve')
|
|
11934
|
+
refuse('unexpected provider');
|
|
11935
|
+
if (preview.access !== 'tailnet')
|
|
11936
|
+
refuse('provider did not guarantee tailnet access');
|
|
11937
|
+
if (preview.lifetime !== 'issue')
|
|
11938
|
+
refuse('provider did not guarantee issue-scoped lifetime');
|
|
11939
|
+
if (preview.namespace !== expected.namespace ||
|
|
11940
|
+
preview.owner !== expected.owner ||
|
|
11941
|
+
preview.service !== expected.service ||
|
|
11942
|
+
preview.repo !== expected.repo ||
|
|
11943
|
+
preview.startCommand !== expected.startCommand ||
|
|
11944
|
+
(preview.configuredTargetPort ?? preview.targetPort) !== expected.targetPort ||
|
|
11945
|
+
preview.targetPort < expected.targetPort ||
|
|
11946
|
+
preview.targetPort >= expected.targetPort + expected.portSpan) {
|
|
11947
|
+
refuse('provider returned a reference for a different dispatch identity');
|
|
11948
|
+
}
|
|
11949
|
+
if (expected.preferredHttpsPort !== undefined && preview.httpsPort !== expected.preferredHttpsPort) {
|
|
11950
|
+
refuse('provider ignored the configured HTTPS port');
|
|
11951
|
+
}
|
|
11952
|
+
if (expected.requireNode && (!preview.node || preview.node === 'self')) {
|
|
11953
|
+
refuse('remote provider did not identify the placement node');
|
|
11954
|
+
}
|
|
11955
|
+
const managedProcess = preview.process;
|
|
11956
|
+
if (!managedProcess ||
|
|
11957
|
+
!Number.isInteger(managedProcess.pid) ||
|
|
11958
|
+
managedProcess.pid <= 0 ||
|
|
11959
|
+
!managedProcess.startTime ||
|
|
11960
|
+
!managedProcess.cmdline ||
|
|
11961
|
+
!managedProcess.cwd ||
|
|
11962
|
+
!managedProcess.marker) {
|
|
11963
|
+
refuse('provider did not return an identity-checked managed process');
|
|
11964
|
+
}
|
|
11965
|
+
if (!expected.requireNode && managedProcess.cwd !== expected.checkoutPath) {
|
|
11966
|
+
refuse('provider started the managed process in a different checkout');
|
|
11967
|
+
}
|
|
11968
|
+
const url = (() => {
|
|
11969
|
+
try {
|
|
11970
|
+
return new URL(preview.url);
|
|
11971
|
+
}
|
|
11972
|
+
catch {
|
|
11973
|
+
return refuse('provider returned an invalid URL');
|
|
11974
|
+
}
|
|
11975
|
+
})();
|
|
11976
|
+
if (url.protocol !== 'https:' || url.username || url.password || !url.hostname) {
|
|
11977
|
+
refuse('provider URL is not credential-free HTTPS');
|
|
11978
|
+
}
|
|
11979
|
+
if (!url.hostname.endsWith('.ts.net')) {
|
|
11980
|
+
refuse('provider URL is not a Tailscale HTTPS name');
|
|
11981
|
+
}
|
|
11982
|
+
const urlPort = url.port ? Number(url.port) : 443;
|
|
11983
|
+
if (urlPort !== preview.httpsPort)
|
|
11984
|
+
refuse('provider URL does not match its guarded HTTPS route');
|
|
11985
|
+
}
|
|
11986
|
+
function specWithPreview(spec, preview) {
|
|
11987
|
+
if (!preview)
|
|
11988
|
+
return { ...spec };
|
|
11989
|
+
return {
|
|
11990
|
+
...spec,
|
|
11991
|
+
preview,
|
|
11992
|
+
// The provider route forwards to loopback on its placement node. Pin every
|
|
11993
|
+
// agent using that checkout to the same node so its dev server is reachable.
|
|
11994
|
+
...(preview.node ? { node: preview.node } : {}),
|
|
11995
|
+
};
|
|
11996
|
+
}
|
|
10773
11997
|
function authoritativeRoutedDecision(triaged, routed) {
|
|
10774
11998
|
if (triaged.confidence !== 'low' || triaged.routes.length > 0 || routed.routes.length === 0) {
|
|
10775
11999
|
return routed;
|
|
@@ -11141,6 +12365,8 @@ const routeForSpec = (decision, spec) => {
|
|
|
11141
12365
|
};
|
|
11142
12366
|
};
|
|
11143
12367
|
const previewUrlFromSpec = (spec) => {
|
|
12368
|
+
if (spec.preview?.url.trim())
|
|
12369
|
+
return spec.preview.url.trim();
|
|
11144
12370
|
const previewUrl = spec.previewUrl;
|
|
11145
12371
|
return typeof previewUrl === 'string' && previewUrl.trim() ? previewUrl.trim() : undefined;
|
|
11146
12372
|
};
|
|
@@ -11933,6 +13159,17 @@ const decodeGithubPathSegment = (value) => {
|
|
|
11933
13159
|
const validGithubRepo = (repo) => /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})\/[A-Za-z0-9_.-]{1,100}$/u.test(repo);
|
|
11934
13160
|
const validPrNumber = (value) => Number.isInteger(value) && value > 0;
|
|
11935
13161
|
const githubPrIdentity = (repo, prNumber) => validGithubRepo(repo) && validPrNumber(prNumber) ? `${repo.toLowerCase()}#${prNumber}` : undefined;
|
|
13162
|
+
// This is the public Relayfile stable identity for a GitHub pull request. It
|
|
13163
|
+
// deliberately comes from the PR's repo/number ownership record, never by
|
|
13164
|
+
// transforming an incoming canonical path (whose title slug can be renamed).
|
|
13165
|
+
const babysitterResourceRef = (repo, prNumber) => {
|
|
13166
|
+
if (!validGithubRepo(repo) || !validPrNumber(prNumber)) {
|
|
13167
|
+
throw new Error('Cannot create a durable babysitter subscription for an invalid GitHub PR identity');
|
|
13168
|
+
}
|
|
13169
|
+
const [owner, name] = repo.split('/');
|
|
13170
|
+
return `/github/repos/${owner}__${name}/pulls/by-id/${prNumber}.json`;
|
|
13171
|
+
};
|
|
13172
|
+
const babysitterSubscriberId = (issue) => `factory-babysitter:${issue.uuid}`;
|
|
11936
13173
|
const babysitterOwnershipKey = (issue, ref) => `${issueKey(issue)}:${githubPrIdentity(ref.repo, ref.prNumber) ?? 'invalid'}`;
|
|
11937
13174
|
const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
11938
13175
|
if (!validGithubRepo(eventRepo))
|
|
@@ -12266,12 +13503,17 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
12266
13503
|
}
|
|
12267
13504
|
return `${owner}/${repo}`;
|
|
12268
13505
|
};
|
|
12269
|
-
const githubPullRequestBody = (issue) => [
|
|
13506
|
+
const githubPullRequestBody = (issue, preview) => [
|
|
12270
13507
|
issue.description,
|
|
12271
13508
|
'',
|
|
12272
13509
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
12273
13510
|
? `Fixes #${issue.key}`
|
|
12274
13511
|
: `Factory issue ${issue.key}`,
|
|
13512
|
+
...(preview ? [
|
|
13513
|
+
'',
|
|
13514
|
+
`Live preview: ${preview.url}`,
|
|
13515
|
+
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
13516
|
+
] : []),
|
|
12275
13517
|
].join('\n').trim();
|
|
12276
13518
|
// The broker rejects re-registering a name it never released on exit
|
|
12277
13519
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
@@ -12729,6 +13971,20 @@ const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
|
|
|
12729
13971
|
`Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
|
|
12730
13972
|
].filter((line) => Boolean(line)).join('\n');
|
|
12731
13973
|
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
13974
|
+
const formatByteCount = (bytes) => {
|
|
13975
|
+
if (bytes < 1024)
|
|
13976
|
+
return `${bytes} B`;
|
|
13977
|
+
const units = ['KiB', 'MiB', 'GiB', 'TiB'];
|
|
13978
|
+
let value = bytes;
|
|
13979
|
+
let unit = 'B';
|
|
13980
|
+
for (const candidate of units) {
|
|
13981
|
+
value /= 1024;
|
|
13982
|
+
unit = candidate;
|
|
13983
|
+
if (value < 1024)
|
|
13984
|
+
break;
|
|
13985
|
+
}
|
|
13986
|
+
return `${value.toFixed(value >= 10 ? 1 : 2)} ${unit}`;
|
|
13987
|
+
};
|
|
12732
13988
|
const unrefDelay = (ms) => new Promise((resolve) => {
|
|
12733
13989
|
const timer = setTimeout(resolve, ms);
|
|
12734
13990
|
timer.unref?.();
|