@agent-relay/factory 0.1.50 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -11
- package/dist/cli/fleet.d.ts +4 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +87 -42
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +1671 -10
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +83 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/dispatch/templates.d.ts +6 -0
- package/dist/dispatch/templates.d.ts.map +1 -1
- package/dist/dispatch/templates.js +16 -0
- package/dist/dispatch/templates.js.map +1 -1
- package/dist/environments/connection-registry.d.ts +617 -0
- package/dist/environments/connection-registry.d.ts.map +1 -0
- package/dist/environments/connection-registry.js +174 -0
- package/dist/environments/connection-registry.js.map +1 -0
- package/dist/environments/index.d.ts +21 -0
- package/dist/environments/index.d.ts.map +1 -0
- package/dist/environments/index.js +11 -0
- package/dist/environments/index.js.map +1 -0
- package/dist/environments/k6-job.d.ts +64 -0
- package/dist/environments/k6-job.d.ts.map +1 -0
- package/dist/environments/k6-job.js +376 -0
- package/dist/environments/k6-job.js.map +1 -0
- package/dist/environments/kubernetes-client.d.ts +72 -0
- package/dist/environments/kubernetes-client.d.ts.map +1 -0
- package/dist/environments/kubernetes-client.js +337 -0
- package/dist/environments/kubernetes-client.js.map +1 -0
- package/dist/environments/kubernetes-command.d.ts +30 -0
- package/dist/environments/kubernetes-command.d.ts.map +1 -0
- package/dist/environments/kubernetes-command.js +108 -0
- package/dist/environments/kubernetes-command.js.map +1 -0
- package/dist/environments/kubernetes-environment.d.ts +38 -0
- package/dist/environments/kubernetes-environment.d.ts.map +1 -0
- package/dist/environments/kubernetes-environment.js +333 -0
- package/dist/environments/kubernetes-environment.js.map +1 -0
- package/dist/environments/kubernetes-provider.d.ts +83 -0
- package/dist/environments/kubernetes-provider.d.ts.map +1 -0
- package/dist/environments/kubernetes-provider.js +956 -0
- package/dist/environments/kubernetes-provider.js.map +1 -0
- package/dist/environments/load-harness.d.ts +199 -0
- package/dist/environments/load-harness.d.ts.map +1 -0
- package/dist/environments/load-harness.js +259 -0
- package/dist/environments/load-harness.js.map +1 -0
- package/dist/environments/load-profile.d.ts +294 -0
- package/dist/environments/load-profile.d.ts.map +1 -0
- package/dist/environments/load-profile.js +125 -0
- package/dist/environments/load-profile.js.map +1 -0
- package/dist/environments/stack-deployer.d.ts +22 -0
- package/dist/environments/stack-deployer.d.ts.map +1 -0
- package/dist/environments/stack-deployer.js +27 -0
- package/dist/environments/stack-deployer.js.map +1 -0
- package/dist/environments/stack-descriptor.d.ts +243 -0
- package/dist/environments/stack-descriptor.d.ts.map +1 -0
- package/dist/environments/stack-descriptor.js +88 -0
- package/dist/environments/stack-descriptor.js.map +1 -0
- package/dist/environments/verification-pipeline.d.ts +117 -0
- package/dist/environments/verification-pipeline.d.ts.map +1 -0
- package/dist/environments/verification-pipeline.js +643 -0
- package/dist/environments/verification-pipeline.js.map +1 -0
- package/dist/environments/verification-stack-deployer.d.ts +73 -0
- package/dist/environments/verification-stack-deployer.d.ts.map +1 -0
- package/dist/environments/verification-stack-deployer.js +484 -0
- package/dist/environments/verification-stack-deployer.js.map +1 -0
- package/dist/environments/verification-stack-descriptor.d.ts +1134 -0
- package/dist/environments/verification-stack-descriptor.d.ts.map +1 -0
- package/dist/environments/verification-stack-descriptor.js +275 -0
- package/dist/environments/verification-stack-descriptor.js.map +1 -0
- package/dist/environments/verification-stack.d.ts +29 -0
- package/dist/environments/verification-stack.d.ts.map +1 -0
- package/dist/environments/verification-stack.js +43 -0
- package/dist/environments/verification-stack.js.map +1 -0
- package/dist/featuremap/generate.d.ts +72 -0
- package/dist/featuremap/generate.d.ts.map +1 -0
- package/dist/featuremap/generate.js +854 -0
- package/dist/featuremap/generate.js.map +1 -0
- package/dist/featuremap/index.d.ts +2 -0
- package/dist/featuremap/index.d.ts.map +1 -1
- package/dist/featuremap/index.js +1 -0
- package/dist/featuremap/index.js.map +1 -1
- package/dist/featuremap/validate.d.ts +6 -0
- package/dist/featuremap/validate.d.ts.map +1 -1
- package/dist/featuremap/validate.js +95 -13
- package/dist/featuremap/validate.js.map +1 -1
- package/dist/fleet/create-fleet.d.ts +2 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -1
- package/dist/fleet/create-fleet.js +1 -0
- package/dist/fleet/create-fleet.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +8 -1
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +31 -6
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +4 -1
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +142 -1
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/git/agent-worktree.d.ts +4 -1
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +151 -8
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/index.d.ts +32 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -3
- package/dist/index.js.map +1 -1
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +6 -2
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +8 -2
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +11 -1
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/node/factory-node.d.ts +2 -0
- package/dist/node/factory-node.d.ts.map +1 -1
- package/dist/node/factory-node.js +107 -2
- package/dist/node/factory-node.js.map +1 -1
- package/dist/node/preview-process.d.ts +65 -0
- package/dist/node/preview-process.d.ts.map +1 -0
- package/dist/node/preview-process.js +431 -0
- package/dist/node/preview-process.js.map +1 -0
- package/dist/node/tailscale-preview.d.ts +45 -0
- package/dist/node/tailscale-preview.d.ts.map +1 -0
- package/dist/node/tailscale-preview.js +748 -0
- package/dist/node/tailscale-preview.js.map +1 -0
- package/dist/observability/events.d.ts +1838 -28
- package/dist/observability/events.d.ts.map +1 -1
- package/dist/observability/events.js +52 -0
- package/dist/observability/events.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1362 -65
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/failure-classifier.d.ts +13 -0
- package/dist/orchestrator/failure-classifier.d.ts.map +1 -0
- package/dist/orchestrator/failure-classifier.js +14 -0
- package/dist/orchestrator/failure-classifier.js.map +1 -0
- package/dist/orchestrator/index.d.ts +2 -1
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/reaper.d.ts +24 -0
- package/dist/orchestrator/reaper.d.ts.map +1 -1
- package/dist/orchestrator/reaper.js +117 -0
- package/dist/orchestrator/reaper.js.map +1 -1
- package/dist/ports/environment.d.ts +83 -0
- package/dist/ports/environment.d.ts.map +1 -0
- package/dist/ports/environment.js +2 -0
- package/dist/ports/environment.js.map +1 -0
- package/dist/ports/fleet.d.ts +68 -1
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +3 -2
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +9 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/ports/worktree.d.ts +14 -0
- package/dist/ports/worktree.d.ts.map +1 -1
- package/dist/state/file-state-store.js +28 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/subscriptions/index.d.ts +2 -0
- package/dist/subscriptions/index.d.ts.map +1 -1
- package/dist/subscriptions/index.js +1 -0
- package/dist/subscriptions/index.js.map +1 -1
- package/dist/subscriptions/resource-subscriptions.d.ts +77 -0
- package/dist/subscriptions/resource-subscriptions.d.ts.map +1 -0
- package/dist/subscriptions/resource-subscriptions.js +91 -0
- package/dist/subscriptions/resource-subscriptions.js.map +1 -0
- package/dist/testing/fakes.d.ts +9 -1
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +40 -0
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +32 -32
- package/dist/types.d.ts +4 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +18 -4
- package/schemas/kubernetes-environment-stack.schema.json +86 -0
- package/schemas/verification-stack.schema.json +292 -0
|
@@ -5,7 +5,8 @@ import { FactoryConfigSchema } from '../config/schema.js';
|
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
8
|
-
import {
|
|
8
|
+
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
|
+
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
9
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
10
11
|
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
@@ -16,6 +17,7 @@ import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/temp
|
|
|
16
17
|
import { resolveTestGuidance } from '../dispatch/test-guidance.js';
|
|
17
18
|
import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
|
|
18
19
|
import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
|
|
20
|
+
import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
|
|
19
21
|
import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
|
|
20
22
|
import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
|
|
21
23
|
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
|
|
@@ -44,6 +46,7 @@ const LIVE_DEDUPE_LIMIT = 5_000;
|
|
|
44
46
|
const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
45
47
|
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
46
48
|
const COMPLETION_SWEEP_BATCH_SIZE = 2;
|
|
49
|
+
const PREVIEW_SWEEP_INTERVAL_MS = 60_000;
|
|
47
50
|
const PROBE_PR_GH_BACKOFF_MS = 60_000;
|
|
48
51
|
const PROBE_PR_GH_CANDIDATE_LIMIT = 200;
|
|
49
52
|
const PUBLISHED_PR_CONFIRM_ATTEMPTS = 20;
|
|
@@ -66,6 +69,24 @@ const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
|
66
69
|
const INJECTION_MAX_ATTEMPTS = 6;
|
|
67
70
|
const BABYSITTER_EVENT_COALESCE_MS = 750;
|
|
68
71
|
const BABYSITTER_EVENT_RETRY_MS = 1_000;
|
|
72
|
+
const BABYSITTER_SUBSCRIPTION_TTL_SECONDS = 60 * 60;
|
|
73
|
+
// Relayfile receives provider-native GitHub events, not the materialized file
|
|
74
|
+
// changes that the legacy local router consumed. `closed` is separately
|
|
75
|
+
// indexed as a terminal event below, so terminal delivery does not need a
|
|
76
|
+
// broad normal-event subscription.
|
|
77
|
+
const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
78
|
+
'pull_request.opened',
|
|
79
|
+
'pull_request.reopened',
|
|
80
|
+
'pull_request.synchronize',
|
|
81
|
+
'pull_request.ready_for_review',
|
|
82
|
+
'pull_request_review.submitted',
|
|
83
|
+
'pull_request_review_comment.created',
|
|
84
|
+
'issue_comment.created',
|
|
85
|
+
'check_run.completed',
|
|
86
|
+
];
|
|
87
|
+
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
88
|
+
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
89
|
+
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
69
90
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
70
91
|
// recipient unavailable) both when a freshly spawned agent has not finished
|
|
71
92
|
// enrolling AND when an agent is up but its relay identity never becomes
|
|
@@ -138,6 +159,7 @@ export class FactoryLoop {
|
|
|
138
159
|
#githubWritebackProvided;
|
|
139
160
|
#slack;
|
|
140
161
|
#mergeGate;
|
|
162
|
+
#verificationGate;
|
|
141
163
|
#probeCloser;
|
|
142
164
|
#probePrResolver;
|
|
143
165
|
#customProbePrResolver;
|
|
@@ -246,6 +268,8 @@ export class FactoryLoop {
|
|
|
246
268
|
#deferLiveEventDrain = false;
|
|
247
269
|
#completionSweepTimer;
|
|
248
270
|
#completionSweepActive = false;
|
|
271
|
+
#previewSweepTimer;
|
|
272
|
+
#previewSweepInFlight;
|
|
249
273
|
#completionInFlight = new Set();
|
|
250
274
|
#agentExitsInFlight = new Map();
|
|
251
275
|
#reconciledAgentExitsActive = 0;
|
|
@@ -262,6 +286,14 @@ export class FactoryLoop {
|
|
|
262
286
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
263
287
|
#babysitterPr = new Map();
|
|
264
288
|
#babysitterIssueRefs = new Map();
|
|
289
|
+
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
290
|
+
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
291
|
+
// owning babysitter.
|
|
292
|
+
#babysitterSubscriptionOwners = new Map();
|
|
293
|
+
#babysitterResourceSubscriptionFault = false;
|
|
294
|
+
#babysitterResourceSubscriptionUnavailable = false;
|
|
295
|
+
#babysitterResourceDeliveryRetryTimer;
|
|
296
|
+
#babysitterResourceSubscriptionRenewTimer;
|
|
265
297
|
#babysitterReady = new Set();
|
|
266
298
|
#babysitterWakeStates = new Map();
|
|
267
299
|
// A babysitter announces this fence before invoking destructive git tooling
|
|
@@ -269,6 +301,8 @@ export class FactoryLoop {
|
|
|
269
301
|
// is active, but the PTY submit must never land in that critical window.
|
|
270
302
|
#babysitterCriticalAgents = new Set();
|
|
271
303
|
#publishedPullRequests = new Map();
|
|
304
|
+
#previewReferences = new Map();
|
|
305
|
+
#removedPreviewIds = new Set();
|
|
272
306
|
#probePrGhBackoffUntilMs = new Map();
|
|
273
307
|
#probePrResolvedCache = new Map();
|
|
274
308
|
// GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion
|
|
@@ -308,6 +342,17 @@ export class FactoryLoop {
|
|
|
308
342
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
309
343
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
310
344
|
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
345
|
+
this.#verificationGate = ports.verificationGate ?? (config.verification.enabled
|
|
346
|
+
? new VerificationPipeline({
|
|
347
|
+
descriptorPath: config.verification.descriptorPath,
|
|
348
|
+
reporter: ports.reporter,
|
|
349
|
+
logger: ports.logger,
|
|
350
|
+
maxConcurrentEnvironments: config.verification.maxConcurrentEnvironments,
|
|
351
|
+
maxRunTimeoutMs: config.verification.maxRunTimeoutMs,
|
|
352
|
+
maxEnvironmentTtlMs: config.verification.maxEnvironmentTtlMs,
|
|
353
|
+
maxTeardownTimeoutMs: config.verification.maxTeardownTimeoutMs,
|
|
354
|
+
})
|
|
355
|
+
: undefined);
|
|
311
356
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
312
357
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
313
358
|
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
@@ -533,12 +578,15 @@ export class FactoryLoop {
|
|
|
533
578
|
this.#wireFleetEvents();
|
|
534
579
|
await this.#adoptInFlightAgents(legacyRegistry);
|
|
535
580
|
this.#startupAgentAdoptionActive = false;
|
|
581
|
+
if (opts.mode !== 'dispatch-owner')
|
|
582
|
+
await this.#reapOrphanedWorktreesOnStartup(legacyRegistry);
|
|
536
583
|
if (this.#config.babysitter.enabled) {
|
|
537
584
|
// Re-run the idempotent receipt fold after adoption returns. This
|
|
538
585
|
// catches records restored by lifecycle work that completed while the
|
|
539
586
|
// startup roster drain was in progress.
|
|
540
587
|
await this.#reconcileRestoredBabysitterReceipts();
|
|
541
588
|
}
|
|
589
|
+
await this.#reapPreviewOrphans();
|
|
542
590
|
}
|
|
543
591
|
catch (error) {
|
|
544
592
|
this.#startupAgentAdoptionActive = false;
|
|
@@ -548,6 +596,7 @@ export class FactoryLoop {
|
|
|
548
596
|
}
|
|
549
597
|
if (opts.mode === 'dispatch-owner') {
|
|
550
598
|
this.#started = true;
|
|
599
|
+
this.#schedulePreviewSweep();
|
|
551
600
|
this.#scheduleDispatchLifecycleRenewal();
|
|
552
601
|
// A replacement one-shot owner must also recover a team parked for
|
|
553
602
|
// human input; it intentionally does not subscribe to the full issue
|
|
@@ -559,6 +608,7 @@ export class FactoryLoop {
|
|
|
559
608
|
}
|
|
560
609
|
if (live) {
|
|
561
610
|
this.#started = true;
|
|
611
|
+
this.#schedulePreviewSweep();
|
|
562
612
|
try {
|
|
563
613
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
564
614
|
await this.#rearmSlackReplyWatchers();
|
|
@@ -598,6 +648,7 @@ export class FactoryLoop {
|
|
|
598
648
|
void this.#handleChange(path);
|
|
599
649
|
});
|
|
600
650
|
this.#started = true;
|
|
651
|
+
this.#schedulePreviewSweep();
|
|
601
652
|
await this.#rearmSlackReplyWatchers();
|
|
602
653
|
await this.#drainReadyClarificationWake();
|
|
603
654
|
await this.#rearmGithubIssueCommentWatchers();
|
|
@@ -606,6 +657,12 @@ export class FactoryLoop {
|
|
|
606
657
|
async stop() {
|
|
607
658
|
this.#started = false;
|
|
608
659
|
this.#stopping = true;
|
|
660
|
+
if (this.#babysitterResourceDeliveryRetryTimer)
|
|
661
|
+
clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
|
|
662
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
663
|
+
if (this.#babysitterResourceSubscriptionRenewTimer)
|
|
664
|
+
clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
|
|
665
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
609
666
|
if (this.#dispatchLifecycleRenewTimer)
|
|
610
667
|
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
611
668
|
this.#dispatchLifecycleRenewTimer = undefined;
|
|
@@ -617,6 +674,10 @@ export class FactoryLoop {
|
|
|
617
674
|
if (this.#completionSweepTimer)
|
|
618
675
|
clearTimeout(this.#completionSweepTimer);
|
|
619
676
|
this.#completionSweepTimer = undefined;
|
|
677
|
+
if (this.#previewSweepTimer)
|
|
678
|
+
clearTimeout(this.#previewSweepTimer);
|
|
679
|
+
this.#previewSweepTimer = undefined;
|
|
680
|
+
await this.#previewSweepInFlight;
|
|
620
681
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
621
682
|
try {
|
|
622
683
|
// Relinquish durable ownership before waiting on mount-backed lifecycle
|
|
@@ -657,6 +718,7 @@ export class FactoryLoop {
|
|
|
657
718
|
this.#babysitterSpawned.clear();
|
|
658
719
|
this.#babysitterPr.clear();
|
|
659
720
|
this.#babysitterIssueRefs.clear();
|
|
721
|
+
this.#babysitterSubscriptionOwners.clear();
|
|
660
722
|
this.#babysitterReady.clear();
|
|
661
723
|
this.#babysitterCriticalAgents.clear();
|
|
662
724
|
const subscription = this.#subscription;
|
|
@@ -2095,9 +2157,6 @@ export class FactoryLoop {
|
|
|
2095
2157
|
};
|
|
2096
2158
|
}
|
|
2097
2159
|
this.#clearDependencyPark(batch, dispatchDecision.issue);
|
|
2098
|
-
// Full task rendering is part of the durable spawn specification. It must
|
|
2099
|
-
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2100
|
-
// recover a persisted minimal triage task after a crash in this gap.
|
|
2101
2160
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
2102
2161
|
// Local dispatches need the same deterministic branch identity as remote
|
|
2103
2162
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -2109,10 +2168,17 @@ export class FactoryLoop {
|
|
|
2109
2168
|
isolateLocalWorktree,
|
|
2110
2169
|
});
|
|
2111
2170
|
}
|
|
2171
|
+
// Full task rendering is part of the durable spawn specification. It must
|
|
2172
|
+
// happen before a remote lifecycle is first claimed so takeover cannot
|
|
2173
|
+
// recover a persisted minimal triage task after a crash in this gap. The
|
|
2174
|
+
// task is rendered again below after preview provisioning adds its URL.
|
|
2112
2175
|
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2176
|
+
let claimedLifecycle;
|
|
2177
|
+
let recoveredRecord;
|
|
2113
2178
|
if (durableDispatch) {
|
|
2114
2179
|
const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun, lifecycleRunId);
|
|
2115
2180
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2181
|
+
claimedLifecycle = lifecycleClaim.lifecycle;
|
|
2116
2182
|
dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision);
|
|
2117
2183
|
if (lifecycleClaim.lifecycle.phase === 'waiting-for-human') {
|
|
2118
2184
|
return lifecycleClaim.lifecycle.result ?? { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
@@ -2130,16 +2196,70 @@ export class FactoryLoop {
|
|
|
2130
2196
|
};
|
|
2131
2197
|
}
|
|
2132
2198
|
if (!lifecycleClaim.created) {
|
|
2133
|
-
|
|
2134
|
-
if (
|
|
2135
|
-
return
|
|
2199
|
+
recoveredRecord = inFlightRecordFromLifecycle(lifecycleClaim.lifecycle);
|
|
2200
|
+
if (recoveredRecord.result)
|
|
2201
|
+
return recoveredRecord.result;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
// External preview creation must happen only after the durable lease is
|
|
2205
|
+
// acquired and capacity admission has promoted the lifecycle. Persist the
|
|
2206
|
+
// fully rendered, preview-bearing decision before any worker can spawn so
|
|
2207
|
+
// takeover never recovers a minimal triage task.
|
|
2208
|
+
const previouslyPersistedPreviewIds = new Set(dispatchSpecs(dispatchDecision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
2209
|
+
try {
|
|
2210
|
+
if (!dryRun) {
|
|
2211
|
+
dispatchDecision = await this.#withPreviewReferences(dispatchDecision);
|
|
2212
|
+
}
|
|
2213
|
+
dispatchDecision = await this.#withRenderedDispatchTasks(dispatchDecision, liveIssue);
|
|
2214
|
+
if (durableDispatch && claimedLifecycle) {
|
|
2215
|
+
const stagedRecord = inFlightRecordFromLifecycle({
|
|
2216
|
+
...claimedLifecycle,
|
|
2217
|
+
decision: structuredClone(dispatchDecision),
|
|
2218
|
+
});
|
|
2219
|
+
if (!await this.#saveDispatchLifecycle(stagedRecord, 'dispatching')) {
|
|
2220
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${dispatchDecision.issue.key}`);
|
|
2221
|
+
}
|
|
2222
|
+
if (recoveredRecord) {
|
|
2223
|
+
recoveredRecord.decision = structuredClone(dispatchDecision);
|
|
2224
|
+
recoveredRecord = batch.restore(recoveredRecord);
|
|
2225
|
+
}
|
|
2136
2226
|
}
|
|
2137
2227
|
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
const newlyCreated = uniquePreviewReferences([
|
|
2230
|
+
...dispatchSpecs(dispatchDecision).map((spec) => spec.preview),
|
|
2231
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue)) ?? []),
|
|
2232
|
+
]).filter((preview) => !previouslyPersistedPreviewIds.has(preview.id));
|
|
2233
|
+
// Once the durable fence is lost, the successor may already have
|
|
2234
|
+
// adopted this deterministic issue preview. Leave cleanup to the
|
|
2235
|
+
// identity-aware sweep instead of letting a stale owner tear down the
|
|
2236
|
+
// successor's route.
|
|
2237
|
+
const mayRollback = !claimedLifecycle ||
|
|
2238
|
+
await this.#assertIssueDispatchLifecycleOwner(dispatchDecision.issue);
|
|
2239
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
2240
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
2241
|
+
this.#logger.warn?.('[factory] failed to roll back preview provisioning', {
|
|
2242
|
+
issue: dispatchDecision.issue.key,
|
|
2243
|
+
error: describeError(cleanupError).errorMessage,
|
|
2244
|
+
});
|
|
2245
|
+
});
|
|
2246
|
+
}
|
|
2247
|
+
if (mayRollback)
|
|
2248
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2249
|
+
if (claimedLifecycle) {
|
|
2250
|
+
this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(claimedLifecycle));
|
|
2251
|
+
}
|
|
2252
|
+
throw error;
|
|
2253
|
+
}
|
|
2138
2254
|
if (!durableDispatch)
|
|
2139
2255
|
this.#consumePendingDispatchClarifications(dispatchDecision.issue);
|
|
2140
2256
|
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
2141
|
-
const record = batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2257
|
+
const record = recoveredRecord ?? batch.start(dispatchDecision, dryRun, dependencyAdmission);
|
|
2142
2258
|
if (!record) {
|
|
2259
|
+
if (!dryRun) {
|
|
2260
|
+
await this.#teardownPreviewReferences(dispatchSpecs(dispatchDecision).map((spec) => spec.preview));
|
|
2261
|
+
this.#previewReferences.delete(issueKey(dispatchDecision.issue));
|
|
2262
|
+
}
|
|
2143
2263
|
await this.#clearDispatchInFlight(dispatchDecision.issue);
|
|
2144
2264
|
this.#increment('queued');
|
|
2145
2265
|
this.#emit('issue-queued', { issue: dispatchDecision.issue });
|
|
@@ -2148,7 +2268,9 @@ export class FactoryLoop {
|
|
|
2148
2268
|
if (record.result) {
|
|
2149
2269
|
return record.result;
|
|
2150
2270
|
}
|
|
2151
|
-
await this.#saveDispatchLifecycle(record, 'dispatching')
|
|
2271
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
2272
|
+
throw new Error(`Dispatch lifecycle ownership lost immediately before spawning ${dispatchDecision.issue.key}`);
|
|
2273
|
+
}
|
|
2152
2274
|
if (!dryRun)
|
|
2153
2275
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
2154
2276
|
const spawnedForReaperHandoff = [];
|
|
@@ -2203,6 +2325,9 @@ export class FactoryLoop {
|
|
|
2203
2325
|
agents,
|
|
2204
2326
|
comments: [comment],
|
|
2205
2327
|
stateId: implementingStateId,
|
|
2328
|
+
...(this.#previewReferences.get(issueKey(dispatchDecision.issue))?.length
|
|
2329
|
+
? { previews: this.#previewReferences.get(issueKey(dispatchDecision.issue)) }
|
|
2330
|
+
: {}),
|
|
2206
2331
|
dryRun,
|
|
2207
2332
|
};
|
|
2208
2333
|
record.result = result;
|
|
@@ -2223,6 +2348,14 @@ export class FactoryLoop {
|
|
|
2223
2348
|
const liveStateChanged = error instanceof LiveDispatchStateChangedError;
|
|
2224
2349
|
const cancellationReason = factoryCloudDispatchCancellationReason(error);
|
|
2225
2350
|
const cleanupReason = liveStateChanged ? 'live dispatch state changed' : 'dispatch failed';
|
|
2351
|
+
let failedState;
|
|
2352
|
+
if (!liveStateChanged) {
|
|
2353
|
+
await this.#recordDispatchFailure(decision.issue);
|
|
2354
|
+
failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key);
|
|
2355
|
+
}
|
|
2356
|
+
const terminalFailure = liveStateChanged || Boolean(failedState?.terminal);
|
|
2357
|
+
if (terminalFailure && !await this.#saveDispatchLifecycle(record, 'abandoning', undefined, cleanupReason, new Set(), { cancellationReason }))
|
|
2358
|
+
throw error;
|
|
2226
2359
|
let worktreesTornDown = await this.#teardownFailedDispatchWorktrees(failureHandoffs, cleanupReason);
|
|
2227
2360
|
if (liveStateChanged && !failureHandoffs.some((handoff) => handoff.worktree)) {
|
|
2228
2361
|
const failed = await this.#releaseAndTerminateAgents(failureHandoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion');
|
|
@@ -2233,16 +2366,43 @@ export class FactoryLoop {
|
|
|
2233
2366
|
worktreesTornDown = failureHandoffs.length > 0;
|
|
2234
2367
|
}
|
|
2235
2368
|
}
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2369
|
+
if (terminalFailure) {
|
|
2370
|
+
try {
|
|
2371
|
+
await this.#teardownPreviews(record);
|
|
2372
|
+
}
|
|
2373
|
+
catch (previewError) {
|
|
2374
|
+
this.#logger.warn?.('[factory] failed to tear down previews after terminal dispatch failure', {
|
|
2375
|
+
issue: record.issue.key,
|
|
2376
|
+
error: describeError(previewError).errorMessage,
|
|
2377
|
+
});
|
|
2378
|
+
// Do not commit a terminal lifecycle while an externally reachable
|
|
2379
|
+
// preview remains. The abandonment driver retries the identity-
|
|
2380
|
+
// checked teardown and commits terminal state only after it succeeds.
|
|
2381
|
+
this.#scheduleAbandonedDispatchRetry(record, cleanupReason);
|
|
2382
|
+
if (!liveStateChanged)
|
|
2383
|
+
this.#error(error, decision.issue);
|
|
2384
|
+
if (worktreesTornDown) {
|
|
2385
|
+
await this.#writeInFlightRegistry().catch((registryError) => {
|
|
2386
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after dispatch worktree teardown', {
|
|
2387
|
+
issue: record.issue,
|
|
2388
|
+
error: describeError(registryError).errorMessage,
|
|
2389
|
+
});
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, undefined, new Set(), { cancellationReason }))
|
|
2395
|
+
throw error;
|
|
2396
|
+
if (liveStateChanged)
|
|
2397
|
+
await this.#clearDispatchInFlight(decision.issue);
|
|
2398
|
+
else
|
|
2399
|
+
await this.#recordDispatchTerminal(decision.issue);
|
|
2400
|
+
if (liveStateChanged)
|
|
2401
|
+
this.#increment('dispatchLiveStateRaces');
|
|
2241
2402
|
}
|
|
2242
2403
|
else {
|
|
2243
|
-
await this.#
|
|
2244
|
-
|
|
2245
|
-
await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable', undefined, undefined, new Set(), { cancellationReason: failedState?.terminal ? cancellationReason : undefined });
|
|
2404
|
+
if (!await this.#saveDispatchLifecycle(record, 'retryable'))
|
|
2405
|
+
throw error;
|
|
2246
2406
|
}
|
|
2247
2407
|
batch.abandon(decision.issue);
|
|
2248
2408
|
if (!liveStateChanged && !failedState?.terminal)
|
|
@@ -2378,6 +2538,12 @@ export class FactoryLoop {
|
|
|
2378
2538
|
for (const [key, lifecycle] of durableLifecycles) {
|
|
2379
2539
|
if (isTerminalDispatchLifecycle(lifecycle))
|
|
2380
2540
|
continue;
|
|
2541
|
+
const previews = uniquePreviewReferences([
|
|
2542
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
2543
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
2544
|
+
]);
|
|
2545
|
+
if (previews.length > 0)
|
|
2546
|
+
this.#previewReferences.set(issueKey(lifecycle.issue), previews);
|
|
2381
2547
|
hasNonterminalDurableLifecycle = true;
|
|
2382
2548
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2383
2549
|
if (!claim.acquired || !claim.lease) {
|
|
@@ -2391,6 +2557,33 @@ export class FactoryLoop {
|
|
|
2391
2557
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2392
2558
|
continue;
|
|
2393
2559
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
2560
|
+
if (!durableRecord.dryRun &&
|
|
2561
|
+
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2562
|
+
claim.lifecycle.phase !== 'releasing') {
|
|
2563
|
+
const liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2564
|
+
// A babysat Linear issue already at Done may have merged while this
|
|
2565
|
+
// process was down. Let authoritative PR restoration drive the
|
|
2566
|
+
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
2567
|
+
const deferDoneToBabysitterRecovery = Boolean(liveIssue &&
|
|
2568
|
+
!isGithubIssue(liveIssue) &&
|
|
2569
|
+
this.#states.roleOf(liveIssue.stateId) === 'done' &&
|
|
2570
|
+
this.#config.babysitter.enabled &&
|
|
2571
|
+
await this.#hasRestorableMergedBabysitterSession(durableRecord.issue));
|
|
2572
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue) && !deferDoneToBabysitterRecovery) {
|
|
2573
|
+
const restored = batch.restore(durableRecord);
|
|
2574
|
+
try {
|
|
2575
|
+
await this.#abandonDurableResume(restored, 'source issue is already terminal during startup recovery');
|
|
2576
|
+
}
|
|
2577
|
+
catch (error) {
|
|
2578
|
+
this.#logger.warn?.('[factory] terminal source preview cleanup will retry after startup', {
|
|
2579
|
+
issue: durableRecord.issue.key,
|
|
2580
|
+
error: describeError(error).errorMessage,
|
|
2581
|
+
});
|
|
2582
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2583
|
+
}
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2394
2587
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2395
2588
|
? durableRecord
|
|
2396
2589
|
: batch.restore(durableRecord);
|
|
@@ -2928,14 +3121,62 @@ export class FactoryLoop {
|
|
|
2928
3121
|
await this.#abandonStuckDispatch(record, abandonedReason);
|
|
2929
3122
|
return;
|
|
2930
3123
|
}
|
|
3124
|
+
if (lifecycle.phase === 'running' && !record.dryRun) {
|
|
3125
|
+
const liveIssue = await this.#readIssue(record.issue.path);
|
|
3126
|
+
if (liveIssue && this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3127
|
+
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3128
|
+
return;
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
2931
3131
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
2932
|
-
|
|
2933
|
-
|
|
3132
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
3133
|
+
if (this.#fleet.hydrateTracked && activeAgents.length > 0) {
|
|
3134
|
+
const hydrated = activeAgents.map((agent) => ({
|
|
2934
3135
|
name: agent.name,
|
|
2935
3136
|
invocationId: agent.tracked.spec.invocationId,
|
|
2936
3137
|
node: agent.tracked.result?.node,
|
|
2937
|
-
}))
|
|
2938
|
-
|
|
3138
|
+
}));
|
|
3139
|
+
this.#fleet.hydrateTracked(hydrated);
|
|
3140
|
+
try {
|
|
3141
|
+
await this.#fleet.reconcileTrackedAgents?.();
|
|
3142
|
+
const online = new Set((await this.#fleet.roster()).agents.map((agent) => agent.name));
|
|
3143
|
+
const fleetTracked = this.#fleet.trackedAgents?.();
|
|
3144
|
+
const missing = activeAgents.filter((agent) => (!online.has(agent.name) || (fleetTracked !== undefined && !fleetTracked.has(agent.name))) &&
|
|
3145
|
+
!this.#agentExitsInFlight.has(agent.name));
|
|
3146
|
+
missing.sort((left, right) => {
|
|
3147
|
+
const priority = (role) => role === 'implementer' ? 0 : role === 'babysitter' ? 1 : 2;
|
|
3148
|
+
return priority(left.tracked.spec.role) - priority(right.tracked.spec.role) ||
|
|
3149
|
+
left.name.localeCompare(right.name);
|
|
3150
|
+
});
|
|
3151
|
+
for (const agent of missing) {
|
|
3152
|
+
this.#fleet.markAgentTerminal?.(agent.name, 'reconciled-missing');
|
|
3153
|
+
this.#queueAgentExit(agent.name, 'reconciled-missing');
|
|
3154
|
+
}
|
|
3155
|
+
if (missing.length > 0) {
|
|
3156
|
+
this.#increment('takeoverRosterMissingExitsSynthesized', missing.length);
|
|
3157
|
+
this.#logger.info?.('[factory] synthesized missing durable takeover roster exits', {
|
|
3158
|
+
issue: lifecycle.issue.key,
|
|
3159
|
+
agents: missing.map((agent) => agent.name),
|
|
3160
|
+
});
|
|
3161
|
+
}
|
|
3162
|
+
const exitNames = new Set(activeAgents.map((agent) => agent.name));
|
|
3163
|
+
const drained = await this.#drainAgentExitsInFlight(exitNames, this.#startMode === 'live' ? this.#startupAgentExitDrainTimeoutMs : undefined);
|
|
3164
|
+
if (!drained) {
|
|
3165
|
+
this.#increment('takeoverAgentExitDrainTimeouts');
|
|
3166
|
+
this.#logger.warn?.('[factory] durable takeover exit reconciliation is still running', {
|
|
3167
|
+
issue: lifecycle.issue.key,
|
|
3168
|
+
timeoutMs: this.#startupAgentExitDrainTimeoutMs,
|
|
3169
|
+
pendingExits: [...exitNames].filter((name) => this.#agentExitsInFlight.has(name)),
|
|
3170
|
+
});
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3173
|
+
catch (error) {
|
|
3174
|
+
// Force the retry path back through takeover reconciliation. Keeping
|
|
3175
|
+
// the epoch cached here would turn the next drive into an ordinary
|
|
3176
|
+
// running lifecycle and permanently skip the authoritative roster.
|
|
3177
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
3178
|
+
throw error;
|
|
3179
|
+
}
|
|
2939
3180
|
}
|
|
2940
3181
|
if (this.#config.babysitter.enabled)
|
|
2941
3182
|
await this.#reconcileRestoredBabysitterReceipts(record);
|
|
@@ -2949,6 +3190,10 @@ export class FactoryLoop {
|
|
|
2949
3190
|
await this.#finishClarificationPark(waiting, true);
|
|
2950
3191
|
return;
|
|
2951
3192
|
}
|
|
3193
|
+
if (lifecycle.phase === 'abandoning') {
|
|
3194
|
+
await this.#abandonStuckDispatch(record, lifecycle.releaseReason ?? 'dispatch failed');
|
|
3195
|
+
return;
|
|
3196
|
+
}
|
|
2952
3197
|
if (lifecycle.phase === 'dispatching' || lifecycle.phase === 'retryable') {
|
|
2953
3198
|
await this.#resumeDurableDispatch(record);
|
|
2954
3199
|
return;
|
|
@@ -3007,15 +3252,39 @@ export class FactoryLoop {
|
|
|
3007
3252
|
}
|
|
3008
3253
|
}
|
|
3009
3254
|
async #resumeDurableDispatch(record) {
|
|
3255
|
+
let liveIssue;
|
|
3010
3256
|
if (!record.dryRun) {
|
|
3011
|
-
|
|
3012
|
-
if (!
|
|
3257
|
+
liveIssue = await this.#readIssue(record.issue.path);
|
|
3258
|
+
if (!liveIssue) {
|
|
3013
3259
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
|
|
3014
3260
|
}
|
|
3015
|
-
if (
|
|
3016
|
-
await this.#abandonDurableResume(record, 'live
|
|
3261
|
+
if (this.#isIssueExternallyTerminal(liveIssue)) {
|
|
3262
|
+
await this.#abandonDurableResume(record, 'live source issue is already terminal');
|
|
3017
3263
|
return;
|
|
3018
3264
|
}
|
|
3265
|
+
const persistedPreviewIds = new Set(dispatchSpecs(record.decision).map((spec) => spec.preview?.id).filter((id) => Boolean(id)));
|
|
3266
|
+
try {
|
|
3267
|
+
record.decision = await this.#withPreviewReferences(record.decision);
|
|
3268
|
+
record.decision = await this.#withRenderedDispatchTasks(record.decision, liveIssue);
|
|
3269
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching'))
|
|
3270
|
+
return;
|
|
3271
|
+
}
|
|
3272
|
+
catch (error) {
|
|
3273
|
+
const newlyCreated = uniquePreviewReferences([
|
|
3274
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3275
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
3276
|
+
]).filter((preview) => !persistedPreviewIds.has(preview.id));
|
|
3277
|
+
const mayRollback = await this.#assertDispatchLifecycleOwner(record);
|
|
3278
|
+
if (newlyCreated.length > 0 && mayRollback) {
|
|
3279
|
+
await this.#teardownPreviewReferences(newlyCreated).catch((cleanupError) => {
|
|
3280
|
+
this.#logger.warn?.('[factory] failed to roll back recovered preview provisioning', {
|
|
3281
|
+
issue: record.issue.key,
|
|
3282
|
+
error: describeError(cleanupError).errorMessage,
|
|
3283
|
+
});
|
|
3284
|
+
});
|
|
3285
|
+
}
|
|
3286
|
+
throw error;
|
|
3287
|
+
}
|
|
3019
3288
|
}
|
|
3020
3289
|
const agents = [];
|
|
3021
3290
|
const specs = dispatchSpecs(record.decision);
|
|
@@ -3032,7 +3301,7 @@ export class FactoryLoop {
|
|
|
3032
3301
|
}
|
|
3033
3302
|
await this.#writeInFlightRegistry();
|
|
3034
3303
|
if (!record.dryRun) {
|
|
3035
|
-
const issue = await this.#readIssue(record.issue.path);
|
|
3304
|
+
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3036
3305
|
if (!issue)
|
|
3037
3306
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3038
3307
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
@@ -3043,15 +3312,24 @@ export class FactoryLoop {
|
|
|
3043
3312
|
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3044
3313
|
}
|
|
3045
3314
|
}
|
|
3315
|
+
const recoveredPreviews = uniquePreviewReferences([
|
|
3316
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
3317
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
3318
|
+
]);
|
|
3046
3319
|
record.result ??= {
|
|
3047
3320
|
issue: record.issue,
|
|
3048
3321
|
agents,
|
|
3049
3322
|
comments: [dispatchComment(record.decision, agents)],
|
|
3323
|
+
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3050
3324
|
dryRun: record.dryRun,
|
|
3051
3325
|
};
|
|
3326
|
+
if (recoveredPreviews.length > 0 && !record.result.previews?.length) {
|
|
3327
|
+
record.result = { ...record.result, previews: recoveredPreviews };
|
|
3328
|
+
}
|
|
3052
3329
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3053
3330
|
return;
|
|
3054
3331
|
if (!record.dryRun) {
|
|
3332
|
+
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3055
3333
|
for (const tracked of record.agents.values()) {
|
|
3056
3334
|
const owned = tracked.spec.ownedPullRequest;
|
|
3057
3335
|
if (tracked.spec.role !== 'babysitter' || !owned)
|
|
@@ -3076,6 +3354,31 @@ export class FactoryLoop {
|
|
|
3076
3354
|
labels.has('factory:in-progress') &&
|
|
3077
3355
|
!labels.has('factory:human-review');
|
|
3078
3356
|
}
|
|
3357
|
+
#isIssueExternallyTerminal(issue) {
|
|
3358
|
+
if (isGithubIssue(issue)) {
|
|
3359
|
+
if (githubFactoryIssueIsClosed(issue))
|
|
3360
|
+
return true;
|
|
3361
|
+
return issue.labels.some((label) => label.trim().toLowerCase() === 'factory:human-review');
|
|
3362
|
+
}
|
|
3363
|
+
const role = this.#states.roleOf(issue.stateId);
|
|
3364
|
+
return role === 'humanReview' || role === 'done';
|
|
3365
|
+
}
|
|
3366
|
+
async #hasRestorableMergedBabysitterSession(issue) {
|
|
3367
|
+
const wanted = issueKey(issue);
|
|
3368
|
+
for (const [, session] of await this.#state.listBabysitterSessions(this.#workspaceId)) {
|
|
3369
|
+
if (issueKey(session.issue) !== wanted ||
|
|
3370
|
+
!validGithubRepo(session.repo) ||
|
|
3371
|
+
!validPrNumber(session.prNumber) ||
|
|
3372
|
+
!session.agentName)
|
|
3373
|
+
continue;
|
|
3374
|
+
const snapshot = await this.#readPrSnapshot(session);
|
|
3375
|
+
if (snapshot &&
|
|
3376
|
+
prMetaShowsMerged(snapshot) &&
|
|
3377
|
+
prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30)
|
|
3378
|
+
return true;
|
|
3379
|
+
}
|
|
3380
|
+
return false;
|
|
3381
|
+
}
|
|
3079
3382
|
async #abandonDurableResume(record, reason) {
|
|
3080
3383
|
const handoffs = this.#dispatchFailureHandoffs(record, [...record.agents].map(([name, tracked]) => ({
|
|
3081
3384
|
issue: record.issue,
|
|
@@ -3084,25 +3387,32 @@ export class FactoryLoop {
|
|
|
3084
3387
|
persistedAtMs: this.#clock.now(),
|
|
3085
3388
|
})));
|
|
3086
3389
|
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
batch.abandon(record.issue);
|
|
3092
|
-
for (const [name] of record.agents) {
|
|
3093
|
-
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3094
|
-
}
|
|
3390
|
+
// Keep the lifecycle nonterminal until every externally reachable route
|
|
3391
|
+
// is confirmed gone. A restart can then retry cleanup instead of treating
|
|
3392
|
+
// an abandoned row as finished and leaking its issue preview forever.
|
|
3393
|
+
await this.#teardownPreviews(record);
|
|
3095
3394
|
if (handoffs.some((handoff) => handoff.worktree)) {
|
|
3096
|
-
await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')
|
|
3395
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'live dispatch state changed')) {
|
|
3396
|
+
throw new Error(`Unable to finish stale dispatch worktree teardown for ${record.issue.key}`);
|
|
3397
|
+
}
|
|
3097
3398
|
}
|
|
3098
3399
|
else if (handoffs.length > 0) {
|
|
3099
3400
|
const failed = new Set(await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), 'live dispatch state changed', 'completion'));
|
|
3401
|
+
if (failed.size > 0) {
|
|
3402
|
+
throw new Error(`Unable to release stale dispatch agents for ${record.issue.key}: ${[...failed].join(', ')}`);
|
|
3403
|
+
}
|
|
3100
3404
|
for (const handoff of handoffs) {
|
|
3101
|
-
if (failed.has(handoff.name))
|
|
3102
|
-
continue;
|
|
3103
3405
|
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
3104
3406
|
}
|
|
3105
3407
|
}
|
|
3408
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'source_state_changed' }))
|
|
3409
|
+
return;
|
|
3410
|
+
await this.#recordDispatchTerminal(record.issue);
|
|
3411
|
+
const batch = await this.#batch();
|
|
3412
|
+
batch.abandon(record.issue);
|
|
3413
|
+
for (const [name] of record.agents) {
|
|
3414
|
+
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
3415
|
+
}
|
|
3106
3416
|
await this.#stopSlackWatcher(record.issue);
|
|
3107
3417
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
3108
3418
|
await this.#writeInFlightRegistry();
|
|
@@ -3117,6 +3427,16 @@ export class FactoryLoop {
|
|
|
3117
3427
|
const batch = await this.#batch();
|
|
3118
3428
|
const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done');
|
|
3119
3429
|
const releaseKey = issueKey(record.issue);
|
|
3430
|
+
// Terminal writeback has already been acknowledged before this method is
|
|
3431
|
+
// entered. Remove externally reachable routes first so a stuck agent
|
|
3432
|
+
// release cannot leave a preview live after Human Review or Done.
|
|
3433
|
+
try {
|
|
3434
|
+
await this.#teardownPreviews(record);
|
|
3435
|
+
}
|
|
3436
|
+
catch {
|
|
3437
|
+
this.#scheduleReleaseRetry(record, reason);
|
|
3438
|
+
return false;
|
|
3439
|
+
}
|
|
3120
3440
|
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, releaseKey);
|
|
3121
3441
|
const released = new Set(lifecycle?.agents
|
|
3122
3442
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
@@ -5013,7 +5333,7 @@ export class FactoryLoop {
|
|
|
5013
5333
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
5014
5334
|
baseRef,
|
|
5015
5335
|
title: `${issue.key}: ${issue.title}`,
|
|
5016
|
-
body: githubPullRequestBody(issue),
|
|
5336
|
+
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
5017
5337
|
});
|
|
5018
5338
|
const published = result.author
|
|
5019
5339
|
? result
|
|
@@ -5196,9 +5516,42 @@ export class FactoryLoop {
|
|
|
5196
5516
|
if (worktree)
|
|
5197
5517
|
unique.set(worktree.worktreePath, worktree);
|
|
5198
5518
|
}
|
|
5519
|
+
const issueSlug = factoryWorktreeIssueSlug(record.issue.key);
|
|
5199
5520
|
const failures = [];
|
|
5521
|
+
for (const repository of this.#worktreeRepositories(record)) {
|
|
5522
|
+
try {
|
|
5523
|
+
const candidates = await this.#worktrees.listWorktrees(repository);
|
|
5524
|
+
for (const candidate of candidates) {
|
|
5525
|
+
if (factoryWorktreeIssueSlug(candidate.issueKey) === issueSlug) {
|
|
5526
|
+
unique.set(candidate.worktreePath, candidate);
|
|
5527
|
+
}
|
|
5528
|
+
}
|
|
5529
|
+
}
|
|
5530
|
+
catch (error) {
|
|
5531
|
+
const message = `${repository.baseClonePath}: ${describeError(error).errorMessage}`;
|
|
5532
|
+
failures.push(message);
|
|
5533
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5534
|
+
this.#logger.warn?.('[factory] failed to enumerate completed issue worktrees', {
|
|
5535
|
+
issue: record.issue.key,
|
|
5536
|
+
repo: repository.repo,
|
|
5537
|
+
baseClonePath: repository.baseClonePath,
|
|
5538
|
+
error: describeError(error).errorMessage,
|
|
5539
|
+
});
|
|
5540
|
+
}
|
|
5541
|
+
}
|
|
5200
5542
|
for (const worktree of unique.values()) {
|
|
5201
5543
|
try {
|
|
5544
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5545
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5546
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5547
|
+
this.#logger.warn?.('[factory] retained completed issue worktree with local state', {
|
|
5548
|
+
issue: record.issue.key,
|
|
5549
|
+
repo: worktree.repo,
|
|
5550
|
+
worktreePath: worktree.worktreePath,
|
|
5551
|
+
retentionReasons: inspection.retentionReasons,
|
|
5552
|
+
});
|
|
5553
|
+
continue;
|
|
5554
|
+
}
|
|
5202
5555
|
await this.#worktrees.cleanup(worktree);
|
|
5203
5556
|
this.#increment('agentWorktreesCleaned');
|
|
5204
5557
|
}
|
|
@@ -5217,6 +5570,132 @@ export class FactoryLoop {
|
|
|
5217
5570
|
throw new Error(`Factory worktree cleanup incomplete for ${record.issue.key}: ${failures.join('; ')}`);
|
|
5218
5571
|
}
|
|
5219
5572
|
}
|
|
5573
|
+
#worktreeRepositories(record) {
|
|
5574
|
+
const repositories = new Map();
|
|
5575
|
+
const add = (repo, baseClonePath) => {
|
|
5576
|
+
if (!baseClonePath)
|
|
5577
|
+
return;
|
|
5578
|
+
const key = `${repo}\u0000${resolve(baseClonePath)}`;
|
|
5579
|
+
repositories.set(key, { repo, baseClonePath });
|
|
5580
|
+
};
|
|
5581
|
+
if (record) {
|
|
5582
|
+
for (const route of record.decision.routes) {
|
|
5583
|
+
add(route.repo, this.#config.repos.clonePaths[route.repo] ?? route.clonePath);
|
|
5584
|
+
}
|
|
5585
|
+
for (const tracked of record.agents.values()) {
|
|
5586
|
+
add(tracked.spec.repo, tracked.spec.baseClonePath);
|
|
5587
|
+
}
|
|
5588
|
+
for (const spec of record.decision.implementers) {
|
|
5589
|
+
add(spec.repo, spec.baseClonePath);
|
|
5590
|
+
}
|
|
5591
|
+
}
|
|
5592
|
+
else {
|
|
5593
|
+
for (const [repo, baseClonePath] of Object.entries(this.#config.repos.clonePaths)) {
|
|
5594
|
+
add(repo, baseClonePath);
|
|
5595
|
+
}
|
|
5596
|
+
}
|
|
5597
|
+
return [...repositories.values()];
|
|
5598
|
+
}
|
|
5599
|
+
async #reapOrphanedWorktreesOnStartup(legacyRegistry) {
|
|
5600
|
+
if (!this.#worktrees)
|
|
5601
|
+
return;
|
|
5602
|
+
const legacyAgents = (legacyRegistry?.agents ?? []).filter((agent) => agent.issue);
|
|
5603
|
+
let durableLifecycles;
|
|
5604
|
+
let waitingClarifications;
|
|
5605
|
+
let onlineAgentNames;
|
|
5606
|
+
try {
|
|
5607
|
+
const [lifecycles, clarifications, roster] = await Promise.all([
|
|
5608
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
5609
|
+
this.#state.listWaitingClarifications(this.#workspaceId),
|
|
5610
|
+
legacyAgents.length > 0 ? this.#fleet.roster() : undefined,
|
|
5611
|
+
]);
|
|
5612
|
+
durableLifecycles = lifecycles;
|
|
5613
|
+
waitingClarifications = clarifications;
|
|
5614
|
+
onlineAgentNames = new Set((roster?.agents ?? []).map((agent) => agent.name));
|
|
5615
|
+
}
|
|
5616
|
+
catch (error) {
|
|
5617
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5618
|
+
this.#logger.warn?.('[factory] startup worktree reaper skipped because active lifecycle or roster state could not be loaded', {
|
|
5619
|
+
error: describeError(error).errorMessage,
|
|
5620
|
+
});
|
|
5621
|
+
return;
|
|
5622
|
+
}
|
|
5623
|
+
const activeIssueSlugs = new Set((await this.#batch()).inFlight.map((record) => factoryWorktreeIssueSlug(record.issue.key)));
|
|
5624
|
+
for (const [, lifecycle] of durableLifecycles) {
|
|
5625
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
5626
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(lifecycle.issue.key));
|
|
5627
|
+
}
|
|
5628
|
+
}
|
|
5629
|
+
for (const [, waiting] of waitingClarifications) {
|
|
5630
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(waiting.issue.key));
|
|
5631
|
+
}
|
|
5632
|
+
for (const agent of legacyAgents) {
|
|
5633
|
+
if (agent.issue && onlineAgentNames.has(agent.name)) {
|
|
5634
|
+
activeIssueSlugs.add(factoryWorktreeIssueSlug(agent.issue.key));
|
|
5635
|
+
}
|
|
5636
|
+
}
|
|
5637
|
+
const candidates = new Map();
|
|
5638
|
+
let reaped = 0;
|
|
5639
|
+
let reclaimedBytes = 0;
|
|
5640
|
+
let retained = 0;
|
|
5641
|
+
let failures = 0;
|
|
5642
|
+
for (const repository of this.#worktreeRepositories()) {
|
|
5643
|
+
try {
|
|
5644
|
+
for (const candidate of await this.#worktrees.listWorktrees(repository)) {
|
|
5645
|
+
candidates.set(candidate.worktreePath, candidate);
|
|
5646
|
+
}
|
|
5647
|
+
}
|
|
5648
|
+
catch (error) {
|
|
5649
|
+
failures += 1;
|
|
5650
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5651
|
+
this.#logger.warn?.('[factory] startup worktree reaper failed to enumerate repository', {
|
|
5652
|
+
repo: repository.repo,
|
|
5653
|
+
baseClonePath: repository.baseClonePath,
|
|
5654
|
+
error: describeError(error).errorMessage,
|
|
5655
|
+
});
|
|
5656
|
+
}
|
|
5657
|
+
}
|
|
5658
|
+
for (const worktree of candidates.values()) {
|
|
5659
|
+
if (activeIssueSlugs.has(factoryWorktreeIssueSlug(worktree.issueKey)))
|
|
5660
|
+
continue;
|
|
5661
|
+
try {
|
|
5662
|
+
const inspection = await this.#worktrees.inspectForCleanup(worktree);
|
|
5663
|
+
if (inspection.retentionReasons.length > 0) {
|
|
5664
|
+
retained += 1;
|
|
5665
|
+
this.#increment('agentWorktreeCleanupRetained');
|
|
5666
|
+
this.#logger.warn?.('[factory] retained startup orphan worktree with local state', {
|
|
5667
|
+
issue: worktree.issueKey,
|
|
5668
|
+
repo: worktree.repo,
|
|
5669
|
+
worktreePath: worktree.worktreePath,
|
|
5670
|
+
retentionReasons: inspection.retentionReasons,
|
|
5671
|
+
});
|
|
5672
|
+
continue;
|
|
5673
|
+
}
|
|
5674
|
+
await this.#worktrees.cleanup(worktree);
|
|
5675
|
+
reaped += 1;
|
|
5676
|
+
reclaimedBytes += inspection.bytes;
|
|
5677
|
+
this.#increment('agentWorktreesCleaned');
|
|
5678
|
+
this.#increment('agentWorktreesReapedOnStartup');
|
|
5679
|
+
}
|
|
5680
|
+
catch (error) {
|
|
5681
|
+
failures += 1;
|
|
5682
|
+
this.#increment('agentWorktreeCleanupFailures');
|
|
5683
|
+
this.#logger.warn?.('[factory] startup worktree reaper retained checkout after cleanup failure', {
|
|
5684
|
+
issue: worktree.issueKey,
|
|
5685
|
+
repo: worktree.repo,
|
|
5686
|
+
worktreePath: worktree.worktreePath,
|
|
5687
|
+
error: describeError(error).errorMessage,
|
|
5688
|
+
});
|
|
5689
|
+
}
|
|
5690
|
+
}
|
|
5691
|
+
this.#logger.info?.('[factory] startup worktree reaper completed', {
|
|
5692
|
+
reaped,
|
|
5693
|
+
reclaimedBytes,
|
|
5694
|
+
reclaimed: formatByteCount(reclaimedBytes),
|
|
5695
|
+
retained,
|
|
5696
|
+
failures,
|
|
5697
|
+
});
|
|
5698
|
+
}
|
|
5220
5699
|
async #confirmPublishedRemotePullRequest(repo, result, expectedHeadRef) {
|
|
5221
5700
|
const parts = githubRepoParts(repo);
|
|
5222
5701
|
if (!parts)
|
|
@@ -5363,6 +5842,28 @@ export class FactoryLoop {
|
|
|
5363
5842
|
async #abandonStuckDispatch(record, reason) {
|
|
5364
5843
|
const key = issueKey(record.issue);
|
|
5365
5844
|
this.#abandonedDispatchReasons.set(key, reason);
|
|
5845
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5846
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5847
|
+
// The generic durable retry can recover the in-memory reason in this
|
|
5848
|
+
// process; after restart the persisted `abandoning` phase is the fence.
|
|
5849
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5850
|
+
return;
|
|
5851
|
+
}
|
|
5852
|
+
try {
|
|
5853
|
+
// Remove externally reachable routes before releasing the agents that
|
|
5854
|
+
// could still be serving the upstream. A failed provider teardown keeps
|
|
5855
|
+
// the durable lifecycle retryable instead of terminalizing a leak.
|
|
5856
|
+
await this.#teardownPreviews(record);
|
|
5857
|
+
}
|
|
5858
|
+
catch (error) {
|
|
5859
|
+
this.#increment('abandonedDispatchReleaseRetries');
|
|
5860
|
+
this.#logger.warn?.('[factory] abandoned dispatch preview teardown failed; retrying', {
|
|
5861
|
+
issue: record.issue.key,
|
|
5862
|
+
error: describeError(error).errorMessage,
|
|
5863
|
+
});
|
|
5864
|
+
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5865
|
+
return;
|
|
5866
|
+
}
|
|
5366
5867
|
const agents = [...record.agents];
|
|
5367
5868
|
for (const [agentName, tracked] of agents) {
|
|
5368
5869
|
if (tracked.spec.role === 'implementer')
|
|
@@ -5396,16 +5897,13 @@ export class FactoryLoop {
|
|
|
5396
5897
|
return;
|
|
5397
5898
|
}
|
|
5398
5899
|
// Batch completion alone only frees the process-local slot. Durable
|
|
5399
|
-
// capacity is computed from lifecycle phases, so
|
|
5400
|
-
//
|
|
5401
|
-
|
|
5402
|
-
// before promoting the next issue.
|
|
5403
|
-
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason)) {
|
|
5900
|
+
// capacity is computed from lifecycle phases, so commit the terminal phase
|
|
5901
|
+
// only after the preview, agents, and worktrees have all been cleaned up.
|
|
5902
|
+
if (!await this.#saveDispatchLifecycle(record, 'abandoned', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
5404
5903
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
5405
5904
|
// #saveDispatchLifecycle already schedules the generic durable retry. The
|
|
5406
|
-
// pending reason makes
|
|
5407
|
-
//
|
|
5408
|
-
// slot. The abandoned-specific scheduler remains the non-durable fallback.
|
|
5905
|
+
// pending reason makes an in-process retry return here, while the durable
|
|
5906
|
+
// `abandoning` phase provides the same recovery guarantee after restart.
|
|
5409
5907
|
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
5410
5908
|
await this.#writeInFlightRegistry();
|
|
5411
5909
|
return;
|
|
@@ -7034,11 +7532,12 @@ export class FactoryLoop {
|
|
|
7034
7532
|
const integrationInstructions = await this.#resolveIntegrationInstructions();
|
|
7035
7533
|
const render = async (spec) => {
|
|
7036
7534
|
const route = routeForSpec(decision, spec);
|
|
7535
|
+
const previewUrl = previewUrlFromSpec(spec);
|
|
7037
7536
|
const testGuidance = await resolveTestGuidance({
|
|
7038
7537
|
repoPath: route.clonePath,
|
|
7039
7538
|
issue: templateIssue,
|
|
7040
7539
|
route,
|
|
7041
|
-
previewUrl
|
|
7540
|
+
previewUrl,
|
|
7042
7541
|
});
|
|
7043
7542
|
return {
|
|
7044
7543
|
...spec,
|
|
@@ -7055,6 +7554,11 @@ export class FactoryLoop {
|
|
|
7055
7554
|
branchName: spec.branch ?? decision.implementers.find((candidate) => candidate.repo === spec.repo)?.branch,
|
|
7056
7555
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
7057
7556
|
agentName: spec.name,
|
|
7557
|
+
...(previewUrl ? {
|
|
7558
|
+
previewUrl,
|
|
7559
|
+
previewTargetPort: spec.preview?.targetPort,
|
|
7560
|
+
previewStartCommand: spec.preview?.startCommand,
|
|
7561
|
+
} : {}),
|
|
7058
7562
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
7059
7563
|
}),
|
|
7060
7564
|
};
|
|
@@ -7066,6 +7570,193 @@ export class FactoryLoop {
|
|
|
7066
7570
|
reviewer: rendered[decision.implementers.length],
|
|
7067
7571
|
};
|
|
7068
7572
|
}
|
|
7573
|
+
async #withPreviewReferences(decision) {
|
|
7574
|
+
if (!this.#config.preview || decision.scope === 'workflow')
|
|
7575
|
+
return decision;
|
|
7576
|
+
if (!this.#fleet.createPreview) {
|
|
7577
|
+
throw new Error('Preview services are configured but the selected fleet backend cannot create previews');
|
|
7578
|
+
}
|
|
7579
|
+
const owner = issueKey(decision.issue);
|
|
7580
|
+
const byRepo = new Map();
|
|
7581
|
+
for (const implementer of decision.implementers) {
|
|
7582
|
+
const service = previewServiceForRepo(this.#config, implementer.repo);
|
|
7583
|
+
if (byRepo.has(implementer.repo))
|
|
7584
|
+
continue;
|
|
7585
|
+
const persisted = implementer.preview;
|
|
7586
|
+
if (!service) {
|
|
7587
|
+
if (persisted)
|
|
7588
|
+
byRepo.set(implementer.repo, persisted);
|
|
7589
|
+
continue;
|
|
7590
|
+
}
|
|
7591
|
+
if (!implementer.clonePath) {
|
|
7592
|
+
throw new Error(`Preview service ${service.name} requires a configured checkout for ${implementer.repo}`);
|
|
7593
|
+
}
|
|
7594
|
+
await this.#preparePreviewCheckout(decision, implementer);
|
|
7595
|
+
const preview = await this.#fleet.createPreview({
|
|
7596
|
+
namespace: this.#workspaceId,
|
|
7597
|
+
owner,
|
|
7598
|
+
issueKey: decision.issue.key,
|
|
7599
|
+
service: service.name,
|
|
7600
|
+
repo: implementer.repo,
|
|
7601
|
+
targetPort: service.config.port,
|
|
7602
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7603
|
+
startCommand: service.config.startCommand,
|
|
7604
|
+
checkoutPath: implementer.clonePath,
|
|
7605
|
+
node: persisted?.node ?? implementer.node,
|
|
7606
|
+
});
|
|
7607
|
+
this.#removedPreviewIds.delete(preview.id);
|
|
7608
|
+
byRepo.set(implementer.repo, preview);
|
|
7609
|
+
// Make the provider identity visible immediately. If validation or the
|
|
7610
|
+
// following durable save fails, the caller can roll it back while it
|
|
7611
|
+
// still owns the lifecycle; after fence loss the startup sweep owns it.
|
|
7612
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7613
|
+
assertPublishablePreview(preview, {
|
|
7614
|
+
namespace: this.#workspaceId,
|
|
7615
|
+
owner,
|
|
7616
|
+
service: service.name,
|
|
7617
|
+
repo: implementer.repo,
|
|
7618
|
+
targetPort: service.config.port,
|
|
7619
|
+
portSpan: service.config.portSpan ?? 100,
|
|
7620
|
+
preferredHttpsPort: service.config.httpsPort,
|
|
7621
|
+
startCommand: service.config.startCommand,
|
|
7622
|
+
checkoutPath: implementer.clonePath,
|
|
7623
|
+
requireNode: this.#fleet.placementLocality === 'remote',
|
|
7624
|
+
});
|
|
7625
|
+
}
|
|
7626
|
+
this.#previewReferences.set(owner, [...byRepo.values()]);
|
|
7627
|
+
return {
|
|
7628
|
+
...decision,
|
|
7629
|
+
implementers: decision.implementers.map((spec) => specWithPreview(spec, byRepo.get(spec.repo))),
|
|
7630
|
+
reviewer: specWithPreview(decision.reviewer, byRepo.get(decision.reviewer.repo)),
|
|
7631
|
+
};
|
|
7632
|
+
}
|
|
7633
|
+
async #preparePreviewCheckout(decision, spec) {
|
|
7634
|
+
if (!this.#worktrees ||
|
|
7635
|
+
!spec.baseClonePath ||
|
|
7636
|
+
!spec.clonePath ||
|
|
7637
|
+
spec.baseClonePath === spec.clonePath ||
|
|
7638
|
+
!spec.branch)
|
|
7639
|
+
return;
|
|
7640
|
+
try {
|
|
7641
|
+
await this.#worktrees.prepare({
|
|
7642
|
+
repo: spec.repo,
|
|
7643
|
+
issueKey: decision.issue.key,
|
|
7644
|
+
baseClonePath: spec.baseClonePath,
|
|
7645
|
+
worktreePath: spec.clonePath,
|
|
7646
|
+
branch: spec.branch,
|
|
7647
|
+
...(spec.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
7648
|
+
});
|
|
7649
|
+
this.#increment('agentWorktreesPrepared');
|
|
7650
|
+
}
|
|
7651
|
+
catch (error) {
|
|
7652
|
+
throw contextualError(`Unable to prepare preview checkout for ${decision.issue.key}/${spec.repo} at ${spec.clonePath}`, error);
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7655
|
+
async #teardownPreviews(record) {
|
|
7656
|
+
const previews = uniquePreviewReferences([
|
|
7657
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7658
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7659
|
+
...(this.#previewReferences.get(issueKey(record.issue)) ?? []),
|
|
7660
|
+
]);
|
|
7661
|
+
if (previews.length === 0)
|
|
7662
|
+
return;
|
|
7663
|
+
await this.#teardownPreviewReferences(previews);
|
|
7664
|
+
this.#previewReferences.delete(issueKey(record.issue));
|
|
7665
|
+
}
|
|
7666
|
+
async #teardownPreviewReferences(references) {
|
|
7667
|
+
const previews = uniquePreviewReferences(references)
|
|
7668
|
+
.filter((preview) => !this.#removedPreviewIds.has(preview.id));
|
|
7669
|
+
if (previews.length === 0)
|
|
7670
|
+
return;
|
|
7671
|
+
if (!this.#fleet.removePreview) {
|
|
7672
|
+
throw new Error('Fleet backend cannot remove its configured previews');
|
|
7673
|
+
}
|
|
7674
|
+
const results = await Promise.allSettled(previews.map(async (preview) => await this.#fleet.removePreview(preview)));
|
|
7675
|
+
const failures = results.flatMap((result, index) => {
|
|
7676
|
+
const preview = previews[index];
|
|
7677
|
+
if (result.status === 'rejected')
|
|
7678
|
+
return [{ preview, reason: result.reason }];
|
|
7679
|
+
if (!result.value) {
|
|
7680
|
+
return [{
|
|
7681
|
+
preview,
|
|
7682
|
+
reason: new Error(`Preview provider could not confirm removal of ${preview.id}`),
|
|
7683
|
+
}];
|
|
7684
|
+
}
|
|
7685
|
+
this.#removedPreviewIds.add(preview.id);
|
|
7686
|
+
return [];
|
|
7687
|
+
});
|
|
7688
|
+
if (failures.length > 0) {
|
|
7689
|
+
this.#logger.warn?.('[factory] preview teardown failed', {
|
|
7690
|
+
owners: [...new Set(failures.map(({ preview }) => preview.owner))],
|
|
7691
|
+
previews: failures.map(({ preview }) => preview.id),
|
|
7692
|
+
});
|
|
7693
|
+
throw new AggregateError(failures.map(({ reason }) => reason), 'Unable to tear down every issue preview');
|
|
7694
|
+
}
|
|
7695
|
+
}
|
|
7696
|
+
async #reapPreviewOrphans() {
|
|
7697
|
+
if (!this.#config.preview || !this.#fleet.reapPreviews)
|
|
7698
|
+
return;
|
|
7699
|
+
const [lifecycles, batch] = await Promise.all([
|
|
7700
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
7701
|
+
this.#batch(),
|
|
7702
|
+
]);
|
|
7703
|
+
const activePreviewIds = new Set();
|
|
7704
|
+
const activeOwners = new Set(lifecycles
|
|
7705
|
+
.map(([, lifecycle]) => lifecycle)
|
|
7706
|
+
.filter((lifecycle) => !isTerminalDispatchLifecycle(lifecycle))
|
|
7707
|
+
.map((lifecycle) => {
|
|
7708
|
+
for (const preview of uniquePreviewReferences([
|
|
7709
|
+
...dispatchSpecs(lifecycle.decision).map((spec) => spec.preview),
|
|
7710
|
+
...lifecycle.agents.map((agent) => agent.tracked.spec.preview),
|
|
7711
|
+
]))
|
|
7712
|
+
activePreviewIds.add(preview.id);
|
|
7713
|
+
return issueKey(lifecycle.issue);
|
|
7714
|
+
}));
|
|
7715
|
+
for (const record of batch.inFlight) {
|
|
7716
|
+
activeOwners.add(issueKey(record.issue));
|
|
7717
|
+
for (const preview of uniquePreviewReferences([
|
|
7718
|
+
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
7719
|
+
...[...record.agents.values()].map((tracked) => tracked.spec.preview),
|
|
7720
|
+
]))
|
|
7721
|
+
activePreviewIds.add(preview.id);
|
|
7722
|
+
}
|
|
7723
|
+
const report = await this.#fleet.reapPreviews({
|
|
7724
|
+
namespace: this.#workspaceId,
|
|
7725
|
+
activeOwners: [...activeOwners],
|
|
7726
|
+
activePreviewIds: [...activePreviewIds],
|
|
7727
|
+
});
|
|
7728
|
+
if (report.reaped.length > 0 || report.skipped.length > 0) {
|
|
7729
|
+
this.#logger.info?.('[factory] preview orphan sweep completed', {
|
|
7730
|
+
reaped: report.reaped.map((preview) => preview.id),
|
|
7731
|
+
skipped: report.skipped,
|
|
7732
|
+
});
|
|
7733
|
+
}
|
|
7734
|
+
}
|
|
7735
|
+
#schedulePreviewSweep(delayMs = PREVIEW_SWEEP_INTERVAL_MS) {
|
|
7736
|
+
if (this.#stopping ||
|
|
7737
|
+
!this.#started ||
|
|
7738
|
+
!this.#config.preview ||
|
|
7739
|
+
!this.#fleet.reapPreviews ||
|
|
7740
|
+
this.#previewSweepTimer ||
|
|
7741
|
+
this.#previewSweepInFlight)
|
|
7742
|
+
return;
|
|
7743
|
+
this.#previewSweepTimer = setTimeout(() => {
|
|
7744
|
+
this.#previewSweepTimer = undefined;
|
|
7745
|
+
if (this.#stopping || !this.#started)
|
|
7746
|
+
return;
|
|
7747
|
+
this.#previewSweepInFlight = this.#reapPreviewOrphans()
|
|
7748
|
+
.catch((error) => {
|
|
7749
|
+
this.#logger.warn?.('[factory] periodic preview orphan sweep failed', {
|
|
7750
|
+
error: describeError(error).errorMessage,
|
|
7751
|
+
});
|
|
7752
|
+
})
|
|
7753
|
+
.finally(() => {
|
|
7754
|
+
this.#previewSweepInFlight = undefined;
|
|
7755
|
+
this.#schedulePreviewSweep();
|
|
7756
|
+
});
|
|
7757
|
+
}, delayMs);
|
|
7758
|
+
this.#previewSweepTimer.unref?.();
|
|
7759
|
+
}
|
|
7069
7760
|
#consumePendingDispatchClarifications(issue) {
|
|
7070
7761
|
const key = issueKey(issue);
|
|
7071
7762
|
this.#pendingSlackClarifications.delete(key);
|
|
@@ -7182,8 +7873,13 @@ export class FactoryLoop {
|
|
|
7182
7873
|
prNumber: session.prNumber,
|
|
7183
7874
|
path: session.path,
|
|
7184
7875
|
agentName: session.agentName,
|
|
7876
|
+
resourceSubscription: session.resourceSubscription,
|
|
7877
|
+
pendingDeliveryClaims: session.pendingDeliveryClaims,
|
|
7185
7878
|
};
|
|
7186
7879
|
this.#babysitterPr.set(ownershipKey, ref);
|
|
7880
|
+
if (ref.resourceSubscription) {
|
|
7881
|
+
this.#babysitterSubscriptionOwners.set(ref.resourceSubscription.subscriptionId, ownershipKey);
|
|
7882
|
+
}
|
|
7187
7883
|
this.#babysitterIssueRefs.set(ownershipKey, { ...session.issue });
|
|
7188
7884
|
this.#babysitterSpawned.add(ownershipKey);
|
|
7189
7885
|
if (session.critical)
|
|
@@ -7195,10 +7891,21 @@ export class FactoryLoop {
|
|
|
7195
7891
|
this.#increment('babysitterOwnershipRestored');
|
|
7196
7892
|
const pendingKinds = session.pendingKinds.filter(isBabysitterWakeKind);
|
|
7197
7893
|
if (pendingKinds.length > 0) {
|
|
7198
|
-
|
|
7894
|
+
// A terminal marker can coexist with the one wake persisted before
|
|
7895
|
+
// acknowledgement. Rehydrate that durable hand-off once, while later
|
|
7896
|
+
// raw events remain quarantined by #queueBabysitterWake.
|
|
7897
|
+
await this.#queueBabysitterWake(session.issue, ref, pendingKinds, tracked, { allowTerminal: true });
|
|
7199
7898
|
this.#increment('babysitterPendingWakesRestored');
|
|
7200
7899
|
}
|
|
7900
|
+
await this.#ensureBabysitterResourceSubscription(session.issue, ref, tracked);
|
|
7201
7901
|
}
|
|
7902
|
+
// A crash after the local queue write but before (or just after) the
|
|
7903
|
+
// remote acceptance leaves an ID in state. Accept is idempotent once the
|
|
7904
|
+
// lease was accepted, so settle those durable hand-offs before claiming
|
|
7905
|
+
// new work; lease expiry is retried below when a server has not released
|
|
7906
|
+
// the original claim yet.
|
|
7907
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
7908
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
7202
7909
|
}
|
|
7203
7910
|
async #reconcileRestoredBabysitterReceipts(onlyRecord) {
|
|
7204
7911
|
const records = onlyRecord ? [onlyRecord] : (await this.#batch()).inFlight;
|
|
@@ -7297,6 +8004,29 @@ export class FactoryLoop {
|
|
|
7297
8004
|
this.#babysitterWakeStates.delete(key);
|
|
7298
8005
|
this.#babysitterCriticalAgents.delete(state.agentName);
|
|
7299
8006
|
}
|
|
8007
|
+
if (mayClearDurable && ref?.resourceSubscription) {
|
|
8008
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
8009
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8010
|
+
if (subscriptions) {
|
|
8011
|
+
try {
|
|
8012
|
+
await subscriptions.cancel(this.#workspaceId, {
|
|
8013
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
8014
|
+
});
|
|
8015
|
+
this.#increment('babysitterResourceSubscriptionsCancelled');
|
|
8016
|
+
}
|
|
8017
|
+
catch (error) {
|
|
8018
|
+
// Cancellation is deliberately idempotent. A terminal acceptance may
|
|
8019
|
+
// have retired this record already; an outage leaves the bounded TTL
|
|
8020
|
+
// as the leak backstop and must not prevent local session cleanup.
|
|
8021
|
+
this.#increment('babysitterResourceSubscriptionCancelFailures');
|
|
8022
|
+
this.#logger.warn?.('[factory] could not cancel durable babysitter resource subscription', {
|
|
8023
|
+
issue: issue?.key,
|
|
8024
|
+
subscriptionId: ref.resourceSubscription.subscriptionId,
|
|
8025
|
+
error: describeError(error).errorMessage,
|
|
8026
|
+
});
|
|
8027
|
+
}
|
|
8028
|
+
}
|
|
8029
|
+
}
|
|
7300
8030
|
this.#babysitterPr.delete(ownershipKey);
|
|
7301
8031
|
this.#babysitterIssueRefs.delete(ownershipKey);
|
|
7302
8032
|
this.#babysitterSpawned.delete(ownershipKey);
|
|
@@ -7311,7 +8041,323 @@ export class FactoryLoop {
|
|
|
7311
8041
|
.map(([key]) => key);
|
|
7312
8042
|
await Promise.all(keys.map(async (key) => this.#cancelBabysitterWake(key)));
|
|
7313
8043
|
}
|
|
8044
|
+
async #ensureBabysitterResourceSubscription(issue, ref, tracked) {
|
|
8045
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8046
|
+
if (!subscriptions || !ref.agentName || !await this.#assertIssueDispatchLifecycleOwner(issue))
|
|
8047
|
+
return;
|
|
8048
|
+
// A terminal claim is persisted before Relayfile acceptance so a crash in
|
|
8049
|
+
// that gap can never renew a retired record into a fresh generation.
|
|
8050
|
+
if (ref.resourceSubscription?.terminal)
|
|
8051
|
+
return;
|
|
8052
|
+
const resourceRef = babysitterResourceRef(ref.repo, ref.prNumber);
|
|
8053
|
+
const subscriberId = babysitterSubscriberId(issue);
|
|
8054
|
+
try {
|
|
8055
|
+
const subscription = await subscriptions.createOrRenew(this.#workspaceId, {
|
|
8056
|
+
provider: 'github',
|
|
8057
|
+
resourceRef,
|
|
8058
|
+
eventTypes: [...BABYSITTER_SUBSCRIPTION_EVENT_TYPES],
|
|
8059
|
+
terminalEventTypes: [...BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES],
|
|
8060
|
+
subscriberId,
|
|
8061
|
+
ttlSeconds: BABYSITTER_SUBSCRIPTION_TTL_SECONDS,
|
|
8062
|
+
});
|
|
8063
|
+
if (!subscription.subscriptionId ||
|
|
8064
|
+
subscription.provider !== 'github' ||
|
|
8065
|
+
subscription.resourceRef !== resourceRef ||
|
|
8066
|
+
subscription.subscriberId !== subscriberId ||
|
|
8067
|
+
!subscription.ownerId ||
|
|
8068
|
+
!subscription.expiresAt ||
|
|
8069
|
+
!subscription.terminalEventTypes?.includes('pull_request.closed')) {
|
|
8070
|
+
throw new Error('Relayfile returned an invalid durable resource subscription');
|
|
8071
|
+
}
|
|
8072
|
+
if (ref.resourceSubscription?.subscriptionId && ref.resourceSubscription.subscriptionId !== subscription.subscriptionId) {
|
|
8073
|
+
this.#babysitterSubscriptionOwners.delete(ref.resourceSubscription.subscriptionId);
|
|
8074
|
+
}
|
|
8075
|
+
ref.resourceSubscription = {
|
|
8076
|
+
subscriptionId: subscription.subscriptionId,
|
|
8077
|
+
provider: subscription.provider,
|
|
8078
|
+
resourceRef: subscription.resourceRef,
|
|
8079
|
+
subscriberId: subscription.subscriberId,
|
|
8080
|
+
ownerId: subscription.ownerId,
|
|
8081
|
+
expiresAt: subscription.expiresAt,
|
|
8082
|
+
};
|
|
8083
|
+
this.#babysitterSubscriptionOwners.set(subscription.subscriptionId, babysitterOwnershipKey(issue, ref));
|
|
8084
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8085
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8086
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8087
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8088
|
+
this.#increment('babysitterResourceSubscriptionsRenewed');
|
|
8089
|
+
}
|
|
8090
|
+
catch (error) {
|
|
8091
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8092
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8093
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8094
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8095
|
+
return;
|
|
8096
|
+
}
|
|
8097
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8098
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8099
|
+
this.#increment('babysitterResourceSubscriptionRenewFailures');
|
|
8100
|
+
this.#logger.warn?.('[factory] could not create or renew durable babysitter resource subscription', {
|
|
8101
|
+
issue: issue.key,
|
|
8102
|
+
repo: ref.repo,
|
|
8103
|
+
prNumber: ref.prNumber,
|
|
8104
|
+
error: describeError(error).errorMessage,
|
|
8105
|
+
});
|
|
8106
|
+
}
|
|
8107
|
+
}
|
|
8108
|
+
async #routeDurableBabysitterDeliveries() {
|
|
8109
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8110
|
+
if (!subscriptions || !this.#config.babysitter.enabled || this.#stopping)
|
|
8111
|
+
return false;
|
|
8112
|
+
// Do not bypass the proven local router until every active babysitter has
|
|
8113
|
+
// completed its own create/renew. This closes the rollout and transient
|
|
8114
|
+
// provisioning gap without making a successful API response for some
|
|
8115
|
+
// other subscription suppress an unregistered PR's wake.
|
|
8116
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.agentName && !ref.resourceSubscription)) {
|
|
8117
|
+
return false;
|
|
8118
|
+
}
|
|
8119
|
+
let claims;
|
|
8120
|
+
try {
|
|
8121
|
+
claims = await subscriptions.claimDeliveryClaims(this.#workspaceId);
|
|
8122
|
+
}
|
|
8123
|
+
catch (error) {
|
|
8124
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8125
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8126
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8127
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8128
|
+
}
|
|
8129
|
+
else {
|
|
8130
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8131
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8132
|
+
this.#increment('babysitterResourceDeliveryLookupFailures');
|
|
8133
|
+
this.#logger.warn?.('[factory] durable babysitter delivery-claim lookup failed; retaining durable delivery retry', {
|
|
8134
|
+
error: describeError(error).errorMessage,
|
|
8135
|
+
});
|
|
8136
|
+
}
|
|
8137
|
+
return !isResourceSubscriptionsUnavailable(error);
|
|
8138
|
+
}
|
|
8139
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8140
|
+
this.#babysitterResourceSubscriptionUnavailable = false;
|
|
8141
|
+
for (const claim of claims) {
|
|
8142
|
+
const issueIdentity = this.#babysitterSubscriptionOwners.get(claim.subscriptionId);
|
|
8143
|
+
const issue = issueIdentity ? this.#babysitterIssueRefs.get(issueIdentity) : undefined;
|
|
8144
|
+
const ref = issueIdentity ? this.#babysitterPr.get(issueIdentity) : undefined;
|
|
8145
|
+
const subscription = ref?.resourceSubscription;
|
|
8146
|
+
if (!issue ||
|
|
8147
|
+
!ref ||
|
|
8148
|
+
!subscription ||
|
|
8149
|
+
subscription.subscriptionId !== claim.subscriptionId ||
|
|
8150
|
+
subscription.provider !== claim.provider ||
|
|
8151
|
+
subscription.resourceRef !== claim.resourceRef ||
|
|
8152
|
+
subscription.subscriberId !== claim.subscriberId ||
|
|
8153
|
+
subscription.ownerId !== claim.ownerId) {
|
|
8154
|
+
// The service is owner-isolated, but Factory may have just retired a
|
|
8155
|
+
// local owner. Never route a stale or other-session claim by resource.
|
|
8156
|
+
this.#increment('babysitterResourceDeliveriesIgnoredUnowned');
|
|
8157
|
+
continue;
|
|
8158
|
+
}
|
|
8159
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
8160
|
+
this.#increment('babysitterResourceDeliveriesIgnoredNonOwner');
|
|
8161
|
+
continue;
|
|
8162
|
+
}
|
|
8163
|
+
// A terminal delivery may be reclaimed after a process crash before its
|
|
8164
|
+
// remote acceptance. Only that already-persisted delivery may finish;
|
|
8165
|
+
// no later claim is allowed to wake or re-open the retired session.
|
|
8166
|
+
if (subscription.terminal && !ref.pendingDeliveryClaims?.some((pending) => pending.deliveryId === claim.deliveryId)) {
|
|
8167
|
+
this.#increment('babysitterResourceDeliveriesIgnoredTerminal');
|
|
8168
|
+
continue;
|
|
8169
|
+
}
|
|
8170
|
+
const batch = await this.#batch();
|
|
8171
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8172
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8173
|
+
?? durableBabysitterTrackedAgent({
|
|
8174
|
+
issue,
|
|
8175
|
+
repo: ref.repo,
|
|
8176
|
+
prNumber: ref.prNumber,
|
|
8177
|
+
path: ref.path,
|
|
8178
|
+
agentName: ref.agentName,
|
|
8179
|
+
critical: false,
|
|
8180
|
+
pendingKinds: [],
|
|
8181
|
+
resourceSubscription: subscription,
|
|
8182
|
+
pendingDeliveryClaims: ref.pendingDeliveryClaims,
|
|
8183
|
+
});
|
|
8184
|
+
const pendingClaim = ref.pendingDeliveryClaims?.find((pending) => pending.deliveryId === claim.deliveryId);
|
|
8185
|
+
const alreadyQueued = Boolean(pendingClaim);
|
|
8186
|
+
if (!alreadyQueued) {
|
|
8187
|
+
const queued = await this.#queueBabysitterWake(issue, ref, ['pull-request-state'], tracked);
|
|
8188
|
+
if (!queued)
|
|
8189
|
+
continue;
|
|
8190
|
+
}
|
|
8191
|
+
if (!pendingClaim || pendingClaim.claimToken !== claim.claimToken) {
|
|
8192
|
+
ref.pendingDeliveryClaims = [
|
|
8193
|
+
...(ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId),
|
|
8194
|
+
{ deliveryId: claim.deliveryId, claimToken: claim.claimToken },
|
|
8195
|
+
];
|
|
8196
|
+
// The claim lease joins Factory's durable pending-wake state before
|
|
8197
|
+
// the external acceptance. A crash after this point can retry the
|
|
8198
|
+
// exact hand-off without delivering the same wake a second time.
|
|
8199
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8200
|
+
}
|
|
8201
|
+
try {
|
|
8202
|
+
if (claim.terminal && !subscription.terminal) {
|
|
8203
|
+
subscription.terminal = true;
|
|
8204
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8205
|
+
}
|
|
8206
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, {
|
|
8207
|
+
deliveryId: claim.deliveryId,
|
|
8208
|
+
claimToken: claim.claimToken,
|
|
8209
|
+
});
|
|
8210
|
+
if (accepted.deliveryId !== claim.deliveryId || accepted.subscriptionId !== claim.subscriptionId) {
|
|
8211
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8212
|
+
}
|
|
8213
|
+
if (accepted.terminal || claim.terminal) {
|
|
8214
|
+
// Keep the terminal marker and subscription identity locally until
|
|
8215
|
+
// normal PR/session teardown. That quarantines the babysitter from
|
|
8216
|
+
// both legacy fallback and a restart-time create-or-renew.
|
|
8217
|
+
subscription.terminal = true;
|
|
8218
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8219
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8220
|
+
this.#increment('babysitterResourceSubscriptionsRetiredTerminal');
|
|
8221
|
+
}
|
|
8222
|
+
else {
|
|
8223
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== claim.deliveryId);
|
|
8224
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8225
|
+
}
|
|
8226
|
+
this.#increment('babysitterResourceDeliveriesAccepted');
|
|
8227
|
+
}
|
|
8228
|
+
catch (error) {
|
|
8229
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8230
|
+
this.#logger.warn?.('[factory] durable babysitter delivery claim remains pending after wake queue', {
|
|
8231
|
+
issue: issue.key,
|
|
8232
|
+
subscriptionId: claim.subscriptionId,
|
|
8233
|
+
deliveryId: claim.deliveryId,
|
|
8234
|
+
error: describeError(error).errorMessage,
|
|
8235
|
+
});
|
|
8236
|
+
}
|
|
8237
|
+
}
|
|
8238
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8239
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8240
|
+
}
|
|
8241
|
+
return true;
|
|
8242
|
+
}
|
|
8243
|
+
async #retryPendingBabysitterDeliveryAcceptances() {
|
|
8244
|
+
const subscriptions = this.#mount.resourceSubscriptions;
|
|
8245
|
+
if (!subscriptions)
|
|
8246
|
+
return;
|
|
8247
|
+
const retrySubscriptionRenewal = this.#babysitterResourceSubscriptionFault;
|
|
8248
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8249
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8250
|
+
if (issue && ref.agentName && !ref.resourceSubscription?.terminal && (!ref.resourceSubscription || retrySubscriptionRenewal)) {
|
|
8251
|
+
const batch = await this.#batch();
|
|
8252
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8253
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8254
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8255
|
+
}
|
|
8256
|
+
const subscription = ref.resourceSubscription;
|
|
8257
|
+
const pendingDeliveryClaims = [...(ref.pendingDeliveryClaims ?? [])];
|
|
8258
|
+
if (!subscription || !issue || pendingDeliveryClaims.length === 0)
|
|
8259
|
+
continue;
|
|
8260
|
+
const batch = await this.#batch();
|
|
8261
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8262
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter')
|
|
8263
|
+
?? durableBabysitterTrackedAgent({
|
|
8264
|
+
issue,
|
|
8265
|
+
repo: ref.repo,
|
|
8266
|
+
prNumber: ref.prNumber,
|
|
8267
|
+
path: ref.path,
|
|
8268
|
+
agentName: ref.agentName,
|
|
8269
|
+
critical: false,
|
|
8270
|
+
pendingKinds: [],
|
|
8271
|
+
resourceSubscription: subscription,
|
|
8272
|
+
pendingDeliveryClaims,
|
|
8273
|
+
});
|
|
8274
|
+
for (const { deliveryId, claimToken } of pendingDeliveryClaims) {
|
|
8275
|
+
try {
|
|
8276
|
+
const accepted = await subscriptions.acceptDelivery(this.#workspaceId, { deliveryId, claimToken });
|
|
8277
|
+
if (accepted.deliveryId !== deliveryId || accepted.subscriptionId !== subscription.subscriptionId) {
|
|
8278
|
+
throw new Error('Relayfile accepted a different durable delivery claim');
|
|
8279
|
+
}
|
|
8280
|
+
if (accepted.terminal)
|
|
8281
|
+
subscription.terminal = true;
|
|
8282
|
+
ref.pendingDeliveryClaims = (ref.pendingDeliveryClaims ?? []).filter((pending) => pending.deliveryId !== deliveryId);
|
|
8283
|
+
await this.#persistBabysitterSession(issue, ref, tracked);
|
|
8284
|
+
this.#increment('babysitterResourceDeliveriesAcceptedAfterRestore');
|
|
8285
|
+
}
|
|
8286
|
+
catch (error) {
|
|
8287
|
+
if (isResourceSubscriptionsUnavailable(error)) {
|
|
8288
|
+
this.#babysitterResourceSubscriptionFault = false;
|
|
8289
|
+
this.#babysitterResourceSubscriptionUnavailable = true;
|
|
8290
|
+
this.#increment('babysitterResourceSubscriptionUnavailable');
|
|
8291
|
+
}
|
|
8292
|
+
else {
|
|
8293
|
+
this.#babysitterResourceSubscriptionFault = true;
|
|
8294
|
+
this.#increment('babysitterResourceDeliveryAcceptFailures');
|
|
8295
|
+
this.#logger.warn?.('[factory] durable babysitter delivery acceptance remains pending after restore', {
|
|
8296
|
+
issue: issue.key,
|
|
8297
|
+
subscriptionId: subscription.subscriptionId,
|
|
8298
|
+
deliveryId,
|
|
8299
|
+
error: describeError(error).errorMessage,
|
|
8300
|
+
});
|
|
8301
|
+
}
|
|
8302
|
+
}
|
|
8303
|
+
}
|
|
8304
|
+
}
|
|
8305
|
+
if ([...this.#babysitterPr.values()].some((ref) => ref.pendingDeliveryClaims?.length)) {
|
|
8306
|
+
this.#scheduleDurableBabysitterDeliveryRetry();
|
|
8307
|
+
}
|
|
8308
|
+
}
|
|
8309
|
+
#scheduleBabysitterResourceSubscriptionRenewal() {
|
|
8310
|
+
if (this.#babysitterResourceSubscriptionRenewTimer ||
|
|
8311
|
+
this.#stopping ||
|
|
8312
|
+
!this.#mount.resourceSubscriptions ||
|
|
8313
|
+
![...this.#babysitterPr.values()].some((ref) => ref.resourceSubscription && !ref.resourceSubscription.terminal))
|
|
8314
|
+
return;
|
|
8315
|
+
this.#babysitterResourceSubscriptionRenewTimer = setTimeout(() => {
|
|
8316
|
+
this.#babysitterResourceSubscriptionRenewTimer = undefined;
|
|
8317
|
+
void (async () => {
|
|
8318
|
+
const batch = await this.#batch();
|
|
8319
|
+
for (const [issueIdentity, ref] of this.#babysitterPr) {
|
|
8320
|
+
const issue = this.#babysitterIssueRefs.get(issueIdentity);
|
|
8321
|
+
if (!issue || !ref.resourceSubscription || ref.resourceSubscription.terminal)
|
|
8322
|
+
continue;
|
|
8323
|
+
const tracked = batch.getIssue(issue)?.agents.get(ref.agentName)
|
|
8324
|
+
?? [...(batch.getIssue(issue)?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter');
|
|
8325
|
+
await this.#ensureBabysitterResourceSubscription(issue, ref, tracked);
|
|
8326
|
+
}
|
|
8327
|
+
})().catch((error) => {
|
|
8328
|
+
this.#logger.warn?.('[factory] durable babysitter subscription renewal rejected', {
|
|
8329
|
+
error: describeError(error).errorMessage,
|
|
8330
|
+
});
|
|
8331
|
+
}).finally(() => {
|
|
8332
|
+
this.#scheduleBabysitterResourceSubscriptionRenewal();
|
|
8333
|
+
});
|
|
8334
|
+
}, BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS);
|
|
8335
|
+
this.#babysitterResourceSubscriptionRenewTimer.unref?.();
|
|
8336
|
+
}
|
|
8337
|
+
#scheduleDurableBabysitterDeliveryRetry() {
|
|
8338
|
+
if (this.#babysitterResourceDeliveryRetryTimer || this.#stopping || !this.#mount.resourceSubscriptions)
|
|
8339
|
+
return;
|
|
8340
|
+
this.#babysitterResourceDeliveryRetryTimer = setTimeout(() => {
|
|
8341
|
+
this.#babysitterResourceDeliveryRetryTimer = undefined;
|
|
8342
|
+
void (async () => {
|
|
8343
|
+
await this.#retryPendingBabysitterDeliveryAcceptances();
|
|
8344
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8345
|
+
})().catch((error) => {
|
|
8346
|
+
this.#logger.warn?.('[factory] durable babysitter delivery retry rejected', {
|
|
8347
|
+
error: describeError(error).errorMessage,
|
|
8348
|
+
});
|
|
8349
|
+
});
|
|
8350
|
+
}, BABYSITTER_RESOURCE_DELIVERY_RETRY_MS);
|
|
8351
|
+
this.#babysitterResourceDeliveryRetryTimer.unref?.();
|
|
8352
|
+
}
|
|
7314
8353
|
async #routeBabysitterEvent(path, extraKinds = []) {
|
|
8354
|
+
// A successful Relayfile claim lookup is the new exact demux. While some
|
|
8355
|
+
// owners are still unregistered, the legacy path router remains available
|
|
8356
|
+
// only to those owners. Registered owners retain and retry service claims,
|
|
8357
|
+
// so a mixed rollout or transient create failure neither double-delivers a
|
|
8358
|
+
// registered PR nor drops an event for an unregistered PR.
|
|
8359
|
+
if (await this.#routeDurableBabysitterDeliveries())
|
|
8360
|
+
return;
|
|
7315
8361
|
const event = githubBabysitterEventPathParts(path);
|
|
7316
8362
|
if (!event || !this.#config.babysitter.enabled || this.#stopping)
|
|
7317
8363
|
return;
|
|
@@ -7344,6 +8390,12 @@ export class FactoryLoop {
|
|
|
7344
8390
|
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', { ...event, prNumber: target.prNumber });
|
|
7345
8391
|
continue;
|
|
7346
8392
|
}
|
|
8393
|
+
if (this.#mount.resourceSubscriptions &&
|
|
8394
|
+
!this.#babysitterResourceSubscriptionUnavailable &&
|
|
8395
|
+
owner.ref.resourceSubscription) {
|
|
8396
|
+
this.#increment('babysitterEventsDeferredToDurableSubscription');
|
|
8397
|
+
continue;
|
|
8398
|
+
}
|
|
7347
8399
|
if (!await this.#assertIssueDispatchLifecycleOwner(owner.issue)) {
|
|
7348
8400
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7349
8401
|
continue;
|
|
@@ -7402,10 +8454,14 @@ export class FactoryLoop {
|
|
|
7402
8454
|
return;
|
|
7403
8455
|
}
|
|
7404
8456
|
}
|
|
7405
|
-
async #queueBabysitterWake(issue, ref, kinds, tracked) {
|
|
8457
|
+
async #queueBabysitterWake(issue, ref, kinds, tracked, options = {}) {
|
|
8458
|
+
if (ref.resourceSubscription?.terminal && !options.allowTerminal) {
|
|
8459
|
+
this.#increment('babysitterEventsIgnoredTerminal');
|
|
8460
|
+
return false;
|
|
8461
|
+
}
|
|
7406
8462
|
if (!await this.#assertIssueDispatchLifecycleOwner(issue)) {
|
|
7407
8463
|
this.#increment('babysitterEventsIgnoredNonOwner');
|
|
7408
|
-
return;
|
|
8464
|
+
return false;
|
|
7409
8465
|
}
|
|
7410
8466
|
// Owner lookup and queueing straddle async mount/state reads. Revalidate
|
|
7411
8467
|
// the exact composite owner so a concurrent close/merge cancellation can
|
|
@@ -7416,7 +8472,7 @@ export class FactoryLoop {
|
|
|
7416
8472
|
current.agentName !== ref.agentName ||
|
|
7417
8473
|
githubPrIdentity(current.repo, current.prNumber) !== githubPrIdentity(ref.repo, ref.prNumber)) {
|
|
7418
8474
|
this.#increment('babysitterEventsIgnoredStaleOwner');
|
|
7419
|
-
return;
|
|
8475
|
+
return false;
|
|
7420
8476
|
}
|
|
7421
8477
|
// Any new event invalidates a prior readiness assertion for this exact PR.
|
|
7422
8478
|
this.#babysitterReady.delete(ownershipKey);
|
|
@@ -7446,9 +8502,10 @@ export class FactoryLoop {
|
|
|
7446
8502
|
});
|
|
7447
8503
|
if (state.deferredSubmitTargets || state.inFlight || this.#babysitterCriticalAgents.has(state.agentName)) {
|
|
7448
8504
|
this.#increment('babysitterEventWakesDeferred');
|
|
7449
|
-
return;
|
|
8505
|
+
return true;
|
|
7450
8506
|
}
|
|
7451
8507
|
this.#scheduleBabysitterWake(state, BABYSITTER_EVENT_COALESCE_MS);
|
|
8508
|
+
return true;
|
|
7452
8509
|
}
|
|
7453
8510
|
async #recordPendingBabysitterWake(state) {
|
|
7454
8511
|
const kinds = new Set([
|
|
@@ -7484,6 +8541,8 @@ export class FactoryLoop {
|
|
|
7484
8541
|
path: ref.path,
|
|
7485
8542
|
critical: this.#babysitterCriticalAgents.has(ref.agentName),
|
|
7486
8543
|
pendingKinds: pending?.kinds.filter(isBabysitterWakeKind).sort(compareBabysitterWakeKinds) ?? [],
|
|
8544
|
+
...(ref.resourceSubscription ? { resourceSubscription: { ...ref.resourceSubscription } } : {}),
|
|
8545
|
+
...(ref.pendingDeliveryClaims?.length ? { pendingDeliveryClaims: structuredClone(ref.pendingDeliveryClaims) } : {}),
|
|
7487
8546
|
});
|
|
7488
8547
|
}
|
|
7489
8548
|
#scheduleBabysitterWake(state, delayMs) {
|
|
@@ -7769,11 +8828,21 @@ export class FactoryLoop {
|
|
|
7769
8828
|
if (!this.#config.babysitter.enabled)
|
|
7770
8829
|
return;
|
|
7771
8830
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8831
|
+
// A provider close produces a separately indexed terminal claim. Keep
|
|
8832
|
+
// the durable owner until it has been accepted (or a transient retry
|
|
8833
|
+
// has claimed it); do not let closed-state cleanup erase that hand-off.
|
|
8834
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8835
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8836
|
+
return;
|
|
7772
8837
|
await this.#cancelBabysitterWake(owned.key);
|
|
7773
8838
|
return;
|
|
7774
8839
|
}
|
|
7775
8840
|
if (snapshot.draft)
|
|
7776
8841
|
this.#increment('babysitterDraftPrSkipped');
|
|
8842
|
+
// PR meta events are also the normal renewal heartbeat for the durable
|
|
8843
|
+
// record. The store's identity makes this a create-or-renew, never a
|
|
8844
|
+
// second subscription for the same babysitter.
|
|
8845
|
+
await this.#ensureBabysitterResourceSubscription(owned.issue, owned.ref, owned.tracked);
|
|
7777
8846
|
await this.#routeBabysitterEvent(path, babysitterWakeKindsFromSnapshot(snapshot));
|
|
7778
8847
|
return;
|
|
7779
8848
|
}
|
|
@@ -7821,6 +8890,13 @@ export class FactoryLoop {
|
|
|
7821
8890
|
return;
|
|
7822
8891
|
}
|
|
7823
8892
|
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
8893
|
+
// `pull_request.closed` is a separately indexed Relayfile terminal
|
|
8894
|
+
// event. Claim and accept its durable hand-off before the local closed
|
|
8895
|
+
// PR cleanup drops the subscription owner. On a transient service fault,
|
|
8896
|
+
// retain the owner so the retry loop can claim it without local fallback.
|
|
8897
|
+
await this.#routeDurableBabysitterDeliveries();
|
|
8898
|
+
if (this.#babysitterResourceSubscriptionFault)
|
|
8899
|
+
return;
|
|
7824
8900
|
if (babysitterKey && existing)
|
|
7825
8901
|
await this.#cancelBabysitterWake(babysitterKey);
|
|
7826
8902
|
return;
|
|
@@ -8018,6 +9094,11 @@ export class FactoryLoop {
|
|
|
8018
9094
|
const settled = this.#babysitterPr.get(babysitterKey);
|
|
8019
9095
|
if (settled && prRef.path)
|
|
8020
9096
|
settled.path = prRef.path;
|
|
9097
|
+
if (settled) {
|
|
9098
|
+
const tracked = record.agents.get(settled.agentName)
|
|
9099
|
+
?? [...record.agents.values()].find((agent) => agent.spec.role === 'babysitter');
|
|
9100
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, settled, tracked);
|
|
9101
|
+
}
|
|
8021
9102
|
return;
|
|
8022
9103
|
}
|
|
8023
9104
|
const wantedPr = githubPrIdentity(prRef.repo, prRef.prNumber);
|
|
@@ -8038,7 +9119,9 @@ export class FactoryLoop {
|
|
|
8038
9119
|
agentName: tracked.result?.name ?? trackedName,
|
|
8039
9120
|
});
|
|
8040
9121
|
this.#babysitterSpawned.add(babysitterKey);
|
|
8041
|
-
|
|
9122
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9123
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9124
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
8042
9125
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
8043
9126
|
return;
|
|
8044
9127
|
}
|
|
@@ -8068,9 +9151,13 @@ export class FactoryLoop {
|
|
|
8068
9151
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath)
|
|
8069
9152
|
?? record.decision.implementers
|
|
8070
9153
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.baseClonePath && candidate.clonePath);
|
|
9154
|
+
const preview = [...record.agents.values()]
|
|
9155
|
+
.map((agent) => agent.spec)
|
|
9156
|
+
.find((candidate) => candidate.repo === initialSpec.repo && candidate.preview)?.preview
|
|
9157
|
+
?? record.decision.implementers.find((candidate) => candidate.repo === initialSpec.repo)?.preview;
|
|
8071
9158
|
const implementerBranch = prRef.headRef ?? record.decision.implementers
|
|
8072
9159
|
.find((candidate) => candidate.repo === initialSpec.repo && candidate.branch)?.branch;
|
|
8073
|
-
const
|
|
9160
|
+
const checkoutSpec = sharedCheckout
|
|
8074
9161
|
? {
|
|
8075
9162
|
...initialSpec,
|
|
8076
9163
|
baseClonePath: sharedCheckout.baseClonePath,
|
|
@@ -8079,6 +9166,7 @@ export class FactoryLoop {
|
|
|
8079
9166
|
...(sharedCheckout.existingPullRequestBranch ? { existingPullRequestBranch: true } : {}),
|
|
8080
9167
|
}
|
|
8081
9168
|
: initialSpec;
|
|
9169
|
+
const spec = specWithPreview(checkoutSpec, preview);
|
|
8082
9170
|
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
8083
9171
|
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name
|
|
8084
9172
|
?? agentNameForRole(issue, 'review', { repo: route?.repo ?? prRef.repo });
|
|
@@ -8111,6 +9199,11 @@ export class FactoryLoop {
|
|
|
8111
9199
|
branchName: spec.branch,
|
|
8112
9200
|
branchPrepared: Boolean(spec.baseClonePath && spec.clonePath && spec.baseClonePath !== spec.clonePath),
|
|
8113
9201
|
agentName: spec.name,
|
|
9202
|
+
...(spec.preview ? {
|
|
9203
|
+
previewUrl: spec.preview.url,
|
|
9204
|
+
previewTargetPort: spec.preview.targetPort,
|
|
9205
|
+
previewStartCommand: spec.preview.startCommand,
|
|
9206
|
+
} : {}),
|
|
8114
9207
|
...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
|
|
8115
9208
|
});
|
|
8116
9209
|
const spawned = await this.#spawnAgent(record, {
|
|
@@ -8125,7 +9218,9 @@ export class FactoryLoop {
|
|
|
8125
9218
|
path: prRef.path,
|
|
8126
9219
|
agentName: tracked?.result?.name ?? spawned.name,
|
|
8127
9220
|
});
|
|
8128
|
-
|
|
9221
|
+
const ref = this.#babysitterPr.get(babysitterKey);
|
|
9222
|
+
await this.#persistBabysitterSession(record.issue, ref, tracked);
|
|
9223
|
+
await this.#ensureBabysitterResourceSubscription(record.issue, ref, tracked);
|
|
8129
9224
|
await this.#retargetSlackConversationToBabysitter(record);
|
|
8130
9225
|
await this.#writeInFlightRegistry();
|
|
8131
9226
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
@@ -8428,6 +9523,7 @@ export class FactoryLoop {
|
|
|
8428
9523
|
// state.
|
|
8429
9524
|
const issueTeam = issue?.team;
|
|
8430
9525
|
const githubIssue = issue ? isGithubIssue(issue) : false;
|
|
9526
|
+
const syntheticProbe = issue ? this.#isSyntheticProbeIssue(issue) : false;
|
|
8431
9527
|
const configuredHumanReview = opts.targetState !== 'done' &&
|
|
8432
9528
|
this.#config.terminalState === 'human-review' &&
|
|
8433
9529
|
(githubIssue || this.#states.hasHumanReview(issueTeam));
|
|
@@ -8435,7 +9531,7 @@ export class FactoryLoop {
|
|
|
8435
9531
|
if (issue && githubIssue && !configuredHumanReview && !githubMerged) {
|
|
8436
9532
|
githubMerged = await this.#githubPrObservedMerged(record, issue);
|
|
8437
9533
|
if (!githubMerged && opts.runMergeGate !== false) {
|
|
8438
|
-
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue);
|
|
9534
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
8439
9535
|
// A successful merge command can mean queued/auto-merge rather than
|
|
8440
9536
|
// merged. Only mounted PR state or a merged webhook may prove merge.
|
|
8441
9537
|
if (mergeCommandAccepted) {
|
|
@@ -8443,6 +9539,21 @@ export class FactoryLoop {
|
|
|
8443
9539
|
}
|
|
8444
9540
|
}
|
|
8445
9541
|
}
|
|
9542
|
+
// Linear-backed issues historically wrote Done before invoking the
|
|
9543
|
+
// guarded merge. With live verification in that guard, a red verdict
|
|
9544
|
+
// would therefore release the team and make the failure terminal even
|
|
9545
|
+
// though the PR remained open. Require the entire merge gate (including
|
|
9546
|
+
// verification) to accept the merge before applying terminal writeback.
|
|
9547
|
+
if (issue &&
|
|
9548
|
+
!githubIssue &&
|
|
9549
|
+
!syntheticProbe &&
|
|
9550
|
+
!configuredHumanReview &&
|
|
9551
|
+
opts.runMergeGate !== false &&
|
|
9552
|
+
this.#config.mergePolicy === 'on-green-with-review') {
|
|
9553
|
+
const mergeCommandAccepted = await this.#runCompletionMergeGate(issue, record);
|
|
9554
|
+
if (!mergeCommandAccepted)
|
|
9555
|
+
return;
|
|
9556
|
+
}
|
|
8446
9557
|
// A GitHub issue only closes after its PR merges. If a configured done
|
|
8447
9558
|
// path cannot merge (including mergePolicy: never), park it for a human
|
|
8448
9559
|
// instead of closing the issue while its PR remains open.
|
|
@@ -8497,11 +9608,11 @@ export class FactoryLoop {
|
|
|
8497
9608
|
this.#markSlackWritebackFailure('completion-thread', error);
|
|
8498
9609
|
}
|
|
8499
9610
|
}
|
|
8500
|
-
//
|
|
8501
|
-
//
|
|
8502
|
-
//
|
|
8503
|
-
if (issue &&
|
|
8504
|
-
await this.#runCompletionMergeGate(issue);
|
|
9611
|
+
// Synthetic canaries are cleanup probes, not merge candidates. Preserve
|
|
9612
|
+
// their close-before-release path under every merge policy without
|
|
9613
|
+
// subjecting them to the required feature verification gate.
|
|
9614
|
+
if (issue && syntheticProbe && !githubIssue && opts.runMergeGate !== false) {
|
|
9615
|
+
await this.#runCompletionMergeGate(issue, record);
|
|
8505
9616
|
}
|
|
8506
9617
|
const releaseReason = humanReview ? 'issue-human-review' : 'issue-done';
|
|
8507
9618
|
releaseReasonForRetry = releaseReason;
|
|
@@ -8531,6 +9642,8 @@ export class FactoryLoop {
|
|
|
8531
9642
|
const stateKey = issueStateKey(record.issue);
|
|
8532
9643
|
this.#probePrGhBackoffUntilMs.delete(stateKey);
|
|
8533
9644
|
this.#probePrResolvedCache.delete(stateKey);
|
|
9645
|
+
// Cancellation must see the subscription identity so it can issue the
|
|
9646
|
+
// idempotent Relayfile DELETE before clearing the local owner maps.
|
|
8534
9647
|
await this.#cancelBabysittersForIssue(record.issue);
|
|
8535
9648
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined);
|
|
8536
9649
|
if (!this.#usesDurableDispatchLifecycle() || (durable && isTerminalDispatchLifecycle(durable))) {
|
|
@@ -8869,6 +9982,10 @@ export class FactoryLoop {
|
|
|
8869
9982
|
await this.#rearmSlackWatcher(record, existingThread, {
|
|
8870
9983
|
replayConversationReplies: Boolean(durableConversation),
|
|
8871
9984
|
});
|
|
9985
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
9986
|
+
if (previews.length > 0) {
|
|
9987
|
+
await this.#slack.reply(existingThread, previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join('\n'));
|
|
9988
|
+
}
|
|
8872
9989
|
}
|
|
8873
9990
|
return;
|
|
8874
9991
|
}
|
|
@@ -8889,12 +10006,15 @@ export class FactoryLoop {
|
|
|
8889
10006
|
if (!this.#slack || !this.#config.slack) {
|
|
8890
10007
|
return;
|
|
8891
10008
|
}
|
|
10009
|
+
const previews = uniquePreviewReferences(result.previews ?? []);
|
|
8892
10010
|
const root = await this.#slack.postThread({
|
|
8893
10011
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
8894
10012
|
text: [
|
|
8895
10013
|
`${record.issue.key}: factory agents dispatched.`,
|
|
8896
|
-
`State: ${result.stateId ?? 'dispatching'}`,
|
|
8897
|
-
|
|
10014
|
+
`State: ${result.stateId ?? 'dispatching'} · Agents: ${result.agents.map((agent) => agent.name).join(', ') || 'none'}`,
|
|
10015
|
+
...(previews.length > 0
|
|
10016
|
+
? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
|
|
10017
|
+
: []),
|
|
8898
10018
|
].join('\n'),
|
|
8899
10019
|
});
|
|
8900
10020
|
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
@@ -10366,7 +11486,7 @@ export class FactoryLoop {
|
|
|
10366
11486
|
? { channel, threadId, mountRoot: this.#integrationsMountRoot() }
|
|
10367
11487
|
: undefined;
|
|
10368
11488
|
}
|
|
10369
|
-
async #runCompletionMergeGate(issue) {
|
|
11489
|
+
async #runCompletionMergeGate(issue, record) {
|
|
10370
11490
|
if (this.#isSyntheticProbeIssue(issue)) {
|
|
10371
11491
|
await this.#closeSyntheticProbeIfPresent(issue);
|
|
10372
11492
|
return false;
|
|
@@ -10391,6 +11511,48 @@ export class FactoryLoop {
|
|
|
10391
11511
|
this.#increment('mergeGateNotMerged');
|
|
10392
11512
|
return false;
|
|
10393
11513
|
}
|
|
11514
|
+
if (this.#verificationGate) {
|
|
11515
|
+
const repositoryPath = this.#verificationRepositoryPath(record, pr.repo);
|
|
11516
|
+
if (!repositoryPath) {
|
|
11517
|
+
this.#logger.warn?.('[factory] verification gate has no feature checkout for merge candidate', {
|
|
11518
|
+
issue: issue.key,
|
|
11519
|
+
repo: pr.repo,
|
|
11520
|
+
prNumber: pr.prNumber,
|
|
11521
|
+
});
|
|
11522
|
+
this.#increment('verificationGateMissingRepository');
|
|
11523
|
+
return false;
|
|
11524
|
+
}
|
|
11525
|
+
try {
|
|
11526
|
+
const verification = await this.#verificationGate.verify({
|
|
11527
|
+
repository: pr.repo,
|
|
11528
|
+
repositoryPath,
|
|
11529
|
+
issueKey: issue.key,
|
|
11530
|
+
expectedHeadSha: headSha,
|
|
11531
|
+
});
|
|
11532
|
+
if (!verification.passed) {
|
|
11533
|
+
this.#logger.warn?.('[factory] verification gate blocked merge', {
|
|
11534
|
+
issue: issue.key,
|
|
11535
|
+
repo: pr.repo,
|
|
11536
|
+
prNumber: pr.prNumber,
|
|
11537
|
+
environmentId: verification.evidence.environmentId,
|
|
11538
|
+
reason: verification.reason,
|
|
11539
|
+
});
|
|
11540
|
+
this.#increment('verificationGateFailed');
|
|
11541
|
+
return false;
|
|
11542
|
+
}
|
|
11543
|
+
this.#increment('verificationGatePassed');
|
|
11544
|
+
}
|
|
11545
|
+
catch (error) {
|
|
11546
|
+
this.#logger.warn?.('[factory] verification gate failed closed', {
|
|
11547
|
+
issue: issue.key,
|
|
11548
|
+
repo: pr.repo,
|
|
11549
|
+
prNumber: pr.prNumber,
|
|
11550
|
+
error: describeError(error).errorMessage,
|
|
11551
|
+
});
|
|
11552
|
+
this.#increment('verificationGateFailed');
|
|
11553
|
+
return false;
|
|
11554
|
+
}
|
|
11555
|
+
}
|
|
10394
11556
|
const result = await this.#mergeGate.merge({
|
|
10395
11557
|
repo: pr.repo,
|
|
10396
11558
|
number: pr.prNumber,
|
|
@@ -10416,6 +11578,14 @@ export class FactoryLoop {
|
|
|
10416
11578
|
this.#increment('mergeGateMerged');
|
|
10417
11579
|
return true;
|
|
10418
11580
|
}
|
|
11581
|
+
#verificationRepositoryPath(record, repo) {
|
|
11582
|
+
const normalized = repo.toLowerCase();
|
|
11583
|
+
const active = [...record.agents.values()]
|
|
11584
|
+
.map((tracked) => tracked.spec)
|
|
11585
|
+
.find((spec) => spec.role === 'implementer' && spec.repo.toLowerCase() === normalized && spec.clonePath)
|
|
11586
|
+
?? record.decision.implementers.find((spec) => spec.repo.toLowerCase() === normalized && spec.clonePath);
|
|
11587
|
+
return active?.clonePath ?? this.#config.repos.clonePaths[repo];
|
|
11588
|
+
}
|
|
10419
11589
|
async #closeSyntheticProbeIfPresent(issue) {
|
|
10420
11590
|
const probe = this.#customProbePrResolver
|
|
10421
11591
|
? await this.#probePrResolver(issue)
|
|
@@ -10770,6 +11940,101 @@ function dispatchSpecs(decision) {
|
|
|
10770
11940
|
}
|
|
10771
11941
|
return [...decision.implementers, decision.reviewer];
|
|
10772
11942
|
}
|
|
11943
|
+
function previewServiceForRepo(config, repo) {
|
|
11944
|
+
const services = config.preview?.services;
|
|
11945
|
+
if (!services)
|
|
11946
|
+
return undefined;
|
|
11947
|
+
const normalizedRepo = repo.replace(/^\/+|\/+$/gu, '').toLowerCase();
|
|
11948
|
+
const basename = normalizedRepo.slice(normalizedRepo.lastIndexOf('/') + 1);
|
|
11949
|
+
const entries = Object.entries(services).map(([name, service]) => ({
|
|
11950
|
+
name,
|
|
11951
|
+
service,
|
|
11952
|
+
normalized: name.replace(/^\/+|\/+$/gu, '').toLowerCase(),
|
|
11953
|
+
}));
|
|
11954
|
+
const exact = entries.find((entry) => entry.normalized === normalizedRepo);
|
|
11955
|
+
if (exact)
|
|
11956
|
+
return { name: exact.name, config: exact.service };
|
|
11957
|
+
const basenameMatches = entries.filter((entry) => entry.normalized.slice(entry.normalized.lastIndexOf('/') + 1) === basename);
|
|
11958
|
+
return basenameMatches.length === 1
|
|
11959
|
+
? { name: basenameMatches[0].name, config: basenameMatches[0].service }
|
|
11960
|
+
: undefined;
|
|
11961
|
+
}
|
|
11962
|
+
function uniquePreviewReferences(previews) {
|
|
11963
|
+
const unique = new Map();
|
|
11964
|
+
for (const preview of previews) {
|
|
11965
|
+
if (preview)
|
|
11966
|
+
unique.set(preview.id, preview);
|
|
11967
|
+
}
|
|
11968
|
+
return [...unique.values()];
|
|
11969
|
+
}
|
|
11970
|
+
function assertPublishablePreview(preview, expected) {
|
|
11971
|
+
const refuse = (reason) => {
|
|
11972
|
+
throw new Error(`Refusing insecure preview for ${expected.repo}: ${reason}`);
|
|
11973
|
+
};
|
|
11974
|
+
if (preview.provider !== 'tailscale-serve')
|
|
11975
|
+
refuse('unexpected provider');
|
|
11976
|
+
if (preview.access !== 'tailnet')
|
|
11977
|
+
refuse('provider did not guarantee tailnet access');
|
|
11978
|
+
if (preview.lifetime !== 'issue')
|
|
11979
|
+
refuse('provider did not guarantee issue-scoped lifetime');
|
|
11980
|
+
if (preview.namespace !== expected.namespace ||
|
|
11981
|
+
preview.owner !== expected.owner ||
|
|
11982
|
+
preview.service !== expected.service ||
|
|
11983
|
+
preview.repo !== expected.repo ||
|
|
11984
|
+
preview.startCommand !== expected.startCommand ||
|
|
11985
|
+
(preview.configuredTargetPort ?? preview.targetPort) !== expected.targetPort ||
|
|
11986
|
+
preview.targetPort < expected.targetPort ||
|
|
11987
|
+
preview.targetPort >= expected.targetPort + expected.portSpan) {
|
|
11988
|
+
refuse('provider returned a reference for a different dispatch identity');
|
|
11989
|
+
}
|
|
11990
|
+
if (expected.preferredHttpsPort !== undefined && preview.httpsPort !== expected.preferredHttpsPort) {
|
|
11991
|
+
refuse('provider ignored the configured HTTPS port');
|
|
11992
|
+
}
|
|
11993
|
+
if (expected.requireNode && (!preview.node || preview.node === 'self')) {
|
|
11994
|
+
refuse('remote provider did not identify the placement node');
|
|
11995
|
+
}
|
|
11996
|
+
const managedProcess = preview.process;
|
|
11997
|
+
if (!managedProcess ||
|
|
11998
|
+
!Number.isInteger(managedProcess.pid) ||
|
|
11999
|
+
managedProcess.pid <= 0 ||
|
|
12000
|
+
!managedProcess.startTime ||
|
|
12001
|
+
!managedProcess.cmdline ||
|
|
12002
|
+
!managedProcess.cwd ||
|
|
12003
|
+
!managedProcess.marker) {
|
|
12004
|
+
refuse('provider did not return an identity-checked managed process');
|
|
12005
|
+
}
|
|
12006
|
+
if (!expected.requireNode && managedProcess.cwd !== expected.checkoutPath) {
|
|
12007
|
+
refuse('provider started the managed process in a different checkout');
|
|
12008
|
+
}
|
|
12009
|
+
const url = (() => {
|
|
12010
|
+
try {
|
|
12011
|
+
return new URL(preview.url);
|
|
12012
|
+
}
|
|
12013
|
+
catch {
|
|
12014
|
+
return refuse('provider returned an invalid URL');
|
|
12015
|
+
}
|
|
12016
|
+
})();
|
|
12017
|
+
if (url.protocol !== 'https:' || url.username || url.password || !url.hostname) {
|
|
12018
|
+
refuse('provider URL is not credential-free HTTPS');
|
|
12019
|
+
}
|
|
12020
|
+
if (!url.hostname.endsWith('.ts.net')) {
|
|
12021
|
+
refuse('provider URL is not a Tailscale HTTPS name');
|
|
12022
|
+
}
|
|
12023
|
+
const urlPort = url.port ? Number(url.port) : 443;
|
|
12024
|
+
if (urlPort !== preview.httpsPort)
|
|
12025
|
+
refuse('provider URL does not match its guarded HTTPS route');
|
|
12026
|
+
}
|
|
12027
|
+
function specWithPreview(spec, preview) {
|
|
12028
|
+
if (!preview)
|
|
12029
|
+
return { ...spec };
|
|
12030
|
+
return {
|
|
12031
|
+
...spec,
|
|
12032
|
+
preview,
|
|
12033
|
+
// The provider route forwards to loopback on its placement node. Pin every
|
|
12034
|
+
// agent using that checkout to the same node so its dev server is reachable.
|
|
12035
|
+
...(preview.node ? { node: preview.node } : {}),
|
|
12036
|
+
};
|
|
12037
|
+
}
|
|
10773
12038
|
function authoritativeRoutedDecision(triaged, routed) {
|
|
10774
12039
|
if (triaged.confidence !== 'low' || triaged.routes.length > 0 || routed.routes.length === 0) {
|
|
10775
12040
|
return routed;
|
|
@@ -11141,6 +12406,8 @@ const routeForSpec = (decision, spec) => {
|
|
|
11141
12406
|
};
|
|
11142
12407
|
};
|
|
11143
12408
|
const previewUrlFromSpec = (spec) => {
|
|
12409
|
+
if (spec.preview?.url.trim())
|
|
12410
|
+
return spec.preview.url.trim();
|
|
11144
12411
|
const previewUrl = spec.previewUrl;
|
|
11145
12412
|
return typeof previewUrl === 'string' && previewUrl.trim() ? previewUrl.trim() : undefined;
|
|
11146
12413
|
};
|
|
@@ -11933,6 +13200,17 @@ const decodeGithubPathSegment = (value) => {
|
|
|
11933
13200
|
const validGithubRepo = (repo) => /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})\/[A-Za-z0-9_.-]{1,100}$/u.test(repo);
|
|
11934
13201
|
const validPrNumber = (value) => Number.isInteger(value) && value > 0;
|
|
11935
13202
|
const githubPrIdentity = (repo, prNumber) => validGithubRepo(repo) && validPrNumber(prNumber) ? `${repo.toLowerCase()}#${prNumber}` : undefined;
|
|
13203
|
+
// This is the public Relayfile stable identity for a GitHub pull request. It
|
|
13204
|
+
// deliberately comes from the PR's repo/number ownership record, never by
|
|
13205
|
+
// transforming an incoming canonical path (whose title slug can be renamed).
|
|
13206
|
+
const babysitterResourceRef = (repo, prNumber) => {
|
|
13207
|
+
if (!validGithubRepo(repo) || !validPrNumber(prNumber)) {
|
|
13208
|
+
throw new Error('Cannot create a durable babysitter subscription for an invalid GitHub PR identity');
|
|
13209
|
+
}
|
|
13210
|
+
const [owner, name] = repo.split('/');
|
|
13211
|
+
return `/github/repos/${owner}__${name}/pulls/by-id/${prNumber}.json`;
|
|
13212
|
+
};
|
|
13213
|
+
const babysitterSubscriberId = (issue) => `factory-babysitter:${issue.uuid}`;
|
|
11936
13214
|
const babysitterOwnershipKey = (issue, ref) => `${issueKey(issue)}:${githubPrIdentity(ref.repo, ref.prNumber) ?? 'invalid'}`;
|
|
11937
13215
|
const recordMatchesGithubRepo = (record, eventRepo, defaultOwner) => {
|
|
11938
13216
|
if (!validGithubRepo(eventRepo))
|
|
@@ -12266,12 +13544,17 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
12266
13544
|
}
|
|
12267
13545
|
return `${owner}/${repo}`;
|
|
12268
13546
|
};
|
|
12269
|
-
const githubPullRequestBody = (issue) => [
|
|
13547
|
+
const githubPullRequestBody = (issue, preview) => [
|
|
12270
13548
|
issue.description,
|
|
12271
13549
|
'',
|
|
12272
13550
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
12273
13551
|
? `Fixes #${issue.key}`
|
|
12274
13552
|
: `Factory issue ${issue.key}`,
|
|
13553
|
+
...(preview ? [
|
|
13554
|
+
'',
|
|
13555
|
+
`Live preview: ${preview.url}`,
|
|
13556
|
+
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
13557
|
+
] : []),
|
|
12275
13558
|
].join('\n').trim();
|
|
12276
13559
|
// The broker rejects re-registering a name it never released on exit
|
|
12277
13560
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
@@ -12729,6 +14012,20 @@ const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
|
|
|
12729
14012
|
`Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
|
|
12730
14013
|
].filter((line) => Boolean(line)).join('\n');
|
|
12731
14014
|
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
14015
|
+
const formatByteCount = (bytes) => {
|
|
14016
|
+
if (bytes < 1024)
|
|
14017
|
+
return `${bytes} B`;
|
|
14018
|
+
const units = ['KiB', 'MiB', 'GiB', 'TiB'];
|
|
14019
|
+
let value = bytes;
|
|
14020
|
+
let unit = 'B';
|
|
14021
|
+
for (const candidate of units) {
|
|
14022
|
+
value /= 1024;
|
|
14023
|
+
unit = candidate;
|
|
14024
|
+
if (value < 1024)
|
|
14025
|
+
break;
|
|
14026
|
+
}
|
|
14027
|
+
return `${value.toFixed(value >= 10 ? 1 : 2)} ${unit}`;
|
|
14028
|
+
};
|
|
12732
14029
|
const unrefDelay = (ms) => new Promise((resolve) => {
|
|
12733
14030
|
const timer = setTimeout(resolve, ms);
|
|
12734
14031
|
timer.unref?.();
|