@agent-relay/factory 0.1.58 → 0.1.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -1
- package/dist/cli/fleet.d.ts +8 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +172 -16
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +796 -273
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +44 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/delivery/ticket-dispatch.d.ts +15 -0
- package/dist/delivery/ticket-dispatch.d.ts.map +1 -0
- package/dist/delivery/ticket-dispatch.js +20 -0
- package/dist/delivery/ticket-dispatch.js.map +1 -0
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +4 -0
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/issue-key-match.d.ts +6 -0
- package/dist/issue-key-match.d.ts.map +1 -1
- package/dist/issue-key-match.js +14 -0
- package/dist/issue-key-match.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +6 -3
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts +7 -1
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +2 -0
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1940 -153
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +1 -1
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/reaper.d.ts +16 -2
- package/dist/orchestrator/reaper.d.ts.map +1 -1
- package/dist/orchestrator/reaper.js +79 -5
- package/dist/orchestrator/reaper.js.map +1 -1
- package/dist/ports/fleet.d.ts +4 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +2 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +60 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +9 -1
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +160 -2
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +7 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +71 -0
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/triage/schema.d.ts +36 -0
- package/dist/triage/schema.d.ts.map +1 -1
- package/dist/triage/schema.js +2 -0
- package/dist/triage/schema.js.map +1 -1
- package/dist/types.d.ts +64 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/version-info.d.ts +21 -0
- package/dist/version-info.d.ts.map +1 -0
- package/dist/version-info.js +146 -0
- package/dist/version-info.js.map +1 -0
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +12 -0
- package/dist/writeback/github.js.map +1 -1
- package/package.json +1 -1
|
@@ -8,7 +8,7 @@ import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
|
8
8
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
9
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
10
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
11
|
-
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
|
+
import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue } from '../issue-key-match.js';
|
|
12
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
13
13
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
14
14
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
@@ -28,6 +28,7 @@ import { findAgentProcessByName, readProcessIdentity } from './process-identity.
|
|
|
28
28
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
29
29
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
30
30
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
31
|
+
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
31
32
|
class ClarificationWakeLeaseLostError extends Error {
|
|
32
33
|
}
|
|
33
34
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -86,6 +87,29 @@ const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
|
86
87
|
'check_run.completed',
|
|
87
88
|
];
|
|
88
89
|
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
90
|
+
// The PR-open snapshot read is the single gate that decides whether a PR ever
|
|
91
|
+
// gets a babysitter, so a transient mount fetch failure must not be allowed to
|
|
92
|
+
// settle that question. Retry in-line, then dead-letter for the reconcile sweep.
|
|
93
|
+
const BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS = 3;
|
|
94
|
+
const BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS = 250;
|
|
95
|
+
// Escalate from warn to error once in-line retries plus this many sweep retries
|
|
96
|
+
// have all failed: the PR is durably unreadable, not momentarily unlucky.
|
|
97
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER = 3;
|
|
98
|
+
// Bound the dead-letter book so a sustained mount outage cannot grow it without
|
|
99
|
+
// limit. Oldest entries are evicted first; the reconcile sweep still adopts
|
|
100
|
+
// their PRs through the ownerless-record path.
|
|
101
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT = 256;
|
|
102
|
+
// Retries carry backoff sleeps, so a sweep drains a slice rather than the whole
|
|
103
|
+
// book; failed entries rotate to the end so successive sweeps cover all of it.
|
|
104
|
+
const BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP = 16;
|
|
105
|
+
// A durably faulted mount would otherwise log an error per path per sweep.
|
|
106
|
+
const BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY = 20;
|
|
107
|
+
// Adoption probes an issue's open PR through the (already gh-backed-off) probe
|
|
108
|
+
// resolver, so it runs on its own slower cadence than the completion sweep.
|
|
109
|
+
const BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS = 60_000;
|
|
110
|
+
// Warn once per PR identity that arrives unowned, then fall back to debug. The
|
|
111
|
+
// counter carries the true volume; the log only has to make it discoverable.
|
|
112
|
+
const BABYSITTER_UNOWNED_PR_WARN_LIMIT = 64;
|
|
89
113
|
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
90
114
|
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
91
115
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
@@ -111,6 +135,10 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
111
135
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
112
136
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
113
137
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
138
|
+
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
139
|
+
const DISPATCH_WRITEBACK_RETRY_MS = 250;
|
|
140
|
+
const HELD_PAST_DEADLINE_RELEASE_REASON = 'held-past-deadline';
|
|
141
|
+
const HELD_DEADLINE_OVERDUE_RETRY_MS = 1_000;
|
|
114
142
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
115
143
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
116
144
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
@@ -124,6 +152,11 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
|
|
|
124
152
|
const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
125
153
|
const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
|
|
126
154
|
const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
|
|
155
|
+
const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
|
|
156
|
+
const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
157
|
+
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
158
|
+
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
159
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
127
160
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
128
161
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
129
162
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -191,6 +224,7 @@ export class FactoryLoop {
|
|
|
191
224
|
#mount;
|
|
192
225
|
#states;
|
|
193
226
|
#fleet;
|
|
227
|
+
#ticketDispatchDelivery;
|
|
194
228
|
#triage;
|
|
195
229
|
#linear;
|
|
196
230
|
#github;
|
|
@@ -268,6 +302,7 @@ export class FactoryLoop {
|
|
|
268
302
|
#clarificationWakeRetryTimers = new Map();
|
|
269
303
|
#clarificationWakeOwner = `${process.pid}:${randomUUID()}`;
|
|
270
304
|
#dispatchLifecycleOwner = `${process.pid}:${randomUUID()}`;
|
|
305
|
+
#discoverySweepOwner = `${process.pid}:${randomUUID()}`;
|
|
271
306
|
#dispatchLifecycleEpochs = new Map();
|
|
272
307
|
#dispatchTerminalWaiters = new Map();
|
|
273
308
|
#dispatchLifecycleRetryTimers = new Map();
|
|
@@ -275,8 +310,12 @@ export class FactoryLoop {
|
|
|
275
310
|
#abandonedDispatchReasons = new Map();
|
|
276
311
|
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
277
312
|
#dispatchLifecycleOwnershipWaitLogged = new Set();
|
|
313
|
+
#dispatchClaimStatuses = new Map();
|
|
278
314
|
#localReleaseCheckpoints = new Map();
|
|
279
315
|
#dispatchLifecycleRenewTimer;
|
|
316
|
+
#heldAgentDeadlineTimer;
|
|
317
|
+
#heldAgentDeadlineDueAtMs;
|
|
318
|
+
#heldAgentDeadlineSweepInFlight;
|
|
280
319
|
#clarificationSweepTimer;
|
|
281
320
|
#clarificationSweepDueAtMs;
|
|
282
321
|
#clarificationSweepInFlight;
|
|
@@ -305,6 +344,15 @@ export class FactoryLoop {
|
|
|
305
344
|
#liveHeartbeatRefresh;
|
|
306
345
|
#liveHeartbeatLastWriteMs = 0;
|
|
307
346
|
#stoppingHeartbeatRefreshActive = false;
|
|
347
|
+
#readinessReconcileTimer;
|
|
348
|
+
#readinessReconcileInFlight;
|
|
349
|
+
#readinessReconcileIntervalMs = 60_000;
|
|
350
|
+
#readinessReconcileConsecutiveFailures = 0;
|
|
351
|
+
#readinessReconcileLastDurationMs;
|
|
352
|
+
#readinessReconcileLastStartedAtMs;
|
|
353
|
+
#readinessReconcileLastCompletedAtMs;
|
|
354
|
+
#readinessReconcileLastFailureAtMs;
|
|
355
|
+
#readinessReconcileLastError;
|
|
308
356
|
#liveEventQueue = [];
|
|
309
357
|
#liveEventDrainScheduled = false;
|
|
310
358
|
#liveEventDrainActive = false;
|
|
@@ -335,6 +383,18 @@ export class FactoryLoop {
|
|
|
335
383
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
336
384
|
#babysitterPr = new Map();
|
|
337
385
|
#babysitterIssueRefs = new Map();
|
|
386
|
+
// PR meta paths whose snapshot read exhausted its in-line retries. Without
|
|
387
|
+
// this book a single failed read permanently orphans the PR: no babysitter is
|
|
388
|
+
// spawned, so no owner exists, so every later event for it is discarded. The
|
|
389
|
+
// reconcile sweep drains this and clears entries that recover.
|
|
390
|
+
#babysitterPrSnapshotDeadLetters = new Map();
|
|
391
|
+
// PR identities already warned about as unowned, so the routing log reports
|
|
392
|
+
// the condition at default level without repeating per event.
|
|
393
|
+
#babysitterUnownedPrWarned = new Set();
|
|
394
|
+
#babysitterOrphanSweepActive = false;
|
|
395
|
+
#babysitterOrphanSweepLastRunMs = 0;
|
|
396
|
+
#babysitterOrphanReportedFailures = 0;
|
|
397
|
+
#babysitterOrphanReportedUnowned = 0;
|
|
338
398
|
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
339
399
|
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
340
400
|
// owning babysitter.
|
|
@@ -367,6 +427,15 @@ export class FactoryLoop {
|
|
|
367
427
|
// undefined = readiness not yet probed; resolved lazily so the standalone
|
|
368
428
|
// runOnce() path (which skips #start) still ingests when the mount is present.
|
|
369
429
|
#githubIngestionEnabled;
|
|
430
|
+
#runOnceInFlight;
|
|
431
|
+
#runOnceInFlightDryRun;
|
|
432
|
+
#discoverySession;
|
|
433
|
+
#discoverySweepEpoch;
|
|
434
|
+
#discoverySweepStartedAtMs;
|
|
435
|
+
#discoverySweepRenewTimer;
|
|
436
|
+
#discoverySweepRenewalInFlight;
|
|
437
|
+
#discoverySweepLeaseLost = false;
|
|
438
|
+
#discoveryOverloadError;
|
|
370
439
|
#resolvedIssueSource;
|
|
371
440
|
#integrationInstructions;
|
|
372
441
|
#integrationInstructionsRefresh;
|
|
@@ -384,6 +453,9 @@ export class FactoryLoop {
|
|
|
384
453
|
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
|
|
385
454
|
installFactoryDraftPredicate(this.#mount, config);
|
|
386
455
|
this.#fleet = ports.fleet;
|
|
456
|
+
this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
|
|
457
|
+
mountRoot: config.localMountRoot,
|
|
458
|
+
});
|
|
387
459
|
this.#triage = ports.triage ?? new TieredTriage(new HeuristicTriage());
|
|
388
460
|
this.#linear = ports.linear ?? MountLinearWriteback(ports.mount, {
|
|
389
461
|
safety: config.safety,
|
|
@@ -481,7 +553,7 @@ export class FactoryLoop {
|
|
|
481
553
|
}
|
|
482
554
|
},
|
|
483
555
|
listPaths: async (prefix) => {
|
|
484
|
-
return this.#
|
|
556
|
+
return this.#listRelayfileTree(prefix, 'integration descriptor discovery');
|
|
485
557
|
},
|
|
486
558
|
};
|
|
487
559
|
const descriptors = await deriveDescriptorsFromMount(reader);
|
|
@@ -674,6 +746,7 @@ export class FactoryLoop {
|
|
|
674
746
|
await this.#rearmSlackReplyWatchers();
|
|
675
747
|
await this.#drainReadyClarificationWake();
|
|
676
748
|
await this.#rearmGithubIssueCommentWatchers();
|
|
749
|
+
this.#scheduleReadinessReconcile();
|
|
677
750
|
this.#scheduleCompletionSweep(0);
|
|
678
751
|
return;
|
|
679
752
|
}
|
|
@@ -726,6 +799,11 @@ export class FactoryLoop {
|
|
|
726
799
|
if (this.#dispatchLifecycleRenewTimer)
|
|
727
800
|
clearInterval(this.#dispatchLifecycleRenewTimer);
|
|
728
801
|
this.#dispatchLifecycleRenewTimer = undefined;
|
|
802
|
+
if (this.#heldAgentDeadlineTimer)
|
|
803
|
+
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
804
|
+
this.#heldAgentDeadlineTimer = undefined;
|
|
805
|
+
this.#heldAgentDeadlineDueAtMs = undefined;
|
|
806
|
+
await this.#heldAgentDeadlineSweepInFlight;
|
|
729
807
|
for (const timer of this.#dispatchLifecycleRetryTimers.values())
|
|
730
808
|
clearTimeout(timer);
|
|
731
809
|
this.#dispatchLifecycleRetryTimers.clear();
|
|
@@ -734,9 +812,13 @@ export class FactoryLoop {
|
|
|
734
812
|
if (this.#completionSweepTimer)
|
|
735
813
|
clearTimeout(this.#completionSweepTimer);
|
|
736
814
|
this.#completionSweepTimer = undefined;
|
|
815
|
+
if (this.#readinessReconcileTimer)
|
|
816
|
+
clearTimeout(this.#readinessReconcileTimer);
|
|
817
|
+
this.#readinessReconcileTimer = undefined;
|
|
737
818
|
if (this.#previewSweepTimer)
|
|
738
819
|
clearTimeout(this.#previewSweepTimer);
|
|
739
820
|
this.#previewSweepTimer = undefined;
|
|
821
|
+
await this.#readinessReconcileInFlight;
|
|
740
822
|
await this.#previewSweepInFlight;
|
|
741
823
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
742
824
|
try {
|
|
@@ -781,6 +863,9 @@ export class FactoryLoop {
|
|
|
781
863
|
this.#babysitterSubscriptionOwners.clear();
|
|
782
864
|
this.#babysitterReady.clear();
|
|
783
865
|
this.#babysitterCriticalAgents.clear();
|
|
866
|
+
this.#babysitterPrSnapshotDeadLetters.clear();
|
|
867
|
+
this.#babysitterUnownedPrWarned.clear();
|
|
868
|
+
this.#babysitterOrphanSweepLastRunMs = 0;
|
|
784
869
|
const subscription = this.#subscription;
|
|
785
870
|
this.#subscription = undefined;
|
|
786
871
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
@@ -875,6 +960,7 @@ export class FactoryLoop {
|
|
|
875
960
|
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
876
961
|
const options = this.#liveOptions(overrides);
|
|
877
962
|
this.#liveTransport = options.transport;
|
|
963
|
+
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
878
964
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
879
965
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
880
966
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1005,6 +1091,7 @@ export class FactoryLoop {
|
|
|
1005
1091
|
pollIntervalMs: overrides.pollIntervalMs ?? this.#config.liveSubscription.pollIntervalMs,
|
|
1006
1092
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
1007
1093
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1094
|
+
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1008
1095
|
};
|
|
1009
1096
|
}
|
|
1010
1097
|
async #currentEventCursor(limit) {
|
|
@@ -1031,6 +1118,64 @@ export class FactoryLoop {
|
|
|
1031
1118
|
};
|
|
1032
1119
|
}
|
|
1033
1120
|
}
|
|
1121
|
+
#scheduleReadinessReconcile(delayMs = this.#readinessReconcileIntervalMs) {
|
|
1122
|
+
if (!this.#started ||
|
|
1123
|
+
this.#stopping ||
|
|
1124
|
+
this.#readinessReconcileTimer ||
|
|
1125
|
+
this.#readinessReconcileInFlight)
|
|
1126
|
+
return;
|
|
1127
|
+
this.#readinessReconcileTimer = setTimeout(() => {
|
|
1128
|
+
this.#readinessReconcileTimer = undefined;
|
|
1129
|
+
if (!this.#started || this.#stopping)
|
|
1130
|
+
return;
|
|
1131
|
+
const sweep = this.#reconcileReadyIssues();
|
|
1132
|
+
this.#readinessReconcileInFlight = sweep;
|
|
1133
|
+
void sweep.finally(() => {
|
|
1134
|
+
if (this.#readinessReconcileInFlight === sweep) {
|
|
1135
|
+
this.#readinessReconcileInFlight = undefined;
|
|
1136
|
+
}
|
|
1137
|
+
if (this.#started && !this.#stopping)
|
|
1138
|
+
this.#scheduleReadinessReconcile();
|
|
1139
|
+
});
|
|
1140
|
+
}, delayMs);
|
|
1141
|
+
this.#readinessReconcileTimer.unref?.();
|
|
1142
|
+
}
|
|
1143
|
+
async #reconcileReadyIssues() {
|
|
1144
|
+
const startedAtMs = this.#clock.now();
|
|
1145
|
+
this.#readinessReconcileLastStartedAtMs = startedAtMs;
|
|
1146
|
+
this.#increment('readinessReconcileSweeps');
|
|
1147
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1148
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1149
|
+
});
|
|
1150
|
+
try {
|
|
1151
|
+
const report = await this.runOnce();
|
|
1152
|
+
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1153
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1154
|
+
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1155
|
+
this.#readinessReconcileLastError = undefined;
|
|
1156
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1157
|
+
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1158
|
+
candidates: report.pulled.length,
|
|
1159
|
+
dispatched: report.dispatched.length,
|
|
1160
|
+
skipped: report.skipped.length,
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
catch (error) {
|
|
1164
|
+
const errorMessage = describeError(error).errorMessage;
|
|
1165
|
+
this.#readinessReconcileConsecutiveFailures += 1;
|
|
1166
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1167
|
+
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1168
|
+
this.#readinessReconcileLastError = errorMessage;
|
|
1169
|
+
this.#increment('readinessReconcileErrors');
|
|
1170
|
+
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1171
|
+
error: errorMessage,
|
|
1172
|
+
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1173
|
+
consecutiveFailures: this.#readinessReconcileConsecutiveFailures,
|
|
1174
|
+
degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
await this.#refreshLiveHeartbeat();
|
|
1178
|
+
}
|
|
1034
1179
|
#scheduleLivePoll(delayMs, options) {
|
|
1035
1180
|
if (this.#livePollTimer || !this.#started)
|
|
1036
1181
|
return;
|
|
@@ -1340,6 +1485,10 @@ export class FactoryLoop {
|
|
|
1340
1485
|
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
1341
1486
|
// webhook-driven rather than polled.
|
|
1342
1487
|
if (this.#config.babysitter.enabled) {
|
|
1488
|
+
// Webhook-driven is not the same as one-shot: a PR-open event that was
|
|
1489
|
+
// missed or whose snapshot read failed leaves its PR with no shepherd
|
|
1490
|
+
// forever. This is the reconciliation that recovers those.
|
|
1491
|
+
await this.#sweepOrphanedBabysitterPrs(reason);
|
|
1343
1492
|
return;
|
|
1344
1493
|
}
|
|
1345
1494
|
if (this.#completionSweepActive) {
|
|
@@ -1401,6 +1550,207 @@ export class FactoryLoop {
|
|
|
1401
1550
|
this.#completionSweepActive = false;
|
|
1402
1551
|
}
|
|
1403
1552
|
}
|
|
1553
|
+
// Reconciliation for the babysitter's one-shot PR-open path. Two failure
|
|
1554
|
+
// modes converge on the same end state — an open Factory PR that no
|
|
1555
|
+
// babysitter owns, whose reviews, CI results and conflicts are then silently
|
|
1556
|
+
// discarded by #routeBabysitterEvent:
|
|
1557
|
+
// 1. the PR-open snapshot read failed and exhausted its retries, or
|
|
1558
|
+
// 2. the PR-open event never arrived at all (restart, drain loss, gap).
|
|
1559
|
+
// Neither is self-healing from events alone, so this sweep re-reads the
|
|
1560
|
+
// dead-lettered paths and adopts still-ownerless in-flight records.
|
|
1561
|
+
async #sweepOrphanedBabysitterPrs(reason = 'forced') {
|
|
1562
|
+
if (!this.#config.babysitter.enabled || this.#stopping || this.#babysitterOrphanSweepActive) {
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
const now = this.#clock.now();
|
|
1566
|
+
// Adoption probes every ownerless in-flight record, so it runs on the slow
|
|
1567
|
+
// cadence. A dead-lettered read is a PR we already know is unshepherded —
|
|
1568
|
+
// it retries at every sweep opportunity rather than waiting out that cadence.
|
|
1569
|
+
const adoptionDue = reason === 'forced' ||
|
|
1570
|
+
now - this.#babysitterOrphanSweepLastRunMs >= BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS;
|
|
1571
|
+
if (!adoptionDue && this.#babysitterPrSnapshotDeadLetters.size === 0) {
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
this.#babysitterOrphanSweepActive = true;
|
|
1575
|
+
try {
|
|
1576
|
+
this.#increment('babysitterOrphanSweepRuns');
|
|
1577
|
+
await this.#retryDeadLetteredPrSnapshots();
|
|
1578
|
+
if (adoptionDue) {
|
|
1579
|
+
this.#babysitterOrphanSweepLastRunMs = now;
|
|
1580
|
+
await this.#adoptOrphanedBabysitterPrs();
|
|
1581
|
+
}
|
|
1582
|
+
this.#reportBabysitterOrphanHealth();
|
|
1583
|
+
}
|
|
1584
|
+
finally {
|
|
1585
|
+
this.#babysitterOrphanSweepActive = false;
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
async #retryDeadLetteredPrSnapshots() {
|
|
1589
|
+
// Each retry costs up to BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS reads with
|
|
1590
|
+
// backoff sleeps between them, so drain a bounded slice per sweep rather
|
|
1591
|
+
// than stalling the sweep behind a full book during a mount outage. A
|
|
1592
|
+
// failed retry re-inserts its entry at the end, so successive sweeps
|
|
1593
|
+
// rotate through the whole book.
|
|
1594
|
+
const batch = [...this.#babysitterPrSnapshotDeadLetters.keys()].slice(0, BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP);
|
|
1595
|
+
if (this.#babysitterPrSnapshotDeadLetters.size > batch.length) {
|
|
1596
|
+
this.#logger.warn?.('[factory] babysitter dead-letter drain is rate limited', {
|
|
1597
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
1598
|
+
retryingThisSweep: batch.length,
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1601
|
+
for (const path of batch) {
|
|
1602
|
+
if (this.#stopping)
|
|
1603
|
+
return;
|
|
1604
|
+
this.#increment('babysitterPrSnapshotDeadLettersRetried');
|
|
1605
|
+
// #handlePrChange clears the entry itself on a successful read and
|
|
1606
|
+
// re-records the failure (bumping `failures`, escalating the log) if the
|
|
1607
|
+
// mount is still faulted, so the whole retry contract lives in one place.
|
|
1608
|
+
await this.#handlePrChange(path);
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
async #adoptOrphanedBabysitterPrs() {
|
|
1612
|
+
const batch = await this.#batch();
|
|
1613
|
+
for (const record of batch.inFlight) {
|
|
1614
|
+
if (this.#stopping)
|
|
1615
|
+
return;
|
|
1616
|
+
if (record.dryRun || this.#hasBabysitterForIssue(record.issue))
|
|
1617
|
+
continue;
|
|
1618
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
1619
|
+
continue;
|
|
1620
|
+
this.#increment('babysitterOrphanCandidatesScanned');
|
|
1621
|
+
try {
|
|
1622
|
+
for (const path of await this.#orphanedPrMetaPaths(record)) {
|
|
1623
|
+
if (this.#stopping || this.#hasBabysitterForIssue(record.issue))
|
|
1624
|
+
break;
|
|
1625
|
+
// Replay the PR meta through the normal PR-open handler rather than
|
|
1626
|
+
// spawning directly. Adoption must be subject to exactly the same
|
|
1627
|
+
// ownership, weak-match and path/payload-identity guards as a live
|
|
1628
|
+
// PR-open event — a reconcile sweep is a second delivery of a missed
|
|
1629
|
+
// event, never a way around the checks that event would have faced.
|
|
1630
|
+
await this.#handlePrChange(path);
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
catch (error) {
|
|
1634
|
+
this.#increment('babysitterOrphanAdoptionErrors');
|
|
1635
|
+
this.#logger.warn?.('[factory] babysitter orphan adoption failed', {
|
|
1636
|
+
issue: record.issue.key,
|
|
1637
|
+
error: describeError(error).errorMessage,
|
|
1638
|
+
});
|
|
1639
|
+
continue;
|
|
1640
|
+
}
|
|
1641
|
+
if (!this.#hasBabysitterForIssue(record.issue))
|
|
1642
|
+
continue;
|
|
1643
|
+
this.#increment('babysitterOrphanedPrsAdopted');
|
|
1644
|
+
this.#logger.warn?.('[factory] adopted an orphaned PR that no babysitter owned', {
|
|
1645
|
+
issue: record.issue.key,
|
|
1646
|
+
...this.#babysitterPrForIssue(record.issue),
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
// Candidate PR meta paths for an ownerless record, newest PR first. Resolution
|
|
1651
|
+
// reuses the durable dispatch receipts and then the probe resolver (which
|
|
1652
|
+
// carries its own gh backoff, so a record whose implementer has not opened a
|
|
1653
|
+
// PR yet costs no more than a cached miss).
|
|
1654
|
+
async #orphanedPrMetaPaths(record) {
|
|
1655
|
+
const wanted = [];
|
|
1656
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
1657
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
1658
|
+
const receipts = lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : []);
|
|
1659
|
+
for (const receipt of receipts)
|
|
1660
|
+
wanted.push({ repo: receipt.repo, prNumber: receipt.number });
|
|
1661
|
+
}
|
|
1662
|
+
if (wanted.length === 0) {
|
|
1663
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1664
|
+
const pr = issue ? await this.#openPrForIssue(issue) : undefined;
|
|
1665
|
+
if (pr)
|
|
1666
|
+
wanted.push({ repo: pr.repo, prNumber: pr.prNumber });
|
|
1667
|
+
}
|
|
1668
|
+
const paths = [];
|
|
1669
|
+
for (const { repo, prNumber } of wanted) {
|
|
1670
|
+
for (const root of githubPullRoots(repo)) {
|
|
1671
|
+
let candidates;
|
|
1672
|
+
try {
|
|
1673
|
+
candidates = await this.#mount.listTree(root);
|
|
1674
|
+
}
|
|
1675
|
+
catch {
|
|
1676
|
+
continue;
|
|
1677
|
+
}
|
|
1678
|
+
const match = candidates.find((path) => {
|
|
1679
|
+
const parts = githubPullPathParts(path);
|
|
1680
|
+
return parts?.number === prNumber &&
|
|
1681
|
+
`${parts.owner}/${parts.repo}`.toLowerCase() === repo.toLowerCase();
|
|
1682
|
+
});
|
|
1683
|
+
if (match && !paths.includes(match)) {
|
|
1684
|
+
paths.push(match);
|
|
1685
|
+
break;
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
if (paths.length === 0 && wanted.length > 0) {
|
|
1690
|
+
// The PR exists but its meta has not materialized in the mount yet. Leave
|
|
1691
|
+
// it for the next sweep rather than spawning off an unvalidated receipt.
|
|
1692
|
+
this.#increment('babysitterOrphanPrMetaUnavailable');
|
|
1693
|
+
}
|
|
1694
|
+
return paths;
|
|
1695
|
+
}
|
|
1696
|
+
#hasBabysitterForIssue(issue) {
|
|
1697
|
+
return this.#babysitterPrForIssue(issue) !== undefined;
|
|
1698
|
+
}
|
|
1699
|
+
#babysitterPrForIssue(issue) {
|
|
1700
|
+
const wanted = issueKey(issue);
|
|
1701
|
+
for (const [key, ref] of this.#babysitterPr) {
|
|
1702
|
+
// Ownership keys are composite (issue + PR), so resolve the issue the same
|
|
1703
|
+
// way #babysitterOwnerFor does: the recorded ref, else the key itself for
|
|
1704
|
+
// the legacy issue-keyed form. Never fall back to the issue being tested,
|
|
1705
|
+
// which would make every entry match.
|
|
1706
|
+
const owningIssue = this.#babysitterIssueRefs.get(key);
|
|
1707
|
+
if ((owningIssue ? issueKey(owningIssue) : key) !== wanted)
|
|
1708
|
+
continue;
|
|
1709
|
+
// An empty agentName is the pre-spawn reservation #ensureBabysitter takes
|
|
1710
|
+
// before its first await; that is ownership in progress, not an orphan.
|
|
1711
|
+
return { repo: ref.repo, prNumber: ref.prNumber };
|
|
1712
|
+
}
|
|
1713
|
+
return undefined;
|
|
1714
|
+
}
|
|
1715
|
+
// The two counters that mark a PR losing its shepherd are otherwise only
|
|
1716
|
+
// visible by diffing `status().counters`. Report the deltas so the condition
|
|
1717
|
+
// reaches an operator reading logs at default level.
|
|
1718
|
+
#reportBabysitterOrphanHealth() {
|
|
1719
|
+
const readFailures = this.#counters.babysitterPrSnapshotReadFailures ?? 0;
|
|
1720
|
+
const unownedEvents = this.#counters.babysitterEventsIgnoredUnownedPr ?? 0;
|
|
1721
|
+
const newReadFailures = readFailures - this.#babysitterOrphanReportedFailures;
|
|
1722
|
+
const newUnownedEvents = unownedEvents - this.#babysitterOrphanReportedUnowned;
|
|
1723
|
+
const deadLettered = this.#babysitterPrSnapshotDeadLetters.size;
|
|
1724
|
+
this.#counters.babysitterPrSnapshotDeadLetterDepth = deadLettered;
|
|
1725
|
+
if (newReadFailures <= 0 && newUnownedEvents <= 0 && deadLettered === 0) {
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
this.#babysitterOrphanReportedFailures = readFailures;
|
|
1729
|
+
this.#babysitterOrphanReportedUnowned = unownedEvents;
|
|
1730
|
+
// The oldest stuck entry is what an operator needs to act on: it names the
|
|
1731
|
+
// PR and how long it has been unshepherded.
|
|
1732
|
+
let oldest;
|
|
1733
|
+
for (const [path, entry] of this.#babysitterPrSnapshotDeadLetters) {
|
|
1734
|
+
if (!oldest || entry.firstFailedAtMs < oldest.entry.firstFailedAtMs)
|
|
1735
|
+
oldest = { path, entry };
|
|
1736
|
+
}
|
|
1737
|
+
this.#logger.warn?.('[factory] babysitter PR routing is dropping work', {
|
|
1738
|
+
prSnapshotReadFailures: readFailures,
|
|
1739
|
+
newPrSnapshotReadFailures: Math.max(0, newReadFailures),
|
|
1740
|
+
unownedPrEventsIgnored: unownedEvents,
|
|
1741
|
+
newUnownedPrEventsIgnored: Math.max(0, newUnownedEvents),
|
|
1742
|
+
deadLetteredPrSnapshots: deadLettered,
|
|
1743
|
+
flatEventsUnreadable: this.#counters.babysitterFlatEventsUnreadable ?? 0,
|
|
1744
|
+
orphanedPrsAdopted: this.#counters.babysitterOrphanedPrsAdopted ?? 0,
|
|
1745
|
+
...(oldest
|
|
1746
|
+
? {
|
|
1747
|
+
oldestDeadLetterPath: oldest.path,
|
|
1748
|
+
oldestDeadLetterError: oldest.entry.lastErrorMessage,
|
|
1749
|
+
oldestDeadLetterAgeMs: Math.max(0, this.#clock.now() - oldest.entry.firstFailedAtMs),
|
|
1750
|
+
}
|
|
1751
|
+
: {}),
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1404
1754
|
async #completionPrForIssue(issue) {
|
|
1405
1755
|
if (this.#customProbePrResolver) {
|
|
1406
1756
|
return this.#probePrResolver(issue);
|
|
@@ -1426,7 +1776,7 @@ export class FactoryLoop {
|
|
|
1426
1776
|
if (cached && cached.expiresAtMs > now) {
|
|
1427
1777
|
return cached.pr;
|
|
1428
1778
|
}
|
|
1429
|
-
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts);
|
|
1779
|
+
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts, (prefix) => this.#listRelayfileTree(prefix, 'PR probe resolution'));
|
|
1430
1780
|
if (mountPr) {
|
|
1431
1781
|
return mountPr;
|
|
1432
1782
|
}
|
|
@@ -1449,6 +1799,158 @@ export class FactoryLoop {
|
|
|
1449
1799
|
return undefined;
|
|
1450
1800
|
}
|
|
1451
1801
|
async runOnce(opts = {}) {
|
|
1802
|
+
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1803
|
+
// Loop rather than a single check-then-wait: while this call was waiting
|
|
1804
|
+
// out a mismatched sweep, another mismatched caller may have raced it to
|
|
1805
|
+
// start the *next* sweep first — in which case that one might match us
|
|
1806
|
+
// and we should coalesce onto it rather than starting a third.
|
|
1807
|
+
for (;;) {
|
|
1808
|
+
if (this.#runOnceInFlight && this.#runOnceInFlightDryRun === dryRun) {
|
|
1809
|
+
this.#increment('discoverySweepsCoalesced');
|
|
1810
|
+
this.#logger.info?.('[factory] coalesced overlapping discovery request into the in-flight sweep');
|
|
1811
|
+
return await this.#runOnceInFlight;
|
|
1812
|
+
}
|
|
1813
|
+
// A mismatched dryRun cannot coalesce onto the in-flight sweep (it
|
|
1814
|
+
// would hand a live caller a dry-run report, or vice versa), but it
|
|
1815
|
+
// still must not race it — wait for it to settle before either
|
|
1816
|
+
// re-checking for a newly-started matching sweep or claiming the lease.
|
|
1817
|
+
if (!this.#runOnceInFlight)
|
|
1818
|
+
break;
|
|
1819
|
+
await this.#runOnceInFlight.catch(() => undefined);
|
|
1820
|
+
}
|
|
1821
|
+
const sweep = this.#runOnceWithDiscoveryFence({ dryRun });
|
|
1822
|
+
this.#runOnceInFlight = sweep;
|
|
1823
|
+
this.#runOnceInFlightDryRun = dryRun;
|
|
1824
|
+
try {
|
|
1825
|
+
return await sweep;
|
|
1826
|
+
}
|
|
1827
|
+
finally {
|
|
1828
|
+
if (this.#runOnceInFlight === sweep) {
|
|
1829
|
+
this.#runOnceInFlight = undefined;
|
|
1830
|
+
this.#runOnceInFlightDryRun = undefined;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
async #runOnceWithDiscoveryFence(opts) {
|
|
1835
|
+
const sweepStartedAtMs = this.#clock.now();
|
|
1836
|
+
let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1837
|
+
if (!claim.acquired && claim.reason === 'backoff') {
|
|
1838
|
+
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
1839
|
+
this.#increment('discoveryBackoffWaits');
|
|
1840
|
+
this.#logger.warn?.('[factory] discovery sweep waiting for Relayfile overload backoff', {
|
|
1841
|
+
delayMs,
|
|
1842
|
+
backoffUntilMs: claim.state.backoffUntilMs,
|
|
1843
|
+
consecutiveOverloads: claim.state.consecutiveOverloads,
|
|
1844
|
+
});
|
|
1845
|
+
await this.#clock.sleep(delayMs);
|
|
1846
|
+
claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1847
|
+
}
|
|
1848
|
+
if (!claim.acquired || !claim.lease) {
|
|
1849
|
+
this.#increment('discoverySweepsSkippedInFlight');
|
|
1850
|
+
this.#logger.info?.('[factory] skipped discovery because another process owns the sweep lease', {
|
|
1851
|
+
owner: claim.state.lease?.owner,
|
|
1852
|
+
epoch: claim.state.lease?.epoch,
|
|
1853
|
+
leaseUntilMs: claim.state.lease?.leaseUntilMs,
|
|
1854
|
+
});
|
|
1855
|
+
return {
|
|
1856
|
+
pulled: [],
|
|
1857
|
+
triaged: [],
|
|
1858
|
+
dispatched: [],
|
|
1859
|
+
skipped: [],
|
|
1860
|
+
dryRun: opts.dryRun ?? this.#config.dryRun,
|
|
1861
|
+
discoveryDeferred: 'sweep-in-flight',
|
|
1862
|
+
};
|
|
1863
|
+
}
|
|
1864
|
+
if (claim.reclaimedLease) {
|
|
1865
|
+
this.#increment('discoverySweepOrphanTakeovers');
|
|
1866
|
+
this.#logger.warn?.('[factory] reclaimed discovery sweep lease from a stopped process', {
|
|
1867
|
+
owner: claim.lease.owner,
|
|
1868
|
+
epoch: claim.lease.epoch,
|
|
1869
|
+
previousOwner: claim.reclaimedLease.owner,
|
|
1870
|
+
previousEpoch: claim.reclaimedLease.epoch,
|
|
1871
|
+
previousLeaseUntilMs: claim.reclaimedLease.leaseUntilMs,
|
|
1872
|
+
});
|
|
1873
|
+
}
|
|
1874
|
+
this.#logger.info?.('[factory] discovery sweep lease claimed', {
|
|
1875
|
+
owner: claim.lease.owner,
|
|
1876
|
+
epoch: claim.lease.epoch,
|
|
1877
|
+
leaseUntilMs: claim.lease.leaseUntilMs,
|
|
1878
|
+
});
|
|
1879
|
+
this.#discoverySweepEpoch = claim.lease.epoch;
|
|
1880
|
+
this.#discoverySweepStartedAtMs = sweepStartedAtMs;
|
|
1881
|
+
this.#discoverySweepLeaseLost = false;
|
|
1882
|
+
this.#discoveryOverloadError = undefined;
|
|
1883
|
+
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
1884
|
+
let leaseReleased = false;
|
|
1885
|
+
try {
|
|
1886
|
+
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
1887
|
+
const report = await this.#performRunOnce(opts);
|
|
1888
|
+
if (this.#discoveryOverloadError)
|
|
1889
|
+
throw this.#discoveryOverloadError;
|
|
1890
|
+
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
1891
|
+
// Do not clear the durable lease while a renewal can still be waiting on
|
|
1892
|
+
// the same state-file lock. A late renewal that observes the completed
|
|
1893
|
+
// (lease-less) checkpoint is a false lease-loss signal and can poison an
|
|
1894
|
+
// otherwise successful reconcile cycle.
|
|
1895
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1896
|
+
if (this.#discoverySweepLeaseLost) {
|
|
1897
|
+
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
1898
|
+
}
|
|
1899
|
+
const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
|
|
1900
|
+
leaseReleased = completed;
|
|
1901
|
+
if (!completed)
|
|
1902
|
+
throw new Error('discovery sweep lease was lost before completion');
|
|
1903
|
+
this.#logger.info?.('[factory] discovery sweep checkpoint committed', {
|
|
1904
|
+
owner: claim.lease.owner,
|
|
1905
|
+
epoch: claim.lease.epoch,
|
|
1906
|
+
durationMs: this.#elapsedSince(sweepStartedAtMs),
|
|
1907
|
+
checkpointUpdatedAtMs: checkpoint?.updatedAtMs,
|
|
1908
|
+
});
|
|
1909
|
+
return report;
|
|
1910
|
+
}
|
|
1911
|
+
catch (error) {
|
|
1912
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1913
|
+
const overload = relayfileOverload(error);
|
|
1914
|
+
if (overload) {
|
|
1915
|
+
const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
|
|
1916
|
+
const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
|
|
1917
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
1918
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
1919
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
1920
|
+
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
1921
|
+
status: overload.status,
|
|
1922
|
+
reason: overload.reason,
|
|
1923
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
1924
|
+
delayMs,
|
|
1925
|
+
backoffUntilMs,
|
|
1926
|
+
consecutiveOverloads,
|
|
1927
|
+
});
|
|
1928
|
+
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
1929
|
+
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
1930
|
+
// here too would additionally block this failing call (and every
|
|
1931
|
+
// caller coalesced onto it) for up to the full backoff window while
|
|
1932
|
+
// still holding the coalescing slot.
|
|
1933
|
+
}
|
|
1934
|
+
throw error;
|
|
1935
|
+
}
|
|
1936
|
+
finally {
|
|
1937
|
+
await this.#stopDiscoverySweepRenewal();
|
|
1938
|
+
this.#discoverySession = undefined;
|
|
1939
|
+
this.#discoverySweepEpoch = undefined;
|
|
1940
|
+
this.#discoverySweepStartedAtMs = undefined;
|
|
1941
|
+
this.#discoveryOverloadError = undefined;
|
|
1942
|
+
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
1943
|
+
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
1944
|
+
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
1945
|
+
// merge-advance scan) throw "lease was lost" until the next sweep
|
|
1946
|
+
// happens to run and reset it at the top of this method.
|
|
1947
|
+
this.#discoverySweepLeaseLost = false;
|
|
1948
|
+
if (!leaseReleased) {
|
|
1949
|
+
await this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch);
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
async #performRunOnce(opts = {}) {
|
|
1452
1954
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1453
1955
|
const startedAtMs = this.#clock.now();
|
|
1454
1956
|
const relayfileWaitWarningsAtStart = this.#counters.relayfileOperationWaitWarnings ?? 0;
|
|
@@ -1479,6 +1981,14 @@ export class FactoryLoop {
|
|
|
1479
1981
|
const triaged = [];
|
|
1480
1982
|
const dispatched = [];
|
|
1481
1983
|
const skipped = [];
|
|
1984
|
+
const recordSkip = (entry) => {
|
|
1985
|
+
skipped.push(entry);
|
|
1986
|
+
this.#logger.info?.('[factory] readiness reconciliation skipped dispatch', {
|
|
1987
|
+
issue: entry.issue.key,
|
|
1988
|
+
path: entry.issue.path,
|
|
1989
|
+
reason: entry.reason,
|
|
1990
|
+
});
|
|
1991
|
+
};
|
|
1482
1992
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
1483
1993
|
let readyIssueReads = 0;
|
|
1484
1994
|
const issueEntries = [];
|
|
@@ -1523,7 +2033,6 @@ export class FactoryLoop {
|
|
|
1523
2033
|
const mayRecoverGithubOrphan = !wasReady &&
|
|
1524
2034
|
!dryRun &&
|
|
1525
2035
|
issueSource === 'github' &&
|
|
1526
|
-
Boolean(orphanRecovery) &&
|
|
1527
2036
|
Boolean(requiredLabel) &&
|
|
1528
2037
|
Boolean(labels?.has(requiredLabel)) &&
|
|
1529
2038
|
Boolean(labels?.has('factory:in-progress')) &&
|
|
@@ -1531,26 +2040,31 @@ export class FactoryLoop {
|
|
|
1531
2040
|
if (!mayRecoverGithubOrphan) {
|
|
1532
2041
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1533
2042
|
if (dispatchBlock) {
|
|
1534
|
-
|
|
2043
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1535
2044
|
continue;
|
|
1536
2045
|
}
|
|
1537
2046
|
}
|
|
2047
|
+
const orphanResult = mayRecoverGithubOrphan
|
|
2048
|
+
? await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun)
|
|
2049
|
+
: { recovered: false };
|
|
2050
|
+
const recoveredOrphan = orphanResult.recovered;
|
|
1538
2051
|
const batch = await this.#batch();
|
|
1539
2052
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
1540
|
-
|
|
2053
|
+
recordSkip({ issue: issueRef(issue), reason: orphanResult.reason ?? 'already tracked' });
|
|
1541
2054
|
continue;
|
|
1542
2055
|
}
|
|
1543
|
-
const recoveredOrphan = mayRecoverGithubOrphan &&
|
|
1544
|
-
await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun);
|
|
1545
2056
|
if (!wasReady && !recoveredOrphan) {
|
|
1546
2057
|
if (mayRecoverGithubOrphan) {
|
|
1547
2058
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1548
2059
|
if (dispatchBlock) {
|
|
1549
|
-
|
|
2060
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1550
2061
|
continue;
|
|
1551
2062
|
}
|
|
1552
2063
|
}
|
|
1553
|
-
|
|
2064
|
+
recordSkip({
|
|
2065
|
+
issue: issueRef(issue),
|
|
2066
|
+
reason: orphanResult.reason ?? 'live state is not ready-for-agent',
|
|
2067
|
+
});
|
|
1554
2068
|
continue;
|
|
1555
2069
|
}
|
|
1556
2070
|
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
@@ -1558,16 +2072,16 @@ export class FactoryLoop {
|
|
|
1558
2072
|
if (recoveredOrphan) {
|
|
1559
2073
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1560
2074
|
if (dispatchBlock) {
|
|
1561
|
-
|
|
2075
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1562
2076
|
continue;
|
|
1563
2077
|
}
|
|
1564
2078
|
}
|
|
1565
2079
|
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
1566
|
-
|
|
2080
|
+
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
1567
2081
|
continue;
|
|
1568
2082
|
}
|
|
1569
2083
|
if (!isDispatchableIssue(issue)) {
|
|
1570
|
-
|
|
2084
|
+
recordSkip({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
|
|
1571
2085
|
continue;
|
|
1572
2086
|
}
|
|
1573
2087
|
const decision = await this.triageIssue(issue);
|
|
@@ -1579,7 +2093,7 @@ export class FactoryLoop {
|
|
|
1579
2093
|
catch (error) {
|
|
1580
2094
|
if (!(error instanceof LiveDispatchStateChangedError))
|
|
1581
2095
|
throw error;
|
|
1582
|
-
|
|
2096
|
+
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
1583
2097
|
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
1584
2098
|
issue: decision.issue.key,
|
|
1585
2099
|
});
|
|
@@ -1591,7 +2105,7 @@ export class FactoryLoop {
|
|
|
1591
2105
|
: result.hold?.kind === 'dependency'
|
|
1592
2106
|
? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
|
|
1593
2107
|
: 'queued or escalated';
|
|
1594
|
-
|
|
2108
|
+
recordSkip({ issue: decision.issue, reason });
|
|
1595
2109
|
}
|
|
1596
2110
|
else {
|
|
1597
2111
|
dispatched.push(result);
|
|
@@ -1630,6 +2144,257 @@ export class FactoryLoop {
|
|
|
1630
2144
|
}
|
|
1631
2145
|
}
|
|
1632
2146
|
}
|
|
2147
|
+
#startDiscoverySweepRenewal(epoch) {
|
|
2148
|
+
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2149
|
+
if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
|
|
2150
|
+
return;
|
|
2151
|
+
const renewal = this.#renewDiscoverySweepLease(epoch);
|
|
2152
|
+
this.#discoverySweepRenewalInFlight = renewal;
|
|
2153
|
+
void renewal.finally(() => {
|
|
2154
|
+
if (this.#discoverySweepRenewalInFlight === renewal) {
|
|
2155
|
+
this.#discoverySweepRenewalInFlight = undefined;
|
|
2156
|
+
}
|
|
2157
|
+
});
|
|
2158
|
+
}, DISCOVERY_SWEEP_RENEW_MS);
|
|
2159
|
+
this.#discoverySweepRenewTimer.unref?.();
|
|
2160
|
+
}
|
|
2161
|
+
async #renewDiscoverySweepLease(epoch) {
|
|
2162
|
+
const requestedAtMs = this.#clock.now();
|
|
2163
|
+
try {
|
|
2164
|
+
let renewal;
|
|
2165
|
+
if (this.#state.renewDiscoverySweepWithDetails) {
|
|
2166
|
+
renewal = await this.#state.renewDiscoverySweepWithDetails(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
|
|
2167
|
+
}
|
|
2168
|
+
else {
|
|
2169
|
+
const renewed = await this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
|
|
2170
|
+
renewal = renewed
|
|
2171
|
+
? {
|
|
2172
|
+
renewed: true,
|
|
2173
|
+
lease: {
|
|
2174
|
+
owner: this.#discoverySweepOwner,
|
|
2175
|
+
epoch,
|
|
2176
|
+
leaseUntilMs: requestedAtMs + DISCOVERY_SWEEP_LEASE_MS,
|
|
2177
|
+
},
|
|
2178
|
+
}
|
|
2179
|
+
: { renewed: false, reason: 'unknown' };
|
|
2180
|
+
}
|
|
2181
|
+
if (!renewal.renewed && this.#discoverySweepEpoch === epoch) {
|
|
2182
|
+
this.#discoverySweepLeaseLost = true;
|
|
2183
|
+
this.#increment('discoverySweepLeaseLosses');
|
|
2184
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests', {
|
|
2185
|
+
reason: renewal.reason,
|
|
2186
|
+
requestedOwner: this.#discoverySweepOwner,
|
|
2187
|
+
requestedEpoch: epoch,
|
|
2188
|
+
requestedAtMs,
|
|
2189
|
+
elapsedMs: this.#elapsedSince(this.#discoverySweepStartedAtMs ?? requestedAtMs),
|
|
2190
|
+
observedOwner: renewal.observedLease?.owner,
|
|
2191
|
+
observedEpoch: renewal.observedLease?.epoch,
|
|
2192
|
+
observedLeaseUntilMs: renewal.observedLease?.leaseUntilMs,
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
catch (error) {
|
|
2197
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
|
|
2198
|
+
owner: this.#discoverySweepOwner,
|
|
2199
|
+
epoch,
|
|
2200
|
+
requestedAtMs,
|
|
2201
|
+
error: describeError(error).errorMessage,
|
|
2202
|
+
});
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
async #stopDiscoverySweepRenewal() {
|
|
2206
|
+
if (this.#discoverySweepRenewTimer)
|
|
2207
|
+
clearInterval(this.#discoverySweepRenewTimer);
|
|
2208
|
+
this.#discoverySweepRenewTimer = undefined;
|
|
2209
|
+
await this.#discoverySweepRenewalInFlight;
|
|
2210
|
+
}
|
|
2211
|
+
async #prepareDiscoverySession(claim) {
|
|
2212
|
+
const checkpoint = structuredClone(claim.state.checkpoint ?? {
|
|
2213
|
+
trees: {},
|
|
2214
|
+
updatedAtMs: 0,
|
|
2215
|
+
});
|
|
2216
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2217
|
+
if (!watermark.available) {
|
|
2218
|
+
this.#increment('discoveryIncrementalUnavailable');
|
|
2219
|
+
return {
|
|
2220
|
+
checkpoint,
|
|
2221
|
+
mode: 'full',
|
|
2222
|
+
highWatermarkAvailable: false,
|
|
2223
|
+
listedPrefixes: new Set(),
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
if (claim.state.checkpoint) {
|
|
2227
|
+
if (watermark.highWatermark === checkpoint.highWatermark) {
|
|
2228
|
+
this.#increment('discoveryIncrementalUnchangedSweeps');
|
|
2229
|
+
this.#logger.info?.('[factory] discovery checkpoint unchanged; reusing cached issue trees', {
|
|
2230
|
+
highWatermark: watermark.highWatermark,
|
|
2231
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2232
|
+
});
|
|
2233
|
+
return {
|
|
2234
|
+
checkpoint,
|
|
2235
|
+
mode: 'incremental',
|
|
2236
|
+
highWatermarkAvailable: true,
|
|
2237
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2238
|
+
listedPrefixes: new Set(),
|
|
2239
|
+
};
|
|
2240
|
+
}
|
|
2241
|
+
const changes = await this.#discoveryEventsSince(checkpoint.highWatermark, watermark.highWatermark);
|
|
2242
|
+
if (changes) {
|
|
2243
|
+
this.#applyDiscoveryEvents(checkpoint, changes);
|
|
2244
|
+
this.#increment('discoveryIncrementalChangedSweeps');
|
|
2245
|
+
this.#logger.info?.('[factory] updated discovery checkpoint from Relayfile changes', {
|
|
2246
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2247
|
+
highWatermark: watermark.highWatermark,
|
|
2248
|
+
changes: changes.length,
|
|
2249
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2250
|
+
});
|
|
2251
|
+
return {
|
|
2252
|
+
checkpoint,
|
|
2253
|
+
mode: 'incremental',
|
|
2254
|
+
highWatermarkAvailable: true,
|
|
2255
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2256
|
+
listedPrefixes: new Set(),
|
|
2257
|
+
};
|
|
2258
|
+
}
|
|
2259
|
+
this.#increment('discoveryIncrementalGapFallbacks');
|
|
2260
|
+
this.#logger.warn?.('[factory] discovery change window did not cover the prior checkpoint; refreshing tree prefixes once', {
|
|
2261
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2262
|
+
highWatermark: watermark.highWatermark,
|
|
2263
|
+
});
|
|
2264
|
+
}
|
|
2265
|
+
// A missing change window means no retained prefix can be trusted. Start
|
|
2266
|
+
// the fallback snapshot empty so prefixes not touched by the current
|
|
2267
|
+
// configuration cannot resurface later with stale membership.
|
|
2268
|
+
checkpoint.trees = {};
|
|
2269
|
+
return {
|
|
2270
|
+
checkpoint,
|
|
2271
|
+
mode: 'full',
|
|
2272
|
+
highWatermarkAvailable: true,
|
|
2273
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2274
|
+
listedPrefixes: new Set(),
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
async #finalizeDiscoveryCheckpoint() {
|
|
2278
|
+
const session = this.#discoverySession;
|
|
2279
|
+
if (!session?.highWatermarkAvailable)
|
|
2280
|
+
return undefined;
|
|
2281
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2282
|
+
if (!watermark.available)
|
|
2283
|
+
return undefined;
|
|
2284
|
+
if (watermark.highWatermark !== session.observedHighWatermark) {
|
|
2285
|
+
const changes = await this.#discoveryEventsSince(session.observedHighWatermark, watermark.highWatermark);
|
|
2286
|
+
if (!changes) {
|
|
2287
|
+
this.#increment('discoveryCheckpointCommitGaps');
|
|
2288
|
+
this.#logger.warn?.('[factory] discovery checkpoint not saved because its change window overflowed during the sweep');
|
|
2289
|
+
return undefined;
|
|
2290
|
+
}
|
|
2291
|
+
this.#applyDiscoveryEvents(session.checkpoint, changes);
|
|
2292
|
+
}
|
|
2293
|
+
session.checkpoint.highWatermark = watermark.highWatermark;
|
|
2294
|
+
session.checkpoint.updatedAtMs = this.#clock.now();
|
|
2295
|
+
for (const [prefix, paths] of Object.entries(session.checkpoint.trees)) {
|
|
2296
|
+
session.checkpoint.trees[prefix] = [...new Set(paths)].sort();
|
|
2297
|
+
}
|
|
2298
|
+
return structuredClone(session.checkpoint);
|
|
2299
|
+
}
|
|
2300
|
+
async #discoveryHighWatermark() {
|
|
2301
|
+
if (!this.#mount.getEventHighWatermark)
|
|
2302
|
+
return { available: false };
|
|
2303
|
+
try {
|
|
2304
|
+
const highWatermark = await this.#mount.getEventHighWatermark();
|
|
2305
|
+
// `undefined` also means that an older Relayfile client cannot expose a
|
|
2306
|
+
// watermark. Treat it as unavailable so a compatibility fallback never
|
|
2307
|
+
// turns into an indefinitely stale cache.
|
|
2308
|
+
return highWatermark === undefined
|
|
2309
|
+
? { available: false }
|
|
2310
|
+
: { available: true, highWatermark };
|
|
2311
|
+
}
|
|
2312
|
+
catch (error) {
|
|
2313
|
+
if (relayfileOverload(error))
|
|
2314
|
+
throw error;
|
|
2315
|
+
this.#increment('discoveryHighWatermarkFailures');
|
|
2316
|
+
this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
|
|
2317
|
+
error: describeError(error).errorMessage,
|
|
2318
|
+
});
|
|
2319
|
+
return { available: false };
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
async #discoveryEventsSince(previousHighWatermark, currentHighWatermark) {
|
|
2323
|
+
if (previousHighWatermark === currentHighWatermark)
|
|
2324
|
+
return [];
|
|
2325
|
+
if (!currentHighWatermark)
|
|
2326
|
+
return undefined;
|
|
2327
|
+
let events;
|
|
2328
|
+
try {
|
|
2329
|
+
const page = await this.#mount.getEvents({
|
|
2330
|
+
last: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2331
|
+
limit: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2332
|
+
});
|
|
2333
|
+
events = [...page.events].sort(compareDiscoveryEvents);
|
|
2334
|
+
}
|
|
2335
|
+
catch (error) {
|
|
2336
|
+
if (relayfileOverload(error))
|
|
2337
|
+
throw error;
|
|
2338
|
+
this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
|
|
2339
|
+
error: describeError(error).errorMessage,
|
|
2340
|
+
});
|
|
2341
|
+
return undefined;
|
|
2342
|
+
}
|
|
2343
|
+
const currentIndex = events.findIndex((event) => discoveryEventId(event) === currentHighWatermark);
|
|
2344
|
+
if (currentIndex < 0)
|
|
2345
|
+
return undefined;
|
|
2346
|
+
if (previousHighWatermark === undefined) {
|
|
2347
|
+
return events.length < DISCOVERY_CHANGE_EVENT_LIMIT
|
|
2348
|
+
? events.slice(0, currentIndex + 1)
|
|
2349
|
+
: undefined;
|
|
2350
|
+
}
|
|
2351
|
+
const previousIndex = events.findIndex((event) => discoveryEventId(event) === previousHighWatermark);
|
|
2352
|
+
if (previousIndex >= 0 && previousIndex <= currentIndex) {
|
|
2353
|
+
return events.slice(previousIndex + 1, currentIndex + 1);
|
|
2354
|
+
}
|
|
2355
|
+
const previousSequence = eventSequenceNumber(previousHighWatermark);
|
|
2356
|
+
const currentSequence = eventSequenceNumber(currentHighWatermark);
|
|
2357
|
+
const sequenced = events.map((event) => ({ event, sequence: eventSequenceNumber(discoveryEventId(event) ?? '') }));
|
|
2358
|
+
if (previousSequence !== undefined &&
|
|
2359
|
+
currentSequence !== undefined &&
|
|
2360
|
+
sequenced.every((entry) => entry.sequence !== undefined) &&
|
|
2361
|
+
sequenced.some((entry) => entry.sequence <= previousSequence)) {
|
|
2362
|
+
return sequenced
|
|
2363
|
+
.filter((entry) => entry.sequence > previousSequence && entry.sequence <= currentSequence)
|
|
2364
|
+
.map((entry) => entry.event);
|
|
2365
|
+
}
|
|
2366
|
+
return undefined;
|
|
2367
|
+
}
|
|
2368
|
+
#applyDiscoveryEvents(checkpoint, events) {
|
|
2369
|
+
for (const event of events) {
|
|
2370
|
+
const path = changeEventPath(event);
|
|
2371
|
+
if (!path)
|
|
2372
|
+
continue;
|
|
2373
|
+
const deletion = discoveryEventIsDeletion(event);
|
|
2374
|
+
const indexRepoRoots = githubIssueIndexRepoRoots(path);
|
|
2375
|
+
for (const [prefix, cachedPaths] of Object.entries(checkpoint.trees)) {
|
|
2376
|
+
if (indexRepoRoots.some((root) => prefix === root || prefix.startsWith(`${root}/`))) {
|
|
2377
|
+
delete checkpoint.trees[prefix];
|
|
2378
|
+
continue;
|
|
2379
|
+
}
|
|
2380
|
+
// A plain startsWith would let `/…/issues-archive/…` match the
|
|
2381
|
+
// cached prefix `/…/issues`, folding an unrelated path into it.
|
|
2382
|
+
if (path !== prefix && !path.startsWith(prefix.endsWith('/') ? prefix : `${prefix}/`))
|
|
2383
|
+
continue;
|
|
2384
|
+
const paths = new Set(cachedPaths);
|
|
2385
|
+
if (deletion) {
|
|
2386
|
+
for (const candidate of paths) {
|
|
2387
|
+
if (candidate === path || candidate.startsWith(`${path}/`))
|
|
2388
|
+
paths.delete(candidate);
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
else {
|
|
2392
|
+
paths.add(path);
|
|
2393
|
+
}
|
|
2394
|
+
checkpoint.trees[prefix] = [...paths];
|
|
2395
|
+
}
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
1633
2398
|
async #githubOrphanRecoveryContext() {
|
|
1634
2399
|
try {
|
|
1635
2400
|
const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
|
|
@@ -1641,18 +2406,42 @@ export class FactoryLoop {
|
|
|
1641
2406
|
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1642
2407
|
const activeIssueIdentities = new Set();
|
|
1643
2408
|
const legacyUnownedAgentsByIssue = new Map();
|
|
1644
|
-
|
|
1645
|
-
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1646
|
-
continue;
|
|
1647
|
-
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
1648
|
-
if (identity)
|
|
1649
|
-
activeIssueIdentities.add(identity);
|
|
1650
|
-
}
|
|
2409
|
+
const orphanedLifecycleClaimsByIssue = new Map();
|
|
1651
2410
|
for (const [, waiting] of waitingClarifications) {
|
|
1652
2411
|
const identity = githubIssueRefIdentity(waiting.issue);
|
|
1653
2412
|
if (identity)
|
|
1654
2413
|
activeIssueIdentities.add(identity);
|
|
1655
2414
|
}
|
|
2415
|
+
for (const [key, lifecycle] of lifecycles) {
|
|
2416
|
+
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
2417
|
+
if (!identity)
|
|
2418
|
+
continue;
|
|
2419
|
+
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
2420
|
+
if (!activeIssueIdentities.has(identity)) {
|
|
2421
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2422
|
+
}
|
|
2423
|
+
continue;
|
|
2424
|
+
}
|
|
2425
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2426
|
+
const hasLiveAgent = activeAgents.some((agent) => onlineAgents.has(agent.name));
|
|
2427
|
+
const exitRecoveryActive = activeAgents.some((agent) => this.#agentExitsInFlight.has(agent.name));
|
|
2428
|
+
const dispatchCallActive = this.#dispatchInFlight.has(issueKey(lifecycle.issue));
|
|
2429
|
+
// The provider's `factory:in-progress` transition happens immediately
|
|
2430
|
+
// before the durable lifecycle advances from dispatching to running.
|
|
2431
|
+
// A crash can therefore leave any nonterminal phase behind while the
|
|
2432
|
+
// external claim survives. The caller still verifies provider status,
|
|
2433
|
+
// open-PR absence, a second roster, and the lifecycle lease before it
|
|
2434
|
+
// releases this orphan-shaped row.
|
|
2435
|
+
if (!activeIssueIdentities.has(identity) &&
|
|
2436
|
+
!hasLiveAgent &&
|
|
2437
|
+
!exitRecoveryActive &&
|
|
2438
|
+
!dispatchCallActive) {
|
|
2439
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2440
|
+
}
|
|
2441
|
+
else {
|
|
2442
|
+
activeIssueIdentities.add(identity);
|
|
2443
|
+
}
|
|
2444
|
+
}
|
|
1656
2445
|
for (const agent of registry?.agents ?? []) {
|
|
1657
2446
|
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1658
2447
|
continue;
|
|
@@ -1677,6 +2466,7 @@ export class FactoryLoop {
|
|
|
1677
2466
|
activeIssueIdentities,
|
|
1678
2467
|
onlineAgentNames: onlineAgents,
|
|
1679
2468
|
legacyUnownedAgentsByIssue,
|
|
2469
|
+
orphanedLifecycleClaimsByIssue,
|
|
1680
2470
|
};
|
|
1681
2471
|
}
|
|
1682
2472
|
catch (error) {
|
|
@@ -1688,15 +2478,19 @@ export class FactoryLoop {
|
|
|
1688
2478
|
}
|
|
1689
2479
|
}
|
|
1690
2480
|
async #reconcileOrphanedGithubInProgress(issue, context, dryRun) {
|
|
1691
|
-
if (dryRun
|
|
1692
|
-
return false;
|
|
2481
|
+
if (dryRun)
|
|
2482
|
+
return { recovered: false, reason: 'dry run does not release an in-progress claim' };
|
|
2483
|
+
if (!context)
|
|
2484
|
+
return { recovered: false, reason: 'orphan-recovery safety context is unavailable' };
|
|
2485
|
+
if (!isGithubIssue(issue))
|
|
2486
|
+
return { recovered: false, reason: 'issue is not GitHub-native' };
|
|
1693
2487
|
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
1694
2488
|
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
1695
2489
|
if (!required ||
|
|
1696
2490
|
!labels.has(required) ||
|
|
1697
2491
|
!labels.has('factory:in-progress') ||
|
|
1698
2492
|
labels.has('factory:human-review'))
|
|
1699
|
-
return false;
|
|
2493
|
+
return { recovered: false, reason: 'issue is not an orphan-recovery candidate' };
|
|
1700
2494
|
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1701
2495
|
const legacyUnownedAgents = identity
|
|
1702
2496
|
? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
|
|
@@ -1707,12 +2501,12 @@ export class FactoryLoop {
|
|
|
1707
2501
|
context.activeIssueIdentities.has(identity) ||
|
|
1708
2502
|
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
|
|
1709
2503
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1710
|
-
return false;
|
|
2504
|
+
return { recovered: false, reason: 'active dispatch claim or live agent still owns the issue' };
|
|
1711
2505
|
}
|
|
1712
2506
|
const getProviderStatus = this.#githubWriteback.getIssueStatus;
|
|
1713
2507
|
if (!getProviderStatus) {
|
|
1714
2508
|
this.#increment('githubOrphanRecoveryStatusLookupUnavailable');
|
|
1715
|
-
return false;
|
|
2509
|
+
return { recovered: false, reason: 'provider-authoritative issue status lookup is unavailable' };
|
|
1716
2510
|
}
|
|
1717
2511
|
let providerStatus;
|
|
1718
2512
|
try {
|
|
@@ -1724,11 +2518,16 @@ export class FactoryLoop {
|
|
|
1724
2518
|
issue: issue.key,
|
|
1725
2519
|
error: describeError(error).errorMessage,
|
|
1726
2520
|
});
|
|
1727
|
-
return false;
|
|
2521
|
+
return { recovered: false, reason: 'provider-authoritative issue status could not be verified' };
|
|
1728
2522
|
}
|
|
1729
2523
|
if (!providerStatus || providerStatus === 'human-review') {
|
|
1730
2524
|
this.#increment('githubOrphanRecoveriesBlockedProviderStatus');
|
|
1731
|
-
return
|
|
2525
|
+
return {
|
|
2526
|
+
recovered: false,
|
|
2527
|
+
reason: providerStatus === 'human-review'
|
|
2528
|
+
? 'provider-authoritative issue status is human-review'
|
|
2529
|
+
: 'provider-authoritative issue status is unavailable',
|
|
2530
|
+
};
|
|
1732
2531
|
}
|
|
1733
2532
|
let openPr;
|
|
1734
2533
|
try {
|
|
@@ -1740,7 +2539,7 @@ export class FactoryLoop {
|
|
|
1740
2539
|
issue: issue.key,
|
|
1741
2540
|
error: describeError(error).errorMessage,
|
|
1742
2541
|
});
|
|
1743
|
-
return false;
|
|
2542
|
+
return { recovered: false, reason: 'matching open-PR absence could not be verified' };
|
|
1744
2543
|
}
|
|
1745
2544
|
if (openPr) {
|
|
1746
2545
|
let adopted = false;
|
|
@@ -1764,7 +2563,12 @@ export class FactoryLoop {
|
|
|
1764
2563
|
repo: openPr.repo,
|
|
1765
2564
|
prNumber: openPr.prNumber,
|
|
1766
2565
|
});
|
|
1767
|
-
return
|
|
2566
|
+
return {
|
|
2567
|
+
recovered: false,
|
|
2568
|
+
reason: adopted
|
|
2569
|
+
? 'matching open PR was adopted'
|
|
2570
|
+
: 'matching open PR still owns the issue',
|
|
2571
|
+
};
|
|
1768
2572
|
}
|
|
1769
2573
|
// A pre-durable local Factory may have left live, registry-proven workers
|
|
1770
2574
|
// without a lifecycle record. They are safe to adopt only once their open
|
|
@@ -1772,7 +2576,11 @@ export class FactoryLoop {
|
|
|
1772
2576
|
// and workers instead of redispatching duplicate agents.
|
|
1773
2577
|
if (legacyUnownedAgents.length > 0) {
|
|
1774
2578
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1775
|
-
return false;
|
|
2579
|
+
return { recovered: false, reason: 'legacy registry-proven agents still own the issue' };
|
|
2580
|
+
}
|
|
2581
|
+
const lifecycleClaim = context.orphanedLifecycleClaimsByIssue.get(identity);
|
|
2582
|
+
if (lifecycleClaim && !await this.#releaseOrphanedGithubLifecycle(issue, identity, lifecycleClaim)) {
|
|
2583
|
+
return { recovered: false, reason: 'orphaned durable lifecycle claim could not be safely released' };
|
|
1776
2584
|
}
|
|
1777
2585
|
try {
|
|
1778
2586
|
if (providerStatus === 'in-progress') {
|
|
@@ -1781,14 +2589,25 @@ export class FactoryLoop {
|
|
|
1781
2589
|
// A crashed dispatch may leave its durable attempt marked in-flight even
|
|
1782
2590
|
// after every agent and lifecycle disappeared. Only clear that stale bit
|
|
1783
2591
|
// after all provider, agent, lifecycle, and open-PR safety checks pass.
|
|
1784
|
-
await this.#
|
|
2592
|
+
const attempt = await this.#state.getDispatchAttempts(this.#workspaceId, issueStateKey(issue));
|
|
2593
|
+
if (attempt?.terminal) {
|
|
2594
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issueStateKey(issue), {
|
|
2595
|
+
attempts: 0,
|
|
2596
|
+
inFlight: false,
|
|
2597
|
+
terminal: false,
|
|
2598
|
+
backoffUntilMs: 0,
|
|
2599
|
+
});
|
|
2600
|
+
}
|
|
2601
|
+
else {
|
|
2602
|
+
await this.#clearDispatchInFlight(issue);
|
|
2603
|
+
}
|
|
1785
2604
|
this.#reconciledGithubInProgress.add(identity);
|
|
1786
2605
|
this.#increment('githubOrphanedInProgressRecovered');
|
|
1787
2606
|
this.#logger.warn?.('[factory] recovered orphaned GitHub in-progress issue for redispatch', {
|
|
1788
2607
|
issue: issue.key,
|
|
1789
2608
|
path: issue.path,
|
|
1790
2609
|
});
|
|
1791
|
-
return true;
|
|
2610
|
+
return { recovered: true };
|
|
1792
2611
|
}
|
|
1793
2612
|
catch (error) {
|
|
1794
2613
|
this.#increment('githubOrphanRecoveryWritebackFailures');
|
|
@@ -1796,6 +2615,116 @@ export class FactoryLoop {
|
|
|
1796
2615
|
issue: issue.key,
|
|
1797
2616
|
error: describeError(error).errorMessage,
|
|
1798
2617
|
});
|
|
2618
|
+
return { recovered: false, reason: 'orphaned provider lifecycle status could not be cleared' };
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
async #releaseOrphanedGithubLifecycle(issue, identity, candidate) {
|
|
2622
|
+
let key = candidate.key;
|
|
2623
|
+
let lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2624
|
+
if (!lifecycle)
|
|
2625
|
+
return true;
|
|
2626
|
+
if (lifecycle.runId !== candidate.lifecycle.runId ||
|
|
2627
|
+
githubIssueRefIdentity(lifecycle.issue) !== identity) {
|
|
2628
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because its durable lifecycle changed', {
|
|
2629
|
+
issue: issue.key,
|
|
2630
|
+
lifecycleKey: key,
|
|
2631
|
+
});
|
|
2632
|
+
return false;
|
|
2633
|
+
}
|
|
2634
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2635
|
+
const roster = await this.#fleet.roster();
|
|
2636
|
+
const online = new Set(roster.agents.map((agent) => agent.name));
|
|
2637
|
+
if (activeAgents.some((agent) => online.has(agent.name)) ||
|
|
2638
|
+
roster.agents.some((agent) => githubAgentNameMatchesIssue(agent.name, issue))) {
|
|
2639
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
2640
|
+
return false;
|
|
2641
|
+
}
|
|
2642
|
+
let epoch;
|
|
2643
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
2644
|
+
epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
2645
|
+
if (epoch !== undefined) {
|
|
2646
|
+
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2647
|
+
if (!renewed) {
|
|
2648
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2649
|
+
return false;
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
else {
|
|
2653
|
+
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2654
|
+
if (!claim.acquired || !claim.lease) {
|
|
2655
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because another publisher still owns its lease', {
|
|
2656
|
+
issue: issue.key,
|
|
2657
|
+
owner: claim.lifecycle.lease?.owner,
|
|
2658
|
+
leaseUntilMs: claim.lifecycle.lease?.leaseUntilMs,
|
|
2659
|
+
});
|
|
2660
|
+
return false;
|
|
2661
|
+
}
|
|
2662
|
+
key = claim.key ?? key;
|
|
2663
|
+
lifecycle = claim.lifecycle;
|
|
2664
|
+
epoch = claim.lease.epoch;
|
|
2665
|
+
this.#dispatchLifecycleEpochs.set(key, epoch);
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
try {
|
|
2669
|
+
for (const agent of lifecycle.agents.filter((entry) => entry.releasedAtMs === undefined)) {
|
|
2670
|
+
try {
|
|
2671
|
+
await this.#fleet.release(agent.name, 'orphaned-claim');
|
|
2672
|
+
}
|
|
2673
|
+
catch (error) {
|
|
2674
|
+
// The roster was checked twice before fencing this lifecycle, so a
|
|
2675
|
+
// missing/dead worker is the expected crash-recovery shape. Do not
|
|
2676
|
+
// let a control-plane "agent not found" response make the durable
|
|
2677
|
+
// claim immortal; a failed fresh spawn remains visible to the normal
|
|
2678
|
+
// dispatch retry machinery.
|
|
2679
|
+
this.#increment('githubOrphanedLifecycleAgentReleaseFailures');
|
|
2680
|
+
this.#logger.warn?.('[factory] dead claim agent release failed; clearing fenced lifecycle anyway', {
|
|
2681
|
+
issue: issue.key,
|
|
2682
|
+
agent: agent.name,
|
|
2683
|
+
error: describeError(error).errorMessage,
|
|
2684
|
+
});
|
|
2685
|
+
}
|
|
2686
|
+
this.#fleet.markAgentTerminal?.(agent.name, 'orphaned-claim');
|
|
2687
|
+
}
|
|
2688
|
+
if (epoch !== undefined && !await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS)) {
|
|
2689
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2690
|
+
return false;
|
|
2691
|
+
}
|
|
2692
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
2693
|
+
if (retryTimer)
|
|
2694
|
+
clearTimeout(retryTimer);
|
|
2695
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2696
|
+
const batch = await this.#batch();
|
|
2697
|
+
batch.abandon(lifecycle.issue);
|
|
2698
|
+
await this.#state.clearDispatchLifecycle(this.#workspaceId, key);
|
|
2699
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
|
|
2700
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2701
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
2702
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
2703
|
+
await this.#writeInFlightRegistry().catch((error) => {
|
|
2704
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
|
|
2705
|
+
issue: issue.key,
|
|
2706
|
+
error: describeError(error).errorMessage,
|
|
2707
|
+
});
|
|
2708
|
+
});
|
|
2709
|
+
this.#increment('githubOrphanedLifecycleClaimsReleased');
|
|
2710
|
+
this.#logger.warn?.('[factory] released orphaned GitHub dispatch claim with no live agents', {
|
|
2711
|
+
issue: issue.key,
|
|
2712
|
+
lifecycleKey: key,
|
|
2713
|
+
runId: lifecycle.runId,
|
|
2714
|
+
agents: activeAgents.map((agent) => agent.name),
|
|
2715
|
+
});
|
|
2716
|
+
return true;
|
|
2717
|
+
}
|
|
2718
|
+
catch (error) {
|
|
2719
|
+
if (epoch !== undefined) {
|
|
2720
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch).catch(() => undefined);
|
|
2721
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2722
|
+
}
|
|
2723
|
+
this.#logger.warn?.('[factory] failed to release orphaned GitHub dispatch claim; preserving it', {
|
|
2724
|
+
issue: issue.key,
|
|
2725
|
+
lifecycleKey: key,
|
|
2726
|
+
error: describeError(error).errorMessage,
|
|
2727
|
+
});
|
|
1799
2728
|
return false;
|
|
1800
2729
|
}
|
|
1801
2730
|
}
|
|
@@ -1954,13 +2883,60 @@ export class FactoryLoop {
|
|
|
1954
2883
|
this.#increment('githubOrphanedPullRequestsAdopted');
|
|
1955
2884
|
return true;
|
|
1956
2885
|
}
|
|
1957
|
-
|
|
1958
|
-
|
|
2886
|
+
// `cache` is opt-in and must stay limited to call sites that walk a full
|
|
2887
|
+
// repository/issue root to enumerate everything under it — the sweep this
|
|
2888
|
+
// whole feature exists to make incremental. A cached listing can lag the
|
|
2889
|
+
// live tree by up to one sweep, which is fine for enumeration but wrong
|
|
2890
|
+
// for a point lookup that needs the current state: a PR-confirmation poll
|
|
2891
|
+
// that expects a specific PR to appear, or an escalation/comment-replay
|
|
2892
|
+
// scan that must not miss a marker or reply that landed after the cache
|
|
2893
|
+
// was populated. Those callers must omit `cache` and pay for a fresh list.
|
|
2894
|
+
async #listRelayfileTree(prefix, phase, opts = {}) {
|
|
2895
|
+
if (this.#discoverySweepLeaseLost) {
|
|
2896
|
+
throw new Error('discovery sweep lease was lost; refusing another tree request');
|
|
2897
|
+
}
|
|
2898
|
+
const cached = opts.cache ? this.#cachedDiscoveryTree(prefix) : undefined;
|
|
2899
|
+
if (cached) {
|
|
2900
|
+
this.#increment('discoveryTreeCacheHits');
|
|
2901
|
+
this.#logger.debug?.('[factory] reused cached Relayfile tree', { phase, prefix, count: cached.length });
|
|
2902
|
+
return cached;
|
|
2903
|
+
}
|
|
2904
|
+
if (opts.cache)
|
|
2905
|
+
this.#increment('discoveryTreeCacheMisses');
|
|
2906
|
+
const paths = await this.#withRelayfileOperation('listTree', { phase, prefix }, () => this.#mount.listTree(prefix), {
|
|
1959
2907
|
count: (paths) => paths.length,
|
|
1960
2908
|
logFailure: true,
|
|
1961
2909
|
logStart: true,
|
|
1962
2910
|
logComplete: true,
|
|
1963
2911
|
});
|
|
2912
|
+
if (opts.cache)
|
|
2913
|
+
await this.#rememberDiscoveryTree(prefix, paths);
|
|
2914
|
+
return paths;
|
|
2915
|
+
}
|
|
2916
|
+
#cachedDiscoveryTree(prefix) {
|
|
2917
|
+
const session = this.#discoverySession;
|
|
2918
|
+
if (!session)
|
|
2919
|
+
return undefined;
|
|
2920
|
+
if (session.mode === 'full' && !session.listedPrefixes.has(prefix))
|
|
2921
|
+
return undefined;
|
|
2922
|
+
const paths = session.checkpoint.trees[prefix];
|
|
2923
|
+
return paths ? [...paths] : undefined;
|
|
2924
|
+
}
|
|
2925
|
+
async #rememberDiscoveryTree(prefix, paths) {
|
|
2926
|
+
const session = this.#discoverySession;
|
|
2927
|
+
if (!session)
|
|
2928
|
+
return;
|
|
2929
|
+
const uniquePaths = new Set();
|
|
2930
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
2931
|
+
uniquePaths.add(paths[index]);
|
|
2932
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
2933
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2934
|
+
if (index + 1 < paths.length)
|
|
2935
|
+
await liveEventYield();
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
session.checkpoint.trees[prefix] = [...uniquePaths].sort();
|
|
2939
|
+
session.listedPrefixes.add(prefix);
|
|
1964
2940
|
}
|
|
1965
2941
|
async #ensureRelayfileSubRoot(prefix, phase, opts) {
|
|
1966
2942
|
return this.#withRelayfileOperation('ensureSubRoot', { phase, prefix }, () => this.#mount.ensureSubRoot(prefix, opts), {
|
|
@@ -2013,6 +2989,9 @@ export class FactoryLoop {
|
|
|
2013
2989
|
return result;
|
|
2014
2990
|
}
|
|
2015
2991
|
catch (error) {
|
|
2992
|
+
if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
|
|
2993
|
+
this.#discoveryOverloadError ??= error;
|
|
2994
|
+
}
|
|
2016
2995
|
if (opts.logFailure || waitWarnings > 0) {
|
|
2017
2996
|
this.#increment('relayfileOperationFailures');
|
|
2018
2997
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
@@ -2235,7 +3214,10 @@ export class FactoryLoop {
|
|
|
2235
3214
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
2236
3215
|
// and concurrent issues can switch each other back to the base branch.
|
|
2237
3216
|
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
2238
|
-
|
|
3217
|
+
// Every live dispatch gets an issue-owned branch, including legacy local
|
|
3218
|
+
// fleets without a durable lifecycle or worktree manager. The publication
|
|
3219
|
+
// boundary uses this exact ref to reject a stale shared checkout.
|
|
3220
|
+
const lifecycleRunId = !dryRun ? randomUUID() : undefined;
|
|
2239
3221
|
if (lifecycleRunId) {
|
|
2240
3222
|
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
2241
3223
|
isolateLocalWorktree,
|
|
@@ -2370,6 +3352,13 @@ export class FactoryLoop {
|
|
|
2370
3352
|
}
|
|
2371
3353
|
agents.push({ name: spawned.name, role: spec.role });
|
|
2372
3354
|
}
|
|
3355
|
+
if (!dryRun) {
|
|
3356
|
+
record.dispatchClaim = {
|
|
3357
|
+
state: 'pending',
|
|
3358
|
+
updatedAtMs: this.#clock.now(),
|
|
3359
|
+
};
|
|
3360
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
3361
|
+
}
|
|
2373
3362
|
await this.#writeInFlightRegistry();
|
|
2374
3363
|
const comment = dispatchComment(dispatchDecision, agents);
|
|
2375
3364
|
let implementingStateId;
|
|
@@ -2378,19 +3367,7 @@ export class FactoryLoop {
|
|
|
2378
3367
|
if (!issue || !this.#isIssueReady(issue)) {
|
|
2379
3368
|
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
2380
3369
|
}
|
|
2381
|
-
|
|
2382
|
-
await this.#postIssueComment(issue, comment);
|
|
2383
|
-
}
|
|
2384
|
-
catch (error) {
|
|
2385
|
-
this.#logger.warn?.('[factory] comment writeback skipped', error);
|
|
2386
|
-
}
|
|
2387
|
-
if (isGithubIssue(issue)) {
|
|
2388
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
2389
|
-
}
|
|
2390
|
-
else {
|
|
2391
|
-
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
2392
|
-
await this.#linear.setState(issue, implementingStateId);
|
|
2393
|
-
}
|
|
3370
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
2394
3371
|
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
2395
3372
|
}
|
|
2396
3373
|
const result = {
|
|
@@ -2404,11 +3381,15 @@ export class FactoryLoop {
|
|
|
2404
3381
|
dryRun,
|
|
2405
3382
|
};
|
|
2406
3383
|
record.result = result;
|
|
2407
|
-
await this.#saveDispatchLifecycle(record, 'running')
|
|
3384
|
+
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3385
|
+
return result;
|
|
2408
3386
|
this.#increment('dispatched');
|
|
2409
3387
|
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
3388
|
+
if (!dryRun && this.#config.hooks?.onTicketDispatch) {
|
|
3389
|
+
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
3390
|
+
}
|
|
2410
3391
|
if (!dryRun) {
|
|
2411
|
-
await this.#ensureSlackDispatchThread(record, result);
|
|
3392
|
+
await this.#ensureSlackDispatchThread(record, result, liveIssue);
|
|
2412
3393
|
}
|
|
2413
3394
|
return result;
|
|
2414
3395
|
}
|
|
@@ -2501,8 +3482,28 @@ export class FactoryLoop {
|
|
|
2501
3482
|
}
|
|
2502
3483
|
status() {
|
|
2503
3484
|
const batch = this.#batchView;
|
|
3485
|
+
const nowMs = this.#clock.now();
|
|
3486
|
+
const inFlightDispatches = batch?.inFlight
|
|
3487
|
+
.filter((record) => !record.dryRun)
|
|
3488
|
+
.map((record) => ({
|
|
3489
|
+
issue: { ...record.issue },
|
|
3490
|
+
agents: [...record.agents].map(([name, tracked]) => ({
|
|
3491
|
+
name,
|
|
3492
|
+
role: tracked.spec.role,
|
|
3493
|
+
...(tracked.sessionRef ? { sessionRef: tracked.sessionRef } : {}),
|
|
3494
|
+
...(tracked.spec.invocationId ? { invocationId: tracked.spec.invocationId } : {}),
|
|
3495
|
+
...(tracked.result?.node ? { node: tracked.result.node } : {}),
|
|
3496
|
+
})),
|
|
3497
|
+
claim: {
|
|
3498
|
+
...(record.dispatchClaim ?? this.#dispatchClaimStatuses.get(issueKey(record.issue)) ?? {
|
|
3499
|
+
state: 'pending',
|
|
3500
|
+
updatedAtMs: nowMs,
|
|
3501
|
+
}),
|
|
3502
|
+
},
|
|
3503
|
+
})) ?? [];
|
|
2504
3504
|
return {
|
|
2505
3505
|
inFlight: batch?.inFlight.map((record) => record.issue) ?? [],
|
|
3506
|
+
...(inFlightDispatches.length > 0 ? { inFlightDispatches } : {}),
|
|
2506
3507
|
queued: batch?.queued.map((queued) => queued.issue) ?? [],
|
|
2507
3508
|
parked: batch?.parked.map((parked) => ({
|
|
2508
3509
|
issue: parked.issue,
|
|
@@ -2514,6 +3515,8 @@ export class FactoryLoop {
|
|
|
2514
3515
|
slackDegraded: this.#slackDegraded,
|
|
2515
3516
|
slackDegradedReason: this.#slackDegradedReason,
|
|
2516
3517
|
eventListener: this.#eventListenerStatus(),
|
|
3518
|
+
readinessReconcile: this.#readinessReconcileStatus(),
|
|
3519
|
+
heldAgents: batch?.inFlight.flatMap((record) => heldAgentsForRecord(record, nowMs, this.#config.dispatch.agentHoldTimeoutMs, this.#config.terminalState)) ?? [],
|
|
2517
3520
|
};
|
|
2518
3521
|
}
|
|
2519
3522
|
#eventListenerStatus() {
|
|
@@ -2534,6 +3537,34 @@ export class FactoryLoop {
|
|
|
2534
3537
|
}
|
|
2535
3538
|
return { state: 'starting' };
|
|
2536
3539
|
}
|
|
3540
|
+
#readinessReconcileStatus() {
|
|
3541
|
+
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
3542
|
+
const state = this.#startMode !== 'live'
|
|
3543
|
+
? 'not-running'
|
|
3544
|
+
: consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
|
|
3545
|
+
? 'degraded'
|
|
3546
|
+
: consecutiveFailures > 0
|
|
3547
|
+
? 'retrying'
|
|
3548
|
+
: 'healthy';
|
|
3549
|
+
return {
|
|
3550
|
+
state,
|
|
3551
|
+
consecutiveFailures,
|
|
3552
|
+
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
3553
|
+
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
3554
|
+
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
3555
|
+
: {}),
|
|
3556
|
+
...(this.#readinessReconcileLastStartedAtMs !== undefined
|
|
3557
|
+
? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
|
|
3558
|
+
: {}),
|
|
3559
|
+
...(this.#readinessReconcileLastCompletedAtMs !== undefined
|
|
3560
|
+
? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
|
|
3561
|
+
: {}),
|
|
3562
|
+
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
3563
|
+
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
3564
|
+
: {}),
|
|
3565
|
+
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
3566
|
+
};
|
|
3567
|
+
}
|
|
2537
3568
|
on(event, listener) {
|
|
2538
3569
|
let listeners = this.#listeners.get(event);
|
|
2539
3570
|
if (!listeners) {
|
|
@@ -2653,10 +3684,11 @@ export class FactoryLoop {
|
|
|
2653
3684
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2654
3685
|
continue;
|
|
2655
3686
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
3687
|
+
let liveIssue;
|
|
2656
3688
|
if (!durableRecord.dryRun &&
|
|
2657
3689
|
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2658
3690
|
claim.lifecycle.phase !== 'releasing') {
|
|
2659
|
-
|
|
3691
|
+
liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2660
3692
|
// A babysat Linear issue already at Done may have merged while this
|
|
2661
3693
|
// process was down. Let authoritative PR restoration drive the
|
|
2662
3694
|
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
@@ -2683,6 +3715,18 @@ export class FactoryLoop {
|
|
|
2683
3715
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2684
3716
|
? durableRecord
|
|
2685
3717
|
: batch.restore(durableRecord);
|
|
3718
|
+
if (claim.lifecycle.phase === 'running' &&
|
|
3719
|
+
this.#ticketDispatchNotificationIsPending(claim.lifecycle)) {
|
|
3720
|
+
if (!liveIssue) {
|
|
3721
|
+
this.#dispatchLifecycleEpochs.delete(claim.key ?? key);
|
|
3722
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
3723
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: issue is not currently readable`);
|
|
3724
|
+
}
|
|
3725
|
+
if (!restored.result) {
|
|
3726
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: dispatch result is missing`);
|
|
3727
|
+
}
|
|
3728
|
+
await this.#notifyTicketDispatch(restored.decision, liveIssue, restored, restored.result);
|
|
3729
|
+
}
|
|
2686
3730
|
if (claim.lifecycle.phase !== 'running')
|
|
2687
3731
|
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2688
3732
|
// Parking agents are cleanup-only. Hydrating them makes relay
|
|
@@ -2781,6 +3825,7 @@ export class FactoryLoop {
|
|
|
2781
3825
|
});
|
|
2782
3826
|
}
|
|
2783
3827
|
}
|
|
3828
|
+
this.#rescheduleHeldAgentDeadlineSweep();
|
|
2784
3829
|
}
|
|
2785
3830
|
catch (error) {
|
|
2786
3831
|
this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
|
|
@@ -2885,13 +3930,100 @@ export class FactoryLoop {
|
|
|
2885
3930
|
this.#reconciledAgentExitWaiters.shift()?.();
|
|
2886
3931
|
}
|
|
2887
3932
|
}
|
|
2888
|
-
#scheduleDispatchLifecycleRenewal() {
|
|
2889
|
-
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
3933
|
+
#scheduleDispatchLifecycleRenewal() {
|
|
3934
|
+
if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0)
|
|
3935
|
+
return;
|
|
3936
|
+
this.#dispatchLifecycleRenewTimer = setInterval(() => {
|
|
3937
|
+
void this.#renewDispatchLifecycles();
|
|
3938
|
+
}, DISPATCH_LIFECYCLE_RENEW_MS);
|
|
3939
|
+
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
3940
|
+
}
|
|
3941
|
+
#scheduleHeldAgentDeadline(record) {
|
|
3942
|
+
if (this.#stopping || record.dryRun || record.heldSinceAtMs === undefined || record.agents.size === 0)
|
|
3943
|
+
return;
|
|
3944
|
+
const dueAtMs = record.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs;
|
|
3945
|
+
if (this.#heldAgentDeadlineTimer &&
|
|
3946
|
+
this.#heldAgentDeadlineDueAtMs !== undefined &&
|
|
3947
|
+
this.#heldAgentDeadlineDueAtMs <= dueAtMs)
|
|
3948
|
+
return;
|
|
3949
|
+
if (this.#heldAgentDeadlineTimer)
|
|
3950
|
+
clearTimeout(this.#heldAgentDeadlineTimer);
|
|
3951
|
+
this.#heldAgentDeadlineDueAtMs = dueAtMs;
|
|
3952
|
+
const remainingMs = dueAtMs - this.#clock.now();
|
|
3953
|
+
// An overdue lifecycle can temporarily be fenced by another owner. Avoid
|
|
3954
|
+
// a zero-delay reschedule loop while its lease is being reclaimed.
|
|
3955
|
+
const delayMs = remainingMs <= 0
|
|
3956
|
+
? HELD_DEADLINE_OVERDUE_RETRY_MS
|
|
3957
|
+
: Math.min(remainingMs, 2_147_483_647);
|
|
3958
|
+
this.#heldAgentDeadlineTimer = setTimeout(() => {
|
|
3959
|
+
this.#heldAgentDeadlineTimer = undefined;
|
|
3960
|
+
this.#heldAgentDeadlineDueAtMs = undefined;
|
|
3961
|
+
const sweep = this.#sweepHeldAgentDeadlines()
|
|
3962
|
+
.catch((error) => {
|
|
3963
|
+
this.#logger.warn?.('[factory] held-agent deadline sweep failed; retrying', {
|
|
3964
|
+
error: describeError(error).errorMessage,
|
|
3965
|
+
});
|
|
3966
|
+
})
|
|
3967
|
+
.finally(() => {
|
|
3968
|
+
if (this.#heldAgentDeadlineSweepInFlight === sweep)
|
|
3969
|
+
this.#heldAgentDeadlineSweepInFlight = undefined;
|
|
3970
|
+
this.#rescheduleHeldAgentDeadlineSweep();
|
|
3971
|
+
});
|
|
3972
|
+
this.#heldAgentDeadlineSweepInFlight = sweep;
|
|
3973
|
+
}, delayMs);
|
|
3974
|
+
this.#heldAgentDeadlineTimer.unref?.();
|
|
3975
|
+
}
|
|
3976
|
+
#rescheduleHeldAgentDeadlineSweep() {
|
|
3977
|
+
if (this.#stopping)
|
|
2890
3978
|
return;
|
|
2891
|
-
this.#
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
3979
|
+
for (const record of this.#batchView?.inFlight ?? [])
|
|
3980
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
3981
|
+
}
|
|
3982
|
+
async #sweepHeldAgentDeadlines() {
|
|
3983
|
+
const nowMs = this.#clock.now();
|
|
3984
|
+
const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
|
|
3985
|
+
for (const record of [...(await this.#batch()).inFlight]) {
|
|
3986
|
+
const heldSinceAtMs = record.heldSinceAtMs;
|
|
3987
|
+
if (record.dryRun ||
|
|
3988
|
+
heldSinceAtMs === undefined ||
|
|
3989
|
+
record.agents.size === 0 ||
|
|
3990
|
+
nowMs - heldSinceAtMs < timeoutMs)
|
|
3991
|
+
continue;
|
|
3992
|
+
const key = issueKey(record.issue);
|
|
3993
|
+
if (this.#abandonedDispatchReasons.has(key))
|
|
3994
|
+
continue;
|
|
3995
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
3996
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
3997
|
+
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle))
|
|
3998
|
+
continue;
|
|
3999
|
+
// Terminal writeback already won the race. Finish its normal release
|
|
4000
|
+
// reason instead of relabeling an acknowledged completion as a timeout.
|
|
4001
|
+
if (lifecycle.phase === 'releasing') {
|
|
4002
|
+
await this.#finishDurableRelease(record, lifecycle.releaseReason);
|
|
4003
|
+
continue;
|
|
4004
|
+
}
|
|
4005
|
+
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
4006
|
+
continue;
|
|
4007
|
+
}
|
|
4008
|
+
const heldForMs = Math.max(0, this.#clock.now() - heldSinceAtMs);
|
|
4009
|
+
const details = {
|
|
4010
|
+
issue: record.issue.key,
|
|
4011
|
+
heldForMs,
|
|
4012
|
+
holdTimeoutMs: timeoutMs,
|
|
4013
|
+
waitingForTerminalState: this.#config.terminalState,
|
|
4014
|
+
reason: HELD_PAST_DEADLINE_RELEASE_REASON,
|
|
4015
|
+
agents: [...record.agents.keys()].sort(),
|
|
4016
|
+
};
|
|
4017
|
+
this.#logger.warn?.('[factory] releasing agents held past deadline', details);
|
|
4018
|
+
await this.#abandonStuckDispatch(record, HELD_PAST_DEADLINE_RELEASE_REASON);
|
|
4019
|
+
const lifecycle = this.#usesDurableDispatchLifecycle()
|
|
4020
|
+
? await this.#state.getDispatchLifecycle(this.#workspaceId, key)
|
|
4021
|
+
: undefined;
|
|
4022
|
+
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle)) {
|
|
4023
|
+
this.#increment('heldPastDeadlineReleases');
|
|
4024
|
+
this.#logger.warn?.('[factory] released agents held past deadline', details);
|
|
4025
|
+
}
|
|
4026
|
+
}
|
|
2895
4027
|
}
|
|
2896
4028
|
async #renewDispatchLifecycles() {
|
|
2897
4029
|
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
@@ -3258,6 +4390,7 @@ export class FactoryLoop {
|
|
|
3258
4390
|
});
|
|
3259
4391
|
}
|
|
3260
4392
|
async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
|
|
4393
|
+
record.lifecyclePhase = phase;
|
|
3261
4394
|
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
3262
4395
|
return true;
|
|
3263
4396
|
if (isTerminalDispatchPhase(phase))
|
|
@@ -3277,6 +4410,9 @@ export class FactoryLoop {
|
|
|
3277
4410
|
? this.#finalizeRunCost(record, lifecycleRunId)
|
|
3278
4411
|
: previous?.cost;
|
|
3279
4412
|
const lifecycle = lifecycleFromInFlightRecord(record, lifecycleRunId, phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason, cost);
|
|
4413
|
+
if (previous?.ticketDispatchNotification) {
|
|
4414
|
+
lifecycle.ticketDispatchNotification = structuredClone(previous.ticketDispatchNotification);
|
|
4415
|
+
}
|
|
3280
4416
|
for (const agent of lifecycle.agents) {
|
|
3281
4417
|
const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
|
|
3282
4418
|
if (previousAgent?.releasedAtMs !== undefined)
|
|
@@ -3458,6 +4594,7 @@ export class FactoryLoop {
|
|
|
3458
4594
|
const batch = await this.#batch();
|
|
3459
4595
|
const durableRecord = inFlightRecordFromLifecycle(lifecycle);
|
|
3460
4596
|
const record = lifecycle.phase === 'releasing' ? durableRecord : batch.restore(durableRecord);
|
|
4597
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
3461
4598
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
3462
4599
|
return;
|
|
3463
4600
|
if (acquiredNow && this.#config.babysitter.enabled)
|
|
@@ -3473,6 +4610,15 @@ export class FactoryLoop {
|
|
|
3473
4610
|
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3474
4611
|
return;
|
|
3475
4612
|
}
|
|
4613
|
+
if (this.#ticketDispatchNotificationIsPending(lifecycle)) {
|
|
4614
|
+
if (!liveIssue) {
|
|
4615
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: issue is not currently readable`);
|
|
4616
|
+
}
|
|
4617
|
+
if (!record.result) {
|
|
4618
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: dispatch result is missing`);
|
|
4619
|
+
}
|
|
4620
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
4621
|
+
}
|
|
3476
4622
|
}
|
|
3477
4623
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
3478
4624
|
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
@@ -3649,18 +4795,22 @@ export class FactoryLoop {
|
|
|
3649
4795
|
const spawned = await this.#spawnAgent(record, spec, record.dryRun);
|
|
3650
4796
|
agents.push({ name: spawned.name, role: spec.role });
|
|
3651
4797
|
}
|
|
4798
|
+
const comment = dispatchComment(record.decision, agents);
|
|
4799
|
+
let implementingStateId;
|
|
4800
|
+
if (!record.dryRun) {
|
|
4801
|
+
record.dispatchClaim = {
|
|
4802
|
+
state: 'pending',
|
|
4803
|
+
updatedAtMs: this.#clock.now(),
|
|
4804
|
+
};
|
|
4805
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4806
|
+
}
|
|
3652
4807
|
await this.#writeInFlightRegistry();
|
|
3653
4808
|
if (!record.dryRun) {
|
|
3654
4809
|
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3655
4810
|
if (!issue)
|
|
3656
4811
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3657
4812
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
3658
|
-
|
|
3659
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
3660
|
-
}
|
|
3661
|
-
else {
|
|
3662
|
-
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3663
|
-
}
|
|
4813
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
3664
4814
|
}
|
|
3665
4815
|
const recoveredPreviews = uniquePreviewReferences([
|
|
3666
4816
|
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
@@ -3669,7 +4819,8 @@ export class FactoryLoop {
|
|
|
3669
4819
|
record.result ??= {
|
|
3670
4820
|
issue: record.issue,
|
|
3671
4821
|
agents,
|
|
3672
|
-
comments: [
|
|
4822
|
+
comments: [comment],
|
|
4823
|
+
stateId: implementingStateId,
|
|
3673
4824
|
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3674
4825
|
dryRun: record.dryRun,
|
|
3675
4826
|
};
|
|
@@ -3678,7 +4829,8 @@ export class FactoryLoop {
|
|
|
3678
4829
|
}
|
|
3679
4830
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3680
4831
|
return;
|
|
3681
|
-
if (
|
|
4832
|
+
if (liveIssue) {
|
|
4833
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
3682
4834
|
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3683
4835
|
for (const tracked of record.agents.values()) {
|
|
3684
4836
|
const owned = tracked.spec.ownedPullRequest;
|
|
@@ -3967,6 +5119,97 @@ export class FactoryLoop {
|
|
|
3967
5119
|
}
|
|
3968
5120
|
await this.#linear.postComment(issue, body);
|
|
3969
5121
|
}
|
|
5122
|
+
async #applyDispatchClaim(record, issue, comment) {
|
|
5123
|
+
let implementingStateId;
|
|
5124
|
+
if (isGithubIssue(issue)) {
|
|
5125
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
|
|
5126
|
+
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
5127
|
+
});
|
|
5128
|
+
const commentApplied = this.#githubWriteback.hasCommentMarker
|
|
5129
|
+
? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
|
|
5130
|
+
: undefined;
|
|
5131
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => this.#githubWriteback.postComment(issue, comment), commentApplied);
|
|
5132
|
+
}
|
|
5133
|
+
else {
|
|
5134
|
+
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
5135
|
+
await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
|
|
5136
|
+
await this.#linear.setState(issue, implementingStateId);
|
|
5137
|
+
});
|
|
5138
|
+
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
5139
|
+
await this.#linear.postComment(issue, comment);
|
|
5140
|
+
});
|
|
5141
|
+
}
|
|
5142
|
+
record.dispatchClaim = {
|
|
5143
|
+
state: 'verified',
|
|
5144
|
+
updatedAtMs: this.#clock.now(),
|
|
5145
|
+
};
|
|
5146
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
5147
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
5148
|
+
return implementingStateId;
|
|
5149
|
+
}
|
|
5150
|
+
async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
|
|
5151
|
+
let lastError;
|
|
5152
|
+
for (let attempt = 1; attempt <= DISPATCH_WRITEBACK_MAX_ATTEMPTS; attempt += 1) {
|
|
5153
|
+
try {
|
|
5154
|
+
if (isApplied && await isApplied())
|
|
5155
|
+
return;
|
|
5156
|
+
await apply();
|
|
5157
|
+
if (isApplied && !await isApplied()) {
|
|
5158
|
+
throw new Error(`${write} returned without a provider read-back acknowledgement`);
|
|
5159
|
+
}
|
|
5160
|
+
return;
|
|
5161
|
+
}
|
|
5162
|
+
catch (error) {
|
|
5163
|
+
lastError = error;
|
|
5164
|
+
const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
|
|
5165
|
+
this.#increment('dispatchWritebackFailures');
|
|
5166
|
+
record.dispatchClaim = {
|
|
5167
|
+
state: 'degraded',
|
|
5168
|
+
write,
|
|
5169
|
+
attempts: attempt,
|
|
5170
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
5171
|
+
error: describeError(error).errorMessage,
|
|
5172
|
+
...(deadLettered ? { deadLettered: true } : {}),
|
|
5173
|
+
updatedAtMs: this.#clock.now(),
|
|
5174
|
+
};
|
|
5175
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
5176
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
5177
|
+
if (deadLettered) {
|
|
5178
|
+
this.#increment('dispatchWritebackDeadLetters');
|
|
5179
|
+
this.#logger.error?.('[factory] dispatch writeback dead-lettered after retries exhausted', {
|
|
5180
|
+
issue: issue.key,
|
|
5181
|
+
write,
|
|
5182
|
+
attempts: attempt,
|
|
5183
|
+
error: describeError(error).errorMessage,
|
|
5184
|
+
});
|
|
5185
|
+
throw error;
|
|
5186
|
+
}
|
|
5187
|
+
this.#increment('dispatchWritebackRetries');
|
|
5188
|
+
this.#logger.error?.('[factory] dispatch writeback failed; retrying', {
|
|
5189
|
+
issue: issue.key,
|
|
5190
|
+
write,
|
|
5191
|
+
attempt,
|
|
5192
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
5193
|
+
retryMs: DISPATCH_WRITEBACK_RETRY_MS,
|
|
5194
|
+
error: describeError(error).errorMessage,
|
|
5195
|
+
});
|
|
5196
|
+
await this.#clock.sleep(DISPATCH_WRITEBACK_RETRY_MS);
|
|
5197
|
+
}
|
|
5198
|
+
}
|
|
5199
|
+
throw lastError;
|
|
5200
|
+
}
|
|
5201
|
+
async #writeDispatchClaimRegistry(issue) {
|
|
5202
|
+
try {
|
|
5203
|
+
await this.#writeInFlightRegistry();
|
|
5204
|
+
}
|
|
5205
|
+
catch (error) {
|
|
5206
|
+
this.#increment('dispatchClaimRegistryWriteFailures');
|
|
5207
|
+
this.#logger.error?.('[factory] failed to persist dispatch claim visibility', {
|
|
5208
|
+
issue: issue.key,
|
|
5209
|
+
error: describeError(error).errorMessage,
|
|
5210
|
+
});
|
|
5211
|
+
}
|
|
5212
|
+
}
|
|
3970
5213
|
// Probes the GitHub issue sub-root at most once and caches the verdict so
|
|
3971
5214
|
// repeated iterations skip listTree calls when the mount is absent.
|
|
3972
5215
|
async #ensureGithubIngestionReady() {
|
|
@@ -4024,7 +5267,7 @@ export class FactoryLoop {
|
|
|
4024
5267
|
const candidates = [];
|
|
4025
5268
|
let scanned = 0;
|
|
4026
5269
|
let lastProgressAtMs = startedAtMs;
|
|
4027
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading')) {
|
|
5270
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
|
|
4028
5271
|
await this.#refreshLiveHeartbeatIfDue();
|
|
4029
5272
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
4030
5273
|
continue;
|
|
@@ -4047,18 +5290,35 @@ export class FactoryLoop {
|
|
|
4047
5290
|
try {
|
|
4048
5291
|
const issuePaths = new Map();
|
|
4049
5292
|
for (const { owner, repo } of configuredGithubRepoParts(this.#config)) {
|
|
4050
|
-
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
4051
5293
|
const roots = githubIssueRepoRoots(owner, repo);
|
|
5294
|
+
const cachedBatches = roots.map((root) => this.#cachedDiscoveryTree(root));
|
|
5295
|
+
const allRootsCached = cachedBatches.every((paths) => paths !== undefined);
|
|
5296
|
+
const indexedPaths = allRootsCached
|
|
5297
|
+
? undefined
|
|
5298
|
+
: await this.#githubIssuePathsFromIndex(owner, repo);
|
|
4052
5299
|
// Keep the fallback roots as separate batches. Flattening a very large
|
|
4053
5300
|
// provider result is synchronous work and can starve the durable loop
|
|
4054
5301
|
// heartbeat before the bounded scan below gets a chance to yield.
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
5302
|
+
let pathBatches;
|
|
5303
|
+
if (allRootsCached) {
|
|
5304
|
+
pathBatches = cachedBatches;
|
|
5305
|
+
this.#increment('githubIssueDiscoveryCacheReposUsed');
|
|
5306
|
+
}
|
|
5307
|
+
else if (indexedPaths) {
|
|
5308
|
+
// Do not feed this into the discovery cache: the index only covers
|
|
5309
|
+
// open, labeled issues, so it is a filtered subset of the real
|
|
5310
|
+
// tree (and an empty result for whichever root form the index
|
|
5311
|
+
// doesn't use) — not something a later fresh listTree call may
|
|
5312
|
+
// treat as "the tree", including on the escalation-marker and
|
|
5313
|
+
// comment-replay call sites that share this cache.
|
|
5314
|
+
pathBatches = [indexedPaths];
|
|
4059
5315
|
this.#increment('githubIssueIndexReposUsed');
|
|
4060
5316
|
}
|
|
4061
5317
|
else {
|
|
5318
|
+
pathBatches = [];
|
|
5319
|
+
for (const root of roots) {
|
|
5320
|
+
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
|
|
5321
|
+
}
|
|
4062
5322
|
this.#increment('githubIssueIndexFallbacks');
|
|
4063
5323
|
}
|
|
4064
5324
|
for (const paths of pathBatches) {
|
|
@@ -4103,6 +5363,8 @@ export class FactoryLoop {
|
|
|
4103
5363
|
return [...issuePaths.values()].sort();
|
|
4104
5364
|
}
|
|
4105
5365
|
catch (error) {
|
|
5366
|
+
if (relayfileOverload(error))
|
|
5367
|
+
throw error;
|
|
4106
5368
|
this.#githubIssuePathIndexReady = false;
|
|
4107
5369
|
this.#increment('githubIssueListFailures');
|
|
4108
5370
|
this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
|
|
@@ -4116,7 +5378,9 @@ export class FactoryLoop {
|
|
|
4116
5378
|
const { content } = await this.#readRelayfileFile(indexPath, 'GitHub issue index discovery');
|
|
4117
5379
|
parsed = parseJsonContent(content);
|
|
4118
5380
|
}
|
|
4119
|
-
catch {
|
|
5381
|
+
catch (error) {
|
|
5382
|
+
if (relayfileOverload(error))
|
|
5383
|
+
throw error;
|
|
4120
5384
|
return undefined;
|
|
4121
5385
|
}
|
|
4122
5386
|
if (!Array.isArray(parsed))
|
|
@@ -4203,6 +5467,8 @@ export class FactoryLoop {
|
|
|
4203
5467
|
this.#increment('githubIssueMirrorsCreated');
|
|
4204
5468
|
}
|
|
4205
5469
|
catch (error) {
|
|
5470
|
+
if (relayfileOverload(error))
|
|
5471
|
+
throw error;
|
|
4206
5472
|
this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
|
|
4207
5473
|
}
|
|
4208
5474
|
}
|
|
@@ -4520,7 +5786,7 @@ export class FactoryLoop {
|
|
|
4520
5786
|
// The pass-scoped flag also lets every later blocked issue reuse the
|
|
4521
5787
|
// resulting dependency index instead of rescanning the full tree.
|
|
4522
5788
|
this.#dependencyLinearTreeLoaded = true;
|
|
4523
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery')) {
|
|
5789
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
|
|
4524
5790
|
if (isIssueFilePath(path))
|
|
4525
5791
|
await this.#readIssue(path);
|
|
4526
5792
|
}
|
|
@@ -4575,7 +5841,7 @@ export class FactoryLoop {
|
|
|
4575
5841
|
const pullRequest = await resolveIssuePrFromMount(this.#mount, this.#config, issue, {
|
|
4576
5842
|
allowLegacyGithubBranch: true,
|
|
4577
5843
|
repo,
|
|
4578
|
-
});
|
|
5844
|
+
}, (prefix) => this.#listRelayfileTree(prefix, 'dependency PR probe resolution'));
|
|
4579
5845
|
if (normalizePrState(pullRequest?.state) !== 'MERGED')
|
|
4580
5846
|
return false;
|
|
4581
5847
|
this.#terminalDependencyIdentities.add(identity);
|
|
@@ -4752,6 +6018,7 @@ export class FactoryLoop {
|
|
|
4752
6018
|
updatedAtMs,
|
|
4753
6019
|
registryPath,
|
|
4754
6020
|
eventListener: this.#eventListenerStatus(),
|
|
6021
|
+
readinessReconcile: this.#readinessReconcileStatus(),
|
|
4755
6022
|
};
|
|
4756
6023
|
await mkdir(dirname(path), { recursive: true });
|
|
4757
6024
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
@@ -4898,14 +6165,14 @@ export class FactoryLoop {
|
|
|
4898
6165
|
}
|
|
4899
6166
|
const pathsByKey = new Map();
|
|
4900
6167
|
const canonicalPathsByKey = new Map();
|
|
4901
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery')) {
|
|
6168
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
|
|
4902
6169
|
if (isIssueFilePath(path)) {
|
|
4903
6170
|
const key = keyFromPath(path);
|
|
4904
6171
|
canonicalPathsByKey.set(key, path);
|
|
4905
6172
|
pathsByKey.set(key, path);
|
|
4906
6173
|
}
|
|
4907
6174
|
}
|
|
4908
|
-
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery')) {
|
|
6175
|
+
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
|
|
4909
6176
|
if (isIssueAliasFilePath(path)) {
|
|
4910
6177
|
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
4911
6178
|
if (canonicalPath) {
|
|
@@ -4946,6 +6213,8 @@ export class FactoryLoop {
|
|
|
4946
6213
|
return resolvedIssue;
|
|
4947
6214
|
}
|
|
4948
6215
|
catch (error) {
|
|
6216
|
+
if (relayfileOverload(error))
|
|
6217
|
+
throw error;
|
|
4949
6218
|
if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
|
|
4950
6219
|
this.#increment('phantomSkipped');
|
|
4951
6220
|
this.#logger.debug?.('[factory] skipped missing issue file discovered from issue tree', { path });
|
|
@@ -5174,7 +6443,7 @@ export class FactoryLoop {
|
|
|
5174
6443
|
const updatedAtMs = this.#clock.now();
|
|
5175
6444
|
const agents = [];
|
|
5176
6445
|
const seenAgents = new Set();
|
|
5177
|
-
const appendAgent = async (issue, agentName, tracked) => {
|
|
6446
|
+
const appendAgent = async (issue, agentName, tracked, hold) => {
|
|
5178
6447
|
const key = registryHandoffKey(issue, agentName);
|
|
5179
6448
|
if (seenAgents.has(key)) {
|
|
5180
6449
|
return;
|
|
@@ -5189,6 +6458,7 @@ export class FactoryLoop {
|
|
|
5189
6458
|
}
|
|
5190
6459
|
}
|
|
5191
6460
|
const fleetTracked = this.#fleet.trackedAgents?.().get(agentName);
|
|
6461
|
+
const dispatchClaim = this.#dispatchClaimStatuses.get(issueKey(issue));
|
|
5192
6462
|
agents.push({
|
|
5193
6463
|
name: agentName,
|
|
5194
6464
|
role: tracked.spec.role,
|
|
@@ -5198,14 +6468,24 @@ export class FactoryLoop {
|
|
|
5198
6468
|
processes,
|
|
5199
6469
|
...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
|
|
5200
6470
|
...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
|
|
6471
|
+
...(dispatchClaim ? { dispatchClaim: { ...dispatchClaim } } : {}),
|
|
6472
|
+
...(hold?.heldSinceAtMs !== undefined ? {
|
|
6473
|
+
heldSinceAtMs: hold.heldSinceAtMs,
|
|
6474
|
+
holdDeadlineAtMs: hold.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs,
|
|
6475
|
+
waitingForTerminalState: this.#config.terminalState,
|
|
6476
|
+
...(hold.lifecyclePhase ? { lifecyclePhase: hold.lifecyclePhase } : {}),
|
|
6477
|
+
} : {}),
|
|
5201
6478
|
});
|
|
5202
6479
|
};
|
|
5203
6480
|
if (!empty) {
|
|
5204
6481
|
for (const record of (await this.#batch()).inFlight) {
|
|
5205
6482
|
if (record.dryRun)
|
|
5206
6483
|
continue;
|
|
6484
|
+
if (record.dispatchClaim) {
|
|
6485
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
6486
|
+
}
|
|
5207
6487
|
for (const [agentName, tracked] of record.agents) {
|
|
5208
|
-
await appendAgent(record.issue, agentName, tracked);
|
|
6488
|
+
await appendAgent(record.issue, agentName, tracked, record);
|
|
5209
6489
|
}
|
|
5210
6490
|
}
|
|
5211
6491
|
}
|
|
@@ -5227,6 +6507,7 @@ export class FactoryLoop {
|
|
|
5227
6507
|
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
5228
6508
|
const existing = record.agents.get(spec.name);
|
|
5229
6509
|
if (existing?.result) {
|
|
6510
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
5230
6511
|
return { name: existing.result?.name ?? spec.name };
|
|
5231
6512
|
}
|
|
5232
6513
|
if (!batch.shouldSpawn(record, invocationId)) {
|
|
@@ -5253,6 +6534,7 @@ export class FactoryLoop {
|
|
|
5253
6534
|
const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
|
|
5254
6535
|
if (rosterAgent) {
|
|
5255
6536
|
const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
|
|
6537
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
5256
6538
|
batch.recordSpawn(record, spec, invocationId, {
|
|
5257
6539
|
name: spec.name,
|
|
5258
6540
|
sessionRef: existing?.sessionRef ?? spec.sessionRef,
|
|
@@ -5262,6 +6544,7 @@ export class FactoryLoop {
|
|
|
5262
6544
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
5263
6545
|
throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
|
|
5264
6546
|
}
|
|
6547
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
5265
6548
|
const adopted = record.agents.get(spec.name);
|
|
5266
6549
|
if (adopted)
|
|
5267
6550
|
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
@@ -5294,10 +6577,12 @@ export class FactoryLoop {
|
|
|
5294
6577
|
: 'agent_spawn_failed',
|
|
5295
6578
|
});
|
|
5296
6579
|
}
|
|
6580
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
5297
6581
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
5298
6582
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
5299
6583
|
throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
|
|
5300
6584
|
}
|
|
6585
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
5301
6586
|
const spawned = record.agents.get(result.name);
|
|
5302
6587
|
if (spawned)
|
|
5303
6588
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
@@ -5739,10 +7024,25 @@ export class FactoryLoop {
|
|
|
5739
7024
|
}
|
|
5740
7025
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
5741
7026
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
7027
|
+
const expectedHeadRef = implementer.spec.branch;
|
|
7028
|
+
if (!expectedHeadRef) {
|
|
7029
|
+
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
7030
|
+
}
|
|
7031
|
+
const matchesCurrentConvention = factoryBranchBelongsToIssue(expectedHeadRef, record.issue.key);
|
|
7032
|
+
const matchesAuthorizedLegacyBranch = implementer.spec.existingPullRequestBranch === true &&
|
|
7033
|
+
/^\d+$/u.test(record.issue.key) &&
|
|
7034
|
+
expectedHeadRef.toLowerCase().startsWith(`${record.issue.key.toLowerCase()}-`);
|
|
7035
|
+
if (!matchesCurrentConvention && !matchesAuthorizedLegacyBranch) {
|
|
7036
|
+
throw new Error(`Refusing to publish ${record.issue.key}: branch ${expectedHeadRef} belongs to a different issue`);
|
|
7037
|
+
}
|
|
5742
7038
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
5743
7039
|
const cached = this.#publishedPullRequests.get(key);
|
|
5744
|
-
if (cached)
|
|
7040
|
+
if (cached) {
|
|
7041
|
+
if (cached.headRef !== expectedHeadRef) {
|
|
7042
|
+
throw new Error(`Refusing cached PR for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${cached.headRef}`);
|
|
7043
|
+
}
|
|
5745
7044
|
return cached;
|
|
7045
|
+
}
|
|
5746
7046
|
const { identity, publisher } = this.#githubPullRequestPublisher();
|
|
5747
7047
|
const remoteBranch = implementer.result?.locality === 'remote' && implementer.spec.branch
|
|
5748
7048
|
? implementer.spec.branch
|
|
@@ -5762,11 +7062,13 @@ export class FactoryLoop {
|
|
|
5762
7062
|
: undefined;
|
|
5763
7063
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
5764
7064
|
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
5765
|
-
|
|
5766
|
-
|
|
5767
|
-
|
|
7065
|
+
if (durableReceipt) {
|
|
7066
|
+
if (durableReceipt.headRef !== expectedHeadRef) {
|
|
7067
|
+
throw new Error(`Refusing durable PR receipt for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${durableReceipt.headRef}`);
|
|
7068
|
+
}
|
|
5768
7069
|
return durableReceipt;
|
|
5769
|
-
|
|
7070
|
+
}
|
|
7071
|
+
if (opts.reconcileExisting) {
|
|
5770
7072
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
5771
7073
|
if (existing) {
|
|
5772
7074
|
this.#publishedPullRequests.set(key, existing);
|
|
@@ -5784,6 +7086,7 @@ export class FactoryLoop {
|
|
|
5784
7086
|
const result = await publisher.publishPullRequest({
|
|
5785
7087
|
repo,
|
|
5786
7088
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
7089
|
+
expectedHeadRef,
|
|
5787
7090
|
baseRef,
|
|
5788
7091
|
title: `${issue.key}: ${issue.title}`,
|
|
5789
7092
|
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
@@ -5793,7 +7096,7 @@ export class FactoryLoop {
|
|
|
5793
7096
|
? result
|
|
5794
7097
|
: { ...result, author: identity };
|
|
5795
7098
|
if (published.repo.toLowerCase() !== repo.toLowerCase() ||
|
|
5796
|
-
published.headRef !==
|
|
7099
|
+
published.headRef !== expectedHeadRef ||
|
|
5797
7100
|
!Number.isInteger(published.number) ||
|
|
5798
7101
|
published.number <= 0 ||
|
|
5799
7102
|
!published.url) {
|
|
@@ -5894,9 +7197,11 @@ export class FactoryLoop {
|
|
|
5894
7197
|
for (const root of roots) {
|
|
5895
7198
|
let paths;
|
|
5896
7199
|
try {
|
|
5897
|
-
paths = await this.#
|
|
7200
|
+
paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
|
|
5898
7201
|
}
|
|
5899
|
-
catch {
|
|
7202
|
+
catch (error) {
|
|
7203
|
+
if (relayfileOverload(error))
|
|
7204
|
+
throw error;
|
|
5900
7205
|
continue;
|
|
5901
7206
|
}
|
|
5902
7207
|
for (const path of paths) {
|
|
@@ -6162,9 +7467,11 @@ export class FactoryLoop {
|
|
|
6162
7467
|
for (let attempt = 0; attempt < PUBLISHED_PR_CONFIRM_ATTEMPTS; attempt += 1) {
|
|
6163
7468
|
const paths = (await Promise.all(roots.map(async (root) => {
|
|
6164
7469
|
try {
|
|
6165
|
-
return await this.#
|
|
7470
|
+
return await this.#listRelayfileTree(root, 'published PR confirmation');
|
|
6166
7471
|
}
|
|
6167
|
-
catch {
|
|
7472
|
+
catch (error) {
|
|
7473
|
+
if (relayfileOverload(error))
|
|
7474
|
+
throw error;
|
|
6168
7475
|
return [];
|
|
6169
7476
|
}
|
|
6170
7477
|
}))).flat();
|
|
@@ -6234,7 +7541,10 @@ export class FactoryLoop {
|
|
|
6234
7541
|
try {
|
|
6235
7542
|
const thread = await this.#slackDispatchThreadFor(record);
|
|
6236
7543
|
if (thread && this.#slack) {
|
|
6237
|
-
await this.#
|
|
7544
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
7545
|
+
if (!issue)
|
|
7546
|
+
return;
|
|
7547
|
+
await this.#slack.reply(thread.threadId, `:warning: ${slackIssueSubject(issue, slackNotificationRepos(record.decision))}\nThe implementer exited without opening a PR after a retry; this needs a human look.`);
|
|
6238
7548
|
}
|
|
6239
7549
|
}
|
|
6240
7550
|
catch (error) {
|
|
@@ -6295,6 +7605,8 @@ export class FactoryLoop {
|
|
|
6295
7605
|
// record leaves the batch.
|
|
6296
7606
|
async #abandonStuckDispatch(record, reason) {
|
|
6297
7607
|
const key = issueKey(record.issue);
|
|
7608
|
+
const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
|
|
7609
|
+
const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
|
|
6298
7610
|
this.#abandonedDispatchReasons.set(key, reason);
|
|
6299
7611
|
if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
|
|
6300
7612
|
this.#increment('abandonedDispatchReleaseRetries');
|
|
@@ -6320,9 +7632,9 @@ export class FactoryLoop {
|
|
|
6320
7632
|
}
|
|
6321
7633
|
const agents = [...record.agents];
|
|
6322
7634
|
for (const [agentName, tracked] of agents) {
|
|
6323
|
-
if (tracked.spec.role === 'implementer')
|
|
7635
|
+
if (!heldPastDeadline && tracked.spec.role === 'implementer')
|
|
6324
7636
|
continue;
|
|
6325
|
-
this.#fleet.markAgentTerminal?.(agentName, `implementer-terminal:${reason}`);
|
|
7637
|
+
this.#fleet.markAgentTerminal?.(agentName, heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : `implementer-terminal:${reason}`);
|
|
6326
7638
|
}
|
|
6327
7639
|
const worktreeHandoffs = this.#dispatchFailureHandoffs(record, []);
|
|
6328
7640
|
let cleanupComplete = true;
|
|
@@ -6335,13 +7647,13 @@ export class FactoryLoop {
|
|
|
6335
7647
|
const worktreeAgentNames = new Set(worktreeHandoffs.map((handoff) => handoff.name));
|
|
6336
7648
|
const nonWorktreeAgents = agents.filter(([name]) => !worktreeAgentNames.has(name));
|
|
6337
7649
|
if (nonWorktreeAgents.length > 0) {
|
|
6338
|
-
const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents,
|
|
7650
|
+
const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, agentReleaseReason, 'completion');
|
|
6339
7651
|
cleanupComplete = failed.length === 0;
|
|
6340
7652
|
}
|
|
6341
|
-
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs) && cleanupComplete;
|
|
7653
|
+
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason) && cleanupComplete;
|
|
6342
7654
|
}
|
|
6343
7655
|
else if (agents.length > 0) {
|
|
6344
|
-
const failed = await this.#releaseAndTerminateAgents(agents,
|
|
7656
|
+
const failed = await this.#releaseAndTerminateAgents(agents, agentReleaseReason, 'completion');
|
|
6345
7657
|
cleanupComplete = failed.length === 0;
|
|
6346
7658
|
}
|
|
6347
7659
|
if (!cleanupComplete) {
|
|
@@ -6930,7 +8242,11 @@ export class FactoryLoop {
|
|
|
6930
8242
|
return await this.#postAgentQuestionToGithub(record, question, 'no Slack dispatch thread exists');
|
|
6931
8243
|
}
|
|
6932
8244
|
try {
|
|
6933
|
-
await this.#
|
|
8245
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
8246
|
+
if (!issue) {
|
|
8247
|
+
throw new Error(`Unable to describe Slack question notification for unreadable issue ${record.issue.key}`);
|
|
8248
|
+
}
|
|
8249
|
+
await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
|
|
6934
8250
|
this.#increment('agentQuestionsPostedToSlack');
|
|
6935
8251
|
this.#recordSlackWritebackSuccess('agent-question');
|
|
6936
8252
|
return true;
|
|
@@ -7032,10 +8348,13 @@ export class FactoryLoop {
|
|
|
7032
8348
|
return await this.#deliverClarificationQuestionToGithub(key, claimed, `Slack writeback is degraded${this.#slackDegradedReason ? `: ${this.#slackDegradedReason}` : ''}`);
|
|
7033
8349
|
}
|
|
7034
8350
|
try {
|
|
7035
|
-
|
|
8351
|
+
if (!claimedIssue) {
|
|
8352
|
+
throw new Error(`Unable to describe Slack question notification for unreadable issue ${claimed.issue.key}`);
|
|
8353
|
+
}
|
|
8354
|
+
await this.#slack.reply(claimed.threadId, agentQuestionSlackText(claimedIssue, {
|
|
7036
8355
|
agentName: claimed.askerName,
|
|
7037
8356
|
question: claimed.question,
|
|
7038
|
-
}, this.#config.slack.stakeholderUserIds));
|
|
8357
|
+
}, this.#config.slack.stakeholderUserIds, slackNotificationRepos(claimed.decision)));
|
|
7039
8358
|
const completed = await this.#state.completeClarificationQuestionDelivery(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
|
|
7040
8359
|
if (!completed) {
|
|
7041
8360
|
this.#increment('clarificationQuestionDeliveryOwnershipLost');
|
|
@@ -7233,7 +8552,11 @@ export class FactoryLoop {
|
|
|
7233
8552
|
return;
|
|
7234
8553
|
}
|
|
7235
8554
|
try {
|
|
7236
|
-
await this.#
|
|
8555
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
8556
|
+
if (!issue) {
|
|
8557
|
+
throw new Error(`Unable to describe Slack question mirror for unreadable issue ${record.issue.key}`);
|
|
8558
|
+
}
|
|
8559
|
+
await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
|
|
7237
8560
|
this.#increment('agentQuestionsMirroredToSlack');
|
|
7238
8561
|
this.#recordSlackWritebackSuccess('agent-question-mirror');
|
|
7239
8562
|
}
|
|
@@ -7415,7 +8738,7 @@ export class FactoryLoop {
|
|
|
7415
8738
|
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
7416
8739
|
]) {
|
|
7417
8740
|
try {
|
|
7418
|
-
for (const path of await this.#
|
|
8741
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub escalation marker listing')) {
|
|
7419
8742
|
const parts = githubIssueCommentPathParts(path);
|
|
7420
8743
|
if (parts &&
|
|
7421
8744
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7709,7 +9032,7 @@ export class FactoryLoop {
|
|
|
7709
9032
|
];
|
|
7710
9033
|
for (const prefix of prefixes) {
|
|
7711
9034
|
try {
|
|
7712
|
-
for (const path of await this.#
|
|
9035
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub issue comment replay listing')) {
|
|
7713
9036
|
const parts = githubIssueCommentPathParts(path);
|
|
7714
9037
|
if (parts &&
|
|
7715
9038
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7720,6 +9043,8 @@ export class FactoryLoop {
|
|
|
7720
9043
|
}
|
|
7721
9044
|
}
|
|
7722
9045
|
catch (error) {
|
|
9046
|
+
if (relayfileOverload(error))
|
|
9047
|
+
throw error;
|
|
7723
9048
|
this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
|
|
7724
9049
|
}
|
|
7725
9050
|
}
|
|
@@ -8326,7 +9651,7 @@ export class FactoryLoop {
|
|
|
8326
9651
|
prMetaShowsMerged(snapshot) &&
|
|
8327
9652
|
prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30) {
|
|
8328
9653
|
await this.#state.clearBabysitterSession(this.#workspaceId, persistedKey);
|
|
8329
|
-
await this.#advanceMergedPrToDone(snapshot, record);
|
|
9654
|
+
await this.#advanceMergedPrToDone(snapshot, session.repo, record);
|
|
8330
9655
|
this.#increment('babysitterOwnershipRestoreMerged');
|
|
8331
9656
|
this.#logger.info?.('[factory] completed restored lifecycle whose pull request was already merged', {
|
|
8332
9657
|
issue: session.issue.key,
|
|
@@ -8879,7 +10204,24 @@ export class FactoryLoop {
|
|
|
8879
10204
|
const owner = await this.#babysitterOwnerFor(`${event.owner}/${event.repo}`, target.prNumber);
|
|
8880
10205
|
if (!owner) {
|
|
8881
10206
|
this.#increment('babysitterEventsIgnoredUnownedPr');
|
|
8882
|
-
|
|
10207
|
+
// "Unowned" means no babysitter was ever registered for this PR — the
|
|
10208
|
+
// downstream symptom of a missed or failed PR-open. Warn once per PR so
|
|
10209
|
+
// it is visible at default level, then fall back to debug for the
|
|
10210
|
+
// repeats; #sweepOrphanedBabysitterPrs is what actually recovers it.
|
|
10211
|
+
const identity = githubPrIdentity(`${event.owner}/${event.repo}`, target.prNumber);
|
|
10212
|
+
const detail = { ...event, prNumber: target.prNumber };
|
|
10213
|
+
// The size cap gates the warn itself, not just the bookkeeping: an
|
|
10214
|
+
// unbounded set of identities must not become an unbounded warn stream.
|
|
10215
|
+
const firstForPr = Boolean(identity) &&
|
|
10216
|
+
!this.#babysitterUnownedPrWarned.has(identity) &&
|
|
10217
|
+
this.#babysitterUnownedPrWarned.size < BABYSITTER_UNOWNED_PR_WARN_LIMIT;
|
|
10218
|
+
if (firstForPr) {
|
|
10219
|
+
this.#babysitterUnownedPrWarned.add(identity);
|
|
10220
|
+
this.#logger.warn?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
10221
|
+
}
|
|
10222
|
+
else {
|
|
10223
|
+
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
10224
|
+
}
|
|
8883
10225
|
continue;
|
|
8884
10226
|
}
|
|
8885
10227
|
if (this.#mount.resourceSubscriptions &&
|
|
@@ -9333,19 +10675,92 @@ export class FactoryLoop {
|
|
|
9333
10675
|
// through the durable Relay lifecycle action; the orchestrator never runs `gh`. PR meta events here
|
|
9334
10676
|
// only (a) spawn the babysitter on open and (b) carry the latest open/draft/
|
|
9335
10677
|
// merged state used to guard the final transition.
|
|
10678
|
+
async #readPrSnapshotContent(path) {
|
|
10679
|
+
let lastError;
|
|
10680
|
+
for (let attempt = 1; attempt <= BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS; attempt += 1) {
|
|
10681
|
+
try {
|
|
10682
|
+
const { content } = await this.#mount.readFile(path);
|
|
10683
|
+
if (attempt > 1)
|
|
10684
|
+
this.#increment('babysitterPrSnapshotReadRetrySucceeded');
|
|
10685
|
+
return content;
|
|
10686
|
+
}
|
|
10687
|
+
catch (error) {
|
|
10688
|
+
lastError = error;
|
|
10689
|
+
if (attempt < BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS && !this.#stopping) {
|
|
10690
|
+
this.#increment('babysitterPrSnapshotReadRetries');
|
|
10691
|
+
await this.#clock.sleep(BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS * attempt);
|
|
10692
|
+
continue;
|
|
10693
|
+
}
|
|
10694
|
+
break;
|
|
10695
|
+
}
|
|
10696
|
+
}
|
|
10697
|
+
throw lastError;
|
|
10698
|
+
}
|
|
10699
|
+
#recordBabysitterPrSnapshotFailure(path, error) {
|
|
10700
|
+
const { errorMessage } = describeError(error);
|
|
10701
|
+
const existing = this.#babysitterPrSnapshotDeadLetters.get(path);
|
|
10702
|
+
const failures = (existing?.failures ?? 0) + 1;
|
|
10703
|
+
// Re-insert last so the eviction below stays insertion-ordered by recency.
|
|
10704
|
+
this.#babysitterPrSnapshotDeadLetters.delete(path);
|
|
10705
|
+
this.#babysitterPrSnapshotDeadLetters.set(path, {
|
|
10706
|
+
failures,
|
|
10707
|
+
lastErrorMessage: errorMessage,
|
|
10708
|
+
firstFailedAtMs: existing?.firstFailedAtMs ?? this.#clock.now(),
|
|
10709
|
+
});
|
|
10710
|
+
while (this.#babysitterPrSnapshotDeadLetters.size > BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT) {
|
|
10711
|
+
const oldest = this.#babysitterPrSnapshotDeadLetters.keys().next().value;
|
|
10712
|
+
if (oldest === undefined)
|
|
10713
|
+
break;
|
|
10714
|
+
this.#babysitterPrSnapshotDeadLetters.delete(oldest);
|
|
10715
|
+
this.#increment('babysitterPrSnapshotDeadLettersEvicted');
|
|
10716
|
+
}
|
|
10717
|
+
this.#increment('babysitterPrSnapshotReadFailures');
|
|
10718
|
+
const detail = {
|
|
10719
|
+
path,
|
|
10720
|
+
error: errorMessage,
|
|
10721
|
+
failures,
|
|
10722
|
+
attemptsPerTry: BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS,
|
|
10723
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
10724
|
+
};
|
|
10725
|
+
// A PR whose meta cannot be read is a PR with no shepherd, which is not
|
|
10726
|
+
// debug-level information. Escalate once the sweep retries also fail.
|
|
10727
|
+
if (failures >= BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER) {
|
|
10728
|
+
this.#increment('babysitterPrSnapshotReadFailuresEscalated');
|
|
10729
|
+
// The sweep keeps retrying for as long as the mount stays faulted, so the
|
|
10730
|
+
// escalation logs on the transition and then only periodically. The
|
|
10731
|
+
// counters carry the true volume.
|
|
10732
|
+
const sinceEscalation = failures - BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER;
|
|
10733
|
+
if (sinceEscalation % BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY === 0) {
|
|
10734
|
+
this.#logger.error?.('[factory] babysitter PR snapshot still unreadable after repeated retries', detail);
|
|
10735
|
+
}
|
|
10736
|
+
return;
|
|
10737
|
+
}
|
|
10738
|
+
this.#logger.warn?.('[factory] babysitter could not read PR snapshot', detail);
|
|
10739
|
+
}
|
|
9336
10740
|
async #handlePrChange(path) {
|
|
9337
10741
|
const parts = githubPullPathParts(path);
|
|
9338
10742
|
if (!parts) {
|
|
9339
10743
|
return;
|
|
9340
10744
|
}
|
|
9341
|
-
|
|
10745
|
+
// This read is the only gate between a PR opening and it getting a
|
|
10746
|
+
// babysitter, so it retries rather than treating one `fetch failed` as a
|
|
10747
|
+
// verdict. On exhaustion the path is dead-lettered for the reconcile sweep
|
|
10748
|
+
// (#sweepOrphanedBabysitterPrs) instead of being dropped.
|
|
10749
|
+
let content;
|
|
9342
10750
|
try {
|
|
9343
|
-
|
|
10751
|
+
content = await this.#readPrSnapshotContent(path);
|
|
9344
10752
|
}
|
|
9345
10753
|
catch (error) {
|
|
9346
|
-
this.#
|
|
10754
|
+
this.#recordBabysitterPrSnapshotFailure(path, error);
|
|
9347
10755
|
return;
|
|
9348
10756
|
}
|
|
10757
|
+
if (this.#babysitterPrSnapshotDeadLetters.delete(path)) {
|
|
10758
|
+
this.#increment('babysitterPrSnapshotDeadLettersRecovered');
|
|
10759
|
+
this.#logger.info?.('[factory] babysitter recovered a dead-lettered PR snapshot read', { path });
|
|
10760
|
+
}
|
|
10761
|
+
// parsePullSnapshot never throws; an undefined result is a structurally
|
|
10762
|
+
// mismatched payload, which a retry cannot fix.
|
|
10763
|
+
const snapshot = parsePullSnapshot(content, parts.number);
|
|
9349
10764
|
if (!snapshot) {
|
|
9350
10765
|
return;
|
|
9351
10766
|
}
|
|
@@ -9357,7 +10772,7 @@ export class FactoryLoop {
|
|
|
9357
10772
|
if (owned) {
|
|
9358
10773
|
if (prMetaShowsMerged(snapshot)) {
|
|
9359
10774
|
if (owned.record)
|
|
9360
|
-
await this.#advanceMergedPrToDone(snapshot, owned.record);
|
|
10775
|
+
await this.#advanceMergedPrToDone(snapshot, repo, owned.record);
|
|
9361
10776
|
else
|
|
9362
10777
|
await this.#cancelBabysitterWake(owned.key);
|
|
9363
10778
|
return;
|
|
@@ -9413,7 +10828,7 @@ export class FactoryLoop {
|
|
|
9413
10828
|
return;
|
|
9414
10829
|
}
|
|
9415
10830
|
if (prMetaShowsMerged(snapshot)) {
|
|
9416
|
-
await this.#advanceMergedPrToDone(snapshot, record);
|
|
10831
|
+
await this.#advanceMergedPrToDone(snapshot, repo, record);
|
|
9417
10832
|
return;
|
|
9418
10833
|
}
|
|
9419
10834
|
if (!this.#config.babysitter.enabled) {
|
|
@@ -9471,9 +10886,15 @@ export class FactoryLoop {
|
|
|
9471
10886
|
}
|
|
9472
10887
|
return best?.record;
|
|
9473
10888
|
}
|
|
9474
|
-
async #advanceMergedPrToDone(snapshot, record) {
|
|
10889
|
+
async #advanceMergedPrToDone(snapshot, repo, record) {
|
|
10890
|
+
const mergedPullRequest = { repo, number: snapshot.number, url: snapshot.url };
|
|
9475
10891
|
if (record) {
|
|
9476
|
-
await this.#completeIssue(record, {
|
|
10892
|
+
await this.#completeIssue(record, {
|
|
10893
|
+
targetState: 'done',
|
|
10894
|
+
runMergeGate: false,
|
|
10895
|
+
completionReason: 'pr-merged',
|
|
10896
|
+
mergedPullRequest,
|
|
10897
|
+
});
|
|
9477
10898
|
return;
|
|
9478
10899
|
}
|
|
9479
10900
|
const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
|
|
@@ -9511,11 +10932,13 @@ export class FactoryLoop {
|
|
|
9511
10932
|
const channel = await this.#slackChannelDir();
|
|
9512
10933
|
if (channel) {
|
|
9513
10934
|
const systemOfRecord = githubIssue ? 'GitHub issue closed' : 'Linear state set to Done';
|
|
10935
|
+
const subject = slackIssueSubject(issue, [repo]);
|
|
10936
|
+
const pullRequest = slackPullRequestLink(mergedPullRequest);
|
|
9514
10937
|
const root = await this.#slack.postThread({
|
|
9515
10938
|
channel,
|
|
9516
|
-
text: `${
|
|
10939
|
+
text: `${subject}\nPR merged · ${pullRequest} · ${systemOfRecord}`,
|
|
9517
10940
|
});
|
|
9518
|
-
await this.#slack.reply(root.threadId, `${
|
|
10941
|
+
await this.#slack.reply(root.threadId, `${subject}\n${systemOfRecord} · ${pullRequest}`);
|
|
9519
10942
|
this.#recordSlackWritebackSuccess('merge-done-thread');
|
|
9520
10943
|
}
|
|
9521
10944
|
}
|
|
@@ -9543,7 +10966,7 @@ export class FactoryLoop {
|
|
|
9543
10966
|
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
9544
10967
|
// over title/body references to avoid "related to AR-N" body false positives.
|
|
9545
10968
|
const githubSource = await this.#issueSource() === 'github';
|
|
9546
|
-
const paths = githubSource ? await this.#githubIssuePaths() : await this.#
|
|
10969
|
+
const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
|
|
9547
10970
|
for (const path of paths) {
|
|
9548
10971
|
if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
|
|
9549
10972
|
continue;
|
|
@@ -10009,10 +11432,12 @@ export class FactoryLoop {
|
|
|
10009
11432
|
const found = [];
|
|
10010
11433
|
for (const root of roots) {
|
|
10011
11434
|
try {
|
|
10012
|
-
const tree = await this.#
|
|
11435
|
+
const tree = await this.#listRelayfileTree(root, 'PR meta path discovery');
|
|
10013
11436
|
found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
|
|
10014
11437
|
}
|
|
10015
|
-
catch {
|
|
11438
|
+
catch (error) {
|
|
11439
|
+
if (relayfileOverload(error))
|
|
11440
|
+
throw error;
|
|
10016
11441
|
// try the next root
|
|
10017
11442
|
}
|
|
10018
11443
|
}
|
|
@@ -10119,20 +11544,26 @@ export class FactoryLoop {
|
|
|
10119
11544
|
}
|
|
10120
11545
|
if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
|
|
10121
11546
|
return;
|
|
10122
|
-
if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
11547
|
+
if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
10123
11548
|
try {
|
|
10124
11549
|
const channel = await this.#slackChannelDir();
|
|
10125
11550
|
if (channel) {
|
|
10126
11551
|
const merged = opts.completionReason === 'pr-merged';
|
|
10127
11552
|
const systemOfRecord = githubIssue ? 'GitHub status' : 'Linear state';
|
|
10128
|
-
const
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
const
|
|
10132
|
-
?
|
|
10133
|
-
:
|
|
10134
|
-
|
|
10135
|
-
|
|
11553
|
+
const pullRequests = await this.#slackPullRequestRefs(record, opts.mergedPullRequest ? [opts.mergedPullRequest] : []);
|
|
11554
|
+
const pullRequestLinks = pullRequests.map(slackPullRequestLink).join(' · ');
|
|
11555
|
+
const subject = slackIssueSubject(issue, slackNotificationRepos(record.decision));
|
|
11556
|
+
const stateResult = githubIssue && merged && !humanReview
|
|
11557
|
+
? 'GitHub issue closed'
|
|
11558
|
+
: `${systemOfRecord} set to ${statusLabel}`;
|
|
11559
|
+
const completionText = [
|
|
11560
|
+
subject,
|
|
11561
|
+
merged
|
|
11562
|
+
? `PR merged${pullRequestLinks ? ` · ${pullRequestLinks}` : ''} · ${stateResult}`
|
|
11563
|
+
: `Completed${humanReview ? ' · awaiting human review' : ''}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`,
|
|
11564
|
+
...(!merged ? [`Status: ${statusLabel} · Merge policy: ${this.#config.mergePolicy}`] : []),
|
|
11565
|
+
].join('\n');
|
|
11566
|
+
const stateText = `${subject}\n${stateResult}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`;
|
|
10136
11567
|
const root = await this.#slack.postThread({
|
|
10137
11568
|
channel,
|
|
10138
11569
|
text: completionText,
|
|
@@ -10205,6 +11636,124 @@ export class FactoryLoop {
|
|
|
10205
11636
|
}
|
|
10206
11637
|
}
|
|
10207
11638
|
}
|
|
11639
|
+
async #notifyTicketDispatch(decision, issue, record, result) {
|
|
11640
|
+
const hook = this.#config.hooks?.onTicketDispatch;
|
|
11641
|
+
if (!hook || result.dryRun)
|
|
11642
|
+
return;
|
|
11643
|
+
const agent = result.agents.find((candidate) => candidate.role === 'implementer')
|
|
11644
|
+
?? result.agents.find((candidate) => candidate.role === 'workflow')
|
|
11645
|
+
?? result.agents[0];
|
|
11646
|
+
if (!agent)
|
|
11647
|
+
return;
|
|
11648
|
+
if (!await this.#claimTicketDispatchNotification(record))
|
|
11649
|
+
return;
|
|
11650
|
+
const tracked = record.agents.get(agent.name);
|
|
11651
|
+
const payload = {
|
|
11652
|
+
eventType: 'ticket.dispatched',
|
|
11653
|
+
summary: `Ticket dispatched: ${issue.title} → ${agent.name}.`,
|
|
11654
|
+
issue: {
|
|
11655
|
+
id: issue.uuid,
|
|
11656
|
+
title: issue.title,
|
|
11657
|
+
url: dispatchIssueUrl(issue),
|
|
11658
|
+
},
|
|
11659
|
+
agent: {
|
|
11660
|
+
name: agent.name,
|
|
11661
|
+
sessionRef: tracked?.sessionRef ?? null,
|
|
11662
|
+
},
|
|
11663
|
+
sessionOwner: dispatchSessionOwner(decision) ?? null,
|
|
11664
|
+
timestamp: new Date(this.#clock.now()).toISOString(),
|
|
11665
|
+
};
|
|
11666
|
+
const text = ticketDispatchNotificationText(payload);
|
|
11667
|
+
for (const target of hook.notify) {
|
|
11668
|
+
if (target.surface === 'linear' && !target.commentOnIssue)
|
|
11669
|
+
continue;
|
|
11670
|
+
try {
|
|
11671
|
+
switch (target.surface) {
|
|
11672
|
+
case 'relay':
|
|
11673
|
+
await this.#fleet.sendMessage({
|
|
11674
|
+
to: `#${target.channel.replace(/^#/u, '')}`,
|
|
11675
|
+
text,
|
|
11676
|
+
});
|
|
11677
|
+
break;
|
|
11678
|
+
case 'slack':
|
|
11679
|
+
if (!target.channel && !target.dm) {
|
|
11680
|
+
throw new Error('Slack ticket-dispatch notification needs channel and/or dm');
|
|
11681
|
+
}
|
|
11682
|
+
await this.#ticketDispatchDelivery.slack({
|
|
11683
|
+
channel: target.channel,
|
|
11684
|
+
dm: target.dm,
|
|
11685
|
+
text,
|
|
11686
|
+
});
|
|
11687
|
+
break;
|
|
11688
|
+
case 'telegram':
|
|
11689
|
+
await this.#ticketDispatchDelivery.telegram({ chatId: target.chatId, text });
|
|
11690
|
+
break;
|
|
11691
|
+
case 'linear':
|
|
11692
|
+
if (isGithubIssue(issue)) {
|
|
11693
|
+
throw new Error('Linear ticket-dispatch comments require a Linear issue');
|
|
11694
|
+
}
|
|
11695
|
+
await this.#linear.postComment(issue, text);
|
|
11696
|
+
break;
|
|
11697
|
+
}
|
|
11698
|
+
this.#increment('ticketDispatchHookNotifications');
|
|
11699
|
+
}
|
|
11700
|
+
catch (error) {
|
|
11701
|
+
this.#increment('ticketDispatchHookFailures');
|
|
11702
|
+
this.#logger.warn?.('[factory] onTicketDispatch notification failed', {
|
|
11703
|
+
issue: issue.key,
|
|
11704
|
+
surface: target.surface,
|
|
11705
|
+
error: describeError(error).errorMessage,
|
|
11706
|
+
});
|
|
11707
|
+
}
|
|
11708
|
+
}
|
|
11709
|
+
}
|
|
11710
|
+
async #claimTicketDispatchNotification(record) {
|
|
11711
|
+
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
11712
|
+
return true;
|
|
11713
|
+
const key = issueKey(record.issue);
|
|
11714
|
+
return this.#serializeDispatchLifecyclePersistence(key, async () => {
|
|
11715
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
11716
|
+
if (epoch === undefined) {
|
|
11717
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11718
|
+
return false;
|
|
11719
|
+
}
|
|
11720
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
11721
|
+
if (!lifecycle) {
|
|
11722
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11723
|
+
return false;
|
|
11724
|
+
}
|
|
11725
|
+
const workUnitId = lifecycle.runId;
|
|
11726
|
+
if (lifecycle.ticketDispatchNotification?.workUnitId === workUnitId)
|
|
11727
|
+
return false;
|
|
11728
|
+
// Reserve the work unit before external side effects. Hook delivery is
|
|
11729
|
+
// best-effort per target, so takeover must prefer at-most-once fan-out
|
|
11730
|
+
// over replaying a notification whose provider acknowledgement was lost.
|
|
11731
|
+
// Follow-up for provider-idempotent retries: https://github.com/AgentWorkforce/factory/issues/239
|
|
11732
|
+
const claimedAtMs = this.#clock.now();
|
|
11733
|
+
const claimedLifecycle = {
|
|
11734
|
+
...lifecycle,
|
|
11735
|
+
ticketDispatchNotification: { workUnitId, claimedAtMs },
|
|
11736
|
+
updatedAtMs: claimedAtMs,
|
|
11737
|
+
};
|
|
11738
|
+
const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, claimedAtMs, claimedLifecycle);
|
|
11739
|
+
if (!saved) {
|
|
11740
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
11741
|
+
this.#increment('dispatchLifecycleFencesRejected');
|
|
11742
|
+
await this.#reportLifecycle(claimedLifecycle, 'factory.anomaly', {
|
|
11743
|
+
level: 'error',
|
|
11744
|
+
errorCode: 'fence_rejected',
|
|
11745
|
+
});
|
|
11746
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11747
|
+
return false;
|
|
11748
|
+
}
|
|
11749
|
+
return true;
|
|
11750
|
+
});
|
|
11751
|
+
}
|
|
11752
|
+
#ticketDispatchNotificationIsPending(lifecycle) {
|
|
11753
|
+
return Boolean(this.#config.hooks?.onTicketDispatch) &&
|
|
11754
|
+
!lifecycle.dryRun &&
|
|
11755
|
+
lifecycle.ticketDispatchNotification?.workUnitId !== lifecycle.runId;
|
|
11756
|
+
}
|
|
10208
11757
|
#error(error, issue) {
|
|
10209
11758
|
this.#increment('errors');
|
|
10210
11759
|
const details = describeError(error);
|
|
@@ -10490,7 +12039,30 @@ export class FactoryLoop {
|
|
|
10490
12039
|
this.#logger.warn?.('[factory] cleared invalid persisted Slack thread id', { issue: key });
|
|
10491
12040
|
return undefined;
|
|
10492
12041
|
}
|
|
10493
|
-
async #
|
|
12042
|
+
async #slackPullRequestRefs(record, additional = []) {
|
|
12043
|
+
const lifecycle = await this.#state
|
|
12044
|
+
.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
|
|
12045
|
+
.catch(() => undefined);
|
|
12046
|
+
const lifecycleRefs = publishedPullRequests(lifecycle).map((receipt) => ({
|
|
12047
|
+
repo: receipt.repo,
|
|
12048
|
+
number: receipt.number,
|
|
12049
|
+
url: receipt.url,
|
|
12050
|
+
}));
|
|
12051
|
+
const trackedRefs = [...record.agents.values()]
|
|
12052
|
+
.map((tracked) => tracked.spec.ownedPullRequest)
|
|
12053
|
+
.filter((ref) => Boolean(ref))
|
|
12054
|
+
.map((ref) => ({ repo: ref.repo, number: ref.number }));
|
|
12055
|
+
const babysitterRefs = [...this.#babysitterPr.entries()]
|
|
12056
|
+
.filter(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))
|
|
12057
|
+
.map(([, ref]) => ({ repo: ref.repo, number: ref.prNumber }));
|
|
12058
|
+
return uniqueSlackPullRequestRefs([
|
|
12059
|
+
...additional,
|
|
12060
|
+
...lifecycleRefs,
|
|
12061
|
+
...babysitterRefs,
|
|
12062
|
+
...trackedRefs,
|
|
12063
|
+
]);
|
|
12064
|
+
}
|
|
12065
|
+
async #ensureSlackDispatchThread(record, result, sourceIssue) {
|
|
10494
12066
|
if (!this.#slack || !this.#config.slack || result.dryRun) {
|
|
10495
12067
|
return;
|
|
10496
12068
|
}
|
|
@@ -10526,7 +12098,7 @@ export class FactoryLoop {
|
|
|
10526
12098
|
}
|
|
10527
12099
|
return;
|
|
10528
12100
|
}
|
|
10529
|
-
const start = this.#postAndWatchSlackDispatchThread(record, result);
|
|
12101
|
+
const start = this.#postAndWatchSlackDispatchThread(record, result, sourceIssue);
|
|
10530
12102
|
this.#slackWatcherStarts.set(key, start);
|
|
10531
12103
|
try {
|
|
10532
12104
|
await start;
|
|
@@ -10539,19 +12111,24 @@ export class FactoryLoop {
|
|
|
10539
12111
|
this.#slackWatcherStarts.delete(key);
|
|
10540
12112
|
}
|
|
10541
12113
|
}
|
|
10542
|
-
async #postAndWatchSlackDispatchThread(record, result) {
|
|
12114
|
+
async #postAndWatchSlackDispatchThread(record, result, sourceIssue) {
|
|
10543
12115
|
if (!this.#slack || !this.#config.slack) {
|
|
10544
12116
|
return;
|
|
10545
12117
|
}
|
|
10546
|
-
const
|
|
12118
|
+
const issue = sourceIssue ?? await this.#readIssue(record.issue.path);
|
|
12119
|
+
if (!issue) {
|
|
12120
|
+
throw new Error(`Unable to describe Slack dispatch notification for unreadable issue ${record.issue.key}`);
|
|
12121
|
+
}
|
|
12122
|
+
const previews = uniquePreviewReferences(result.previews ?? this.#previewReferences.get(issueKey(record.issue)) ?? []);
|
|
12123
|
+
const repos = slackNotificationRepos(record.decision);
|
|
10547
12124
|
const root = await this.#slack.postThread({
|
|
10548
12125
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
10549
12126
|
text: [
|
|
10550
|
-
|
|
10551
|
-
`
|
|
12127
|
+
slackIssueSubject(issue, repos),
|
|
12128
|
+
`Dispatched · ${result.agents.map((agent) => agent.name).join(', ') || 'no agents'} · Repos: ${slackRepoList(repos)}`,
|
|
10552
12129
|
...(previews.length > 0
|
|
10553
12130
|
? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
|
|
10554
|
-
: []),
|
|
12131
|
+
: [`State: ${result.stateId ?? 'dispatching'}`]),
|
|
10555
12132
|
].join('\n'),
|
|
10556
12133
|
});
|
|
10557
12134
|
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
@@ -10914,13 +12491,13 @@ export class FactoryLoop {
|
|
|
10914
12491
|
.filter((part) => Boolean(part))
|
|
10915
12492
|
.join(' ');
|
|
10916
12493
|
const replyInstruction = source?.url
|
|
10917
|
-
?
|
|
12494
|
+
? 'Reply on the linked GitHub issue so Factory can resume.'
|
|
10918
12495
|
: 'Reply on the source GitHub issue so Factory can resume.';
|
|
10919
12496
|
const root = await this.#slack.postThread({
|
|
10920
12497
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
10921
12498
|
text: [
|
|
10922
|
-
`${audience ? `${audience} ` : ''}${
|
|
10923
|
-
`Reason: ${reason}`,
|
|
12499
|
+
`${audience ? `${audience} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
|
|
12500
|
+
`Triage blocked · Reason: ${reason}`,
|
|
10924
12501
|
`Question: ${triageEscalationQuestion(decision, issue)} ${replyInstruction}`,
|
|
10925
12502
|
].join('\n'),
|
|
10926
12503
|
});
|
|
@@ -10999,9 +12576,11 @@ export class FactoryLoop {
|
|
|
10999
12576
|
async #findSlackUserIdByIdentity(identity) {
|
|
11000
12577
|
const list = async (prefix) => {
|
|
11001
12578
|
try {
|
|
11002
|
-
return await this.#
|
|
12579
|
+
return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
|
|
11003
12580
|
}
|
|
11004
|
-
catch {
|
|
12581
|
+
catch (error) {
|
|
12582
|
+
if (relayfileOverload(error))
|
|
12583
|
+
throw error;
|
|
11005
12584
|
return [];
|
|
11006
12585
|
}
|
|
11007
12586
|
};
|
|
@@ -11042,12 +12621,15 @@ export class FactoryLoop {
|
|
|
11042
12621
|
return;
|
|
11043
12622
|
}
|
|
11044
12623
|
const issue = await this.#readIssue(decision.issue.path);
|
|
12624
|
+
if (!issue) {
|
|
12625
|
+
throw new Error(`Unable to describe Slack triage escalation for unreadable issue ${decision.issue.key}`);
|
|
12626
|
+
}
|
|
11045
12627
|
const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
|
|
11046
12628
|
const root = await this.#slack.postThread({
|
|
11047
12629
|
channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
|
|
11048
12630
|
text: [
|
|
11049
|
-
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${
|
|
11050
|
-
`Reason: ${reason}`,
|
|
12631
|
+
`${stakeholderMentions ? `${stakeholderMentions} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
|
|
12632
|
+
`Triage blocked · Reason: ${reason}`,
|
|
11051
12633
|
`Question: ${triageEscalationQuestion(decision, issue)}`,
|
|
11052
12634
|
].join('\n'),
|
|
11053
12635
|
});
|
|
@@ -11390,7 +12972,11 @@ export class FactoryLoop {
|
|
|
11390
12972
|
waitingAgeMs,
|
|
11391
12973
|
});
|
|
11392
12974
|
try {
|
|
11393
|
-
await this.#
|
|
12975
|
+
const issue = await this.#readIssue(escalated.issue.path);
|
|
12976
|
+
if (!issue) {
|
|
12977
|
+
throw new Error(`Unable to describe stale Slack clarification for unreadable issue ${escalated.issue.key}`);
|
|
12978
|
+
}
|
|
12979
|
+
await this.#slack.reply(waiting.threadId, clarificationStaleSlackText(escalated, issue, this.#config.slack.stakeholderUserIds));
|
|
11394
12980
|
const completed = await this.#state.completeClarificationEscalation(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
|
|
11395
12981
|
if (!completed) {
|
|
11396
12982
|
this.#increment('clarificationEscalationOwnershipLost');
|
|
@@ -11654,6 +13240,9 @@ export class FactoryLoop {
|
|
|
11654
13240
|
this.#increment('clarificationWakesQueuedForCapacity');
|
|
11655
13241
|
return;
|
|
11656
13242
|
}
|
|
13243
|
+
// A human-answer wake starts a new agent-hold generation. Time spent
|
|
13244
|
+
// parked with the previous team released must not consume its deadline.
|
|
13245
|
+
record.heldSinceAtMs = undefined;
|
|
11657
13246
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
11658
13247
|
batch.complete(waiting.issue);
|
|
11659
13248
|
await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner);
|
|
@@ -11683,8 +13272,10 @@ export class FactoryLoop {
|
|
|
11683
13272
|
}
|
|
11684
13273
|
: await this.#resumeOrColdStartClarificationAgent(parked.name, tracked, waiting);
|
|
11685
13274
|
const invocationId = batch.invocationIdFor(record.issue, tracked.spec);
|
|
13275
|
+
record.heldSinceAtMs ??= this.#clock.now();
|
|
11686
13276
|
batch.recordSpawn(record, tracked.spec, invocationId, result);
|
|
11687
13277
|
await this.#saveDispatchLifecycle(record, 'dispatching');
|
|
13278
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
11688
13279
|
const live = record.agents.get(result.name);
|
|
11689
13280
|
if (live) {
|
|
11690
13281
|
resumed.push([result.name, live]);
|
|
@@ -11978,7 +13569,7 @@ export class FactoryLoop {
|
|
|
11978
13569
|
}
|
|
11979
13570
|
let paths;
|
|
11980
13571
|
try {
|
|
11981
|
-
paths = await this.#
|
|
13572
|
+
paths = await this.#listRelayfileTree('/slack/channels', 'Slack channel resolution');
|
|
11982
13573
|
}
|
|
11983
13574
|
catch (error) {
|
|
11984
13575
|
this.#logger.warn?.('[factory] unable to resolve Slack channel from mount; using configured channel value', {
|
|
@@ -12487,6 +14078,101 @@ function dispatchSpecs(decision) {
|
|
|
12487
14078
|
}
|
|
12488
14079
|
return [...decision.implementers, decision.reviewer];
|
|
12489
14080
|
}
|
|
14081
|
+
function dispatchSessionOwner(decision) {
|
|
14082
|
+
for (const spec of dispatchSpecs(decision)) {
|
|
14083
|
+
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
14084
|
+
if (sessionOwner)
|
|
14085
|
+
return sessionOwner;
|
|
14086
|
+
}
|
|
14087
|
+
return undefined;
|
|
14088
|
+
}
|
|
14089
|
+
function dispatchIssueUrl(issue) {
|
|
14090
|
+
const payload = wrappedPayload(issue.raw);
|
|
14091
|
+
const source = asRecord(payload.source);
|
|
14092
|
+
return stringValue(payload.url)
|
|
14093
|
+
?? stringValue(payload.html_url)
|
|
14094
|
+
?? stringValue(source?.url)
|
|
14095
|
+
?? issue.path;
|
|
14096
|
+
}
|
|
14097
|
+
const SLACK_ISSUE_TITLE_MAX_LENGTH = 120;
|
|
14098
|
+
function slackEscapeText(value) {
|
|
14099
|
+
return value
|
|
14100
|
+
.replace(/&/gu, '&')
|
|
14101
|
+
.replace(/</gu, '<')
|
|
14102
|
+
.replace(/>/gu, '>');
|
|
14103
|
+
}
|
|
14104
|
+
function slackLinkUrl(value) {
|
|
14105
|
+
try {
|
|
14106
|
+
const url = new URL(value);
|
|
14107
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:')
|
|
14108
|
+
return undefined;
|
|
14109
|
+
return url.toString()
|
|
14110
|
+
.replace(/\|/gu, '%7C')
|
|
14111
|
+
.replace(/</gu, '%3C')
|
|
14112
|
+
.replace(/>/gu, '%3E');
|
|
14113
|
+
}
|
|
14114
|
+
catch {
|
|
14115
|
+
return undefined;
|
|
14116
|
+
}
|
|
14117
|
+
}
|
|
14118
|
+
function truncateSlackIssueTitle(title) {
|
|
14119
|
+
const normalized = title.replace(/\s+/gu, ' ').trim() || 'Untitled issue';
|
|
14120
|
+
const characters = Array.from(normalized);
|
|
14121
|
+
if (characters.length <= SLACK_ISSUE_TITLE_MAX_LENGTH)
|
|
14122
|
+
return normalized;
|
|
14123
|
+
return `${characters.slice(0, SLACK_ISSUE_TITLE_MAX_LENGTH - 1).join('').trimEnd()}…`;
|
|
14124
|
+
}
|
|
14125
|
+
function slackRepoName(repo) {
|
|
14126
|
+
const normalized = repo.trim().replace(/^\/+|\/+$/gu, '');
|
|
14127
|
+
return normalized.slice(normalized.lastIndexOf('/') + 1) || normalized || 'unknown repo';
|
|
14128
|
+
}
|
|
14129
|
+
function slackNotificationRepos(decision) {
|
|
14130
|
+
const repos = [
|
|
14131
|
+
...decision.routes.map((route) => route.repo),
|
|
14132
|
+
...decision.implementers.map((implementer) => implementer.repo),
|
|
14133
|
+
];
|
|
14134
|
+
return [...new Map(repos
|
|
14135
|
+
.filter((repo) => Boolean(repo.trim()))
|
|
14136
|
+
.map((repo) => [repo.trim().toLowerCase(), repo.trim()])).values()];
|
|
14137
|
+
}
|
|
14138
|
+
function slackRepoList(repos) {
|
|
14139
|
+
const names = [...new Set(repos.map(slackRepoName))];
|
|
14140
|
+
return slackEscapeText(names.join(', ') || 'unknown repo');
|
|
14141
|
+
}
|
|
14142
|
+
function slackIssueSubject(issue, repos = []) {
|
|
14143
|
+
const githubSource = githubIssueSourceRef(issue);
|
|
14144
|
+
const githubPath = githubIssuePathParts(issue.path);
|
|
14145
|
+
const githubIdentity = githubSource ?? githubPath;
|
|
14146
|
+
const payload = wrappedPayload(issue.raw);
|
|
14147
|
+
const source = asRecord(payload.source);
|
|
14148
|
+
const fallbackIssueUrl = githubIdentity
|
|
14149
|
+
? `https://github.com/${githubIdentity.owner}/${githubIdentity.repo}/issues/${githubIdentity.number}`
|
|
14150
|
+
: `https://linear.app/issue/${encodeURIComponent(issue.key)}`;
|
|
14151
|
+
const issueUrl = slackLinkUrl(githubSource?.url
|
|
14152
|
+
?? stringValue(payload.url)
|
|
14153
|
+
?? stringValue(payload.html_url)
|
|
14154
|
+
?? stringValue(source?.url)
|
|
14155
|
+
?? fallbackIssueUrl) ?? fallbackIssueUrl;
|
|
14156
|
+
const repoNames = [...new Set(repos.map(slackRepoName))];
|
|
14157
|
+
const label = githubIdentity
|
|
14158
|
+
? `${githubIdentity.repo}#${githubIdentity.number}`
|
|
14159
|
+
: `${repoNames.join(', ') || 'unknown repo'} · ${issue.key}`;
|
|
14160
|
+
return `<${issueUrl}|${slackEscapeText(label)}> — ${slackEscapeText(truncateSlackIssueTitle(issue.title))}`;
|
|
14161
|
+
}
|
|
14162
|
+
function slackPullRequestLink(pullRequest) {
|
|
14163
|
+
const fallbackUrl = `https://github.com/${pullRequest.repo}/pull/${pullRequest.number}`;
|
|
14164
|
+
const url = slackLinkUrl(pullRequest.url ?? fallbackUrl) ?? fallbackUrl;
|
|
14165
|
+
const label = `${slackRepoName(pullRequest.repo)}#${pullRequest.number}`;
|
|
14166
|
+
return `<${url}|${slackEscapeText(label)}>`;
|
|
14167
|
+
}
|
|
14168
|
+
function uniqueSlackPullRequestRefs(refs) {
|
|
14169
|
+
return [...new Map(refs
|
|
14170
|
+
.filter((ref) => Boolean(ref.repo.trim()) && Number.isSafeInteger(ref.number) && ref.number > 0)
|
|
14171
|
+
.map((ref) => [`${ref.repo.trim().toLowerCase()}#${ref.number}`, { ...ref, repo: ref.repo.trim() }])).values()];
|
|
14172
|
+
}
|
|
14173
|
+
function ticketDispatchNotificationText(payload) {
|
|
14174
|
+
return `${payload.summary}\n${JSON.stringify(payload)}`;
|
|
14175
|
+
}
|
|
12490
14176
|
function previewServiceForRepo(config, repo) {
|
|
12491
14177
|
const services = config.preview?.services;
|
|
12492
14178
|
if (!services)
|
|
@@ -12819,7 +14505,8 @@ function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
|
12819
14505
|
...(opts.isolateLocalWorktree && baseClonePath && branch ? { baseClonePath, clonePath } : {}),
|
|
12820
14506
|
// The same persisted lifecycle reuses this id after takeover, while a
|
|
12821
14507
|
// genuine reopen gets a new id and cannot replay an old placement ack.
|
|
12822
|
-
invocationId:
|
|
14508
|
+
invocationId: spec.invocationId ??
|
|
14509
|
+
`factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`,
|
|
12823
14510
|
};
|
|
12824
14511
|
return branch ? { ...lifecycleSpec, branch } : lifecycleSpec;
|
|
12825
14512
|
};
|
|
@@ -13226,17 +14913,19 @@ const linearIssueMirrorsGithubIssue = (issue, ghIssue) => {
|
|
|
13226
14913
|
.split(/\r?\n/u)
|
|
13227
14914
|
.some((line) => line.trim() === `${GITHUB_MIRROR_SOURCE_PREFIX}${ghIssue.url}`);
|
|
13228
14915
|
};
|
|
13229
|
-
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
14916
|
+
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix)) => {
|
|
13230
14917
|
const candidates = [];
|
|
13231
14918
|
const listErrors = [];
|
|
13232
14919
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
13233
14920
|
const paths = new Set();
|
|
13234
14921
|
for (const root of githubPullRoots(repo)) {
|
|
13235
14922
|
try {
|
|
13236
|
-
for (const path of await
|
|
14923
|
+
for (const path of await listTree(root))
|
|
13237
14924
|
paths.add(path);
|
|
13238
14925
|
}
|
|
13239
14926
|
catch (error) {
|
|
14927
|
+
if (relayfileOverload(error))
|
|
14928
|
+
throw error;
|
|
13240
14929
|
listErrors.push(error);
|
|
13241
14930
|
}
|
|
13242
14931
|
}
|
|
@@ -13503,8 +15192,7 @@ const issuePrMatchScore = (pr, issue, marker, opts = {}) => {
|
|
|
13503
15192
|
};
|
|
13504
15193
|
const hasTitlePrefix = (title, marker) => title === marker || title.startsWith(`${marker} `);
|
|
13505
15194
|
const factoryBranchMatchesIssue = (headRef, issueKey) => /^\d+$/u.test(issueKey)
|
|
13506
|
-
? headRef
|
|
13507
|
-
headRef.toLowerCase().startsWith(`factory/${issueKey.toLowerCase()}-`)
|
|
15195
|
+
? factoryBranchBelongsToIssue(headRef, issueKey)
|
|
13508
15196
|
: containsIssueKey(headRef, issueKey);
|
|
13509
15197
|
// Legacy Factory runs created GitHub-native branches as `<issue-number>-*`
|
|
13510
15198
|
// before the current `factory/<issue>-*` convention. Keep this matcher out of
|
|
@@ -14068,6 +15756,75 @@ const eventOccurredAtMs = (event) => {
|
|
|
14068
15756
|
const parsed = timestamp ? Date.parse(timestamp) : Number.NaN;
|
|
14069
15757
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
14070
15758
|
};
|
|
15759
|
+
const discoveryEventId = (event) => {
|
|
15760
|
+
const flat = asRecord(event) ?? {};
|
|
15761
|
+
const id = flat.id ?? flat.eventId ?? flat.event_id ?? flat.seq;
|
|
15762
|
+
return typeof id === 'string' || typeof id === 'number' ? String(id) : undefined;
|
|
15763
|
+
};
|
|
15764
|
+
const compareDiscoveryEvents = (left, right) => {
|
|
15765
|
+
const leftOccurredAt = eventOccurredAtMs(left);
|
|
15766
|
+
const rightOccurredAt = eventOccurredAtMs(right);
|
|
15767
|
+
if (leftOccurredAt !== undefined && rightOccurredAt !== undefined && leftOccurredAt !== rightOccurredAt) {
|
|
15768
|
+
return leftOccurredAt - rightOccurredAt;
|
|
15769
|
+
}
|
|
15770
|
+
const leftId = discoveryEventId(left) ?? '';
|
|
15771
|
+
const rightId = discoveryEventId(right) ?? '';
|
|
15772
|
+
const leftSequence = eventSequenceNumber(leftId);
|
|
15773
|
+
const rightSequence = eventSequenceNumber(rightId);
|
|
15774
|
+
if (leftSequence !== undefined && rightSequence !== undefined && leftSequence !== rightSequence) {
|
|
15775
|
+
return leftSequence - rightSequence;
|
|
15776
|
+
}
|
|
15777
|
+
return leftId.localeCompare(rightId);
|
|
15778
|
+
};
|
|
15779
|
+
const discoveryEventIsDeletion = (event) => {
|
|
15780
|
+
const flat = asRecord(event) ?? {};
|
|
15781
|
+
const summary = asRecord(flat.summary) ?? {};
|
|
15782
|
+
const type = stringValue(flat.type)?.toLowerCase();
|
|
15783
|
+
// filesystemEventToChangeEvent normalizes filesystem watcher events to
|
|
15784
|
+
// type: 'relayfile.changed' and stashes the real operation here, so a
|
|
15785
|
+
// deletion arrives as filesystemEventType: 'file.deleted' / 'dir.deleted'
|
|
15786
|
+
// with an outer type that does not itself say "deleted".
|
|
15787
|
+
const filesystemEventType = stringValue(flat.filesystemEventType)?.toLowerCase();
|
|
15788
|
+
const action = stringValue(flat.action)?.toLowerCase();
|
|
15789
|
+
const status = stringValue(summary.status)?.toLowerCase();
|
|
15790
|
+
const digest = stringValue(flat.digest)?.toLowerCase();
|
|
15791
|
+
return type === 'file.deleted' || type === 'dir.deleted' ||
|
|
15792
|
+
filesystemEventType === 'file.deleted' || filesystemEventType === 'dir.deleted' ||
|
|
15793
|
+
action === 'delete' || status === 'deleted' || digest?.startsWith('deleted:') === true;
|
|
15794
|
+
};
|
|
15795
|
+
const githubIssueIndexRepoRoots = (path) => {
|
|
15796
|
+
const slash = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15797
|
+
const flat = /^\/github\/repos\/([^/]+)__([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15798
|
+
const parts = slash ?? flat;
|
|
15799
|
+
if (!parts)
|
|
15800
|
+
return [];
|
|
15801
|
+
const [, owner, repo] = parts;
|
|
15802
|
+
return [
|
|
15803
|
+
`/github/repos/${owner}/${repo}/issues`,
|
|
15804
|
+
`/github/repos/${owner}__${repo}/issues`,
|
|
15805
|
+
];
|
|
15806
|
+
};
|
|
15807
|
+
const relayfileOverload = (error) => {
|
|
15808
|
+
const flat = asRecord(error) ?? {};
|
|
15809
|
+
const response = asRecord(flat.response) ?? {};
|
|
15810
|
+
const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
|
|
15811
|
+
const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
|
|
15812
|
+
const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
|
|
15813
|
+
const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
|
|
15814
|
+
if (status !== 429)
|
|
15815
|
+
return undefined;
|
|
15816
|
+
const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
|
|
15817
|
+
const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
|
|
15818
|
+
const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
|
|
15819
|
+
const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
|
|
15820
|
+
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
15821
|
+
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
15822
|
+
};
|
|
15823
|
+
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
15824
|
+
const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
|
|
15825
|
+
const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
|
|
15826
|
+
return Math.max(retryAfterMs, exponentialMs);
|
|
15827
|
+
};
|
|
14071
15828
|
const eventSequenceNumber = (eventId) => {
|
|
14072
15829
|
const whole = Number(eventId);
|
|
14073
15830
|
if (Number.isFinite(whole))
|
|
@@ -14183,7 +15940,8 @@ const eventIdentity = (event) => {
|
|
|
14183
15940
|
// subscription/poll handler) crash-loops. Returns undefined for such events so
|
|
14184
15941
|
// callers skip them and keep processing the well-formed ones.
|
|
14185
15942
|
export const changeEventPath = (event) => {
|
|
14186
|
-
const
|
|
15943
|
+
const candidate = event;
|
|
15944
|
+
const path = candidate?.resource?.path ?? candidate?.path;
|
|
14187
15945
|
return typeof path === 'string' && path ? path : undefined;
|
|
14188
15946
|
};
|
|
14189
15947
|
const describeError = (error) => {
|
|
@@ -14350,6 +16108,28 @@ const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') =>
|
|
|
14350
16108
|
});
|
|
14351
16109
|
const isTerminalDispatchLifecycle = (lifecycle) => lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned';
|
|
14352
16110
|
const isTerminalDispatchPhase = (phase) => phase === 'complete' || phase === 'abandoned';
|
|
16111
|
+
const heldAgentsForRecord = (record, nowMs, holdTimeoutMs, terminalState) => {
|
|
16112
|
+
if (record.dryRun || record.heldSinceAtMs === undefined)
|
|
16113
|
+
return [];
|
|
16114
|
+
const heldSinceAtMs = record.heldSinceAtMs;
|
|
16115
|
+
const holdDeadlineAtMs = heldSinceAtMs + holdTimeoutMs;
|
|
16116
|
+
const heldForMs = Math.max(0, nowMs - heldSinceAtMs);
|
|
16117
|
+
return [...record.agents]
|
|
16118
|
+
.filter(([, tracked]) => Boolean(tracked.result))
|
|
16119
|
+
.map(([name, tracked]) => ({
|
|
16120
|
+
name,
|
|
16121
|
+
role: tracked.spec.role,
|
|
16122
|
+
issue: { ...record.issue },
|
|
16123
|
+
...(record.lifecyclePhase ? { lifecyclePhase: record.lifecyclePhase } : {}),
|
|
16124
|
+
waitingForTerminalState: terminalState,
|
|
16125
|
+
heldSince: new Date(heldSinceAtMs).toISOString(),
|
|
16126
|
+
heldSinceAtMs,
|
|
16127
|
+
heldForMs,
|
|
16128
|
+
holdDeadline: new Date(holdDeadlineAtMs).toISOString(),
|
|
16129
|
+
holdDeadlineAtMs,
|
|
16130
|
+
pastDeadline: nowMs >= holdDeadlineAtMs,
|
|
16131
|
+
}));
|
|
16132
|
+
};
|
|
14353
16133
|
const costUsageGroupId = (runId, tracked) => JSON.stringify([runId, tracked.spec.invocationId ?? tracked.spec.name]);
|
|
14354
16134
|
const costEntryId = (groupId, model) => JSON.stringify([groupId, model]);
|
|
14355
16135
|
const publishedPullRequests = (lifecycle) => {
|
|
@@ -14390,10 +16170,12 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
14390
16170
|
agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
|
|
14391
16171
|
invocationIds: [...record.invocationIds],
|
|
14392
16172
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
16173
|
+
...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
|
|
14393
16174
|
...(pullRequests.length > 0 ? { pullRequests: pullRequests.map((receipt) => ({ ...receipt })) } : {}),
|
|
14394
16175
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
14395
16176
|
...(releaseReason ? { releaseReason } : {}),
|
|
14396
16177
|
...(cost ? { cost: structuredClone(cost) } : {}),
|
|
16178
|
+
...(record.heldSinceAtMs !== undefined ? { heldSinceAtMs: record.heldSinceAtMs } : {}),
|
|
14397
16179
|
updatedAtMs,
|
|
14398
16180
|
});
|
|
14399
16181
|
const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
@@ -14403,6 +16185,11 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
|
14403
16185
|
agents: new Map(lifecycle.agents.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)])),
|
|
14404
16186
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
14405
16187
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
16188
|
+
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
16189
|
+
heldSinceAtMs: lifecycle.heldSinceAtMs ?? (lifecycle.agents.some((agent) => agent.releasedAtMs === undefined)
|
|
16190
|
+
? lifecycle.updatedAtMs
|
|
16191
|
+
: undefined),
|
|
16192
|
+
lifecyclePhase: lifecycle.phase,
|
|
14406
16193
|
});
|
|
14407
16194
|
const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
|
|
14408
16195
|
issue: { ...lifecycle.issue },
|
|
@@ -14597,14 +16384,14 @@ const slackUserIdMatchingIdentity = (payload, identity) => {
|
|
|
14597
16384
|
? userId
|
|
14598
16385
|
: undefined;
|
|
14599
16386
|
};
|
|
14600
|
-
const agentQuestionSlackText = (issue, question, stakeholderUserIds = []) => [
|
|
14601
|
-
slackMentions(stakeholderUserIds),
|
|
14602
|
-
`${
|
|
16387
|
+
const agentQuestionSlackText = (issue, question, stakeholderUserIds = [], repos = []) => [
|
|
16388
|
+
`${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, repos)}`.trim(),
|
|
16389
|
+
`${question.agentName} needs input.`,
|
|
14603
16390
|
`Question: ${question.question}`,
|
|
14604
16391
|
].filter((line) => Boolean(line)).join('\n');
|
|
14605
|
-
const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
|
|
14606
|
-
slackMentions(stakeholderUserIds),
|
|
14607
|
-
|
|
16392
|
+
const clarificationStaleSlackText = (waiting, issue, stakeholderUserIds = []) => [
|
|
16393
|
+
`${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, slackNotificationRepos(waiting.decision))}`.trim(),
|
|
16394
|
+
'This issue has been parked for seven days without a reply.',
|
|
14608
16395
|
`Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
|
|
14609
16396
|
].filter((line) => Boolean(line)).join('\n');
|
|
14610
16397
|
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|