@agent-relay/factory 0.1.58 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/cli/fleet.d.ts +7 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +126 -10
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +777 -273
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +37 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/delivery/ticket-dispatch.d.ts +15 -0
- package/dist/delivery/ticket-dispatch.d.ts.map +1 -0
- package/dist/delivery/ticket-dispatch.js +20 -0
- package/dist/delivery/ticket-dispatch.js.map +1 -0
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +4 -0
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/issue-key-match.d.ts +6 -0
- package/dist/issue-key-match.d.ts.map +1 -1
- package/dist/issue-key-match.js +14 -0
- package/dist/issue-key-match.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +6 -3
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts +2 -1
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1473 -99
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +4 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +2 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +45 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +6 -1
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +119 -2
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +6 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +62 -0
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/triage/schema.d.ts +36 -0
- package/dist/triage/schema.d.ts.map +1 -1
- package/dist/triage/schema.js +2 -0
- package/dist/triage/schema.js.map +1 -1
- package/dist/types.d.ts +31 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version-info.d.ts +21 -0
- package/dist/version-info.d.ts.map +1 -0
- package/dist/version-info.js +146 -0
- package/dist/version-info.js.map +1 -0
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +12 -0
- package/dist/writeback/github.js.map +1 -1
- package/package.json +1 -1
|
@@ -8,7 +8,7 @@ import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
|
8
8
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
9
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
10
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
11
|
-
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
|
+
import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue } from '../issue-key-match.js';
|
|
12
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
13
13
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
14
14
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
@@ -28,6 +28,7 @@ import { findAgentProcessByName, readProcessIdentity } from './process-identity.
|
|
|
28
28
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
29
29
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
30
30
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
31
|
+
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
31
32
|
class ClarificationWakeLeaseLostError extends Error {
|
|
32
33
|
}
|
|
33
34
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -86,6 +87,29 @@ const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
|
86
87
|
'check_run.completed',
|
|
87
88
|
];
|
|
88
89
|
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
90
|
+
// The PR-open snapshot read is the single gate that decides whether a PR ever
|
|
91
|
+
// gets a babysitter, so a transient mount fetch failure must not be allowed to
|
|
92
|
+
// settle that question. Retry in-line, then dead-letter for the reconcile sweep.
|
|
93
|
+
const BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS = 3;
|
|
94
|
+
const BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS = 250;
|
|
95
|
+
// Escalate from warn to error once in-line retries plus this many sweep retries
|
|
96
|
+
// have all failed: the PR is durably unreadable, not momentarily unlucky.
|
|
97
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER = 3;
|
|
98
|
+
// Bound the dead-letter book so a sustained mount outage cannot grow it without
|
|
99
|
+
// limit. Oldest entries are evicted first; the reconcile sweep still adopts
|
|
100
|
+
// their PRs through the ownerless-record path.
|
|
101
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT = 256;
|
|
102
|
+
// Retries carry backoff sleeps, so a sweep drains a slice rather than the whole
|
|
103
|
+
// book; failed entries rotate to the end so successive sweeps cover all of it.
|
|
104
|
+
const BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP = 16;
|
|
105
|
+
// A durably faulted mount would otherwise log an error per path per sweep.
|
|
106
|
+
const BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY = 20;
|
|
107
|
+
// Adoption probes an issue's open PR through the (already gh-backed-off) probe
|
|
108
|
+
// resolver, so it runs on its own slower cadence than the completion sweep.
|
|
109
|
+
const BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS = 60_000;
|
|
110
|
+
// Warn once per PR identity that arrives unowned, then fall back to debug. The
|
|
111
|
+
// counter carries the true volume; the log only has to make it discoverable.
|
|
112
|
+
const BABYSITTER_UNOWNED_PR_WARN_LIMIT = 64;
|
|
89
113
|
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
90
114
|
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
91
115
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
@@ -111,6 +135,8 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
111
135
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
112
136
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
113
137
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
138
|
+
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
139
|
+
const DISPATCH_WRITEBACK_RETRY_MS = 250;
|
|
114
140
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
115
141
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
116
142
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
@@ -124,6 +150,10 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
|
|
|
124
150
|
const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
125
151
|
const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
|
|
126
152
|
const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
|
|
153
|
+
const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
|
|
154
|
+
const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
155
|
+
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
156
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
127
157
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
128
158
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
129
159
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -191,6 +221,7 @@ export class FactoryLoop {
|
|
|
191
221
|
#mount;
|
|
192
222
|
#states;
|
|
193
223
|
#fleet;
|
|
224
|
+
#ticketDispatchDelivery;
|
|
194
225
|
#triage;
|
|
195
226
|
#linear;
|
|
196
227
|
#github;
|
|
@@ -268,6 +299,7 @@ export class FactoryLoop {
|
|
|
268
299
|
#clarificationWakeRetryTimers = new Map();
|
|
269
300
|
#clarificationWakeOwner = `${process.pid}:${randomUUID()}`;
|
|
270
301
|
#dispatchLifecycleOwner = `${process.pid}:${randomUUID()}`;
|
|
302
|
+
#discoverySweepOwner = `${process.pid}:${randomUUID()}`;
|
|
271
303
|
#dispatchLifecycleEpochs = new Map();
|
|
272
304
|
#dispatchTerminalWaiters = new Map();
|
|
273
305
|
#dispatchLifecycleRetryTimers = new Map();
|
|
@@ -275,6 +307,7 @@ export class FactoryLoop {
|
|
|
275
307
|
#abandonedDispatchReasons = new Map();
|
|
276
308
|
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
277
309
|
#dispatchLifecycleOwnershipWaitLogged = new Set();
|
|
310
|
+
#dispatchClaimStatuses = new Map();
|
|
278
311
|
#localReleaseCheckpoints = new Map();
|
|
279
312
|
#dispatchLifecycleRenewTimer;
|
|
280
313
|
#clarificationSweepTimer;
|
|
@@ -305,6 +338,9 @@ export class FactoryLoop {
|
|
|
305
338
|
#liveHeartbeatRefresh;
|
|
306
339
|
#liveHeartbeatLastWriteMs = 0;
|
|
307
340
|
#stoppingHeartbeatRefreshActive = false;
|
|
341
|
+
#readinessReconcileTimer;
|
|
342
|
+
#readinessReconcileInFlight;
|
|
343
|
+
#readinessReconcileIntervalMs = 60_000;
|
|
308
344
|
#liveEventQueue = [];
|
|
309
345
|
#liveEventDrainScheduled = false;
|
|
310
346
|
#liveEventDrainActive = false;
|
|
@@ -335,6 +371,18 @@ export class FactoryLoop {
|
|
|
335
371
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
336
372
|
#babysitterPr = new Map();
|
|
337
373
|
#babysitterIssueRefs = new Map();
|
|
374
|
+
// PR meta paths whose snapshot read exhausted its in-line retries. Without
|
|
375
|
+
// this book a single failed read permanently orphans the PR: no babysitter is
|
|
376
|
+
// spawned, so no owner exists, so every later event for it is discarded. The
|
|
377
|
+
// reconcile sweep drains this and clears entries that recover.
|
|
378
|
+
#babysitterPrSnapshotDeadLetters = new Map();
|
|
379
|
+
// PR identities already warned about as unowned, so the routing log reports
|
|
380
|
+
// the condition at default level without repeating per event.
|
|
381
|
+
#babysitterUnownedPrWarned = new Set();
|
|
382
|
+
#babysitterOrphanSweepActive = false;
|
|
383
|
+
#babysitterOrphanSweepLastRunMs = 0;
|
|
384
|
+
#babysitterOrphanReportedFailures = 0;
|
|
385
|
+
#babysitterOrphanReportedUnowned = 0;
|
|
338
386
|
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
339
387
|
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
340
388
|
// owning babysitter.
|
|
@@ -367,6 +415,13 @@ export class FactoryLoop {
|
|
|
367
415
|
// undefined = readiness not yet probed; resolved lazily so the standalone
|
|
368
416
|
// runOnce() path (which skips #start) still ingests when the mount is present.
|
|
369
417
|
#githubIngestionEnabled;
|
|
418
|
+
#runOnceInFlight;
|
|
419
|
+
#runOnceInFlightDryRun;
|
|
420
|
+
#discoverySession;
|
|
421
|
+
#discoverySweepEpoch;
|
|
422
|
+
#discoverySweepRenewTimer;
|
|
423
|
+
#discoverySweepLeaseLost = false;
|
|
424
|
+
#discoveryOverloadError;
|
|
370
425
|
#resolvedIssueSource;
|
|
371
426
|
#integrationInstructions;
|
|
372
427
|
#integrationInstructionsRefresh;
|
|
@@ -384,6 +439,9 @@ export class FactoryLoop {
|
|
|
384
439
|
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
|
|
385
440
|
installFactoryDraftPredicate(this.#mount, config);
|
|
386
441
|
this.#fleet = ports.fleet;
|
|
442
|
+
this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
|
|
443
|
+
mountRoot: config.localMountRoot,
|
|
444
|
+
});
|
|
387
445
|
this.#triage = ports.triage ?? new TieredTriage(new HeuristicTriage());
|
|
388
446
|
this.#linear = ports.linear ?? MountLinearWriteback(ports.mount, {
|
|
389
447
|
safety: config.safety,
|
|
@@ -481,7 +539,7 @@ export class FactoryLoop {
|
|
|
481
539
|
}
|
|
482
540
|
},
|
|
483
541
|
listPaths: async (prefix) => {
|
|
484
|
-
return this.#
|
|
542
|
+
return this.#listRelayfileTree(prefix, 'integration descriptor discovery');
|
|
485
543
|
},
|
|
486
544
|
};
|
|
487
545
|
const descriptors = await deriveDescriptorsFromMount(reader);
|
|
@@ -674,6 +732,7 @@ export class FactoryLoop {
|
|
|
674
732
|
await this.#rearmSlackReplyWatchers();
|
|
675
733
|
await this.#drainReadyClarificationWake();
|
|
676
734
|
await this.#rearmGithubIssueCommentWatchers();
|
|
735
|
+
this.#scheduleReadinessReconcile();
|
|
677
736
|
this.#scheduleCompletionSweep(0);
|
|
678
737
|
return;
|
|
679
738
|
}
|
|
@@ -734,9 +793,13 @@ export class FactoryLoop {
|
|
|
734
793
|
if (this.#completionSweepTimer)
|
|
735
794
|
clearTimeout(this.#completionSweepTimer);
|
|
736
795
|
this.#completionSweepTimer = undefined;
|
|
796
|
+
if (this.#readinessReconcileTimer)
|
|
797
|
+
clearTimeout(this.#readinessReconcileTimer);
|
|
798
|
+
this.#readinessReconcileTimer = undefined;
|
|
737
799
|
if (this.#previewSweepTimer)
|
|
738
800
|
clearTimeout(this.#previewSweepTimer);
|
|
739
801
|
this.#previewSweepTimer = undefined;
|
|
802
|
+
await this.#readinessReconcileInFlight;
|
|
740
803
|
await this.#previewSweepInFlight;
|
|
741
804
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
742
805
|
try {
|
|
@@ -781,6 +844,9 @@ export class FactoryLoop {
|
|
|
781
844
|
this.#babysitterSubscriptionOwners.clear();
|
|
782
845
|
this.#babysitterReady.clear();
|
|
783
846
|
this.#babysitterCriticalAgents.clear();
|
|
847
|
+
this.#babysitterPrSnapshotDeadLetters.clear();
|
|
848
|
+
this.#babysitterUnownedPrWarned.clear();
|
|
849
|
+
this.#babysitterOrphanSweepLastRunMs = 0;
|
|
784
850
|
const subscription = this.#subscription;
|
|
785
851
|
this.#subscription = undefined;
|
|
786
852
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
@@ -875,6 +941,7 @@ export class FactoryLoop {
|
|
|
875
941
|
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
876
942
|
const options = this.#liveOptions(overrides);
|
|
877
943
|
this.#liveTransport = options.transport;
|
|
944
|
+
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
878
945
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
879
946
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
880
947
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1005,6 +1072,7 @@ export class FactoryLoop {
|
|
|
1005
1072
|
pollIntervalMs: overrides.pollIntervalMs ?? this.#config.liveSubscription.pollIntervalMs,
|
|
1006
1073
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
1007
1074
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1075
|
+
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1008
1076
|
};
|
|
1009
1077
|
}
|
|
1010
1078
|
async #currentEventCursor(limit) {
|
|
@@ -1031,6 +1099,48 @@ export class FactoryLoop {
|
|
|
1031
1099
|
};
|
|
1032
1100
|
}
|
|
1033
1101
|
}
|
|
1102
|
+
#scheduleReadinessReconcile(delayMs = this.#readinessReconcileIntervalMs) {
|
|
1103
|
+
if (!this.#started ||
|
|
1104
|
+
this.#stopping ||
|
|
1105
|
+
this.#readinessReconcileTimer ||
|
|
1106
|
+
this.#readinessReconcileInFlight)
|
|
1107
|
+
return;
|
|
1108
|
+
this.#readinessReconcileTimer = setTimeout(() => {
|
|
1109
|
+
this.#readinessReconcileTimer = undefined;
|
|
1110
|
+
if (!this.#started || this.#stopping)
|
|
1111
|
+
return;
|
|
1112
|
+
const sweep = this.#reconcileReadyIssues();
|
|
1113
|
+
this.#readinessReconcileInFlight = sweep;
|
|
1114
|
+
void sweep.finally(() => {
|
|
1115
|
+
if (this.#readinessReconcileInFlight === sweep) {
|
|
1116
|
+
this.#readinessReconcileInFlight = undefined;
|
|
1117
|
+
}
|
|
1118
|
+
if (this.#started && !this.#stopping)
|
|
1119
|
+
this.#scheduleReadinessReconcile();
|
|
1120
|
+
});
|
|
1121
|
+
}, delayMs);
|
|
1122
|
+
this.#readinessReconcileTimer.unref?.();
|
|
1123
|
+
}
|
|
1124
|
+
async #reconcileReadyIssues() {
|
|
1125
|
+
this.#increment('readinessReconcileSweeps');
|
|
1126
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1127
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1128
|
+
});
|
|
1129
|
+
try {
|
|
1130
|
+
const report = await this.runOnce();
|
|
1131
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1132
|
+
candidates: report.pulled.length,
|
|
1133
|
+
dispatched: report.dispatched.length,
|
|
1134
|
+
skipped: report.skipped.length,
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
catch (error) {
|
|
1138
|
+
this.#increment('readinessReconcileErrors');
|
|
1139
|
+
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1140
|
+
error: describeError(error).errorMessage,
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1034
1144
|
#scheduleLivePoll(delayMs, options) {
|
|
1035
1145
|
if (this.#livePollTimer || !this.#started)
|
|
1036
1146
|
return;
|
|
@@ -1340,6 +1450,10 @@ export class FactoryLoop {
|
|
|
1340
1450
|
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
1341
1451
|
// webhook-driven rather than polled.
|
|
1342
1452
|
if (this.#config.babysitter.enabled) {
|
|
1453
|
+
// Webhook-driven is not the same as one-shot: a PR-open event that was
|
|
1454
|
+
// missed or whose snapshot read failed leaves its PR with no shepherd
|
|
1455
|
+
// forever. This is the reconciliation that recovers those.
|
|
1456
|
+
await this.#sweepOrphanedBabysitterPrs(reason);
|
|
1343
1457
|
return;
|
|
1344
1458
|
}
|
|
1345
1459
|
if (this.#completionSweepActive) {
|
|
@@ -1401,6 +1515,207 @@ export class FactoryLoop {
|
|
|
1401
1515
|
this.#completionSweepActive = false;
|
|
1402
1516
|
}
|
|
1403
1517
|
}
|
|
1518
|
+
// Reconciliation for the babysitter's one-shot PR-open path. Two failure
|
|
1519
|
+
// modes converge on the same end state — an open Factory PR that no
|
|
1520
|
+
// babysitter owns, whose reviews, CI results and conflicts are then silently
|
|
1521
|
+
// discarded by #routeBabysitterEvent:
|
|
1522
|
+
// 1. the PR-open snapshot read failed and exhausted its retries, or
|
|
1523
|
+
// 2. the PR-open event never arrived at all (restart, drain loss, gap).
|
|
1524
|
+
// Neither is self-healing from events alone, so this sweep re-reads the
|
|
1525
|
+
// dead-lettered paths and adopts still-ownerless in-flight records.
|
|
1526
|
+
async #sweepOrphanedBabysitterPrs(reason = 'forced') {
|
|
1527
|
+
if (!this.#config.babysitter.enabled || this.#stopping || this.#babysitterOrphanSweepActive) {
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1530
|
+
const now = this.#clock.now();
|
|
1531
|
+
// Adoption probes every ownerless in-flight record, so it runs on the slow
|
|
1532
|
+
// cadence. A dead-lettered read is a PR we already know is unshepherded —
|
|
1533
|
+
// it retries at every sweep opportunity rather than waiting out that cadence.
|
|
1534
|
+
const adoptionDue = reason === 'forced' ||
|
|
1535
|
+
now - this.#babysitterOrphanSweepLastRunMs >= BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS;
|
|
1536
|
+
if (!adoptionDue && this.#babysitterPrSnapshotDeadLetters.size === 0) {
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
this.#babysitterOrphanSweepActive = true;
|
|
1540
|
+
try {
|
|
1541
|
+
this.#increment('babysitterOrphanSweepRuns');
|
|
1542
|
+
await this.#retryDeadLetteredPrSnapshots();
|
|
1543
|
+
if (adoptionDue) {
|
|
1544
|
+
this.#babysitterOrphanSweepLastRunMs = now;
|
|
1545
|
+
await this.#adoptOrphanedBabysitterPrs();
|
|
1546
|
+
}
|
|
1547
|
+
this.#reportBabysitterOrphanHealth();
|
|
1548
|
+
}
|
|
1549
|
+
finally {
|
|
1550
|
+
this.#babysitterOrphanSweepActive = false;
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
async #retryDeadLetteredPrSnapshots() {
|
|
1554
|
+
// Each retry costs up to BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS reads with
|
|
1555
|
+
// backoff sleeps between them, so drain a bounded slice per sweep rather
|
|
1556
|
+
// than stalling the sweep behind a full book during a mount outage. A
|
|
1557
|
+
// failed retry re-inserts its entry at the end, so successive sweeps
|
|
1558
|
+
// rotate through the whole book.
|
|
1559
|
+
const batch = [...this.#babysitterPrSnapshotDeadLetters.keys()].slice(0, BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP);
|
|
1560
|
+
if (this.#babysitterPrSnapshotDeadLetters.size > batch.length) {
|
|
1561
|
+
this.#logger.warn?.('[factory] babysitter dead-letter drain is rate limited', {
|
|
1562
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
1563
|
+
retryingThisSweep: batch.length,
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
for (const path of batch) {
|
|
1567
|
+
if (this.#stopping)
|
|
1568
|
+
return;
|
|
1569
|
+
this.#increment('babysitterPrSnapshotDeadLettersRetried');
|
|
1570
|
+
// #handlePrChange clears the entry itself on a successful read and
|
|
1571
|
+
// re-records the failure (bumping `failures`, escalating the log) if the
|
|
1572
|
+
// mount is still faulted, so the whole retry contract lives in one place.
|
|
1573
|
+
await this.#handlePrChange(path);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
async #adoptOrphanedBabysitterPrs() {
|
|
1577
|
+
const batch = await this.#batch();
|
|
1578
|
+
for (const record of batch.inFlight) {
|
|
1579
|
+
if (this.#stopping)
|
|
1580
|
+
return;
|
|
1581
|
+
if (record.dryRun || this.#hasBabysitterForIssue(record.issue))
|
|
1582
|
+
continue;
|
|
1583
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
1584
|
+
continue;
|
|
1585
|
+
this.#increment('babysitterOrphanCandidatesScanned');
|
|
1586
|
+
try {
|
|
1587
|
+
for (const path of await this.#orphanedPrMetaPaths(record)) {
|
|
1588
|
+
if (this.#stopping || this.#hasBabysitterForIssue(record.issue))
|
|
1589
|
+
break;
|
|
1590
|
+
// Replay the PR meta through the normal PR-open handler rather than
|
|
1591
|
+
// spawning directly. Adoption must be subject to exactly the same
|
|
1592
|
+
// ownership, weak-match and path/payload-identity guards as a live
|
|
1593
|
+
// PR-open event — a reconcile sweep is a second delivery of a missed
|
|
1594
|
+
// event, never a way around the checks that event would have faced.
|
|
1595
|
+
await this.#handlePrChange(path);
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
catch (error) {
|
|
1599
|
+
this.#increment('babysitterOrphanAdoptionErrors');
|
|
1600
|
+
this.#logger.warn?.('[factory] babysitter orphan adoption failed', {
|
|
1601
|
+
issue: record.issue.key,
|
|
1602
|
+
error: describeError(error).errorMessage,
|
|
1603
|
+
});
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
if (!this.#hasBabysitterForIssue(record.issue))
|
|
1607
|
+
continue;
|
|
1608
|
+
this.#increment('babysitterOrphanedPrsAdopted');
|
|
1609
|
+
this.#logger.warn?.('[factory] adopted an orphaned PR that no babysitter owned', {
|
|
1610
|
+
issue: record.issue.key,
|
|
1611
|
+
...this.#babysitterPrForIssue(record.issue),
|
|
1612
|
+
});
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
// Candidate PR meta paths for an ownerless record, newest PR first. Resolution
|
|
1616
|
+
// reuses the durable dispatch receipts and then the probe resolver (which
|
|
1617
|
+
// carries its own gh backoff, so a record whose implementer has not opened a
|
|
1618
|
+
// PR yet costs no more than a cached miss).
|
|
1619
|
+
async #orphanedPrMetaPaths(record) {
|
|
1620
|
+
const wanted = [];
|
|
1621
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
1622
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
1623
|
+
const receipts = lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : []);
|
|
1624
|
+
for (const receipt of receipts)
|
|
1625
|
+
wanted.push({ repo: receipt.repo, prNumber: receipt.number });
|
|
1626
|
+
}
|
|
1627
|
+
if (wanted.length === 0) {
|
|
1628
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1629
|
+
const pr = issue ? await this.#openPrForIssue(issue) : undefined;
|
|
1630
|
+
if (pr)
|
|
1631
|
+
wanted.push({ repo: pr.repo, prNumber: pr.prNumber });
|
|
1632
|
+
}
|
|
1633
|
+
const paths = [];
|
|
1634
|
+
for (const { repo, prNumber } of wanted) {
|
|
1635
|
+
for (const root of githubPullRoots(repo)) {
|
|
1636
|
+
let candidates;
|
|
1637
|
+
try {
|
|
1638
|
+
candidates = await this.#mount.listTree(root);
|
|
1639
|
+
}
|
|
1640
|
+
catch {
|
|
1641
|
+
continue;
|
|
1642
|
+
}
|
|
1643
|
+
const match = candidates.find((path) => {
|
|
1644
|
+
const parts = githubPullPathParts(path);
|
|
1645
|
+
return parts?.number === prNumber &&
|
|
1646
|
+
`${parts.owner}/${parts.repo}`.toLowerCase() === repo.toLowerCase();
|
|
1647
|
+
});
|
|
1648
|
+
if (match && !paths.includes(match)) {
|
|
1649
|
+
paths.push(match);
|
|
1650
|
+
break;
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
if (paths.length === 0 && wanted.length > 0) {
|
|
1655
|
+
// The PR exists but its meta has not materialized in the mount yet. Leave
|
|
1656
|
+
// it for the next sweep rather than spawning off an unvalidated receipt.
|
|
1657
|
+
this.#increment('babysitterOrphanPrMetaUnavailable');
|
|
1658
|
+
}
|
|
1659
|
+
return paths;
|
|
1660
|
+
}
|
|
1661
|
+
#hasBabysitterForIssue(issue) {
|
|
1662
|
+
return this.#babysitterPrForIssue(issue) !== undefined;
|
|
1663
|
+
}
|
|
1664
|
+
#babysitterPrForIssue(issue) {
|
|
1665
|
+
const wanted = issueKey(issue);
|
|
1666
|
+
for (const [key, ref] of this.#babysitterPr) {
|
|
1667
|
+
// Ownership keys are composite (issue + PR), so resolve the issue the same
|
|
1668
|
+
// way #babysitterOwnerFor does: the recorded ref, else the key itself for
|
|
1669
|
+
// the legacy issue-keyed form. Never fall back to the issue being tested,
|
|
1670
|
+
// which would make every entry match.
|
|
1671
|
+
const owningIssue = this.#babysitterIssueRefs.get(key);
|
|
1672
|
+
if ((owningIssue ? issueKey(owningIssue) : key) !== wanted)
|
|
1673
|
+
continue;
|
|
1674
|
+
// An empty agentName is the pre-spawn reservation #ensureBabysitter takes
|
|
1675
|
+
// before its first await; that is ownership in progress, not an orphan.
|
|
1676
|
+
return { repo: ref.repo, prNumber: ref.prNumber };
|
|
1677
|
+
}
|
|
1678
|
+
return undefined;
|
|
1679
|
+
}
|
|
1680
|
+
// The two counters that mark a PR losing its shepherd are otherwise only
|
|
1681
|
+
// visible by diffing `status().counters`. Report the deltas so the condition
|
|
1682
|
+
// reaches an operator reading logs at default level.
|
|
1683
|
+
#reportBabysitterOrphanHealth() {
|
|
1684
|
+
const readFailures = this.#counters.babysitterPrSnapshotReadFailures ?? 0;
|
|
1685
|
+
const unownedEvents = this.#counters.babysitterEventsIgnoredUnownedPr ?? 0;
|
|
1686
|
+
const newReadFailures = readFailures - this.#babysitterOrphanReportedFailures;
|
|
1687
|
+
const newUnownedEvents = unownedEvents - this.#babysitterOrphanReportedUnowned;
|
|
1688
|
+
const deadLettered = this.#babysitterPrSnapshotDeadLetters.size;
|
|
1689
|
+
this.#counters.babysitterPrSnapshotDeadLetterDepth = deadLettered;
|
|
1690
|
+
if (newReadFailures <= 0 && newUnownedEvents <= 0 && deadLettered === 0) {
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
this.#babysitterOrphanReportedFailures = readFailures;
|
|
1694
|
+
this.#babysitterOrphanReportedUnowned = unownedEvents;
|
|
1695
|
+
// The oldest stuck entry is what an operator needs to act on: it names the
|
|
1696
|
+
// PR and how long it has been unshepherded.
|
|
1697
|
+
let oldest;
|
|
1698
|
+
for (const [path, entry] of this.#babysitterPrSnapshotDeadLetters) {
|
|
1699
|
+
if (!oldest || entry.firstFailedAtMs < oldest.entry.firstFailedAtMs)
|
|
1700
|
+
oldest = { path, entry };
|
|
1701
|
+
}
|
|
1702
|
+
this.#logger.warn?.('[factory] babysitter PR routing is dropping work', {
|
|
1703
|
+
prSnapshotReadFailures: readFailures,
|
|
1704
|
+
newPrSnapshotReadFailures: Math.max(0, newReadFailures),
|
|
1705
|
+
unownedPrEventsIgnored: unownedEvents,
|
|
1706
|
+
newUnownedPrEventsIgnored: Math.max(0, newUnownedEvents),
|
|
1707
|
+
deadLetteredPrSnapshots: deadLettered,
|
|
1708
|
+
flatEventsUnreadable: this.#counters.babysitterFlatEventsUnreadable ?? 0,
|
|
1709
|
+
orphanedPrsAdopted: this.#counters.babysitterOrphanedPrsAdopted ?? 0,
|
|
1710
|
+
...(oldest
|
|
1711
|
+
? {
|
|
1712
|
+
oldestDeadLetterPath: oldest.path,
|
|
1713
|
+
oldestDeadLetterError: oldest.entry.lastErrorMessage,
|
|
1714
|
+
oldestDeadLetterAgeMs: Math.max(0, this.#clock.now() - oldest.entry.firstFailedAtMs),
|
|
1715
|
+
}
|
|
1716
|
+
: {}),
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1404
1719
|
async #completionPrForIssue(issue) {
|
|
1405
1720
|
if (this.#customProbePrResolver) {
|
|
1406
1721
|
return this.#probePrResolver(issue);
|
|
@@ -1426,7 +1741,7 @@ export class FactoryLoop {
|
|
|
1426
1741
|
if (cached && cached.expiresAtMs > now) {
|
|
1427
1742
|
return cached.pr;
|
|
1428
1743
|
}
|
|
1429
|
-
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts);
|
|
1744
|
+
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts, (prefix) => this.#listRelayfileTree(prefix, 'PR probe resolution'));
|
|
1430
1745
|
if (mountPr) {
|
|
1431
1746
|
return mountPr;
|
|
1432
1747
|
}
|
|
@@ -1449,6 +1764,128 @@ export class FactoryLoop {
|
|
|
1449
1764
|
return undefined;
|
|
1450
1765
|
}
|
|
1451
1766
|
async runOnce(opts = {}) {
|
|
1767
|
+
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1768
|
+
// Loop rather than a single check-then-wait: while this call was waiting
|
|
1769
|
+
// out a mismatched sweep, another mismatched caller may have raced it to
|
|
1770
|
+
// start the *next* sweep first — in which case that one might match us
|
|
1771
|
+
// and we should coalesce onto it rather than starting a third.
|
|
1772
|
+
for (;;) {
|
|
1773
|
+
if (this.#runOnceInFlight && this.#runOnceInFlightDryRun === dryRun) {
|
|
1774
|
+
this.#increment('discoverySweepsCoalesced');
|
|
1775
|
+
this.#logger.info?.('[factory] coalesced overlapping discovery request into the in-flight sweep');
|
|
1776
|
+
return await this.#runOnceInFlight;
|
|
1777
|
+
}
|
|
1778
|
+
// A mismatched dryRun cannot coalesce onto the in-flight sweep (it
|
|
1779
|
+
// would hand a live caller a dry-run report, or vice versa), but it
|
|
1780
|
+
// still must not race it — wait for it to settle before either
|
|
1781
|
+
// re-checking for a newly-started matching sweep or claiming the lease.
|
|
1782
|
+
if (!this.#runOnceInFlight)
|
|
1783
|
+
break;
|
|
1784
|
+
await this.#runOnceInFlight.catch(() => undefined);
|
|
1785
|
+
}
|
|
1786
|
+
const sweep = this.#runOnceWithDiscoveryFence({ dryRun });
|
|
1787
|
+
this.#runOnceInFlight = sweep;
|
|
1788
|
+
this.#runOnceInFlightDryRun = dryRun;
|
|
1789
|
+
try {
|
|
1790
|
+
return await sweep;
|
|
1791
|
+
}
|
|
1792
|
+
finally {
|
|
1793
|
+
if (this.#runOnceInFlight === sweep) {
|
|
1794
|
+
this.#runOnceInFlight = undefined;
|
|
1795
|
+
this.#runOnceInFlightDryRun = undefined;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
async #runOnceWithDiscoveryFence(opts) {
|
|
1800
|
+
let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1801
|
+
if (!claim.acquired && claim.reason === 'backoff') {
|
|
1802
|
+
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
1803
|
+
this.#increment('discoveryBackoffWaits');
|
|
1804
|
+
this.#logger.warn?.('[factory] discovery sweep waiting for Relayfile overload backoff', {
|
|
1805
|
+
delayMs,
|
|
1806
|
+
backoffUntilMs: claim.state.backoffUntilMs,
|
|
1807
|
+
consecutiveOverloads: claim.state.consecutiveOverloads,
|
|
1808
|
+
});
|
|
1809
|
+
await this.#clock.sleep(delayMs);
|
|
1810
|
+
claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1811
|
+
}
|
|
1812
|
+
if (!claim.acquired || !claim.lease) {
|
|
1813
|
+
this.#increment('discoverySweepsSkippedInFlight');
|
|
1814
|
+
this.#logger.info?.('[factory] skipped discovery because another process owns the sweep lease', {
|
|
1815
|
+
leaseUntilMs: claim.state.lease?.leaseUntilMs,
|
|
1816
|
+
});
|
|
1817
|
+
return {
|
|
1818
|
+
pulled: [],
|
|
1819
|
+
triaged: [],
|
|
1820
|
+
dispatched: [],
|
|
1821
|
+
skipped: [],
|
|
1822
|
+
dryRun: opts.dryRun ?? this.#config.dryRun,
|
|
1823
|
+
discoveryDeferred: 'sweep-in-flight',
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
this.#discoverySweepEpoch = claim.lease.epoch;
|
|
1827
|
+
this.#discoverySweepLeaseLost = false;
|
|
1828
|
+
this.#discoveryOverloadError = undefined;
|
|
1829
|
+
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
1830
|
+
let leaseReleased = false;
|
|
1831
|
+
try {
|
|
1832
|
+
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
1833
|
+
const report = await this.#performRunOnce(opts);
|
|
1834
|
+
if (this.#discoveryOverloadError)
|
|
1835
|
+
throw this.#discoveryOverloadError;
|
|
1836
|
+
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
1837
|
+
if (this.#discoverySweepLeaseLost) {
|
|
1838
|
+
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
1839
|
+
}
|
|
1840
|
+
const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
|
|
1841
|
+
leaseReleased = completed;
|
|
1842
|
+
if (!completed)
|
|
1843
|
+
throw new Error('discovery sweep lease was lost before completion');
|
|
1844
|
+
return report;
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
const overload = relayfileOverload(error);
|
|
1848
|
+
if (overload) {
|
|
1849
|
+
const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
|
|
1850
|
+
const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
|
|
1851
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
1852
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
1853
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
1854
|
+
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
1855
|
+
status: overload.status,
|
|
1856
|
+
reason: overload.reason,
|
|
1857
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
1858
|
+
delayMs,
|
|
1859
|
+
backoffUntilMs,
|
|
1860
|
+
consecutiveOverloads,
|
|
1861
|
+
});
|
|
1862
|
+
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
1863
|
+
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
1864
|
+
// here too would additionally block this failing call (and every
|
|
1865
|
+
// caller coalesced onto it) for up to the full backoff window while
|
|
1866
|
+
// still holding the coalescing slot.
|
|
1867
|
+
}
|
|
1868
|
+
throw error;
|
|
1869
|
+
}
|
|
1870
|
+
finally {
|
|
1871
|
+
if (this.#discoverySweepRenewTimer)
|
|
1872
|
+
clearInterval(this.#discoverySweepRenewTimer);
|
|
1873
|
+
this.#discoverySweepRenewTimer = undefined;
|
|
1874
|
+
this.#discoverySession = undefined;
|
|
1875
|
+
this.#discoverySweepEpoch = undefined;
|
|
1876
|
+
this.#discoveryOverloadError = undefined;
|
|
1877
|
+
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
1878
|
+
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
1879
|
+
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
1880
|
+
// merge-advance scan) throw "lease was lost" until the next sweep
|
|
1881
|
+
// happens to run and reset it at the top of this method.
|
|
1882
|
+
this.#discoverySweepLeaseLost = false;
|
|
1883
|
+
if (!leaseReleased) {
|
|
1884
|
+
await this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
async #performRunOnce(opts = {}) {
|
|
1452
1889
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1453
1890
|
const startedAtMs = this.#clock.now();
|
|
1454
1891
|
const relayfileWaitWarningsAtStart = this.#counters.relayfileOperationWaitWarnings ?? 0;
|
|
@@ -1479,6 +1916,14 @@ export class FactoryLoop {
|
|
|
1479
1916
|
const triaged = [];
|
|
1480
1917
|
const dispatched = [];
|
|
1481
1918
|
const skipped = [];
|
|
1919
|
+
const recordSkip = (entry) => {
|
|
1920
|
+
skipped.push(entry);
|
|
1921
|
+
this.#logger.info?.('[factory] readiness reconciliation skipped dispatch', {
|
|
1922
|
+
issue: entry.issue.key,
|
|
1923
|
+
path: entry.issue.path,
|
|
1924
|
+
reason: entry.reason,
|
|
1925
|
+
});
|
|
1926
|
+
};
|
|
1482
1927
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
1483
1928
|
let readyIssueReads = 0;
|
|
1484
1929
|
const issueEntries = [];
|
|
@@ -1523,7 +1968,6 @@ export class FactoryLoop {
|
|
|
1523
1968
|
const mayRecoverGithubOrphan = !wasReady &&
|
|
1524
1969
|
!dryRun &&
|
|
1525
1970
|
issueSource === 'github' &&
|
|
1526
|
-
Boolean(orphanRecovery) &&
|
|
1527
1971
|
Boolean(requiredLabel) &&
|
|
1528
1972
|
Boolean(labels?.has(requiredLabel)) &&
|
|
1529
1973
|
Boolean(labels?.has('factory:in-progress')) &&
|
|
@@ -1531,26 +1975,31 @@ export class FactoryLoop {
|
|
|
1531
1975
|
if (!mayRecoverGithubOrphan) {
|
|
1532
1976
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1533
1977
|
if (dispatchBlock) {
|
|
1534
|
-
|
|
1978
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1535
1979
|
continue;
|
|
1536
1980
|
}
|
|
1537
1981
|
}
|
|
1982
|
+
const orphanResult = mayRecoverGithubOrphan
|
|
1983
|
+
? await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun)
|
|
1984
|
+
: { recovered: false };
|
|
1985
|
+
const recoveredOrphan = orphanResult.recovered;
|
|
1538
1986
|
const batch = await this.#batch();
|
|
1539
1987
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
1540
|
-
|
|
1988
|
+
recordSkip({ issue: issueRef(issue), reason: orphanResult.reason ?? 'already tracked' });
|
|
1541
1989
|
continue;
|
|
1542
1990
|
}
|
|
1543
|
-
const recoveredOrphan = mayRecoverGithubOrphan &&
|
|
1544
|
-
await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun);
|
|
1545
1991
|
if (!wasReady && !recoveredOrphan) {
|
|
1546
1992
|
if (mayRecoverGithubOrphan) {
|
|
1547
1993
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1548
1994
|
if (dispatchBlock) {
|
|
1549
|
-
|
|
1995
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1550
1996
|
continue;
|
|
1551
1997
|
}
|
|
1552
1998
|
}
|
|
1553
|
-
|
|
1999
|
+
recordSkip({
|
|
2000
|
+
issue: issueRef(issue),
|
|
2001
|
+
reason: orphanResult.reason ?? 'live state is not ready-for-agent',
|
|
2002
|
+
});
|
|
1554
2003
|
continue;
|
|
1555
2004
|
}
|
|
1556
2005
|
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
@@ -1558,16 +2007,16 @@ export class FactoryLoop {
|
|
|
1558
2007
|
if (recoveredOrphan) {
|
|
1559
2008
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1560
2009
|
if (dispatchBlock) {
|
|
1561
|
-
|
|
2010
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1562
2011
|
continue;
|
|
1563
2012
|
}
|
|
1564
2013
|
}
|
|
1565
2014
|
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
1566
|
-
|
|
2015
|
+
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
1567
2016
|
continue;
|
|
1568
2017
|
}
|
|
1569
2018
|
if (!isDispatchableIssue(issue)) {
|
|
1570
|
-
|
|
2019
|
+
recordSkip({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
|
|
1571
2020
|
continue;
|
|
1572
2021
|
}
|
|
1573
2022
|
const decision = await this.triageIssue(issue);
|
|
@@ -1579,7 +2028,7 @@ export class FactoryLoop {
|
|
|
1579
2028
|
catch (error) {
|
|
1580
2029
|
if (!(error instanceof LiveDispatchStateChangedError))
|
|
1581
2030
|
throw error;
|
|
1582
|
-
|
|
2031
|
+
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
1583
2032
|
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
1584
2033
|
issue: decision.issue.key,
|
|
1585
2034
|
});
|
|
@@ -1591,7 +2040,7 @@ export class FactoryLoop {
|
|
|
1591
2040
|
: result.hold?.kind === 'dependency'
|
|
1592
2041
|
? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
|
|
1593
2042
|
: 'queued or escalated';
|
|
1594
|
-
|
|
2043
|
+
recordSkip({ issue: decision.issue, reason });
|
|
1595
2044
|
}
|
|
1596
2045
|
else {
|
|
1597
2046
|
dispatched.push(result);
|
|
@@ -1630,6 +2079,215 @@ export class FactoryLoop {
|
|
|
1630
2079
|
}
|
|
1631
2080
|
}
|
|
1632
2081
|
}
|
|
2082
|
+
#startDiscoverySweepRenewal(epoch) {
|
|
2083
|
+
let renewalInFlight = false;
|
|
2084
|
+
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2085
|
+
if (renewalInFlight || this.#discoverySweepLeaseLost)
|
|
2086
|
+
return;
|
|
2087
|
+
renewalInFlight = true;
|
|
2088
|
+
void this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS).then((renewed) => {
|
|
2089
|
+
if (!renewed && this.#discoverySweepEpoch === epoch) {
|
|
2090
|
+
this.#discoverySweepLeaseLost = true;
|
|
2091
|
+
this.#increment('discoverySweepLeaseLosses');
|
|
2092
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests');
|
|
2093
|
+
}
|
|
2094
|
+
}).catch((error) => {
|
|
2095
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
|
|
2096
|
+
error: describeError(error).errorMessage,
|
|
2097
|
+
});
|
|
2098
|
+
}).finally(() => {
|
|
2099
|
+
renewalInFlight = false;
|
|
2100
|
+
});
|
|
2101
|
+
}, DISCOVERY_SWEEP_RENEW_MS);
|
|
2102
|
+
this.#discoverySweepRenewTimer.unref?.();
|
|
2103
|
+
}
|
|
2104
|
+
async #prepareDiscoverySession(claim) {
|
|
2105
|
+
const checkpoint = structuredClone(claim.state.checkpoint ?? {
|
|
2106
|
+
trees: {},
|
|
2107
|
+
updatedAtMs: 0,
|
|
2108
|
+
});
|
|
2109
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2110
|
+
if (!watermark.available) {
|
|
2111
|
+
this.#increment('discoveryIncrementalUnavailable');
|
|
2112
|
+
return {
|
|
2113
|
+
checkpoint,
|
|
2114
|
+
mode: 'full',
|
|
2115
|
+
highWatermarkAvailable: false,
|
|
2116
|
+
listedPrefixes: new Set(),
|
|
2117
|
+
};
|
|
2118
|
+
}
|
|
2119
|
+
if (claim.state.checkpoint) {
|
|
2120
|
+
if (watermark.highWatermark === checkpoint.highWatermark) {
|
|
2121
|
+
this.#increment('discoveryIncrementalUnchangedSweeps');
|
|
2122
|
+
this.#logger.info?.('[factory] discovery checkpoint unchanged; reusing cached issue trees', {
|
|
2123
|
+
highWatermark: watermark.highWatermark,
|
|
2124
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2125
|
+
});
|
|
2126
|
+
return {
|
|
2127
|
+
checkpoint,
|
|
2128
|
+
mode: 'incremental',
|
|
2129
|
+
highWatermarkAvailable: true,
|
|
2130
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2131
|
+
listedPrefixes: new Set(),
|
|
2132
|
+
};
|
|
2133
|
+
}
|
|
2134
|
+
const changes = await this.#discoveryEventsSince(checkpoint.highWatermark, watermark.highWatermark);
|
|
2135
|
+
if (changes) {
|
|
2136
|
+
this.#applyDiscoveryEvents(checkpoint, changes);
|
|
2137
|
+
this.#increment('discoveryIncrementalChangedSweeps');
|
|
2138
|
+
this.#logger.info?.('[factory] updated discovery checkpoint from Relayfile changes', {
|
|
2139
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2140
|
+
highWatermark: watermark.highWatermark,
|
|
2141
|
+
changes: changes.length,
|
|
2142
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2143
|
+
});
|
|
2144
|
+
return {
|
|
2145
|
+
checkpoint,
|
|
2146
|
+
mode: 'incremental',
|
|
2147
|
+
highWatermarkAvailable: true,
|
|
2148
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2149
|
+
listedPrefixes: new Set(),
|
|
2150
|
+
};
|
|
2151
|
+
}
|
|
2152
|
+
this.#increment('discoveryIncrementalGapFallbacks');
|
|
2153
|
+
this.#logger.warn?.('[factory] discovery change window did not cover the prior checkpoint; refreshing tree prefixes once', {
|
|
2154
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2155
|
+
highWatermark: watermark.highWatermark,
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2158
|
+
// A missing change window means no retained prefix can be trusted. Start
|
|
2159
|
+
// the fallback snapshot empty so prefixes not touched by the current
|
|
2160
|
+
// configuration cannot resurface later with stale membership.
|
|
2161
|
+
checkpoint.trees = {};
|
|
2162
|
+
return {
|
|
2163
|
+
checkpoint,
|
|
2164
|
+
mode: 'full',
|
|
2165
|
+
highWatermarkAvailable: true,
|
|
2166
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2167
|
+
listedPrefixes: new Set(),
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
async #finalizeDiscoveryCheckpoint() {
|
|
2171
|
+
const session = this.#discoverySession;
|
|
2172
|
+
if (!session?.highWatermarkAvailable)
|
|
2173
|
+
return undefined;
|
|
2174
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2175
|
+
if (!watermark.available)
|
|
2176
|
+
return undefined;
|
|
2177
|
+
if (watermark.highWatermark !== session.observedHighWatermark) {
|
|
2178
|
+
const changes = await this.#discoveryEventsSince(session.observedHighWatermark, watermark.highWatermark);
|
|
2179
|
+
if (!changes) {
|
|
2180
|
+
this.#increment('discoveryCheckpointCommitGaps');
|
|
2181
|
+
this.#logger.warn?.('[factory] discovery checkpoint not saved because its change window overflowed during the sweep');
|
|
2182
|
+
return undefined;
|
|
2183
|
+
}
|
|
2184
|
+
this.#applyDiscoveryEvents(session.checkpoint, changes);
|
|
2185
|
+
}
|
|
2186
|
+
session.checkpoint.highWatermark = watermark.highWatermark;
|
|
2187
|
+
session.checkpoint.updatedAtMs = this.#clock.now();
|
|
2188
|
+
for (const [prefix, paths] of Object.entries(session.checkpoint.trees)) {
|
|
2189
|
+
session.checkpoint.trees[prefix] = [...new Set(paths)].sort();
|
|
2190
|
+
}
|
|
2191
|
+
return structuredClone(session.checkpoint);
|
|
2192
|
+
}
|
|
2193
|
+
async #discoveryHighWatermark() {
|
|
2194
|
+
if (!this.#mount.getEventHighWatermark)
|
|
2195
|
+
return { available: false };
|
|
2196
|
+
try {
|
|
2197
|
+
const highWatermark = await this.#mount.getEventHighWatermark();
|
|
2198
|
+
// `undefined` also means that an older Relayfile client cannot expose a
|
|
2199
|
+
// watermark. Treat it as unavailable so a compatibility fallback never
|
|
2200
|
+
// turns into an indefinitely stale cache.
|
|
2201
|
+
return highWatermark === undefined
|
|
2202
|
+
? { available: false }
|
|
2203
|
+
: { available: true, highWatermark };
|
|
2204
|
+
}
|
|
2205
|
+
catch (error) {
|
|
2206
|
+
if (relayfileOverload(error))
|
|
2207
|
+
throw error;
|
|
2208
|
+
this.#increment('discoveryHighWatermarkFailures');
|
|
2209
|
+
this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
|
|
2210
|
+
error: describeError(error).errorMessage,
|
|
2211
|
+
});
|
|
2212
|
+
return { available: false };
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
async #discoveryEventsSince(previousHighWatermark, currentHighWatermark) {
|
|
2216
|
+
if (previousHighWatermark === currentHighWatermark)
|
|
2217
|
+
return [];
|
|
2218
|
+
if (!currentHighWatermark)
|
|
2219
|
+
return undefined;
|
|
2220
|
+
let events;
|
|
2221
|
+
try {
|
|
2222
|
+
const page = await this.#mount.getEvents({
|
|
2223
|
+
last: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2224
|
+
limit: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2225
|
+
});
|
|
2226
|
+
events = [...page.events].sort(compareDiscoveryEvents);
|
|
2227
|
+
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
if (relayfileOverload(error))
|
|
2230
|
+
throw error;
|
|
2231
|
+
this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
|
|
2232
|
+
error: describeError(error).errorMessage,
|
|
2233
|
+
});
|
|
2234
|
+
return undefined;
|
|
2235
|
+
}
|
|
2236
|
+
const currentIndex = events.findIndex((event) => discoveryEventId(event) === currentHighWatermark);
|
|
2237
|
+
if (currentIndex < 0)
|
|
2238
|
+
return undefined;
|
|
2239
|
+
if (previousHighWatermark === undefined) {
|
|
2240
|
+
return events.length < DISCOVERY_CHANGE_EVENT_LIMIT
|
|
2241
|
+
? events.slice(0, currentIndex + 1)
|
|
2242
|
+
: undefined;
|
|
2243
|
+
}
|
|
2244
|
+
const previousIndex = events.findIndex((event) => discoveryEventId(event) === previousHighWatermark);
|
|
2245
|
+
if (previousIndex >= 0 && previousIndex <= currentIndex) {
|
|
2246
|
+
return events.slice(previousIndex + 1, currentIndex + 1);
|
|
2247
|
+
}
|
|
2248
|
+
const previousSequence = eventSequenceNumber(previousHighWatermark);
|
|
2249
|
+
const currentSequence = eventSequenceNumber(currentHighWatermark);
|
|
2250
|
+
const sequenced = events.map((event) => ({ event, sequence: eventSequenceNumber(discoveryEventId(event) ?? '') }));
|
|
2251
|
+
if (previousSequence !== undefined &&
|
|
2252
|
+
currentSequence !== undefined &&
|
|
2253
|
+
sequenced.every((entry) => entry.sequence !== undefined) &&
|
|
2254
|
+
sequenced.some((entry) => entry.sequence <= previousSequence)) {
|
|
2255
|
+
return sequenced
|
|
2256
|
+
.filter((entry) => entry.sequence > previousSequence && entry.sequence <= currentSequence)
|
|
2257
|
+
.map((entry) => entry.event);
|
|
2258
|
+
}
|
|
2259
|
+
return undefined;
|
|
2260
|
+
}
|
|
2261
|
+
#applyDiscoveryEvents(checkpoint, events) {
|
|
2262
|
+
for (const event of events) {
|
|
2263
|
+
const path = changeEventPath(event);
|
|
2264
|
+
if (!path)
|
|
2265
|
+
continue;
|
|
2266
|
+
const deletion = discoveryEventIsDeletion(event);
|
|
2267
|
+
const indexRepoRoots = githubIssueIndexRepoRoots(path);
|
|
2268
|
+
for (const [prefix, cachedPaths] of Object.entries(checkpoint.trees)) {
|
|
2269
|
+
if (indexRepoRoots.some((root) => prefix === root || prefix.startsWith(`${root}/`))) {
|
|
2270
|
+
delete checkpoint.trees[prefix];
|
|
2271
|
+
continue;
|
|
2272
|
+
}
|
|
2273
|
+
// A plain startsWith would let `/…/issues-archive/…` match the
|
|
2274
|
+
// cached prefix `/…/issues`, folding an unrelated path into it.
|
|
2275
|
+
if (path !== prefix && !path.startsWith(prefix.endsWith('/') ? prefix : `${prefix}/`))
|
|
2276
|
+
continue;
|
|
2277
|
+
const paths = new Set(cachedPaths);
|
|
2278
|
+
if (deletion) {
|
|
2279
|
+
for (const candidate of paths) {
|
|
2280
|
+
if (candidate === path || candidate.startsWith(`${path}/`))
|
|
2281
|
+
paths.delete(candidate);
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
else {
|
|
2285
|
+
paths.add(path);
|
|
2286
|
+
}
|
|
2287
|
+
checkpoint.trees[prefix] = [...paths];
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
1633
2291
|
async #githubOrphanRecoveryContext() {
|
|
1634
2292
|
try {
|
|
1635
2293
|
const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
|
|
@@ -1641,18 +2299,42 @@ export class FactoryLoop {
|
|
|
1641
2299
|
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1642
2300
|
const activeIssueIdentities = new Set();
|
|
1643
2301
|
const legacyUnownedAgentsByIssue = new Map();
|
|
1644
|
-
|
|
1645
|
-
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1646
|
-
continue;
|
|
1647
|
-
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
1648
|
-
if (identity)
|
|
1649
|
-
activeIssueIdentities.add(identity);
|
|
1650
|
-
}
|
|
2302
|
+
const orphanedLifecycleClaimsByIssue = new Map();
|
|
1651
2303
|
for (const [, waiting] of waitingClarifications) {
|
|
1652
2304
|
const identity = githubIssueRefIdentity(waiting.issue);
|
|
1653
2305
|
if (identity)
|
|
1654
2306
|
activeIssueIdentities.add(identity);
|
|
1655
2307
|
}
|
|
2308
|
+
for (const [key, lifecycle] of lifecycles) {
|
|
2309
|
+
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
2310
|
+
if (!identity)
|
|
2311
|
+
continue;
|
|
2312
|
+
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
2313
|
+
if (!activeIssueIdentities.has(identity)) {
|
|
2314
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2315
|
+
}
|
|
2316
|
+
continue;
|
|
2317
|
+
}
|
|
2318
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2319
|
+
const hasLiveAgent = activeAgents.some((agent) => onlineAgents.has(agent.name));
|
|
2320
|
+
const exitRecoveryActive = activeAgents.some((agent) => this.#agentExitsInFlight.has(agent.name));
|
|
2321
|
+
const dispatchCallActive = this.#dispatchInFlight.has(issueKey(lifecycle.issue));
|
|
2322
|
+
// The provider's `factory:in-progress` transition happens immediately
|
|
2323
|
+
// before the durable lifecycle advances from dispatching to running.
|
|
2324
|
+
// A crash can therefore leave any nonterminal phase behind while the
|
|
2325
|
+
// external claim survives. The caller still verifies provider status,
|
|
2326
|
+
// open-PR absence, a second roster, and the lifecycle lease before it
|
|
2327
|
+
// releases this orphan-shaped row.
|
|
2328
|
+
if (!activeIssueIdentities.has(identity) &&
|
|
2329
|
+
!hasLiveAgent &&
|
|
2330
|
+
!exitRecoveryActive &&
|
|
2331
|
+
!dispatchCallActive) {
|
|
2332
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
activeIssueIdentities.add(identity);
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
1656
2338
|
for (const agent of registry?.agents ?? []) {
|
|
1657
2339
|
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1658
2340
|
continue;
|
|
@@ -1677,6 +2359,7 @@ export class FactoryLoop {
|
|
|
1677
2359
|
activeIssueIdentities,
|
|
1678
2360
|
onlineAgentNames: onlineAgents,
|
|
1679
2361
|
legacyUnownedAgentsByIssue,
|
|
2362
|
+
orphanedLifecycleClaimsByIssue,
|
|
1680
2363
|
};
|
|
1681
2364
|
}
|
|
1682
2365
|
catch (error) {
|
|
@@ -1688,15 +2371,19 @@ export class FactoryLoop {
|
|
|
1688
2371
|
}
|
|
1689
2372
|
}
|
|
1690
2373
|
async #reconcileOrphanedGithubInProgress(issue, context, dryRun) {
|
|
1691
|
-
if (dryRun
|
|
1692
|
-
return false;
|
|
2374
|
+
if (dryRun)
|
|
2375
|
+
return { recovered: false, reason: 'dry run does not release an in-progress claim' };
|
|
2376
|
+
if (!context)
|
|
2377
|
+
return { recovered: false, reason: 'orphan-recovery safety context is unavailable' };
|
|
2378
|
+
if (!isGithubIssue(issue))
|
|
2379
|
+
return { recovered: false, reason: 'issue is not GitHub-native' };
|
|
1693
2380
|
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
1694
2381
|
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
1695
2382
|
if (!required ||
|
|
1696
2383
|
!labels.has(required) ||
|
|
1697
2384
|
!labels.has('factory:in-progress') ||
|
|
1698
2385
|
labels.has('factory:human-review'))
|
|
1699
|
-
return false;
|
|
2386
|
+
return { recovered: false, reason: 'issue is not an orphan-recovery candidate' };
|
|
1700
2387
|
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1701
2388
|
const legacyUnownedAgents = identity
|
|
1702
2389
|
? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
|
|
@@ -1707,12 +2394,12 @@ export class FactoryLoop {
|
|
|
1707
2394
|
context.activeIssueIdentities.has(identity) ||
|
|
1708
2395
|
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
|
|
1709
2396
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1710
|
-
return false;
|
|
2397
|
+
return { recovered: false, reason: 'active dispatch claim or live agent still owns the issue' };
|
|
1711
2398
|
}
|
|
1712
2399
|
const getProviderStatus = this.#githubWriteback.getIssueStatus;
|
|
1713
2400
|
if (!getProviderStatus) {
|
|
1714
2401
|
this.#increment('githubOrphanRecoveryStatusLookupUnavailable');
|
|
1715
|
-
return false;
|
|
2402
|
+
return { recovered: false, reason: 'provider-authoritative issue status lookup is unavailable' };
|
|
1716
2403
|
}
|
|
1717
2404
|
let providerStatus;
|
|
1718
2405
|
try {
|
|
@@ -1724,11 +2411,16 @@ export class FactoryLoop {
|
|
|
1724
2411
|
issue: issue.key,
|
|
1725
2412
|
error: describeError(error).errorMessage,
|
|
1726
2413
|
});
|
|
1727
|
-
return false;
|
|
2414
|
+
return { recovered: false, reason: 'provider-authoritative issue status could not be verified' };
|
|
1728
2415
|
}
|
|
1729
2416
|
if (!providerStatus || providerStatus === 'human-review') {
|
|
1730
2417
|
this.#increment('githubOrphanRecoveriesBlockedProviderStatus');
|
|
1731
|
-
return
|
|
2418
|
+
return {
|
|
2419
|
+
recovered: false,
|
|
2420
|
+
reason: providerStatus === 'human-review'
|
|
2421
|
+
? 'provider-authoritative issue status is human-review'
|
|
2422
|
+
: 'provider-authoritative issue status is unavailable',
|
|
2423
|
+
};
|
|
1732
2424
|
}
|
|
1733
2425
|
let openPr;
|
|
1734
2426
|
try {
|
|
@@ -1740,7 +2432,7 @@ export class FactoryLoop {
|
|
|
1740
2432
|
issue: issue.key,
|
|
1741
2433
|
error: describeError(error).errorMessage,
|
|
1742
2434
|
});
|
|
1743
|
-
return false;
|
|
2435
|
+
return { recovered: false, reason: 'matching open-PR absence could not be verified' };
|
|
1744
2436
|
}
|
|
1745
2437
|
if (openPr) {
|
|
1746
2438
|
let adopted = false;
|
|
@@ -1764,7 +2456,12 @@ export class FactoryLoop {
|
|
|
1764
2456
|
repo: openPr.repo,
|
|
1765
2457
|
prNumber: openPr.prNumber,
|
|
1766
2458
|
});
|
|
1767
|
-
return
|
|
2459
|
+
return {
|
|
2460
|
+
recovered: false,
|
|
2461
|
+
reason: adopted
|
|
2462
|
+
? 'matching open PR was adopted'
|
|
2463
|
+
: 'matching open PR still owns the issue',
|
|
2464
|
+
};
|
|
1768
2465
|
}
|
|
1769
2466
|
// A pre-durable local Factory may have left live, registry-proven workers
|
|
1770
2467
|
// without a lifecycle record. They are safe to adopt only once their open
|
|
@@ -1772,7 +2469,11 @@ export class FactoryLoop {
|
|
|
1772
2469
|
// and workers instead of redispatching duplicate agents.
|
|
1773
2470
|
if (legacyUnownedAgents.length > 0) {
|
|
1774
2471
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1775
|
-
return false;
|
|
2472
|
+
return { recovered: false, reason: 'legacy registry-proven agents still own the issue' };
|
|
2473
|
+
}
|
|
2474
|
+
const lifecycleClaim = context.orphanedLifecycleClaimsByIssue.get(identity);
|
|
2475
|
+
if (lifecycleClaim && !await this.#releaseOrphanedGithubLifecycle(issue, identity, lifecycleClaim)) {
|
|
2476
|
+
return { recovered: false, reason: 'orphaned durable lifecycle claim could not be safely released' };
|
|
1776
2477
|
}
|
|
1777
2478
|
try {
|
|
1778
2479
|
if (providerStatus === 'in-progress') {
|
|
@@ -1781,14 +2482,25 @@ export class FactoryLoop {
|
|
|
1781
2482
|
// A crashed dispatch may leave its durable attempt marked in-flight even
|
|
1782
2483
|
// after every agent and lifecycle disappeared. Only clear that stale bit
|
|
1783
2484
|
// after all provider, agent, lifecycle, and open-PR safety checks pass.
|
|
1784
|
-
await this.#
|
|
2485
|
+
const attempt = await this.#state.getDispatchAttempts(this.#workspaceId, issueStateKey(issue));
|
|
2486
|
+
if (attempt?.terminal) {
|
|
2487
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issueStateKey(issue), {
|
|
2488
|
+
attempts: 0,
|
|
2489
|
+
inFlight: false,
|
|
2490
|
+
terminal: false,
|
|
2491
|
+
backoffUntilMs: 0,
|
|
2492
|
+
});
|
|
2493
|
+
}
|
|
2494
|
+
else {
|
|
2495
|
+
await this.#clearDispatchInFlight(issue);
|
|
2496
|
+
}
|
|
1785
2497
|
this.#reconciledGithubInProgress.add(identity);
|
|
1786
2498
|
this.#increment('githubOrphanedInProgressRecovered');
|
|
1787
2499
|
this.#logger.warn?.('[factory] recovered orphaned GitHub in-progress issue for redispatch', {
|
|
1788
2500
|
issue: issue.key,
|
|
1789
2501
|
path: issue.path,
|
|
1790
2502
|
});
|
|
1791
|
-
return true;
|
|
2503
|
+
return { recovered: true };
|
|
1792
2504
|
}
|
|
1793
2505
|
catch (error) {
|
|
1794
2506
|
this.#increment('githubOrphanRecoveryWritebackFailures');
|
|
@@ -1796,6 +2508,116 @@ export class FactoryLoop {
|
|
|
1796
2508
|
issue: issue.key,
|
|
1797
2509
|
error: describeError(error).errorMessage,
|
|
1798
2510
|
});
|
|
2511
|
+
return { recovered: false, reason: 'orphaned provider lifecycle status could not be cleared' };
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
async #releaseOrphanedGithubLifecycle(issue, identity, candidate) {
|
|
2515
|
+
let key = candidate.key;
|
|
2516
|
+
let lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2517
|
+
if (!lifecycle)
|
|
2518
|
+
return true;
|
|
2519
|
+
if (lifecycle.runId !== candidate.lifecycle.runId ||
|
|
2520
|
+
githubIssueRefIdentity(lifecycle.issue) !== identity) {
|
|
2521
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because its durable lifecycle changed', {
|
|
2522
|
+
issue: issue.key,
|
|
2523
|
+
lifecycleKey: key,
|
|
2524
|
+
});
|
|
2525
|
+
return false;
|
|
2526
|
+
}
|
|
2527
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2528
|
+
const roster = await this.#fleet.roster();
|
|
2529
|
+
const online = new Set(roster.agents.map((agent) => agent.name));
|
|
2530
|
+
if (activeAgents.some((agent) => online.has(agent.name)) ||
|
|
2531
|
+
roster.agents.some((agent) => githubAgentNameMatchesIssue(agent.name, issue))) {
|
|
2532
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
2533
|
+
return false;
|
|
2534
|
+
}
|
|
2535
|
+
let epoch;
|
|
2536
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
2537
|
+
epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
2538
|
+
if (epoch !== undefined) {
|
|
2539
|
+
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2540
|
+
if (!renewed) {
|
|
2541
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2542
|
+
return false;
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
else {
|
|
2546
|
+
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2547
|
+
if (!claim.acquired || !claim.lease) {
|
|
2548
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because another publisher still owns its lease', {
|
|
2549
|
+
issue: issue.key,
|
|
2550
|
+
owner: claim.lifecycle.lease?.owner,
|
|
2551
|
+
leaseUntilMs: claim.lifecycle.lease?.leaseUntilMs,
|
|
2552
|
+
});
|
|
2553
|
+
return false;
|
|
2554
|
+
}
|
|
2555
|
+
key = claim.key ?? key;
|
|
2556
|
+
lifecycle = claim.lifecycle;
|
|
2557
|
+
epoch = claim.lease.epoch;
|
|
2558
|
+
this.#dispatchLifecycleEpochs.set(key, epoch);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
try {
|
|
2562
|
+
for (const agent of lifecycle.agents.filter((entry) => entry.releasedAtMs === undefined)) {
|
|
2563
|
+
try {
|
|
2564
|
+
await this.#fleet.release(agent.name, 'orphaned-claim');
|
|
2565
|
+
}
|
|
2566
|
+
catch (error) {
|
|
2567
|
+
// The roster was checked twice before fencing this lifecycle, so a
|
|
2568
|
+
// missing/dead worker is the expected crash-recovery shape. Do not
|
|
2569
|
+
// let a control-plane "agent not found" response make the durable
|
|
2570
|
+
// claim immortal; a failed fresh spawn remains visible to the normal
|
|
2571
|
+
// dispatch retry machinery.
|
|
2572
|
+
this.#increment('githubOrphanedLifecycleAgentReleaseFailures');
|
|
2573
|
+
this.#logger.warn?.('[factory] dead claim agent release failed; clearing fenced lifecycle anyway', {
|
|
2574
|
+
issue: issue.key,
|
|
2575
|
+
agent: agent.name,
|
|
2576
|
+
error: describeError(error).errorMessage,
|
|
2577
|
+
});
|
|
2578
|
+
}
|
|
2579
|
+
this.#fleet.markAgentTerminal?.(agent.name, 'orphaned-claim');
|
|
2580
|
+
}
|
|
2581
|
+
if (epoch !== undefined && !await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS)) {
|
|
2582
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2583
|
+
return false;
|
|
2584
|
+
}
|
|
2585
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
2586
|
+
if (retryTimer)
|
|
2587
|
+
clearTimeout(retryTimer);
|
|
2588
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2589
|
+
const batch = await this.#batch();
|
|
2590
|
+
batch.abandon(lifecycle.issue);
|
|
2591
|
+
await this.#state.clearDispatchLifecycle(this.#workspaceId, key);
|
|
2592
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
|
|
2593
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2594
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
2595
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
2596
|
+
await this.#writeInFlightRegistry().catch((error) => {
|
|
2597
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
|
|
2598
|
+
issue: issue.key,
|
|
2599
|
+
error: describeError(error).errorMessage,
|
|
2600
|
+
});
|
|
2601
|
+
});
|
|
2602
|
+
this.#increment('githubOrphanedLifecycleClaimsReleased');
|
|
2603
|
+
this.#logger.warn?.('[factory] released orphaned GitHub dispatch claim with no live agents', {
|
|
2604
|
+
issue: issue.key,
|
|
2605
|
+
lifecycleKey: key,
|
|
2606
|
+
runId: lifecycle.runId,
|
|
2607
|
+
agents: activeAgents.map((agent) => agent.name),
|
|
2608
|
+
});
|
|
2609
|
+
return true;
|
|
2610
|
+
}
|
|
2611
|
+
catch (error) {
|
|
2612
|
+
if (epoch !== undefined) {
|
|
2613
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch).catch(() => undefined);
|
|
2614
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2615
|
+
}
|
|
2616
|
+
this.#logger.warn?.('[factory] failed to release orphaned GitHub dispatch claim; preserving it', {
|
|
2617
|
+
issue: issue.key,
|
|
2618
|
+
lifecycleKey: key,
|
|
2619
|
+
error: describeError(error).errorMessage,
|
|
2620
|
+
});
|
|
1799
2621
|
return false;
|
|
1800
2622
|
}
|
|
1801
2623
|
}
|
|
@@ -1954,13 +2776,60 @@ export class FactoryLoop {
|
|
|
1954
2776
|
this.#increment('githubOrphanedPullRequestsAdopted');
|
|
1955
2777
|
return true;
|
|
1956
2778
|
}
|
|
1957
|
-
|
|
1958
|
-
|
|
2779
|
+
// `cache` is opt-in and must stay limited to call sites that walk a full
|
|
2780
|
+
// repository/issue root to enumerate everything under it — the sweep this
|
|
2781
|
+
// whole feature exists to make incremental. A cached listing can lag the
|
|
2782
|
+
// live tree by up to one sweep, which is fine for enumeration but wrong
|
|
2783
|
+
// for a point lookup that needs the current state: a PR-confirmation poll
|
|
2784
|
+
// that expects a specific PR to appear, or an escalation/comment-replay
|
|
2785
|
+
// scan that must not miss a marker or reply that landed after the cache
|
|
2786
|
+
// was populated. Those callers must omit `cache` and pay for a fresh list.
|
|
2787
|
+
async #listRelayfileTree(prefix, phase, opts = {}) {
|
|
2788
|
+
if (this.#discoverySweepLeaseLost) {
|
|
2789
|
+
throw new Error('discovery sweep lease was lost; refusing another tree request');
|
|
2790
|
+
}
|
|
2791
|
+
const cached = opts.cache ? this.#cachedDiscoveryTree(prefix) : undefined;
|
|
2792
|
+
if (cached) {
|
|
2793
|
+
this.#increment('discoveryTreeCacheHits');
|
|
2794
|
+
this.#logger.debug?.('[factory] reused cached Relayfile tree', { phase, prefix, count: cached.length });
|
|
2795
|
+
return cached;
|
|
2796
|
+
}
|
|
2797
|
+
if (opts.cache)
|
|
2798
|
+
this.#increment('discoveryTreeCacheMisses');
|
|
2799
|
+
const paths = await this.#withRelayfileOperation('listTree', { phase, prefix }, () => this.#mount.listTree(prefix), {
|
|
1959
2800
|
count: (paths) => paths.length,
|
|
1960
2801
|
logFailure: true,
|
|
1961
2802
|
logStart: true,
|
|
1962
2803
|
logComplete: true,
|
|
1963
2804
|
});
|
|
2805
|
+
if (opts.cache)
|
|
2806
|
+
await this.#rememberDiscoveryTree(prefix, paths);
|
|
2807
|
+
return paths;
|
|
2808
|
+
}
|
|
2809
|
+
#cachedDiscoveryTree(prefix) {
|
|
2810
|
+
const session = this.#discoverySession;
|
|
2811
|
+
if (!session)
|
|
2812
|
+
return undefined;
|
|
2813
|
+
if (session.mode === 'full' && !session.listedPrefixes.has(prefix))
|
|
2814
|
+
return undefined;
|
|
2815
|
+
const paths = session.checkpoint.trees[prefix];
|
|
2816
|
+
return paths ? [...paths] : undefined;
|
|
2817
|
+
}
|
|
2818
|
+
async #rememberDiscoveryTree(prefix, paths) {
|
|
2819
|
+
const session = this.#discoverySession;
|
|
2820
|
+
if (!session)
|
|
2821
|
+
return;
|
|
2822
|
+
const uniquePaths = new Set();
|
|
2823
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
2824
|
+
uniquePaths.add(paths[index]);
|
|
2825
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
2826
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2827
|
+
if (index + 1 < paths.length)
|
|
2828
|
+
await liveEventYield();
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
session.checkpoint.trees[prefix] = [...uniquePaths].sort();
|
|
2832
|
+
session.listedPrefixes.add(prefix);
|
|
1964
2833
|
}
|
|
1965
2834
|
async #ensureRelayfileSubRoot(prefix, phase, opts) {
|
|
1966
2835
|
return this.#withRelayfileOperation('ensureSubRoot', { phase, prefix }, () => this.#mount.ensureSubRoot(prefix, opts), {
|
|
@@ -2013,6 +2882,9 @@ export class FactoryLoop {
|
|
|
2013
2882
|
return result;
|
|
2014
2883
|
}
|
|
2015
2884
|
catch (error) {
|
|
2885
|
+
if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
|
|
2886
|
+
this.#discoveryOverloadError ??= error;
|
|
2887
|
+
}
|
|
2016
2888
|
if (opts.logFailure || waitWarnings > 0) {
|
|
2017
2889
|
this.#increment('relayfileOperationFailures');
|
|
2018
2890
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
@@ -2235,7 +3107,10 @@ export class FactoryLoop {
|
|
|
2235
3107
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
2236
3108
|
// and concurrent issues can switch each other back to the base branch.
|
|
2237
3109
|
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
2238
|
-
|
|
3110
|
+
// Every live dispatch gets an issue-owned branch, including legacy local
|
|
3111
|
+
// fleets without a durable lifecycle or worktree manager. The publication
|
|
3112
|
+
// boundary uses this exact ref to reject a stale shared checkout.
|
|
3113
|
+
const lifecycleRunId = !dryRun ? randomUUID() : undefined;
|
|
2239
3114
|
if (lifecycleRunId) {
|
|
2240
3115
|
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
2241
3116
|
isolateLocalWorktree,
|
|
@@ -2370,6 +3245,13 @@ export class FactoryLoop {
|
|
|
2370
3245
|
}
|
|
2371
3246
|
agents.push({ name: spawned.name, role: spec.role });
|
|
2372
3247
|
}
|
|
3248
|
+
if (!dryRun) {
|
|
3249
|
+
record.dispatchClaim = {
|
|
3250
|
+
state: 'pending',
|
|
3251
|
+
updatedAtMs: this.#clock.now(),
|
|
3252
|
+
};
|
|
3253
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
3254
|
+
}
|
|
2373
3255
|
await this.#writeInFlightRegistry();
|
|
2374
3256
|
const comment = dispatchComment(dispatchDecision, agents);
|
|
2375
3257
|
let implementingStateId;
|
|
@@ -2378,19 +3260,7 @@ export class FactoryLoop {
|
|
|
2378
3260
|
if (!issue || !this.#isIssueReady(issue)) {
|
|
2379
3261
|
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
2380
3262
|
}
|
|
2381
|
-
|
|
2382
|
-
await this.#postIssueComment(issue, comment);
|
|
2383
|
-
}
|
|
2384
|
-
catch (error) {
|
|
2385
|
-
this.#logger.warn?.('[factory] comment writeback skipped', error);
|
|
2386
|
-
}
|
|
2387
|
-
if (isGithubIssue(issue)) {
|
|
2388
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
2389
|
-
}
|
|
2390
|
-
else {
|
|
2391
|
-
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
2392
|
-
await this.#linear.setState(issue, implementingStateId);
|
|
2393
|
-
}
|
|
3263
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
2394
3264
|
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
2395
3265
|
}
|
|
2396
3266
|
const result = {
|
|
@@ -2404,9 +3274,13 @@ export class FactoryLoop {
|
|
|
2404
3274
|
dryRun,
|
|
2405
3275
|
};
|
|
2406
3276
|
record.result = result;
|
|
2407
|
-
await this.#saveDispatchLifecycle(record, 'running')
|
|
3277
|
+
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3278
|
+
return result;
|
|
2408
3279
|
this.#increment('dispatched');
|
|
2409
3280
|
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
3281
|
+
if (!dryRun && this.#config.hooks?.onTicketDispatch) {
|
|
3282
|
+
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
3283
|
+
}
|
|
2410
3284
|
if (!dryRun) {
|
|
2411
3285
|
await this.#ensureSlackDispatchThread(record, result);
|
|
2412
3286
|
}
|
|
@@ -2501,8 +3375,27 @@ export class FactoryLoop {
|
|
|
2501
3375
|
}
|
|
2502
3376
|
status() {
|
|
2503
3377
|
const batch = this.#batchView;
|
|
3378
|
+
const inFlightDispatches = batch?.inFlight
|
|
3379
|
+
.filter((record) => !record.dryRun)
|
|
3380
|
+
.map((record) => ({
|
|
3381
|
+
issue: { ...record.issue },
|
|
3382
|
+
agents: [...record.agents].map(([name, tracked]) => ({
|
|
3383
|
+
name,
|
|
3384
|
+
role: tracked.spec.role,
|
|
3385
|
+
...(tracked.sessionRef ? { sessionRef: tracked.sessionRef } : {}),
|
|
3386
|
+
...(tracked.spec.invocationId ? { invocationId: tracked.spec.invocationId } : {}),
|
|
3387
|
+
...(tracked.result?.node ? { node: tracked.result.node } : {}),
|
|
3388
|
+
})),
|
|
3389
|
+
claim: {
|
|
3390
|
+
...(record.dispatchClaim ?? this.#dispatchClaimStatuses.get(issueKey(record.issue)) ?? {
|
|
3391
|
+
state: 'pending',
|
|
3392
|
+
updatedAtMs: this.#clock.now(),
|
|
3393
|
+
}),
|
|
3394
|
+
},
|
|
3395
|
+
})) ?? [];
|
|
2504
3396
|
return {
|
|
2505
3397
|
inFlight: batch?.inFlight.map((record) => record.issue) ?? [],
|
|
3398
|
+
...(inFlightDispatches.length > 0 ? { inFlightDispatches } : {}),
|
|
2506
3399
|
queued: batch?.queued.map((queued) => queued.issue) ?? [],
|
|
2507
3400
|
parked: batch?.parked.map((parked) => ({
|
|
2508
3401
|
issue: parked.issue,
|
|
@@ -2653,10 +3546,11 @@ export class FactoryLoop {
|
|
|
2653
3546
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2654
3547
|
continue;
|
|
2655
3548
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
3549
|
+
let liveIssue;
|
|
2656
3550
|
if (!durableRecord.dryRun &&
|
|
2657
3551
|
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2658
3552
|
claim.lifecycle.phase !== 'releasing') {
|
|
2659
|
-
|
|
3553
|
+
liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2660
3554
|
// A babysat Linear issue already at Done may have merged while this
|
|
2661
3555
|
// process was down. Let authoritative PR restoration drive the
|
|
2662
3556
|
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
@@ -2683,6 +3577,18 @@ export class FactoryLoop {
|
|
|
2683
3577
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2684
3578
|
? durableRecord
|
|
2685
3579
|
: batch.restore(durableRecord);
|
|
3580
|
+
if (claim.lifecycle.phase === 'running' &&
|
|
3581
|
+
this.#ticketDispatchNotificationIsPending(claim.lifecycle)) {
|
|
3582
|
+
if (!liveIssue) {
|
|
3583
|
+
this.#dispatchLifecycleEpochs.delete(claim.key ?? key);
|
|
3584
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
3585
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: issue is not currently readable`);
|
|
3586
|
+
}
|
|
3587
|
+
if (!restored.result) {
|
|
3588
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: dispatch result is missing`);
|
|
3589
|
+
}
|
|
3590
|
+
await this.#notifyTicketDispatch(restored.decision, liveIssue, restored, restored.result);
|
|
3591
|
+
}
|
|
2686
3592
|
if (claim.lifecycle.phase !== 'running')
|
|
2687
3593
|
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2688
3594
|
// Parking agents are cleanup-only. Hydrating them makes relay
|
|
@@ -3277,6 +4183,9 @@ export class FactoryLoop {
|
|
|
3277
4183
|
? this.#finalizeRunCost(record, lifecycleRunId)
|
|
3278
4184
|
: previous?.cost;
|
|
3279
4185
|
const lifecycle = lifecycleFromInFlightRecord(record, lifecycleRunId, phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason, cost);
|
|
4186
|
+
if (previous?.ticketDispatchNotification) {
|
|
4187
|
+
lifecycle.ticketDispatchNotification = structuredClone(previous.ticketDispatchNotification);
|
|
4188
|
+
}
|
|
3280
4189
|
for (const agent of lifecycle.agents) {
|
|
3281
4190
|
const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
|
|
3282
4191
|
if (previousAgent?.releasedAtMs !== undefined)
|
|
@@ -3473,6 +4382,15 @@ export class FactoryLoop {
|
|
|
3473
4382
|
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3474
4383
|
return;
|
|
3475
4384
|
}
|
|
4385
|
+
if (this.#ticketDispatchNotificationIsPending(lifecycle)) {
|
|
4386
|
+
if (!liveIssue) {
|
|
4387
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: issue is not currently readable`);
|
|
4388
|
+
}
|
|
4389
|
+
if (!record.result) {
|
|
4390
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: dispatch result is missing`);
|
|
4391
|
+
}
|
|
4392
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
4393
|
+
}
|
|
3476
4394
|
}
|
|
3477
4395
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
3478
4396
|
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
@@ -3649,18 +4567,22 @@ export class FactoryLoop {
|
|
|
3649
4567
|
const spawned = await this.#spawnAgent(record, spec, record.dryRun);
|
|
3650
4568
|
agents.push({ name: spawned.name, role: spec.role });
|
|
3651
4569
|
}
|
|
4570
|
+
const comment = dispatchComment(record.decision, agents);
|
|
4571
|
+
let implementingStateId;
|
|
4572
|
+
if (!record.dryRun) {
|
|
4573
|
+
record.dispatchClaim = {
|
|
4574
|
+
state: 'pending',
|
|
4575
|
+
updatedAtMs: this.#clock.now(),
|
|
4576
|
+
};
|
|
4577
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4578
|
+
}
|
|
3652
4579
|
await this.#writeInFlightRegistry();
|
|
3653
4580
|
if (!record.dryRun) {
|
|
3654
4581
|
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3655
4582
|
if (!issue)
|
|
3656
4583
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3657
4584
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
3658
|
-
|
|
3659
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
3660
|
-
}
|
|
3661
|
-
else {
|
|
3662
|
-
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3663
|
-
}
|
|
4585
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
3664
4586
|
}
|
|
3665
4587
|
const recoveredPreviews = uniquePreviewReferences([
|
|
3666
4588
|
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
@@ -3669,7 +4591,8 @@ export class FactoryLoop {
|
|
|
3669
4591
|
record.result ??= {
|
|
3670
4592
|
issue: record.issue,
|
|
3671
4593
|
agents,
|
|
3672
|
-
comments: [
|
|
4594
|
+
comments: [comment],
|
|
4595
|
+
stateId: implementingStateId,
|
|
3673
4596
|
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3674
4597
|
dryRun: record.dryRun,
|
|
3675
4598
|
};
|
|
@@ -3678,7 +4601,8 @@ export class FactoryLoop {
|
|
|
3678
4601
|
}
|
|
3679
4602
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3680
4603
|
return;
|
|
3681
|
-
if (
|
|
4604
|
+
if (liveIssue) {
|
|
4605
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
3682
4606
|
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3683
4607
|
for (const tracked of record.agents.values()) {
|
|
3684
4608
|
const owned = tracked.spec.ownedPullRequest;
|
|
@@ -3967,6 +4891,97 @@ export class FactoryLoop {
|
|
|
3967
4891
|
}
|
|
3968
4892
|
await this.#linear.postComment(issue, body);
|
|
3969
4893
|
}
|
|
4894
|
+
async #applyDispatchClaim(record, issue, comment) {
|
|
4895
|
+
let implementingStateId;
|
|
4896
|
+
if (isGithubIssue(issue)) {
|
|
4897
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
|
|
4898
|
+
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
4899
|
+
});
|
|
4900
|
+
const commentApplied = this.#githubWriteback.hasCommentMarker
|
|
4901
|
+
? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
|
|
4902
|
+
: undefined;
|
|
4903
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => this.#githubWriteback.postComment(issue, comment), commentApplied);
|
|
4904
|
+
}
|
|
4905
|
+
else {
|
|
4906
|
+
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
4907
|
+
await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
|
|
4908
|
+
await this.#linear.setState(issue, implementingStateId);
|
|
4909
|
+
});
|
|
4910
|
+
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
4911
|
+
await this.#linear.postComment(issue, comment);
|
|
4912
|
+
});
|
|
4913
|
+
}
|
|
4914
|
+
record.dispatchClaim = {
|
|
4915
|
+
state: 'verified',
|
|
4916
|
+
updatedAtMs: this.#clock.now(),
|
|
4917
|
+
};
|
|
4918
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4919
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
4920
|
+
return implementingStateId;
|
|
4921
|
+
}
|
|
4922
|
+
async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
|
|
4923
|
+
let lastError;
|
|
4924
|
+
for (let attempt = 1; attempt <= DISPATCH_WRITEBACK_MAX_ATTEMPTS; attempt += 1) {
|
|
4925
|
+
try {
|
|
4926
|
+
if (isApplied && await isApplied())
|
|
4927
|
+
return;
|
|
4928
|
+
await apply();
|
|
4929
|
+
if (isApplied && !await isApplied()) {
|
|
4930
|
+
throw new Error(`${write} returned without a provider read-back acknowledgement`);
|
|
4931
|
+
}
|
|
4932
|
+
return;
|
|
4933
|
+
}
|
|
4934
|
+
catch (error) {
|
|
4935
|
+
lastError = error;
|
|
4936
|
+
const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
|
|
4937
|
+
this.#increment('dispatchWritebackFailures');
|
|
4938
|
+
record.dispatchClaim = {
|
|
4939
|
+
state: 'degraded',
|
|
4940
|
+
write,
|
|
4941
|
+
attempts: attempt,
|
|
4942
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
4943
|
+
error: describeError(error).errorMessage,
|
|
4944
|
+
...(deadLettered ? { deadLettered: true } : {}),
|
|
4945
|
+
updatedAtMs: this.#clock.now(),
|
|
4946
|
+
};
|
|
4947
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4948
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
4949
|
+
if (deadLettered) {
|
|
4950
|
+
this.#increment('dispatchWritebackDeadLetters');
|
|
4951
|
+
this.#logger.error?.('[factory] dispatch writeback dead-lettered after retries exhausted', {
|
|
4952
|
+
issue: issue.key,
|
|
4953
|
+
write,
|
|
4954
|
+
attempts: attempt,
|
|
4955
|
+
error: describeError(error).errorMessage,
|
|
4956
|
+
});
|
|
4957
|
+
throw error;
|
|
4958
|
+
}
|
|
4959
|
+
this.#increment('dispatchWritebackRetries');
|
|
4960
|
+
this.#logger.error?.('[factory] dispatch writeback failed; retrying', {
|
|
4961
|
+
issue: issue.key,
|
|
4962
|
+
write,
|
|
4963
|
+
attempt,
|
|
4964
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
4965
|
+
retryMs: DISPATCH_WRITEBACK_RETRY_MS,
|
|
4966
|
+
error: describeError(error).errorMessage,
|
|
4967
|
+
});
|
|
4968
|
+
await this.#clock.sleep(DISPATCH_WRITEBACK_RETRY_MS);
|
|
4969
|
+
}
|
|
4970
|
+
}
|
|
4971
|
+
throw lastError;
|
|
4972
|
+
}
|
|
4973
|
+
async #writeDispatchClaimRegistry(issue) {
|
|
4974
|
+
try {
|
|
4975
|
+
await this.#writeInFlightRegistry();
|
|
4976
|
+
}
|
|
4977
|
+
catch (error) {
|
|
4978
|
+
this.#increment('dispatchClaimRegistryWriteFailures');
|
|
4979
|
+
this.#logger.error?.('[factory] failed to persist dispatch claim visibility', {
|
|
4980
|
+
issue: issue.key,
|
|
4981
|
+
error: describeError(error).errorMessage,
|
|
4982
|
+
});
|
|
4983
|
+
}
|
|
4984
|
+
}
|
|
3970
4985
|
// Probes the GitHub issue sub-root at most once and caches the verdict so
|
|
3971
4986
|
// repeated iterations skip listTree calls when the mount is absent.
|
|
3972
4987
|
async #ensureGithubIngestionReady() {
|
|
@@ -4024,7 +5039,7 @@ export class FactoryLoop {
|
|
|
4024
5039
|
const candidates = [];
|
|
4025
5040
|
let scanned = 0;
|
|
4026
5041
|
let lastProgressAtMs = startedAtMs;
|
|
4027
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading')) {
|
|
5042
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
|
|
4028
5043
|
await this.#refreshLiveHeartbeatIfDue();
|
|
4029
5044
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
4030
5045
|
continue;
|
|
@@ -4047,18 +5062,35 @@ export class FactoryLoop {
|
|
|
4047
5062
|
try {
|
|
4048
5063
|
const issuePaths = new Map();
|
|
4049
5064
|
for (const { owner, repo } of configuredGithubRepoParts(this.#config)) {
|
|
4050
|
-
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
4051
5065
|
const roots = githubIssueRepoRoots(owner, repo);
|
|
5066
|
+
const cachedBatches = roots.map((root) => this.#cachedDiscoveryTree(root));
|
|
5067
|
+
const allRootsCached = cachedBatches.every((paths) => paths !== undefined);
|
|
5068
|
+
const indexedPaths = allRootsCached
|
|
5069
|
+
? undefined
|
|
5070
|
+
: await this.#githubIssuePathsFromIndex(owner, repo);
|
|
4052
5071
|
// Keep the fallback roots as separate batches. Flattening a very large
|
|
4053
5072
|
// provider result is synchronous work and can starve the durable loop
|
|
4054
5073
|
// heartbeat before the bounded scan below gets a chance to yield.
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
5074
|
+
let pathBatches;
|
|
5075
|
+
if (allRootsCached) {
|
|
5076
|
+
pathBatches = cachedBatches;
|
|
5077
|
+
this.#increment('githubIssueDiscoveryCacheReposUsed');
|
|
5078
|
+
}
|
|
5079
|
+
else if (indexedPaths) {
|
|
5080
|
+
// Do not feed this into the discovery cache: the index only covers
|
|
5081
|
+
// open, labeled issues, so it is a filtered subset of the real
|
|
5082
|
+
// tree (and an empty result for whichever root form the index
|
|
5083
|
+
// doesn't use) — not something a later fresh listTree call may
|
|
5084
|
+
// treat as "the tree", including on the escalation-marker and
|
|
5085
|
+
// comment-replay call sites that share this cache.
|
|
5086
|
+
pathBatches = [indexedPaths];
|
|
4059
5087
|
this.#increment('githubIssueIndexReposUsed');
|
|
4060
5088
|
}
|
|
4061
5089
|
else {
|
|
5090
|
+
pathBatches = [];
|
|
5091
|
+
for (const root of roots) {
|
|
5092
|
+
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
|
|
5093
|
+
}
|
|
4062
5094
|
this.#increment('githubIssueIndexFallbacks');
|
|
4063
5095
|
}
|
|
4064
5096
|
for (const paths of pathBatches) {
|
|
@@ -4103,6 +5135,8 @@ export class FactoryLoop {
|
|
|
4103
5135
|
return [...issuePaths.values()].sort();
|
|
4104
5136
|
}
|
|
4105
5137
|
catch (error) {
|
|
5138
|
+
if (relayfileOverload(error))
|
|
5139
|
+
throw error;
|
|
4106
5140
|
this.#githubIssuePathIndexReady = false;
|
|
4107
5141
|
this.#increment('githubIssueListFailures');
|
|
4108
5142
|
this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
|
|
@@ -4116,7 +5150,9 @@ export class FactoryLoop {
|
|
|
4116
5150
|
const { content } = await this.#readRelayfileFile(indexPath, 'GitHub issue index discovery');
|
|
4117
5151
|
parsed = parseJsonContent(content);
|
|
4118
5152
|
}
|
|
4119
|
-
catch {
|
|
5153
|
+
catch (error) {
|
|
5154
|
+
if (relayfileOverload(error))
|
|
5155
|
+
throw error;
|
|
4120
5156
|
return undefined;
|
|
4121
5157
|
}
|
|
4122
5158
|
if (!Array.isArray(parsed))
|
|
@@ -4203,6 +5239,8 @@ export class FactoryLoop {
|
|
|
4203
5239
|
this.#increment('githubIssueMirrorsCreated');
|
|
4204
5240
|
}
|
|
4205
5241
|
catch (error) {
|
|
5242
|
+
if (relayfileOverload(error))
|
|
5243
|
+
throw error;
|
|
4206
5244
|
this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
|
|
4207
5245
|
}
|
|
4208
5246
|
}
|
|
@@ -4520,7 +5558,7 @@ export class FactoryLoop {
|
|
|
4520
5558
|
// The pass-scoped flag also lets every later blocked issue reuse the
|
|
4521
5559
|
// resulting dependency index instead of rescanning the full tree.
|
|
4522
5560
|
this.#dependencyLinearTreeLoaded = true;
|
|
4523
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery')) {
|
|
5561
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
|
|
4524
5562
|
if (isIssueFilePath(path))
|
|
4525
5563
|
await this.#readIssue(path);
|
|
4526
5564
|
}
|
|
@@ -4575,7 +5613,7 @@ export class FactoryLoop {
|
|
|
4575
5613
|
const pullRequest = await resolveIssuePrFromMount(this.#mount, this.#config, issue, {
|
|
4576
5614
|
allowLegacyGithubBranch: true,
|
|
4577
5615
|
repo,
|
|
4578
|
-
});
|
|
5616
|
+
}, (prefix) => this.#listRelayfileTree(prefix, 'dependency PR probe resolution'));
|
|
4579
5617
|
if (normalizePrState(pullRequest?.state) !== 'MERGED')
|
|
4580
5618
|
return false;
|
|
4581
5619
|
this.#terminalDependencyIdentities.add(identity);
|
|
@@ -4898,14 +5936,14 @@ export class FactoryLoop {
|
|
|
4898
5936
|
}
|
|
4899
5937
|
const pathsByKey = new Map();
|
|
4900
5938
|
const canonicalPathsByKey = new Map();
|
|
4901
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery')) {
|
|
5939
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
|
|
4902
5940
|
if (isIssueFilePath(path)) {
|
|
4903
5941
|
const key = keyFromPath(path);
|
|
4904
5942
|
canonicalPathsByKey.set(key, path);
|
|
4905
5943
|
pathsByKey.set(key, path);
|
|
4906
5944
|
}
|
|
4907
5945
|
}
|
|
4908
|
-
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery')) {
|
|
5946
|
+
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
|
|
4909
5947
|
if (isIssueAliasFilePath(path)) {
|
|
4910
5948
|
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
4911
5949
|
if (canonicalPath) {
|
|
@@ -4946,6 +5984,8 @@ export class FactoryLoop {
|
|
|
4946
5984
|
return resolvedIssue;
|
|
4947
5985
|
}
|
|
4948
5986
|
catch (error) {
|
|
5987
|
+
if (relayfileOverload(error))
|
|
5988
|
+
throw error;
|
|
4949
5989
|
if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
|
|
4950
5990
|
this.#increment('phantomSkipped');
|
|
4951
5991
|
this.#logger.debug?.('[factory] skipped missing issue file discovered from issue tree', { path });
|
|
@@ -5189,6 +6229,7 @@ export class FactoryLoop {
|
|
|
5189
6229
|
}
|
|
5190
6230
|
}
|
|
5191
6231
|
const fleetTracked = this.#fleet.trackedAgents?.().get(agentName);
|
|
6232
|
+
const dispatchClaim = this.#dispatchClaimStatuses.get(issueKey(issue));
|
|
5192
6233
|
agents.push({
|
|
5193
6234
|
name: agentName,
|
|
5194
6235
|
role: tracked.spec.role,
|
|
@@ -5198,12 +6239,16 @@ export class FactoryLoop {
|
|
|
5198
6239
|
processes,
|
|
5199
6240
|
...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
|
|
5200
6241
|
...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
|
|
6242
|
+
...(dispatchClaim ? { dispatchClaim: { ...dispatchClaim } } : {}),
|
|
5201
6243
|
});
|
|
5202
6244
|
};
|
|
5203
6245
|
if (!empty) {
|
|
5204
6246
|
for (const record of (await this.#batch()).inFlight) {
|
|
5205
6247
|
if (record.dryRun)
|
|
5206
6248
|
continue;
|
|
6249
|
+
if (record.dispatchClaim) {
|
|
6250
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
6251
|
+
}
|
|
5207
6252
|
for (const [agentName, tracked] of record.agents) {
|
|
5208
6253
|
await appendAgent(record.issue, agentName, tracked);
|
|
5209
6254
|
}
|
|
@@ -5739,10 +6784,25 @@ export class FactoryLoop {
|
|
|
5739
6784
|
}
|
|
5740
6785
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
5741
6786
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
6787
|
+
const expectedHeadRef = implementer.spec.branch;
|
|
6788
|
+
if (!expectedHeadRef) {
|
|
6789
|
+
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
6790
|
+
}
|
|
6791
|
+
const matchesCurrentConvention = factoryBranchBelongsToIssue(expectedHeadRef, record.issue.key);
|
|
6792
|
+
const matchesAuthorizedLegacyBranch = implementer.spec.existingPullRequestBranch === true &&
|
|
6793
|
+
/^\d+$/u.test(record.issue.key) &&
|
|
6794
|
+
expectedHeadRef.toLowerCase().startsWith(`${record.issue.key.toLowerCase()}-`);
|
|
6795
|
+
if (!matchesCurrentConvention && !matchesAuthorizedLegacyBranch) {
|
|
6796
|
+
throw new Error(`Refusing to publish ${record.issue.key}: branch ${expectedHeadRef} belongs to a different issue`);
|
|
6797
|
+
}
|
|
5742
6798
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
5743
6799
|
const cached = this.#publishedPullRequests.get(key);
|
|
5744
|
-
if (cached)
|
|
6800
|
+
if (cached) {
|
|
6801
|
+
if (cached.headRef !== expectedHeadRef) {
|
|
6802
|
+
throw new Error(`Refusing cached PR for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${cached.headRef}`);
|
|
6803
|
+
}
|
|
5745
6804
|
return cached;
|
|
6805
|
+
}
|
|
5746
6806
|
const { identity, publisher } = this.#githubPullRequestPublisher();
|
|
5747
6807
|
const remoteBranch = implementer.result?.locality === 'remote' && implementer.spec.branch
|
|
5748
6808
|
? implementer.spec.branch
|
|
@@ -5762,11 +6822,13 @@ export class FactoryLoop {
|
|
|
5762
6822
|
: undefined;
|
|
5763
6823
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
5764
6824
|
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
5765
|
-
|
|
5766
|
-
|
|
5767
|
-
|
|
6825
|
+
if (durableReceipt) {
|
|
6826
|
+
if (durableReceipt.headRef !== expectedHeadRef) {
|
|
6827
|
+
throw new Error(`Refusing durable PR receipt for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${durableReceipt.headRef}`);
|
|
6828
|
+
}
|
|
5768
6829
|
return durableReceipt;
|
|
5769
|
-
|
|
6830
|
+
}
|
|
6831
|
+
if (opts.reconcileExisting) {
|
|
5770
6832
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
5771
6833
|
if (existing) {
|
|
5772
6834
|
this.#publishedPullRequests.set(key, existing);
|
|
@@ -5784,6 +6846,7 @@ export class FactoryLoop {
|
|
|
5784
6846
|
const result = await publisher.publishPullRequest({
|
|
5785
6847
|
repo,
|
|
5786
6848
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
6849
|
+
expectedHeadRef,
|
|
5787
6850
|
baseRef,
|
|
5788
6851
|
title: `${issue.key}: ${issue.title}`,
|
|
5789
6852
|
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
@@ -5793,7 +6856,7 @@ export class FactoryLoop {
|
|
|
5793
6856
|
? result
|
|
5794
6857
|
: { ...result, author: identity };
|
|
5795
6858
|
if (published.repo.toLowerCase() !== repo.toLowerCase() ||
|
|
5796
|
-
published.headRef !==
|
|
6859
|
+
published.headRef !== expectedHeadRef ||
|
|
5797
6860
|
!Number.isInteger(published.number) ||
|
|
5798
6861
|
published.number <= 0 ||
|
|
5799
6862
|
!published.url) {
|
|
@@ -5894,9 +6957,11 @@ export class FactoryLoop {
|
|
|
5894
6957
|
for (const root of roots) {
|
|
5895
6958
|
let paths;
|
|
5896
6959
|
try {
|
|
5897
|
-
paths = await this.#
|
|
6960
|
+
paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
|
|
5898
6961
|
}
|
|
5899
|
-
catch {
|
|
6962
|
+
catch (error) {
|
|
6963
|
+
if (relayfileOverload(error))
|
|
6964
|
+
throw error;
|
|
5900
6965
|
continue;
|
|
5901
6966
|
}
|
|
5902
6967
|
for (const path of paths) {
|
|
@@ -6162,9 +7227,11 @@ export class FactoryLoop {
|
|
|
6162
7227
|
for (let attempt = 0; attempt < PUBLISHED_PR_CONFIRM_ATTEMPTS; attempt += 1) {
|
|
6163
7228
|
const paths = (await Promise.all(roots.map(async (root) => {
|
|
6164
7229
|
try {
|
|
6165
|
-
return await this.#
|
|
7230
|
+
return await this.#listRelayfileTree(root, 'published PR confirmation');
|
|
6166
7231
|
}
|
|
6167
|
-
catch {
|
|
7232
|
+
catch (error) {
|
|
7233
|
+
if (relayfileOverload(error))
|
|
7234
|
+
throw error;
|
|
6168
7235
|
return [];
|
|
6169
7236
|
}
|
|
6170
7237
|
}))).flat();
|
|
@@ -7415,7 +8482,7 @@ export class FactoryLoop {
|
|
|
7415
8482
|
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
7416
8483
|
]) {
|
|
7417
8484
|
try {
|
|
7418
|
-
for (const path of await this.#
|
|
8485
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub escalation marker listing')) {
|
|
7419
8486
|
const parts = githubIssueCommentPathParts(path);
|
|
7420
8487
|
if (parts &&
|
|
7421
8488
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7709,7 +8776,7 @@ export class FactoryLoop {
|
|
|
7709
8776
|
];
|
|
7710
8777
|
for (const prefix of prefixes) {
|
|
7711
8778
|
try {
|
|
7712
|
-
for (const path of await this.#
|
|
8779
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub issue comment replay listing')) {
|
|
7713
8780
|
const parts = githubIssueCommentPathParts(path);
|
|
7714
8781
|
if (parts &&
|
|
7715
8782
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7720,6 +8787,8 @@ export class FactoryLoop {
|
|
|
7720
8787
|
}
|
|
7721
8788
|
}
|
|
7722
8789
|
catch (error) {
|
|
8790
|
+
if (relayfileOverload(error))
|
|
8791
|
+
throw error;
|
|
7723
8792
|
this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
|
|
7724
8793
|
}
|
|
7725
8794
|
}
|
|
@@ -8879,7 +9948,24 @@ export class FactoryLoop {
|
|
|
8879
9948
|
const owner = await this.#babysitterOwnerFor(`${event.owner}/${event.repo}`, target.prNumber);
|
|
8880
9949
|
if (!owner) {
|
|
8881
9950
|
this.#increment('babysitterEventsIgnoredUnownedPr');
|
|
8882
|
-
|
|
9951
|
+
// "Unowned" means no babysitter was ever registered for this PR — the
|
|
9952
|
+
// downstream symptom of a missed or failed PR-open. Warn once per PR so
|
|
9953
|
+
// it is visible at default level, then fall back to debug for the
|
|
9954
|
+
// repeats; #sweepOrphanedBabysitterPrs is what actually recovers it.
|
|
9955
|
+
const identity = githubPrIdentity(`${event.owner}/${event.repo}`, target.prNumber);
|
|
9956
|
+
const detail = { ...event, prNumber: target.prNumber };
|
|
9957
|
+
// The size cap gates the warn itself, not just the bookkeeping: an
|
|
9958
|
+
// unbounded set of identities must not become an unbounded warn stream.
|
|
9959
|
+
const firstForPr = Boolean(identity) &&
|
|
9960
|
+
!this.#babysitterUnownedPrWarned.has(identity) &&
|
|
9961
|
+
this.#babysitterUnownedPrWarned.size < BABYSITTER_UNOWNED_PR_WARN_LIMIT;
|
|
9962
|
+
if (firstForPr) {
|
|
9963
|
+
this.#babysitterUnownedPrWarned.add(identity);
|
|
9964
|
+
this.#logger.warn?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
9965
|
+
}
|
|
9966
|
+
else {
|
|
9967
|
+
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
9968
|
+
}
|
|
8883
9969
|
continue;
|
|
8884
9970
|
}
|
|
8885
9971
|
if (this.#mount.resourceSubscriptions &&
|
|
@@ -9333,19 +10419,92 @@ export class FactoryLoop {
|
|
|
9333
10419
|
// through the durable Relay lifecycle action; the orchestrator never runs `gh`. PR meta events here
|
|
9334
10420
|
// only (a) spawn the babysitter on open and (b) carry the latest open/draft/
|
|
9335
10421
|
// merged state used to guard the final transition.
|
|
10422
|
+
async #readPrSnapshotContent(path) {
|
|
10423
|
+
let lastError;
|
|
10424
|
+
for (let attempt = 1; attempt <= BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS; attempt += 1) {
|
|
10425
|
+
try {
|
|
10426
|
+
const { content } = await this.#mount.readFile(path);
|
|
10427
|
+
if (attempt > 1)
|
|
10428
|
+
this.#increment('babysitterPrSnapshotReadRetrySucceeded');
|
|
10429
|
+
return content;
|
|
10430
|
+
}
|
|
10431
|
+
catch (error) {
|
|
10432
|
+
lastError = error;
|
|
10433
|
+
if (attempt < BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS && !this.#stopping) {
|
|
10434
|
+
this.#increment('babysitterPrSnapshotReadRetries');
|
|
10435
|
+
await this.#clock.sleep(BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS * attempt);
|
|
10436
|
+
continue;
|
|
10437
|
+
}
|
|
10438
|
+
break;
|
|
10439
|
+
}
|
|
10440
|
+
}
|
|
10441
|
+
throw lastError;
|
|
10442
|
+
}
|
|
10443
|
+
#recordBabysitterPrSnapshotFailure(path, error) {
|
|
10444
|
+
const { errorMessage } = describeError(error);
|
|
10445
|
+
const existing = this.#babysitterPrSnapshotDeadLetters.get(path);
|
|
10446
|
+
const failures = (existing?.failures ?? 0) + 1;
|
|
10447
|
+
// Re-insert last so the eviction below stays insertion-ordered by recency.
|
|
10448
|
+
this.#babysitterPrSnapshotDeadLetters.delete(path);
|
|
10449
|
+
this.#babysitterPrSnapshotDeadLetters.set(path, {
|
|
10450
|
+
failures,
|
|
10451
|
+
lastErrorMessage: errorMessage,
|
|
10452
|
+
firstFailedAtMs: existing?.firstFailedAtMs ?? this.#clock.now(),
|
|
10453
|
+
});
|
|
10454
|
+
while (this.#babysitterPrSnapshotDeadLetters.size > BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT) {
|
|
10455
|
+
const oldest = this.#babysitterPrSnapshotDeadLetters.keys().next().value;
|
|
10456
|
+
if (oldest === undefined)
|
|
10457
|
+
break;
|
|
10458
|
+
this.#babysitterPrSnapshotDeadLetters.delete(oldest);
|
|
10459
|
+
this.#increment('babysitterPrSnapshotDeadLettersEvicted');
|
|
10460
|
+
}
|
|
10461
|
+
this.#increment('babysitterPrSnapshotReadFailures');
|
|
10462
|
+
const detail = {
|
|
10463
|
+
path,
|
|
10464
|
+
error: errorMessage,
|
|
10465
|
+
failures,
|
|
10466
|
+
attemptsPerTry: BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS,
|
|
10467
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
10468
|
+
};
|
|
10469
|
+
// A PR whose meta cannot be read is a PR with no shepherd, which is not
|
|
10470
|
+
// debug-level information. Escalate once the sweep retries also fail.
|
|
10471
|
+
if (failures >= BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER) {
|
|
10472
|
+
this.#increment('babysitterPrSnapshotReadFailuresEscalated');
|
|
10473
|
+
// The sweep keeps retrying for as long as the mount stays faulted, so the
|
|
10474
|
+
// escalation logs on the transition and then only periodically. The
|
|
10475
|
+
// counters carry the true volume.
|
|
10476
|
+
const sinceEscalation = failures - BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER;
|
|
10477
|
+
if (sinceEscalation % BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY === 0) {
|
|
10478
|
+
this.#logger.error?.('[factory] babysitter PR snapshot still unreadable after repeated retries', detail);
|
|
10479
|
+
}
|
|
10480
|
+
return;
|
|
10481
|
+
}
|
|
10482
|
+
this.#logger.warn?.('[factory] babysitter could not read PR snapshot', detail);
|
|
10483
|
+
}
|
|
9336
10484
|
async #handlePrChange(path) {
|
|
9337
10485
|
const parts = githubPullPathParts(path);
|
|
9338
10486
|
if (!parts) {
|
|
9339
10487
|
return;
|
|
9340
10488
|
}
|
|
9341
|
-
|
|
10489
|
+
// This read is the only gate between a PR opening and it getting a
|
|
10490
|
+
// babysitter, so it retries rather than treating one `fetch failed` as a
|
|
10491
|
+
// verdict. On exhaustion the path is dead-lettered for the reconcile sweep
|
|
10492
|
+
// (#sweepOrphanedBabysitterPrs) instead of being dropped.
|
|
10493
|
+
let content;
|
|
9342
10494
|
try {
|
|
9343
|
-
|
|
10495
|
+
content = await this.#readPrSnapshotContent(path);
|
|
9344
10496
|
}
|
|
9345
10497
|
catch (error) {
|
|
9346
|
-
this.#
|
|
10498
|
+
this.#recordBabysitterPrSnapshotFailure(path, error);
|
|
9347
10499
|
return;
|
|
9348
10500
|
}
|
|
10501
|
+
if (this.#babysitterPrSnapshotDeadLetters.delete(path)) {
|
|
10502
|
+
this.#increment('babysitterPrSnapshotDeadLettersRecovered');
|
|
10503
|
+
this.#logger.info?.('[factory] babysitter recovered a dead-lettered PR snapshot read', { path });
|
|
10504
|
+
}
|
|
10505
|
+
// parsePullSnapshot never throws; an undefined result is a structurally
|
|
10506
|
+
// mismatched payload, which a retry cannot fix.
|
|
10507
|
+
const snapshot = parsePullSnapshot(content, parts.number);
|
|
9349
10508
|
if (!snapshot) {
|
|
9350
10509
|
return;
|
|
9351
10510
|
}
|
|
@@ -9543,7 +10702,7 @@ export class FactoryLoop {
|
|
|
9543
10702
|
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
9544
10703
|
// over title/body references to avoid "related to AR-N" body false positives.
|
|
9545
10704
|
const githubSource = await this.#issueSource() === 'github';
|
|
9546
|
-
const paths = githubSource ? await this.#githubIssuePaths() : await this.#
|
|
10705
|
+
const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
|
|
9547
10706
|
for (const path of paths) {
|
|
9548
10707
|
if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
|
|
9549
10708
|
continue;
|
|
@@ -10009,10 +11168,12 @@ export class FactoryLoop {
|
|
|
10009
11168
|
const found = [];
|
|
10010
11169
|
for (const root of roots) {
|
|
10011
11170
|
try {
|
|
10012
|
-
const tree = await this.#
|
|
11171
|
+
const tree = await this.#listRelayfileTree(root, 'PR meta path discovery');
|
|
10013
11172
|
found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
|
|
10014
11173
|
}
|
|
10015
|
-
catch {
|
|
11174
|
+
catch (error) {
|
|
11175
|
+
if (relayfileOverload(error))
|
|
11176
|
+
throw error;
|
|
10016
11177
|
// try the next root
|
|
10017
11178
|
}
|
|
10018
11179
|
}
|
|
@@ -10205,6 +11366,124 @@ export class FactoryLoop {
|
|
|
10205
11366
|
}
|
|
10206
11367
|
}
|
|
10207
11368
|
}
|
|
11369
|
+
async #notifyTicketDispatch(decision, issue, record, result) {
|
|
11370
|
+
const hook = this.#config.hooks?.onTicketDispatch;
|
|
11371
|
+
if (!hook || result.dryRun)
|
|
11372
|
+
return;
|
|
11373
|
+
const agent = result.agents.find((candidate) => candidate.role === 'implementer')
|
|
11374
|
+
?? result.agents.find((candidate) => candidate.role === 'workflow')
|
|
11375
|
+
?? result.agents[0];
|
|
11376
|
+
if (!agent)
|
|
11377
|
+
return;
|
|
11378
|
+
if (!await this.#claimTicketDispatchNotification(record))
|
|
11379
|
+
return;
|
|
11380
|
+
const tracked = record.agents.get(agent.name);
|
|
11381
|
+
const payload = {
|
|
11382
|
+
eventType: 'ticket.dispatched',
|
|
11383
|
+
summary: `Ticket dispatched: ${issue.title} → ${agent.name}.`,
|
|
11384
|
+
issue: {
|
|
11385
|
+
id: issue.uuid,
|
|
11386
|
+
title: issue.title,
|
|
11387
|
+
url: dispatchIssueUrl(issue),
|
|
11388
|
+
},
|
|
11389
|
+
agent: {
|
|
11390
|
+
name: agent.name,
|
|
11391
|
+
sessionRef: tracked?.sessionRef ?? null,
|
|
11392
|
+
},
|
|
11393
|
+
sessionOwner: dispatchSessionOwner(decision) ?? null,
|
|
11394
|
+
timestamp: new Date(this.#clock.now()).toISOString(),
|
|
11395
|
+
};
|
|
11396
|
+
const text = ticketDispatchNotificationText(payload);
|
|
11397
|
+
for (const target of hook.notify) {
|
|
11398
|
+
if (target.surface === 'linear' && !target.commentOnIssue)
|
|
11399
|
+
continue;
|
|
11400
|
+
try {
|
|
11401
|
+
switch (target.surface) {
|
|
11402
|
+
case 'relay':
|
|
11403
|
+
await this.#fleet.sendMessage({
|
|
11404
|
+
to: `#${target.channel.replace(/^#/u, '')}`,
|
|
11405
|
+
text,
|
|
11406
|
+
});
|
|
11407
|
+
break;
|
|
11408
|
+
case 'slack':
|
|
11409
|
+
if (!target.channel && !target.dm) {
|
|
11410
|
+
throw new Error('Slack ticket-dispatch notification needs channel and/or dm');
|
|
11411
|
+
}
|
|
11412
|
+
await this.#ticketDispatchDelivery.slack({
|
|
11413
|
+
channel: target.channel,
|
|
11414
|
+
dm: target.dm,
|
|
11415
|
+
text,
|
|
11416
|
+
});
|
|
11417
|
+
break;
|
|
11418
|
+
case 'telegram':
|
|
11419
|
+
await this.#ticketDispatchDelivery.telegram({ chatId: target.chatId, text });
|
|
11420
|
+
break;
|
|
11421
|
+
case 'linear':
|
|
11422
|
+
if (isGithubIssue(issue)) {
|
|
11423
|
+
throw new Error('Linear ticket-dispatch comments require a Linear issue');
|
|
11424
|
+
}
|
|
11425
|
+
await this.#linear.postComment(issue, text);
|
|
11426
|
+
break;
|
|
11427
|
+
}
|
|
11428
|
+
this.#increment('ticketDispatchHookNotifications');
|
|
11429
|
+
}
|
|
11430
|
+
catch (error) {
|
|
11431
|
+
this.#increment('ticketDispatchHookFailures');
|
|
11432
|
+
this.#logger.warn?.('[factory] onTicketDispatch notification failed', {
|
|
11433
|
+
issue: issue.key,
|
|
11434
|
+
surface: target.surface,
|
|
11435
|
+
error: describeError(error).errorMessage,
|
|
11436
|
+
});
|
|
11437
|
+
}
|
|
11438
|
+
}
|
|
11439
|
+
}
|
|
11440
|
+
async #claimTicketDispatchNotification(record) {
|
|
11441
|
+
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
11442
|
+
return true;
|
|
11443
|
+
const key = issueKey(record.issue);
|
|
11444
|
+
return this.#serializeDispatchLifecyclePersistence(key, async () => {
|
|
11445
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
11446
|
+
if (epoch === undefined) {
|
|
11447
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11448
|
+
return false;
|
|
11449
|
+
}
|
|
11450
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
11451
|
+
if (!lifecycle) {
|
|
11452
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11453
|
+
return false;
|
|
11454
|
+
}
|
|
11455
|
+
const workUnitId = lifecycle.runId;
|
|
11456
|
+
if (lifecycle.ticketDispatchNotification?.workUnitId === workUnitId)
|
|
11457
|
+
return false;
|
|
11458
|
+
// Reserve the work unit before external side effects. Hook delivery is
|
|
11459
|
+
// best-effort per target, so takeover must prefer at-most-once fan-out
|
|
11460
|
+
// over replaying a notification whose provider acknowledgement was lost.
|
|
11461
|
+
// Follow-up for provider-idempotent retries: https://github.com/AgentWorkforce/factory/issues/239
|
|
11462
|
+
const claimedAtMs = this.#clock.now();
|
|
11463
|
+
const claimedLifecycle = {
|
|
11464
|
+
...lifecycle,
|
|
11465
|
+
ticketDispatchNotification: { workUnitId, claimedAtMs },
|
|
11466
|
+
updatedAtMs: claimedAtMs,
|
|
11467
|
+
};
|
|
11468
|
+
const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, claimedAtMs, claimedLifecycle);
|
|
11469
|
+
if (!saved) {
|
|
11470
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
11471
|
+
this.#increment('dispatchLifecycleFencesRejected');
|
|
11472
|
+
await this.#reportLifecycle(claimedLifecycle, 'factory.anomaly', {
|
|
11473
|
+
level: 'error',
|
|
11474
|
+
errorCode: 'fence_rejected',
|
|
11475
|
+
});
|
|
11476
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11477
|
+
return false;
|
|
11478
|
+
}
|
|
11479
|
+
return true;
|
|
11480
|
+
});
|
|
11481
|
+
}
|
|
11482
|
+
#ticketDispatchNotificationIsPending(lifecycle) {
|
|
11483
|
+
return Boolean(this.#config.hooks?.onTicketDispatch) &&
|
|
11484
|
+
!lifecycle.dryRun &&
|
|
11485
|
+
lifecycle.ticketDispatchNotification?.workUnitId !== lifecycle.runId;
|
|
11486
|
+
}
|
|
10208
11487
|
#error(error, issue) {
|
|
10209
11488
|
this.#increment('errors');
|
|
10210
11489
|
const details = describeError(error);
|
|
@@ -10999,9 +12278,11 @@ export class FactoryLoop {
|
|
|
10999
12278
|
async #findSlackUserIdByIdentity(identity) {
|
|
11000
12279
|
const list = async (prefix) => {
|
|
11001
12280
|
try {
|
|
11002
|
-
return await this.#
|
|
12281
|
+
return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
|
|
11003
12282
|
}
|
|
11004
|
-
catch {
|
|
12283
|
+
catch (error) {
|
|
12284
|
+
if (relayfileOverload(error))
|
|
12285
|
+
throw error;
|
|
11005
12286
|
return [];
|
|
11006
12287
|
}
|
|
11007
12288
|
};
|
|
@@ -11978,7 +13259,7 @@ export class FactoryLoop {
|
|
|
11978
13259
|
}
|
|
11979
13260
|
let paths;
|
|
11980
13261
|
try {
|
|
11981
|
-
paths = await this.#
|
|
13262
|
+
paths = await this.#listRelayfileTree('/slack/channels', 'Slack channel resolution');
|
|
11982
13263
|
}
|
|
11983
13264
|
catch (error) {
|
|
11984
13265
|
this.#logger.warn?.('[factory] unable to resolve Slack channel from mount; using configured channel value', {
|
|
@@ -12487,6 +13768,25 @@ function dispatchSpecs(decision) {
|
|
|
12487
13768
|
}
|
|
12488
13769
|
return [...decision.implementers, decision.reviewer];
|
|
12489
13770
|
}
|
|
13771
|
+
function dispatchSessionOwner(decision) {
|
|
13772
|
+
for (const spec of dispatchSpecs(decision)) {
|
|
13773
|
+
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
13774
|
+
if (sessionOwner)
|
|
13775
|
+
return sessionOwner;
|
|
13776
|
+
}
|
|
13777
|
+
return undefined;
|
|
13778
|
+
}
|
|
13779
|
+
function dispatchIssueUrl(issue) {
|
|
13780
|
+
const payload = wrappedPayload(issue.raw);
|
|
13781
|
+
const source = asRecord(payload.source);
|
|
13782
|
+
return stringValue(payload.url)
|
|
13783
|
+
?? stringValue(payload.html_url)
|
|
13784
|
+
?? stringValue(source?.url)
|
|
13785
|
+
?? issue.path;
|
|
13786
|
+
}
|
|
13787
|
+
function ticketDispatchNotificationText(payload) {
|
|
13788
|
+
return `${payload.summary}\n${JSON.stringify(payload)}`;
|
|
13789
|
+
}
|
|
12490
13790
|
function previewServiceForRepo(config, repo) {
|
|
12491
13791
|
const services = config.preview?.services;
|
|
12492
13792
|
if (!services)
|
|
@@ -12819,7 +14119,8 @@ function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
|
12819
14119
|
...(opts.isolateLocalWorktree && baseClonePath && branch ? { baseClonePath, clonePath } : {}),
|
|
12820
14120
|
// The same persisted lifecycle reuses this id after takeover, while a
|
|
12821
14121
|
// genuine reopen gets a new id and cannot replay an old placement ack.
|
|
12822
|
-
invocationId:
|
|
14122
|
+
invocationId: spec.invocationId ??
|
|
14123
|
+
`factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`,
|
|
12823
14124
|
};
|
|
12824
14125
|
return branch ? { ...lifecycleSpec, branch } : lifecycleSpec;
|
|
12825
14126
|
};
|
|
@@ -13226,17 +14527,19 @@ const linearIssueMirrorsGithubIssue = (issue, ghIssue) => {
|
|
|
13226
14527
|
.split(/\r?\n/u)
|
|
13227
14528
|
.some((line) => line.trim() === `${GITHUB_MIRROR_SOURCE_PREFIX}${ghIssue.url}`);
|
|
13228
14529
|
};
|
|
13229
|
-
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
14530
|
+
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix)) => {
|
|
13230
14531
|
const candidates = [];
|
|
13231
14532
|
const listErrors = [];
|
|
13232
14533
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
13233
14534
|
const paths = new Set();
|
|
13234
14535
|
for (const root of githubPullRoots(repo)) {
|
|
13235
14536
|
try {
|
|
13236
|
-
for (const path of await
|
|
14537
|
+
for (const path of await listTree(root))
|
|
13237
14538
|
paths.add(path);
|
|
13238
14539
|
}
|
|
13239
14540
|
catch (error) {
|
|
14541
|
+
if (relayfileOverload(error))
|
|
14542
|
+
throw error;
|
|
13240
14543
|
listErrors.push(error);
|
|
13241
14544
|
}
|
|
13242
14545
|
}
|
|
@@ -13503,8 +14806,7 @@ const issuePrMatchScore = (pr, issue, marker, opts = {}) => {
|
|
|
13503
14806
|
};
|
|
13504
14807
|
const hasTitlePrefix = (title, marker) => title === marker || title.startsWith(`${marker} `);
|
|
13505
14808
|
const factoryBranchMatchesIssue = (headRef, issueKey) => /^\d+$/u.test(issueKey)
|
|
13506
|
-
? headRef
|
|
13507
|
-
headRef.toLowerCase().startsWith(`factory/${issueKey.toLowerCase()}-`)
|
|
14809
|
+
? factoryBranchBelongsToIssue(headRef, issueKey)
|
|
13508
14810
|
: containsIssueKey(headRef, issueKey);
|
|
13509
14811
|
// Legacy Factory runs created GitHub-native branches as `<issue-number>-*`
|
|
13510
14812
|
// before the current `factory/<issue>-*` convention. Keep this matcher out of
|
|
@@ -14068,6 +15370,75 @@ const eventOccurredAtMs = (event) => {
|
|
|
14068
15370
|
const parsed = timestamp ? Date.parse(timestamp) : Number.NaN;
|
|
14069
15371
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
14070
15372
|
};
|
|
15373
|
+
const discoveryEventId = (event) => {
|
|
15374
|
+
const flat = asRecord(event) ?? {};
|
|
15375
|
+
const id = flat.id ?? flat.eventId ?? flat.event_id ?? flat.seq;
|
|
15376
|
+
return typeof id === 'string' || typeof id === 'number' ? String(id) : undefined;
|
|
15377
|
+
};
|
|
15378
|
+
const compareDiscoveryEvents = (left, right) => {
|
|
15379
|
+
const leftOccurredAt = eventOccurredAtMs(left);
|
|
15380
|
+
const rightOccurredAt = eventOccurredAtMs(right);
|
|
15381
|
+
if (leftOccurredAt !== undefined && rightOccurredAt !== undefined && leftOccurredAt !== rightOccurredAt) {
|
|
15382
|
+
return leftOccurredAt - rightOccurredAt;
|
|
15383
|
+
}
|
|
15384
|
+
const leftId = discoveryEventId(left) ?? '';
|
|
15385
|
+
const rightId = discoveryEventId(right) ?? '';
|
|
15386
|
+
const leftSequence = eventSequenceNumber(leftId);
|
|
15387
|
+
const rightSequence = eventSequenceNumber(rightId);
|
|
15388
|
+
if (leftSequence !== undefined && rightSequence !== undefined && leftSequence !== rightSequence) {
|
|
15389
|
+
return leftSequence - rightSequence;
|
|
15390
|
+
}
|
|
15391
|
+
return leftId.localeCompare(rightId);
|
|
15392
|
+
};
|
|
15393
|
+
const discoveryEventIsDeletion = (event) => {
|
|
15394
|
+
const flat = asRecord(event) ?? {};
|
|
15395
|
+
const summary = asRecord(flat.summary) ?? {};
|
|
15396
|
+
const type = stringValue(flat.type)?.toLowerCase();
|
|
15397
|
+
// filesystemEventToChangeEvent normalizes filesystem watcher events to
|
|
15398
|
+
// type: 'relayfile.changed' and stashes the real operation here, so a
|
|
15399
|
+
// deletion arrives as filesystemEventType: 'file.deleted' / 'dir.deleted'
|
|
15400
|
+
// with an outer type that does not itself say "deleted".
|
|
15401
|
+
const filesystemEventType = stringValue(flat.filesystemEventType)?.toLowerCase();
|
|
15402
|
+
const action = stringValue(flat.action)?.toLowerCase();
|
|
15403
|
+
const status = stringValue(summary.status)?.toLowerCase();
|
|
15404
|
+
const digest = stringValue(flat.digest)?.toLowerCase();
|
|
15405
|
+
return type === 'file.deleted' || type === 'dir.deleted' ||
|
|
15406
|
+
filesystemEventType === 'file.deleted' || filesystemEventType === 'dir.deleted' ||
|
|
15407
|
+
action === 'delete' || status === 'deleted' || digest?.startsWith('deleted:') === true;
|
|
15408
|
+
};
|
|
15409
|
+
const githubIssueIndexRepoRoots = (path) => {
|
|
15410
|
+
const slash = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15411
|
+
const flat = /^\/github\/repos\/([^/]+)__([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15412
|
+
const parts = slash ?? flat;
|
|
15413
|
+
if (!parts)
|
|
15414
|
+
return [];
|
|
15415
|
+
const [, owner, repo] = parts;
|
|
15416
|
+
return [
|
|
15417
|
+
`/github/repos/${owner}/${repo}/issues`,
|
|
15418
|
+
`/github/repos/${owner}__${repo}/issues`,
|
|
15419
|
+
];
|
|
15420
|
+
};
|
|
15421
|
+
const relayfileOverload = (error) => {
|
|
15422
|
+
const flat = asRecord(error) ?? {};
|
|
15423
|
+
const response = asRecord(flat.response) ?? {};
|
|
15424
|
+
const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
|
|
15425
|
+
const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
|
|
15426
|
+
const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
|
|
15427
|
+
const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
|
|
15428
|
+
if (status !== 429)
|
|
15429
|
+
return undefined;
|
|
15430
|
+
const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
|
|
15431
|
+
const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
|
|
15432
|
+
const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
|
|
15433
|
+
const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
|
|
15434
|
+
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
15435
|
+
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
15436
|
+
};
|
|
15437
|
+
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
15438
|
+
const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
|
|
15439
|
+
const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
|
|
15440
|
+
return Math.max(retryAfterMs, exponentialMs);
|
|
15441
|
+
};
|
|
14071
15442
|
const eventSequenceNumber = (eventId) => {
|
|
14072
15443
|
const whole = Number(eventId);
|
|
14073
15444
|
if (Number.isFinite(whole))
|
|
@@ -14183,7 +15554,8 @@ const eventIdentity = (event) => {
|
|
|
14183
15554
|
// subscription/poll handler) crash-loops. Returns undefined for such events so
|
|
14184
15555
|
// callers skip them and keep processing the well-formed ones.
|
|
14185
15556
|
export const changeEventPath = (event) => {
|
|
14186
|
-
const
|
|
15557
|
+
const candidate = event;
|
|
15558
|
+
const path = candidate?.resource?.path ?? candidate?.path;
|
|
14187
15559
|
return typeof path === 'string' && path ? path : undefined;
|
|
14188
15560
|
};
|
|
14189
15561
|
const describeError = (error) => {
|
|
@@ -14390,6 +15762,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
14390
15762
|
agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
|
|
14391
15763
|
invocationIds: [...record.invocationIds],
|
|
14392
15764
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
15765
|
+
...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
|
|
14393
15766
|
...(pullRequests.length > 0 ? { pullRequests: pullRequests.map((receipt) => ({ ...receipt })) } : {}),
|
|
14394
15767
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
14395
15768
|
...(releaseReason ? { releaseReason } : {}),
|
|
@@ -14403,6 +15776,7 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
|
14403
15776
|
agents: new Map(lifecycle.agents.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)])),
|
|
14404
15777
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
14405
15778
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
15779
|
+
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
14406
15780
|
});
|
|
14407
15781
|
const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
|
|
14408
15782
|
issue: { ...lifecycle.issue },
|