@agent-relay/factory 0.1.57 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -1
- package/dist/cli/fleet.d.ts +20 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +550 -106
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +1778 -515
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +104 -6
- package/dist/config/schema.js.map +1 -1
- package/dist/delivery/ticket-dispatch.d.ts +15 -0
- package/dist/delivery/ticket-dispatch.d.ts.map +1 -0
- package/dist/delivery/ticket-dispatch.js +20 -0
- package/dist/delivery/ticket-dispatch.js.map +1 -0
- package/dist/feature-guardian/conversation.d.ts +251 -0
- package/dist/feature-guardian/conversation.d.ts.map +1 -0
- package/dist/feature-guardian/conversation.js +1893 -0
- package/dist/feature-guardian/conversation.js.map +1 -0
- package/dist/feature-guardian/index.d.ts +2 -0
- package/dist/feature-guardian/index.d.ts.map +1 -0
- package/dist/feature-guardian/index.js +2 -0
- package/dist/feature-guardian/index.js.map +1 -0
- package/dist/git/agent-worktree.d.ts.map +1 -1
- package/dist/git/agent-worktree.js +4 -0
- package/dist/git/agent-worktree.js.map +1 -1
- package/dist/github/index.d.ts +2 -0
- package/dist/github/index.d.ts.map +1 -1
- package/dist/github/index.js +1 -0
- package/dist/github/index.js.map +1 -1
- package/dist/github/routed-pr-babysitter.d.ts +48 -0
- package/dist/github/routed-pr-babysitter.d.ts.map +1 -0
- package/dist/github/routed-pr-babysitter.js +197 -0
- package/dist/github/routed-pr-babysitter.js.map +1 -0
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/intake/index.d.ts +4 -0
- package/dist/intake/index.d.ts.map +1 -0
- package/dist/intake/index.js +4 -0
- package/dist/intake/index.js.map +1 -0
- package/dist/intake/notion-relay-claim.d.ts +32 -0
- package/dist/intake/notion-relay-claim.d.ts.map +1 -0
- package/dist/intake/notion-relay-claim.js +173 -0
- package/dist/intake/notion-relay-claim.js.map +1 -0
- package/dist/intake/notion-relay-contract.d.ts +33 -0
- package/dist/intake/notion-relay-contract.d.ts.map +1 -0
- package/dist/intake/notion-relay-contract.js +163 -0
- package/dist/intake/notion-relay-contract.js.map +1 -0
- package/dist/intake/notion.d.ts +375 -0
- package/dist/intake/notion.d.ts.map +1 -0
- package/dist/intake/notion.js +773 -0
- package/dist/intake/notion.js.map +1 -0
- package/dist/issue-key-match.d.ts +6 -0
- package/dist/issue-key-match.d.ts.map +1 -1
- package/dist/issue-key-match.js +14 -0
- package/dist/issue-key-match.js.map +1 -1
- package/dist/mount/github-api-issue-read.d.ts +28 -0
- package/dist/mount/github-api-issue-read.d.ts.map +1 -0
- package/dist/mount/github-api-issue-read.js +163 -0
- package/dist/mount/github-api-issue-read.js.map +1 -0
- package/dist/mount/local-mount-preflight.d.ts +2 -0
- package/dist/mount/local-mount-preflight.d.ts.map +1 -1
- package/dist/mount/local-mount-preflight.js +59 -9
- package/dist/mount/local-mount-preflight.js.map +1 -1
- package/dist/mount/mount-auth-error.d.ts +41 -0
- package/dist/mount/mount-auth-error.d.ts.map +1 -0
- package/dist/mount/mount-auth-error.js +94 -0
- package/dist/mount/mount-auth-error.js.map +1 -0
- package/dist/mount/relayfile-binary.d.ts +5 -0
- package/dist/mount/relayfile-binary.d.ts.map +1 -1
- package/dist/mount/relayfile-binary.js +16 -2
- package/dist/mount/relayfile-binary.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +18 -3
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +157 -13
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
- package/dist/mount/relayfile-github-connection-write.js +6 -3
- package/dist/mount/relayfile-github-connection-write.js.map +1 -1
- package/dist/mount/workspace-mirror.d.ts +13 -0
- package/dist/mount/workspace-mirror.d.ts.map +1 -0
- package/dist/mount/workspace-mirror.js +94 -0
- package/dist/mount/workspace-mirror.js.map +1 -0
- package/dist/observability/events.d.ts +32 -32
- package/dist/orchestrator/batch-tracker.d.ts +2 -1
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +51 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1736 -116
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +4 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +60 -1
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +66 -1
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +15 -1
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +210 -2
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +15 -1
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +105 -0
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/testing/fakes.d.ts +4 -0
- package/dist/testing/fakes.d.ts.map +1 -1
- package/dist/testing/fakes.js +6 -0
- package/dist/testing/fakes.js.map +1 -1
- package/dist/triage/schema.d.ts +169 -0
- package/dist/triage/schema.d.ts.map +1 -1
- package/dist/triage/schema.js +21 -0
- package/dist/triage/schema.js.map +1 -1
- package/dist/types.d.ts +61 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version-info.d.ts +21 -0
- package/dist/version-info.d.ts.map +1 -0
- package/dist/version-info.js +146 -0
- package/dist/version-info.js.map +1 -0
- package/dist/writeback/github.d.ts.map +1 -1
- package/dist/writeback/github.js +51 -0
- package/dist/writeback/github.js.map +1 -1
- package/package.json +10 -1
|
@@ -8,7 +8,7 @@ import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
|
8
8
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
9
9
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
10
10
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
11
|
-
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match.js';
|
|
11
|
+
import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue } from '../issue-key-match.js';
|
|
12
12
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
13
13
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
14
14
|
import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
|
|
@@ -28,6 +28,7 @@ import { findAgentProcessByName, readProcessIdentity } from './process-identity.
|
|
|
28
28
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
29
29
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
30
30
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
31
|
+
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
31
32
|
class ClarificationWakeLeaseLostError extends Error {
|
|
32
33
|
}
|
|
33
34
|
class ClarificationQuestionDeliveryLeaseLostError extends Error {
|
|
@@ -86,6 +87,29 @@ const BABYSITTER_SUBSCRIPTION_EVENT_TYPES = [
|
|
|
86
87
|
'check_run.completed',
|
|
87
88
|
];
|
|
88
89
|
const BABYSITTER_SUBSCRIPTION_TERMINAL_EVENT_TYPES = ['pull_request.closed'];
|
|
90
|
+
// The PR-open snapshot read is the single gate that decides whether a PR ever
|
|
91
|
+
// gets a babysitter, so a transient mount fetch failure must not be allowed to
|
|
92
|
+
// settle that question. Retry in-line, then dead-letter for the reconcile sweep.
|
|
93
|
+
const BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS = 3;
|
|
94
|
+
const BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS = 250;
|
|
95
|
+
// Escalate from warn to error once in-line retries plus this many sweep retries
|
|
96
|
+
// have all failed: the PR is durably unreadable, not momentarily unlucky.
|
|
97
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER = 3;
|
|
98
|
+
// Bound the dead-letter book so a sustained mount outage cannot grow it without
|
|
99
|
+
// limit. Oldest entries are evicted first; the reconcile sweep still adopts
|
|
100
|
+
// their PRs through the ownerless-record path.
|
|
101
|
+
const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT = 256;
|
|
102
|
+
// Retries carry backoff sleeps, so a sweep drains a slice rather than the whole
|
|
103
|
+
// book; failed entries rotate to the end so successive sweeps cover all of it.
|
|
104
|
+
const BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP = 16;
|
|
105
|
+
// A durably faulted mount would otherwise log an error per path per sweep.
|
|
106
|
+
const BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY = 20;
|
|
107
|
+
// Adoption probes an issue's open PR through the (already gh-backed-off) probe
|
|
108
|
+
// resolver, so it runs on its own slower cadence than the completion sweep.
|
|
109
|
+
const BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS = 60_000;
|
|
110
|
+
// Warn once per PR identity that arrives unowned, then fall back to debug. The
|
|
111
|
+
// counter carries the true volume; the log only has to make it discoverable.
|
|
112
|
+
const BABYSITTER_UNOWNED_PR_WARN_LIMIT = 64;
|
|
89
113
|
const BABYSITTER_RESOURCE_DELIVERY_RETRY_MS = 5_000;
|
|
90
114
|
const BABYSITTER_RESOURCE_SUBSCRIPTION_RENEW_MS = (BABYSITTER_SUBSCRIPTION_TTL_SECONDS * 1_000) / 2;
|
|
91
115
|
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
@@ -111,6 +135,8 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
111
135
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
112
136
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
113
137
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
138
|
+
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
139
|
+
const DISPATCH_WRITEBACK_RETRY_MS = 250;
|
|
114
140
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
115
141
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
116
142
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
@@ -124,6 +150,10 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
|
|
|
124
150
|
const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
125
151
|
const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
|
|
126
152
|
const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
|
|
153
|
+
const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
|
|
154
|
+
const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
155
|
+
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
156
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
127
157
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
128
158
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
129
159
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -132,6 +162,44 @@ const STALE_LOCAL_AGENT_RECLAIM_MAX_ATTEMPTS = 3;
|
|
|
132
162
|
const STALE_LOCAL_AGENT_RECLAIM_BACKOFF_MS = 500;
|
|
133
163
|
export const DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH = '/tmp/factory-run/factory-loop-heartbeat.json';
|
|
134
164
|
export const DEFAULT_FACTORY_LOOP_REGISTRY_PATH = '/tmp/factory-run/factory-loop-registry.json';
|
|
165
|
+
// #githubApiFallbackIssues is append-only in memory for the life of the
|
|
166
|
+
// process; bound it so a long-running `factory start --mode live` daemon
|
|
167
|
+
// doesn't accumulate one entry per fallback dispatch forever.
|
|
168
|
+
const GITHUB_API_FALLBACK_ISSUES_MAX = 2_000;
|
|
169
|
+
// A small, separately-bounded record of recently evicted identities, so a
|
|
170
|
+
// later miss can be reported as "eligibility was evicted" rather than
|
|
171
|
+
// silently folded into the same signal as "never was eligible" — an
|
|
172
|
+
// operator investigating a false phantom-skip needs to tell those apart.
|
|
173
|
+
const GITHUB_API_FALLBACK_ISSUES_EVICTED_MAX = 200;
|
|
174
|
+
/**
|
|
175
|
+
* Adds `identity` to `eligible`, evicting the least-recently-registered
|
|
176
|
+
* entry once at capacity (Sets preserve insertion order; delete+add
|
|
177
|
+
* refreshes an existing identity's recency instead of leaving it at its
|
|
178
|
+
* original position). An evicted identity moves into `evicted` — itself
|
|
179
|
+
* bounded — so a later caller can tell "eligibility was evicted" apart from
|
|
180
|
+
* "never was eligible" instead of collapsing both into the same signal.
|
|
181
|
+
* Exported standalone (pure, no class state) so the bounding and eviction
|
|
182
|
+
* behavior is directly testable without driving thousands of dispatches
|
|
183
|
+
* through a full FactoryLoop to fill the production-sized set.
|
|
184
|
+
*/
|
|
185
|
+
export function rememberBoundedFallbackEligibility(eligible, evicted, identity, maxEligible, maxEvicted) {
|
|
186
|
+
evicted.delete(identity);
|
|
187
|
+
eligible.delete(identity);
|
|
188
|
+
eligible.add(identity);
|
|
189
|
+
if (eligible.size <= maxEligible)
|
|
190
|
+
return;
|
|
191
|
+
const oldest = eligible.values().next().value;
|
|
192
|
+
if (oldest === undefined)
|
|
193
|
+
return;
|
|
194
|
+
eligible.delete(oldest);
|
|
195
|
+
evicted.delete(oldest);
|
|
196
|
+
evicted.add(oldest);
|
|
197
|
+
if (evicted.size <= maxEvicted)
|
|
198
|
+
return;
|
|
199
|
+
const oldestEvicted = evicted.values().next().value;
|
|
200
|
+
if (oldestEvicted !== undefined)
|
|
201
|
+
evicted.delete(oldestEvicted);
|
|
202
|
+
}
|
|
135
203
|
class DispatchLifecycleCapacityError extends Error {
|
|
136
204
|
}
|
|
137
205
|
class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
@@ -153,6 +221,7 @@ export class FactoryLoop {
|
|
|
153
221
|
#mount;
|
|
154
222
|
#states;
|
|
155
223
|
#fleet;
|
|
224
|
+
#ticketDispatchDelivery;
|
|
156
225
|
#triage;
|
|
157
226
|
#linear;
|
|
158
227
|
#github;
|
|
@@ -199,6 +268,8 @@ export class FactoryLoop {
|
|
|
199
268
|
#githubIssueAuthors = new Map();
|
|
200
269
|
#githubIssueAuthorLookups = new Map();
|
|
201
270
|
#githubIssuePreferredPaths = new Map();
|
|
271
|
+
#githubApiFallbackIssues = new Set();
|
|
272
|
+
#githubApiFallbackIssuesEvicted = new Set();
|
|
202
273
|
#githubIssuePathIndexReady = false;
|
|
203
274
|
#slackReporterUserIds = new Map();
|
|
204
275
|
#slackReporterUserIdLookups = new Map();
|
|
@@ -228,6 +299,7 @@ export class FactoryLoop {
|
|
|
228
299
|
#clarificationWakeRetryTimers = new Map();
|
|
229
300
|
#clarificationWakeOwner = `${process.pid}:${randomUUID()}`;
|
|
230
301
|
#dispatchLifecycleOwner = `${process.pid}:${randomUUID()}`;
|
|
302
|
+
#discoverySweepOwner = `${process.pid}:${randomUUID()}`;
|
|
231
303
|
#dispatchLifecycleEpochs = new Map();
|
|
232
304
|
#dispatchTerminalWaiters = new Map();
|
|
233
305
|
#dispatchLifecycleRetryTimers = new Map();
|
|
@@ -235,6 +307,7 @@ export class FactoryLoop {
|
|
|
235
307
|
#abandonedDispatchReasons = new Map();
|
|
236
308
|
#dispatchLifecycleCapacityWaitLogged = new Set();
|
|
237
309
|
#dispatchLifecycleOwnershipWaitLogged = new Set();
|
|
310
|
+
#dispatchClaimStatuses = new Map();
|
|
238
311
|
#localReleaseCheckpoints = new Map();
|
|
239
312
|
#dispatchLifecycleRenewTimer;
|
|
240
313
|
#clarificationSweepTimer;
|
|
@@ -251,6 +324,10 @@ export class FactoryLoop {
|
|
|
251
324
|
#subscription;
|
|
252
325
|
#livePollTimer;
|
|
253
326
|
#livePollInFlight = false;
|
|
327
|
+
// The effective transport includes per-start overrides, which can differ
|
|
328
|
+
// from config.liveSubscription. Persist it so status/heartbeat describes
|
|
329
|
+
// the listener that was actually registered.
|
|
330
|
+
#liveTransport;
|
|
254
331
|
#liveEventCursor;
|
|
255
332
|
#liveEventHighWatermark;
|
|
256
333
|
#liveConnectStartedAtMs = 0;
|
|
@@ -261,6 +338,9 @@ export class FactoryLoop {
|
|
|
261
338
|
#liveHeartbeatRefresh;
|
|
262
339
|
#liveHeartbeatLastWriteMs = 0;
|
|
263
340
|
#stoppingHeartbeatRefreshActive = false;
|
|
341
|
+
#readinessReconcileTimer;
|
|
342
|
+
#readinessReconcileInFlight;
|
|
343
|
+
#readinessReconcileIntervalMs = 60_000;
|
|
264
344
|
#liveEventQueue = [];
|
|
265
345
|
#liveEventDrainScheduled = false;
|
|
266
346
|
#liveEventDrainActive = false;
|
|
@@ -291,6 +371,18 @@ export class FactoryLoop {
|
|
|
291
371
|
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
292
372
|
#babysitterPr = new Map();
|
|
293
373
|
#babysitterIssueRefs = new Map();
|
|
374
|
+
// PR meta paths whose snapshot read exhausted its in-line retries. Without
|
|
375
|
+
// this book a single failed read permanently orphans the PR: no babysitter is
|
|
376
|
+
// spawned, so no owner exists, so every later event for it is discarded. The
|
|
377
|
+
// reconcile sweep drains this and clears entries that recover.
|
|
378
|
+
#babysitterPrSnapshotDeadLetters = new Map();
|
|
379
|
+
// PR identities already warned about as unowned, so the routing log reports
|
|
380
|
+
// the condition at default level without repeating per event.
|
|
381
|
+
#babysitterUnownedPrWarned = new Set();
|
|
382
|
+
#babysitterOrphanSweepActive = false;
|
|
383
|
+
#babysitterOrphanSweepLastRunMs = 0;
|
|
384
|
+
#babysitterOrphanReportedFailures = 0;
|
|
385
|
+
#babysitterOrphanReportedUnowned = 0;
|
|
294
386
|
// Relayfile matches subscription IDs server-side. This direct index means a
|
|
295
387
|
// delivery claim never requires the legacy local repo/PR scan to find an
|
|
296
388
|
// owning babysitter.
|
|
@@ -323,6 +415,13 @@ export class FactoryLoop {
|
|
|
323
415
|
// undefined = readiness not yet probed; resolved lazily so the standalone
|
|
324
416
|
// runOnce() path (which skips #start) still ingests when the mount is present.
|
|
325
417
|
#githubIngestionEnabled;
|
|
418
|
+
#runOnceInFlight;
|
|
419
|
+
#runOnceInFlightDryRun;
|
|
420
|
+
#discoverySession;
|
|
421
|
+
#discoverySweepEpoch;
|
|
422
|
+
#discoverySweepRenewTimer;
|
|
423
|
+
#discoverySweepLeaseLost = false;
|
|
424
|
+
#discoveryOverloadError;
|
|
326
425
|
#resolvedIssueSource;
|
|
327
426
|
#integrationInstructions;
|
|
328
427
|
#integrationInstructionsRefresh;
|
|
@@ -340,6 +439,9 @@ export class FactoryLoop {
|
|
|
340
439
|
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
|
|
341
440
|
installFactoryDraftPredicate(this.#mount, config);
|
|
342
441
|
this.#fleet = ports.fleet;
|
|
442
|
+
this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
|
|
443
|
+
mountRoot: config.localMountRoot,
|
|
444
|
+
});
|
|
343
445
|
this.#triage = ports.triage ?? new TieredTriage(new HeuristicTriage());
|
|
344
446
|
this.#linear = ports.linear ?? MountLinearWriteback(ports.mount, {
|
|
345
447
|
safety: config.safety,
|
|
@@ -437,7 +539,7 @@ export class FactoryLoop {
|
|
|
437
539
|
}
|
|
438
540
|
},
|
|
439
541
|
listPaths: async (prefix) => {
|
|
440
|
-
return this.#
|
|
542
|
+
return this.#listRelayfileTree(prefix, 'integration descriptor discovery');
|
|
441
543
|
},
|
|
442
544
|
};
|
|
443
545
|
const descriptors = await deriveDescriptorsFromMount(reader);
|
|
@@ -622,9 +724,15 @@ export class FactoryLoop {
|
|
|
622
724
|
this.#schedulePreviewSweep();
|
|
623
725
|
try {
|
|
624
726
|
await this.#startLiveSubscription(issueSource, opts.liveSubscription);
|
|
727
|
+
// The initial live heartbeat is intentionally written before startup
|
|
728
|
+
// so crash reapers can see a daemon while it bootstraps. Write again
|
|
729
|
+
// once the listener is actually registered so external `factory
|
|
730
|
+
// status` can distinguish a quiet feed from no listener.
|
|
731
|
+
await this.#writeLiveHeartbeat('running');
|
|
625
732
|
await this.#rearmSlackReplyWatchers();
|
|
626
733
|
await this.#drainReadyClarificationWake();
|
|
627
734
|
await this.#rearmGithubIssueCommentWatchers();
|
|
735
|
+
this.#scheduleReadinessReconcile();
|
|
628
736
|
this.#scheduleCompletionSweep(0);
|
|
629
737
|
return;
|
|
630
738
|
}
|
|
@@ -685,9 +793,13 @@ export class FactoryLoop {
|
|
|
685
793
|
if (this.#completionSweepTimer)
|
|
686
794
|
clearTimeout(this.#completionSweepTimer);
|
|
687
795
|
this.#completionSweepTimer = undefined;
|
|
796
|
+
if (this.#readinessReconcileTimer)
|
|
797
|
+
clearTimeout(this.#readinessReconcileTimer);
|
|
798
|
+
this.#readinessReconcileTimer = undefined;
|
|
688
799
|
if (this.#previewSweepTimer)
|
|
689
800
|
clearTimeout(this.#previewSweepTimer);
|
|
690
801
|
this.#previewSweepTimer = undefined;
|
|
802
|
+
await this.#readinessReconcileInFlight;
|
|
691
803
|
await this.#previewSweepInFlight;
|
|
692
804
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
693
805
|
try {
|
|
@@ -732,6 +844,9 @@ export class FactoryLoop {
|
|
|
732
844
|
this.#babysitterSubscriptionOwners.clear();
|
|
733
845
|
this.#babysitterReady.clear();
|
|
734
846
|
this.#babysitterCriticalAgents.clear();
|
|
847
|
+
this.#babysitterPrSnapshotDeadLetters.clear();
|
|
848
|
+
this.#babysitterUnownedPrWarned.clear();
|
|
849
|
+
this.#babysitterOrphanSweepLastRunMs = 0;
|
|
735
850
|
const subscription = this.#subscription;
|
|
736
851
|
this.#subscription = undefined;
|
|
737
852
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
@@ -825,6 +940,8 @@ export class FactoryLoop {
|
|
|
825
940
|
}
|
|
826
941
|
async #startLiveSubscription(issueSource, overrides = {}) {
|
|
827
942
|
const options = this.#liveOptions(overrides);
|
|
943
|
+
this.#liveTransport = options.transport;
|
|
944
|
+
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
828
945
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
829
946
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
830
947
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -955,6 +1072,7 @@ export class FactoryLoop {
|
|
|
955
1072
|
pollIntervalMs: overrides.pollIntervalMs ?? this.#config.liveSubscription.pollIntervalMs,
|
|
956
1073
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
957
1074
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1075
|
+
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
958
1076
|
};
|
|
959
1077
|
}
|
|
960
1078
|
async #currentEventCursor(limit) {
|
|
@@ -981,6 +1099,48 @@ export class FactoryLoop {
|
|
|
981
1099
|
};
|
|
982
1100
|
}
|
|
983
1101
|
}
|
|
1102
|
+
#scheduleReadinessReconcile(delayMs = this.#readinessReconcileIntervalMs) {
|
|
1103
|
+
if (!this.#started ||
|
|
1104
|
+
this.#stopping ||
|
|
1105
|
+
this.#readinessReconcileTimer ||
|
|
1106
|
+
this.#readinessReconcileInFlight)
|
|
1107
|
+
return;
|
|
1108
|
+
this.#readinessReconcileTimer = setTimeout(() => {
|
|
1109
|
+
this.#readinessReconcileTimer = undefined;
|
|
1110
|
+
if (!this.#started || this.#stopping)
|
|
1111
|
+
return;
|
|
1112
|
+
const sweep = this.#reconcileReadyIssues();
|
|
1113
|
+
this.#readinessReconcileInFlight = sweep;
|
|
1114
|
+
void sweep.finally(() => {
|
|
1115
|
+
if (this.#readinessReconcileInFlight === sweep) {
|
|
1116
|
+
this.#readinessReconcileInFlight = undefined;
|
|
1117
|
+
}
|
|
1118
|
+
if (this.#started && !this.#stopping)
|
|
1119
|
+
this.#scheduleReadinessReconcile();
|
|
1120
|
+
});
|
|
1121
|
+
}, delayMs);
|
|
1122
|
+
this.#readinessReconcileTimer.unref?.();
|
|
1123
|
+
}
|
|
1124
|
+
async #reconcileReadyIssues() {
|
|
1125
|
+
this.#increment('readinessReconcileSweeps');
|
|
1126
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1127
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1128
|
+
});
|
|
1129
|
+
try {
|
|
1130
|
+
const report = await this.runOnce();
|
|
1131
|
+
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1132
|
+
candidates: report.pulled.length,
|
|
1133
|
+
dispatched: report.dispatched.length,
|
|
1134
|
+
skipped: report.skipped.length,
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
catch (error) {
|
|
1138
|
+
this.#increment('readinessReconcileErrors');
|
|
1139
|
+
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1140
|
+
error: describeError(error).errorMessage,
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
984
1144
|
#scheduleLivePoll(delayMs, options) {
|
|
985
1145
|
if (this.#livePollTimer || !this.#started)
|
|
986
1146
|
return;
|
|
@@ -1290,6 +1450,10 @@ export class FactoryLoop {
|
|
|
1290
1450
|
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
1291
1451
|
// webhook-driven rather than polled.
|
|
1292
1452
|
if (this.#config.babysitter.enabled) {
|
|
1453
|
+
// Webhook-driven is not the same as one-shot: a PR-open event that was
|
|
1454
|
+
// missed or whose snapshot read failed leaves its PR with no shepherd
|
|
1455
|
+
// forever. This is the reconciliation that recovers those.
|
|
1456
|
+
await this.#sweepOrphanedBabysitterPrs(reason);
|
|
1293
1457
|
return;
|
|
1294
1458
|
}
|
|
1295
1459
|
if (this.#completionSweepActive) {
|
|
@@ -1351,6 +1515,207 @@ export class FactoryLoop {
|
|
|
1351
1515
|
this.#completionSweepActive = false;
|
|
1352
1516
|
}
|
|
1353
1517
|
}
|
|
1518
|
+
// Reconciliation for the babysitter's one-shot PR-open path. Two failure
|
|
1519
|
+
// modes converge on the same end state — an open Factory PR that no
|
|
1520
|
+
// babysitter owns, whose reviews, CI results and conflicts are then silently
|
|
1521
|
+
// discarded by #routeBabysitterEvent:
|
|
1522
|
+
// 1. the PR-open snapshot read failed and exhausted its retries, or
|
|
1523
|
+
// 2. the PR-open event never arrived at all (restart, drain loss, gap).
|
|
1524
|
+
// Neither is self-healing from events alone, so this sweep re-reads the
|
|
1525
|
+
// dead-lettered paths and adopts still-ownerless in-flight records.
|
|
1526
|
+
async #sweepOrphanedBabysitterPrs(reason = 'forced') {
|
|
1527
|
+
if (!this.#config.babysitter.enabled || this.#stopping || this.#babysitterOrphanSweepActive) {
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1530
|
+
const now = this.#clock.now();
|
|
1531
|
+
// Adoption probes every ownerless in-flight record, so it runs on the slow
|
|
1532
|
+
// cadence. A dead-lettered read is a PR we already know is unshepherded —
|
|
1533
|
+
// it retries at every sweep opportunity rather than waiting out that cadence.
|
|
1534
|
+
const adoptionDue = reason === 'forced' ||
|
|
1535
|
+
now - this.#babysitterOrphanSweepLastRunMs >= BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS;
|
|
1536
|
+
if (!adoptionDue && this.#babysitterPrSnapshotDeadLetters.size === 0) {
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
this.#babysitterOrphanSweepActive = true;
|
|
1540
|
+
try {
|
|
1541
|
+
this.#increment('babysitterOrphanSweepRuns');
|
|
1542
|
+
await this.#retryDeadLetteredPrSnapshots();
|
|
1543
|
+
if (adoptionDue) {
|
|
1544
|
+
this.#babysitterOrphanSweepLastRunMs = now;
|
|
1545
|
+
await this.#adoptOrphanedBabysitterPrs();
|
|
1546
|
+
}
|
|
1547
|
+
this.#reportBabysitterOrphanHealth();
|
|
1548
|
+
}
|
|
1549
|
+
finally {
|
|
1550
|
+
this.#babysitterOrphanSweepActive = false;
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
async #retryDeadLetteredPrSnapshots() {
|
|
1554
|
+
// Each retry costs up to BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS reads with
|
|
1555
|
+
// backoff sleeps between them, so drain a bounded slice per sweep rather
|
|
1556
|
+
// than stalling the sweep behind a full book during a mount outage. A
|
|
1557
|
+
// failed retry re-inserts its entry at the end, so successive sweeps
|
|
1558
|
+
// rotate through the whole book.
|
|
1559
|
+
const batch = [...this.#babysitterPrSnapshotDeadLetters.keys()].slice(0, BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP);
|
|
1560
|
+
if (this.#babysitterPrSnapshotDeadLetters.size > batch.length) {
|
|
1561
|
+
this.#logger.warn?.('[factory] babysitter dead-letter drain is rate limited', {
|
|
1562
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
1563
|
+
retryingThisSweep: batch.length,
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
for (const path of batch) {
|
|
1567
|
+
if (this.#stopping)
|
|
1568
|
+
return;
|
|
1569
|
+
this.#increment('babysitterPrSnapshotDeadLettersRetried');
|
|
1570
|
+
// #handlePrChange clears the entry itself on a successful read and
|
|
1571
|
+
// re-records the failure (bumping `failures`, escalating the log) if the
|
|
1572
|
+
// mount is still faulted, so the whole retry contract lives in one place.
|
|
1573
|
+
await this.#handlePrChange(path);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
async #adoptOrphanedBabysitterPrs() {
|
|
1577
|
+
const batch = await this.#batch();
|
|
1578
|
+
for (const record of batch.inFlight) {
|
|
1579
|
+
if (this.#stopping)
|
|
1580
|
+
return;
|
|
1581
|
+
if (record.dryRun || this.#hasBabysitterForIssue(record.issue))
|
|
1582
|
+
continue;
|
|
1583
|
+
if (!await this.#assertIssueDispatchLifecycleOwner(record.issue))
|
|
1584
|
+
continue;
|
|
1585
|
+
this.#increment('babysitterOrphanCandidatesScanned');
|
|
1586
|
+
try {
|
|
1587
|
+
for (const path of await this.#orphanedPrMetaPaths(record)) {
|
|
1588
|
+
if (this.#stopping || this.#hasBabysitterForIssue(record.issue))
|
|
1589
|
+
break;
|
|
1590
|
+
// Replay the PR meta through the normal PR-open handler rather than
|
|
1591
|
+
// spawning directly. Adoption must be subject to exactly the same
|
|
1592
|
+
// ownership, weak-match and path/payload-identity guards as a live
|
|
1593
|
+
// PR-open event — a reconcile sweep is a second delivery of a missed
|
|
1594
|
+
// event, never a way around the checks that event would have faced.
|
|
1595
|
+
await this.#handlePrChange(path);
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
catch (error) {
|
|
1599
|
+
this.#increment('babysitterOrphanAdoptionErrors');
|
|
1600
|
+
this.#logger.warn?.('[factory] babysitter orphan adoption failed', {
|
|
1601
|
+
issue: record.issue.key,
|
|
1602
|
+
error: describeError(error).errorMessage,
|
|
1603
|
+
});
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
if (!this.#hasBabysitterForIssue(record.issue))
|
|
1607
|
+
continue;
|
|
1608
|
+
this.#increment('babysitterOrphanedPrsAdopted');
|
|
1609
|
+
this.#logger.warn?.('[factory] adopted an orphaned PR that no babysitter owned', {
|
|
1610
|
+
issue: record.issue.key,
|
|
1611
|
+
...this.#babysitterPrForIssue(record.issue),
|
|
1612
|
+
});
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
// Candidate PR meta paths for an ownerless record, newest PR first. Resolution
|
|
1616
|
+
// reuses the durable dispatch receipts and then the probe resolver (which
|
|
1617
|
+
// carries its own gh backoff, so a record whose implementer has not opened a
|
|
1618
|
+
// PR yet costs no more than a cached miss).
|
|
1619
|
+
async #orphanedPrMetaPaths(record) {
|
|
1620
|
+
const wanted = [];
|
|
1621
|
+
if (this.#usesDurableDispatchLifecycle()) {
|
|
1622
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
1623
|
+
const receipts = lifecycle?.pullRequests ?? (lifecycle?.pullRequest ? [lifecycle.pullRequest] : []);
|
|
1624
|
+
for (const receipt of receipts)
|
|
1625
|
+
wanted.push({ repo: receipt.repo, prNumber: receipt.number });
|
|
1626
|
+
}
|
|
1627
|
+
if (wanted.length === 0) {
|
|
1628
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1629
|
+
const pr = issue ? await this.#openPrForIssue(issue) : undefined;
|
|
1630
|
+
if (pr)
|
|
1631
|
+
wanted.push({ repo: pr.repo, prNumber: pr.prNumber });
|
|
1632
|
+
}
|
|
1633
|
+
const paths = [];
|
|
1634
|
+
for (const { repo, prNumber } of wanted) {
|
|
1635
|
+
for (const root of githubPullRoots(repo)) {
|
|
1636
|
+
let candidates;
|
|
1637
|
+
try {
|
|
1638
|
+
candidates = await this.#mount.listTree(root);
|
|
1639
|
+
}
|
|
1640
|
+
catch {
|
|
1641
|
+
continue;
|
|
1642
|
+
}
|
|
1643
|
+
const match = candidates.find((path) => {
|
|
1644
|
+
const parts = githubPullPathParts(path);
|
|
1645
|
+
return parts?.number === prNumber &&
|
|
1646
|
+
`${parts.owner}/${parts.repo}`.toLowerCase() === repo.toLowerCase();
|
|
1647
|
+
});
|
|
1648
|
+
if (match && !paths.includes(match)) {
|
|
1649
|
+
paths.push(match);
|
|
1650
|
+
break;
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
if (paths.length === 0 && wanted.length > 0) {
|
|
1655
|
+
// The PR exists but its meta has not materialized in the mount yet. Leave
|
|
1656
|
+
// it for the next sweep rather than spawning off an unvalidated receipt.
|
|
1657
|
+
this.#increment('babysitterOrphanPrMetaUnavailable');
|
|
1658
|
+
}
|
|
1659
|
+
return paths;
|
|
1660
|
+
}
|
|
1661
|
+
#hasBabysitterForIssue(issue) {
|
|
1662
|
+
return this.#babysitterPrForIssue(issue) !== undefined;
|
|
1663
|
+
}
|
|
1664
|
+
#babysitterPrForIssue(issue) {
|
|
1665
|
+
const wanted = issueKey(issue);
|
|
1666
|
+
for (const [key, ref] of this.#babysitterPr) {
|
|
1667
|
+
// Ownership keys are composite (issue + PR), so resolve the issue the same
|
|
1668
|
+
// way #babysitterOwnerFor does: the recorded ref, else the key itself for
|
|
1669
|
+
// the legacy issue-keyed form. Never fall back to the issue being tested,
|
|
1670
|
+
// which would make every entry match.
|
|
1671
|
+
const owningIssue = this.#babysitterIssueRefs.get(key);
|
|
1672
|
+
if ((owningIssue ? issueKey(owningIssue) : key) !== wanted)
|
|
1673
|
+
continue;
|
|
1674
|
+
// An empty agentName is the pre-spawn reservation #ensureBabysitter takes
|
|
1675
|
+
// before its first await; that is ownership in progress, not an orphan.
|
|
1676
|
+
return { repo: ref.repo, prNumber: ref.prNumber };
|
|
1677
|
+
}
|
|
1678
|
+
return undefined;
|
|
1679
|
+
}
|
|
1680
|
+
// The two counters that mark a PR losing its shepherd are otherwise only
|
|
1681
|
+
// visible by diffing `status().counters`. Report the deltas so the condition
|
|
1682
|
+
// reaches an operator reading logs at default level.
|
|
1683
|
+
#reportBabysitterOrphanHealth() {
|
|
1684
|
+
const readFailures = this.#counters.babysitterPrSnapshotReadFailures ?? 0;
|
|
1685
|
+
const unownedEvents = this.#counters.babysitterEventsIgnoredUnownedPr ?? 0;
|
|
1686
|
+
const newReadFailures = readFailures - this.#babysitterOrphanReportedFailures;
|
|
1687
|
+
const newUnownedEvents = unownedEvents - this.#babysitterOrphanReportedUnowned;
|
|
1688
|
+
const deadLettered = this.#babysitterPrSnapshotDeadLetters.size;
|
|
1689
|
+
this.#counters.babysitterPrSnapshotDeadLetterDepth = deadLettered;
|
|
1690
|
+
if (newReadFailures <= 0 && newUnownedEvents <= 0 && deadLettered === 0) {
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
this.#babysitterOrphanReportedFailures = readFailures;
|
|
1694
|
+
this.#babysitterOrphanReportedUnowned = unownedEvents;
|
|
1695
|
+
// The oldest stuck entry is what an operator needs to act on: it names the
|
|
1696
|
+
// PR and how long it has been unshepherded.
|
|
1697
|
+
let oldest;
|
|
1698
|
+
for (const [path, entry] of this.#babysitterPrSnapshotDeadLetters) {
|
|
1699
|
+
if (!oldest || entry.firstFailedAtMs < oldest.entry.firstFailedAtMs)
|
|
1700
|
+
oldest = { path, entry };
|
|
1701
|
+
}
|
|
1702
|
+
this.#logger.warn?.('[factory] babysitter PR routing is dropping work', {
|
|
1703
|
+
prSnapshotReadFailures: readFailures,
|
|
1704
|
+
newPrSnapshotReadFailures: Math.max(0, newReadFailures),
|
|
1705
|
+
unownedPrEventsIgnored: unownedEvents,
|
|
1706
|
+
newUnownedPrEventsIgnored: Math.max(0, newUnownedEvents),
|
|
1707
|
+
deadLetteredPrSnapshots: deadLettered,
|
|
1708
|
+
flatEventsUnreadable: this.#counters.babysitterFlatEventsUnreadable ?? 0,
|
|
1709
|
+
orphanedPrsAdopted: this.#counters.babysitterOrphanedPrsAdopted ?? 0,
|
|
1710
|
+
...(oldest
|
|
1711
|
+
? {
|
|
1712
|
+
oldestDeadLetterPath: oldest.path,
|
|
1713
|
+
oldestDeadLetterError: oldest.entry.lastErrorMessage,
|
|
1714
|
+
oldestDeadLetterAgeMs: Math.max(0, this.#clock.now() - oldest.entry.firstFailedAtMs),
|
|
1715
|
+
}
|
|
1716
|
+
: {}),
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1354
1719
|
async #completionPrForIssue(issue) {
|
|
1355
1720
|
if (this.#customProbePrResolver) {
|
|
1356
1721
|
return this.#probePrResolver(issue);
|
|
@@ -1376,7 +1741,7 @@ export class FactoryLoop {
|
|
|
1376
1741
|
if (cached && cached.expiresAtMs > now) {
|
|
1377
1742
|
return cached.pr;
|
|
1378
1743
|
}
|
|
1379
|
-
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts);
|
|
1744
|
+
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts, (prefix) => this.#listRelayfileTree(prefix, 'PR probe resolution'));
|
|
1380
1745
|
if (mountPr) {
|
|
1381
1746
|
return mountPr;
|
|
1382
1747
|
}
|
|
@@ -1399,6 +1764,128 @@ export class FactoryLoop {
|
|
|
1399
1764
|
return undefined;
|
|
1400
1765
|
}
|
|
1401
1766
|
async runOnce(opts = {}) {
|
|
1767
|
+
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1768
|
+
// Loop rather than a single check-then-wait: while this call was waiting
|
|
1769
|
+
// out a mismatched sweep, another mismatched caller may have raced it to
|
|
1770
|
+
// start the *next* sweep first — in which case that one might match us
|
|
1771
|
+
// and we should coalesce onto it rather than starting a third.
|
|
1772
|
+
for (;;) {
|
|
1773
|
+
if (this.#runOnceInFlight && this.#runOnceInFlightDryRun === dryRun) {
|
|
1774
|
+
this.#increment('discoverySweepsCoalesced');
|
|
1775
|
+
this.#logger.info?.('[factory] coalesced overlapping discovery request into the in-flight sweep');
|
|
1776
|
+
return await this.#runOnceInFlight;
|
|
1777
|
+
}
|
|
1778
|
+
// A mismatched dryRun cannot coalesce onto the in-flight sweep (it
|
|
1779
|
+
// would hand a live caller a dry-run report, or vice versa), but it
|
|
1780
|
+
// still must not race it — wait for it to settle before either
|
|
1781
|
+
// re-checking for a newly-started matching sweep or claiming the lease.
|
|
1782
|
+
if (!this.#runOnceInFlight)
|
|
1783
|
+
break;
|
|
1784
|
+
await this.#runOnceInFlight.catch(() => undefined);
|
|
1785
|
+
}
|
|
1786
|
+
const sweep = this.#runOnceWithDiscoveryFence({ dryRun });
|
|
1787
|
+
this.#runOnceInFlight = sweep;
|
|
1788
|
+
this.#runOnceInFlightDryRun = dryRun;
|
|
1789
|
+
try {
|
|
1790
|
+
return await sweep;
|
|
1791
|
+
}
|
|
1792
|
+
finally {
|
|
1793
|
+
if (this.#runOnceInFlight === sweep) {
|
|
1794
|
+
this.#runOnceInFlight = undefined;
|
|
1795
|
+
this.#runOnceInFlightDryRun = undefined;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
async #runOnceWithDiscoveryFence(opts) {
|
|
1800
|
+
let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1801
|
+
if (!claim.acquired && claim.reason === 'backoff') {
|
|
1802
|
+
const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
|
|
1803
|
+
this.#increment('discoveryBackoffWaits');
|
|
1804
|
+
this.#logger.warn?.('[factory] discovery sweep waiting for Relayfile overload backoff', {
|
|
1805
|
+
delayMs,
|
|
1806
|
+
backoffUntilMs: claim.state.backoffUntilMs,
|
|
1807
|
+
consecutiveOverloads: claim.state.consecutiveOverloads,
|
|
1808
|
+
});
|
|
1809
|
+
await this.#clock.sleep(delayMs);
|
|
1810
|
+
claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
|
|
1811
|
+
}
|
|
1812
|
+
if (!claim.acquired || !claim.lease) {
|
|
1813
|
+
this.#increment('discoverySweepsSkippedInFlight');
|
|
1814
|
+
this.#logger.info?.('[factory] skipped discovery because another process owns the sweep lease', {
|
|
1815
|
+
leaseUntilMs: claim.state.lease?.leaseUntilMs,
|
|
1816
|
+
});
|
|
1817
|
+
return {
|
|
1818
|
+
pulled: [],
|
|
1819
|
+
triaged: [],
|
|
1820
|
+
dispatched: [],
|
|
1821
|
+
skipped: [],
|
|
1822
|
+
dryRun: opts.dryRun ?? this.#config.dryRun,
|
|
1823
|
+
discoveryDeferred: 'sweep-in-flight',
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
this.#discoverySweepEpoch = claim.lease.epoch;
|
|
1827
|
+
this.#discoverySweepLeaseLost = false;
|
|
1828
|
+
this.#discoveryOverloadError = undefined;
|
|
1829
|
+
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
1830
|
+
let leaseReleased = false;
|
|
1831
|
+
try {
|
|
1832
|
+
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
1833
|
+
const report = await this.#performRunOnce(opts);
|
|
1834
|
+
if (this.#discoveryOverloadError)
|
|
1835
|
+
throw this.#discoveryOverloadError;
|
|
1836
|
+
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
1837
|
+
if (this.#discoverySweepLeaseLost) {
|
|
1838
|
+
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
1839
|
+
}
|
|
1840
|
+
const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
|
|
1841
|
+
leaseReleased = completed;
|
|
1842
|
+
if (!completed)
|
|
1843
|
+
throw new Error('discovery sweep lease was lost before completion');
|
|
1844
|
+
return report;
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
const overload = relayfileOverload(error);
|
|
1848
|
+
if (overload) {
|
|
1849
|
+
const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
|
|
1850
|
+
const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
|
|
1851
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
1852
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
1853
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
1854
|
+
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
1855
|
+
status: overload.status,
|
|
1856
|
+
reason: overload.reason,
|
|
1857
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
1858
|
+
delayMs,
|
|
1859
|
+
backoffUntilMs,
|
|
1860
|
+
consecutiveOverloads,
|
|
1861
|
+
});
|
|
1862
|
+
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
1863
|
+
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
1864
|
+
// here too would additionally block this failing call (and every
|
|
1865
|
+
// caller coalesced onto it) for up to the full backoff window while
|
|
1866
|
+
// still holding the coalescing slot.
|
|
1867
|
+
}
|
|
1868
|
+
throw error;
|
|
1869
|
+
}
|
|
1870
|
+
finally {
|
|
1871
|
+
if (this.#discoverySweepRenewTimer)
|
|
1872
|
+
clearInterval(this.#discoverySweepRenewTimer);
|
|
1873
|
+
this.#discoverySweepRenewTimer = undefined;
|
|
1874
|
+
this.#discoverySession = undefined;
|
|
1875
|
+
this.#discoverySweepEpoch = undefined;
|
|
1876
|
+
this.#discoveryOverloadError = undefined;
|
|
1877
|
+
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
1878
|
+
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
1879
|
+
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
1880
|
+
// merge-advance scan) throw "lease was lost" until the next sweep
|
|
1881
|
+
// happens to run and reset it at the top of this method.
|
|
1882
|
+
this.#discoverySweepLeaseLost = false;
|
|
1883
|
+
if (!leaseReleased) {
|
|
1884
|
+
await this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
async #performRunOnce(opts = {}) {
|
|
1402
1889
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
1403
1890
|
const startedAtMs = this.#clock.now();
|
|
1404
1891
|
const relayfileWaitWarningsAtStart = this.#counters.relayfileOperationWaitWarnings ?? 0;
|
|
@@ -1429,6 +1916,14 @@ export class FactoryLoop {
|
|
|
1429
1916
|
const triaged = [];
|
|
1430
1917
|
const dispatched = [];
|
|
1431
1918
|
const skipped = [];
|
|
1919
|
+
const recordSkip = (entry) => {
|
|
1920
|
+
skipped.push(entry);
|
|
1921
|
+
this.#logger.info?.('[factory] readiness reconciliation skipped dispatch', {
|
|
1922
|
+
issue: entry.issue.key,
|
|
1923
|
+
path: entry.issue.path,
|
|
1924
|
+
reason: entry.reason,
|
|
1925
|
+
});
|
|
1926
|
+
};
|
|
1432
1927
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
1433
1928
|
let readyIssueReads = 0;
|
|
1434
1929
|
const issueEntries = [];
|
|
@@ -1473,7 +1968,6 @@ export class FactoryLoop {
|
|
|
1473
1968
|
const mayRecoverGithubOrphan = !wasReady &&
|
|
1474
1969
|
!dryRun &&
|
|
1475
1970
|
issueSource === 'github' &&
|
|
1476
|
-
Boolean(orphanRecovery) &&
|
|
1477
1971
|
Boolean(requiredLabel) &&
|
|
1478
1972
|
Boolean(labels?.has(requiredLabel)) &&
|
|
1479
1973
|
Boolean(labels?.has('factory:in-progress')) &&
|
|
@@ -1481,26 +1975,31 @@ export class FactoryLoop {
|
|
|
1481
1975
|
if (!mayRecoverGithubOrphan) {
|
|
1482
1976
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1483
1977
|
if (dispatchBlock) {
|
|
1484
|
-
|
|
1978
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1485
1979
|
continue;
|
|
1486
1980
|
}
|
|
1487
1981
|
}
|
|
1982
|
+
const orphanResult = mayRecoverGithubOrphan
|
|
1983
|
+
? await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun)
|
|
1984
|
+
: { recovered: false };
|
|
1985
|
+
const recoveredOrphan = orphanResult.recovered;
|
|
1488
1986
|
const batch = await this.#batch();
|
|
1489
1987
|
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
1490
|
-
|
|
1988
|
+
recordSkip({ issue: issueRef(issue), reason: orphanResult.reason ?? 'already tracked' });
|
|
1491
1989
|
continue;
|
|
1492
1990
|
}
|
|
1493
|
-
const recoveredOrphan = mayRecoverGithubOrphan &&
|
|
1494
|
-
await this.#reconcileOrphanedGithubInProgress(issue, orphanRecovery, dryRun);
|
|
1495
1991
|
if (!wasReady && !recoveredOrphan) {
|
|
1496
1992
|
if (mayRecoverGithubOrphan) {
|
|
1497
1993
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1498
1994
|
if (dispatchBlock) {
|
|
1499
|
-
|
|
1995
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1500
1996
|
continue;
|
|
1501
1997
|
}
|
|
1502
1998
|
}
|
|
1503
|
-
|
|
1999
|
+
recordSkip({
|
|
2000
|
+
issue: issueRef(issue),
|
|
2001
|
+
reason: orphanResult.reason ?? 'live state is not ready-for-agent',
|
|
2002
|
+
});
|
|
1504
2003
|
continue;
|
|
1505
2004
|
}
|
|
1506
2005
|
const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
|
|
@@ -1508,16 +2007,16 @@ export class FactoryLoop {
|
|
|
1508
2007
|
if (recoveredOrphan) {
|
|
1509
2008
|
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
1510
2009
|
if (dispatchBlock) {
|
|
1511
|
-
|
|
2010
|
+
recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
|
|
1512
2011
|
continue;
|
|
1513
2012
|
}
|
|
1514
2013
|
}
|
|
1515
2014
|
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
1516
|
-
|
|
2015
|
+
recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
1517
2016
|
continue;
|
|
1518
2017
|
}
|
|
1519
2018
|
if (!isDispatchableIssue(issue)) {
|
|
1520
|
-
|
|
2019
|
+
recordSkip({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
|
|
1521
2020
|
continue;
|
|
1522
2021
|
}
|
|
1523
2022
|
const decision = await this.triageIssue(issue);
|
|
@@ -1529,7 +2028,7 @@ export class FactoryLoop {
|
|
|
1529
2028
|
catch (error) {
|
|
1530
2029
|
if (!(error instanceof LiveDispatchStateChangedError))
|
|
1531
2030
|
throw error;
|
|
1532
|
-
|
|
2031
|
+
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
1533
2032
|
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
1534
2033
|
issue: decision.issue.key,
|
|
1535
2034
|
});
|
|
@@ -1541,7 +2040,7 @@ export class FactoryLoop {
|
|
|
1541
2040
|
: result.hold?.kind === 'dependency'
|
|
1542
2041
|
? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
|
|
1543
2042
|
: 'queued or escalated';
|
|
1544
|
-
|
|
2043
|
+
recordSkip({ issue: decision.issue, reason });
|
|
1545
2044
|
}
|
|
1546
2045
|
else {
|
|
1547
2046
|
dispatched.push(result);
|
|
@@ -1580,6 +2079,215 @@ export class FactoryLoop {
|
|
|
1580
2079
|
}
|
|
1581
2080
|
}
|
|
1582
2081
|
}
|
|
2082
|
+
#startDiscoverySweepRenewal(epoch) {
|
|
2083
|
+
let renewalInFlight = false;
|
|
2084
|
+
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2085
|
+
if (renewalInFlight || this.#discoverySweepLeaseLost)
|
|
2086
|
+
return;
|
|
2087
|
+
renewalInFlight = true;
|
|
2088
|
+
void this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS).then((renewed) => {
|
|
2089
|
+
if (!renewed && this.#discoverySweepEpoch === epoch) {
|
|
2090
|
+
this.#discoverySweepLeaseLost = true;
|
|
2091
|
+
this.#increment('discoverySweepLeaseLosses');
|
|
2092
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests');
|
|
2093
|
+
}
|
|
2094
|
+
}).catch((error) => {
|
|
2095
|
+
this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
|
|
2096
|
+
error: describeError(error).errorMessage,
|
|
2097
|
+
});
|
|
2098
|
+
}).finally(() => {
|
|
2099
|
+
renewalInFlight = false;
|
|
2100
|
+
});
|
|
2101
|
+
}, DISCOVERY_SWEEP_RENEW_MS);
|
|
2102
|
+
this.#discoverySweepRenewTimer.unref?.();
|
|
2103
|
+
}
|
|
2104
|
+
async #prepareDiscoverySession(claim) {
|
|
2105
|
+
const checkpoint = structuredClone(claim.state.checkpoint ?? {
|
|
2106
|
+
trees: {},
|
|
2107
|
+
updatedAtMs: 0,
|
|
2108
|
+
});
|
|
2109
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2110
|
+
if (!watermark.available) {
|
|
2111
|
+
this.#increment('discoveryIncrementalUnavailable');
|
|
2112
|
+
return {
|
|
2113
|
+
checkpoint,
|
|
2114
|
+
mode: 'full',
|
|
2115
|
+
highWatermarkAvailable: false,
|
|
2116
|
+
listedPrefixes: new Set(),
|
|
2117
|
+
};
|
|
2118
|
+
}
|
|
2119
|
+
if (claim.state.checkpoint) {
|
|
2120
|
+
if (watermark.highWatermark === checkpoint.highWatermark) {
|
|
2121
|
+
this.#increment('discoveryIncrementalUnchangedSweeps');
|
|
2122
|
+
this.#logger.info?.('[factory] discovery checkpoint unchanged; reusing cached issue trees', {
|
|
2123
|
+
highWatermark: watermark.highWatermark,
|
|
2124
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2125
|
+
});
|
|
2126
|
+
return {
|
|
2127
|
+
checkpoint,
|
|
2128
|
+
mode: 'incremental',
|
|
2129
|
+
highWatermarkAvailable: true,
|
|
2130
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2131
|
+
listedPrefixes: new Set(),
|
|
2132
|
+
};
|
|
2133
|
+
}
|
|
2134
|
+
const changes = await this.#discoveryEventsSince(checkpoint.highWatermark, watermark.highWatermark);
|
|
2135
|
+
if (changes) {
|
|
2136
|
+
this.#applyDiscoveryEvents(checkpoint, changes);
|
|
2137
|
+
this.#increment('discoveryIncrementalChangedSweeps');
|
|
2138
|
+
this.#logger.info?.('[factory] updated discovery checkpoint from Relayfile changes', {
|
|
2139
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2140
|
+
highWatermark: watermark.highWatermark,
|
|
2141
|
+
changes: changes.length,
|
|
2142
|
+
cachedPrefixes: Object.keys(checkpoint.trees).length,
|
|
2143
|
+
});
|
|
2144
|
+
return {
|
|
2145
|
+
checkpoint,
|
|
2146
|
+
mode: 'incremental',
|
|
2147
|
+
highWatermarkAvailable: true,
|
|
2148
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2149
|
+
listedPrefixes: new Set(),
|
|
2150
|
+
};
|
|
2151
|
+
}
|
|
2152
|
+
this.#increment('discoveryIncrementalGapFallbacks');
|
|
2153
|
+
this.#logger.warn?.('[factory] discovery change window did not cover the prior checkpoint; refreshing tree prefixes once', {
|
|
2154
|
+
previousHighWatermark: checkpoint.highWatermark,
|
|
2155
|
+
highWatermark: watermark.highWatermark,
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2158
|
+
// A missing change window means no retained prefix can be trusted. Start
|
|
2159
|
+
// the fallback snapshot empty so prefixes not touched by the current
|
|
2160
|
+
// configuration cannot resurface later with stale membership.
|
|
2161
|
+
checkpoint.trees = {};
|
|
2162
|
+
return {
|
|
2163
|
+
checkpoint,
|
|
2164
|
+
mode: 'full',
|
|
2165
|
+
highWatermarkAvailable: true,
|
|
2166
|
+
observedHighWatermark: watermark.highWatermark,
|
|
2167
|
+
listedPrefixes: new Set(),
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
async #finalizeDiscoveryCheckpoint() {
|
|
2171
|
+
const session = this.#discoverySession;
|
|
2172
|
+
if (!session?.highWatermarkAvailable)
|
|
2173
|
+
return undefined;
|
|
2174
|
+
const watermark = await this.#discoveryHighWatermark();
|
|
2175
|
+
if (!watermark.available)
|
|
2176
|
+
return undefined;
|
|
2177
|
+
if (watermark.highWatermark !== session.observedHighWatermark) {
|
|
2178
|
+
const changes = await this.#discoveryEventsSince(session.observedHighWatermark, watermark.highWatermark);
|
|
2179
|
+
if (!changes) {
|
|
2180
|
+
this.#increment('discoveryCheckpointCommitGaps');
|
|
2181
|
+
this.#logger.warn?.('[factory] discovery checkpoint not saved because its change window overflowed during the sweep');
|
|
2182
|
+
return undefined;
|
|
2183
|
+
}
|
|
2184
|
+
this.#applyDiscoveryEvents(session.checkpoint, changes);
|
|
2185
|
+
}
|
|
2186
|
+
session.checkpoint.highWatermark = watermark.highWatermark;
|
|
2187
|
+
session.checkpoint.updatedAtMs = this.#clock.now();
|
|
2188
|
+
for (const [prefix, paths] of Object.entries(session.checkpoint.trees)) {
|
|
2189
|
+
session.checkpoint.trees[prefix] = [...new Set(paths)].sort();
|
|
2190
|
+
}
|
|
2191
|
+
return structuredClone(session.checkpoint);
|
|
2192
|
+
}
|
|
2193
|
+
async #discoveryHighWatermark() {
|
|
2194
|
+
if (!this.#mount.getEventHighWatermark)
|
|
2195
|
+
return { available: false };
|
|
2196
|
+
try {
|
|
2197
|
+
const highWatermark = await this.#mount.getEventHighWatermark();
|
|
2198
|
+
// `undefined` also means that an older Relayfile client cannot expose a
|
|
2199
|
+
// watermark. Treat it as unavailable so a compatibility fallback never
|
|
2200
|
+
// turns into an indefinitely stale cache.
|
|
2201
|
+
return highWatermark === undefined
|
|
2202
|
+
? { available: false }
|
|
2203
|
+
: { available: true, highWatermark };
|
|
2204
|
+
}
|
|
2205
|
+
catch (error) {
|
|
2206
|
+
if (relayfileOverload(error))
|
|
2207
|
+
throw error;
|
|
2208
|
+
this.#increment('discoveryHighWatermarkFailures');
|
|
2209
|
+
this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
|
|
2210
|
+
error: describeError(error).errorMessage,
|
|
2211
|
+
});
|
|
2212
|
+
return { available: false };
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
async #discoveryEventsSince(previousHighWatermark, currentHighWatermark) {
|
|
2216
|
+
if (previousHighWatermark === currentHighWatermark)
|
|
2217
|
+
return [];
|
|
2218
|
+
if (!currentHighWatermark)
|
|
2219
|
+
return undefined;
|
|
2220
|
+
let events;
|
|
2221
|
+
try {
|
|
2222
|
+
const page = await this.#mount.getEvents({
|
|
2223
|
+
last: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2224
|
+
limit: DISCOVERY_CHANGE_EVENT_LIMIT,
|
|
2225
|
+
});
|
|
2226
|
+
events = [...page.events].sort(compareDiscoveryEvents);
|
|
2227
|
+
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
if (relayfileOverload(error))
|
|
2230
|
+
throw error;
|
|
2231
|
+
this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
|
|
2232
|
+
error: describeError(error).errorMessage,
|
|
2233
|
+
});
|
|
2234
|
+
return undefined;
|
|
2235
|
+
}
|
|
2236
|
+
const currentIndex = events.findIndex((event) => discoveryEventId(event) === currentHighWatermark);
|
|
2237
|
+
if (currentIndex < 0)
|
|
2238
|
+
return undefined;
|
|
2239
|
+
if (previousHighWatermark === undefined) {
|
|
2240
|
+
return events.length < DISCOVERY_CHANGE_EVENT_LIMIT
|
|
2241
|
+
? events.slice(0, currentIndex + 1)
|
|
2242
|
+
: undefined;
|
|
2243
|
+
}
|
|
2244
|
+
const previousIndex = events.findIndex((event) => discoveryEventId(event) === previousHighWatermark);
|
|
2245
|
+
if (previousIndex >= 0 && previousIndex <= currentIndex) {
|
|
2246
|
+
return events.slice(previousIndex + 1, currentIndex + 1);
|
|
2247
|
+
}
|
|
2248
|
+
const previousSequence = eventSequenceNumber(previousHighWatermark);
|
|
2249
|
+
const currentSequence = eventSequenceNumber(currentHighWatermark);
|
|
2250
|
+
const sequenced = events.map((event) => ({ event, sequence: eventSequenceNumber(discoveryEventId(event) ?? '') }));
|
|
2251
|
+
if (previousSequence !== undefined &&
|
|
2252
|
+
currentSequence !== undefined &&
|
|
2253
|
+
sequenced.every((entry) => entry.sequence !== undefined) &&
|
|
2254
|
+
sequenced.some((entry) => entry.sequence <= previousSequence)) {
|
|
2255
|
+
return sequenced
|
|
2256
|
+
.filter((entry) => entry.sequence > previousSequence && entry.sequence <= currentSequence)
|
|
2257
|
+
.map((entry) => entry.event);
|
|
2258
|
+
}
|
|
2259
|
+
return undefined;
|
|
2260
|
+
}
|
|
2261
|
+
#applyDiscoveryEvents(checkpoint, events) {
|
|
2262
|
+
for (const event of events) {
|
|
2263
|
+
const path = changeEventPath(event);
|
|
2264
|
+
if (!path)
|
|
2265
|
+
continue;
|
|
2266
|
+
const deletion = discoveryEventIsDeletion(event);
|
|
2267
|
+
const indexRepoRoots = githubIssueIndexRepoRoots(path);
|
|
2268
|
+
for (const [prefix, cachedPaths] of Object.entries(checkpoint.trees)) {
|
|
2269
|
+
if (indexRepoRoots.some((root) => prefix === root || prefix.startsWith(`${root}/`))) {
|
|
2270
|
+
delete checkpoint.trees[prefix];
|
|
2271
|
+
continue;
|
|
2272
|
+
}
|
|
2273
|
+
// A plain startsWith would let `/…/issues-archive/…` match the
|
|
2274
|
+
// cached prefix `/…/issues`, folding an unrelated path into it.
|
|
2275
|
+
if (path !== prefix && !path.startsWith(prefix.endsWith('/') ? prefix : `${prefix}/`))
|
|
2276
|
+
continue;
|
|
2277
|
+
const paths = new Set(cachedPaths);
|
|
2278
|
+
if (deletion) {
|
|
2279
|
+
for (const candidate of paths) {
|
|
2280
|
+
if (candidate === path || candidate.startsWith(`${path}/`))
|
|
2281
|
+
paths.delete(candidate);
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
else {
|
|
2285
|
+
paths.add(path);
|
|
2286
|
+
}
|
|
2287
|
+
checkpoint.trees[prefix] = [...paths];
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
1583
2291
|
async #githubOrphanRecoveryContext() {
|
|
1584
2292
|
try {
|
|
1585
2293
|
const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
|
|
@@ -1591,18 +2299,42 @@ export class FactoryLoop {
|
|
|
1591
2299
|
const onlineAgents = new Set(roster.agents.map((agent) => agent.name));
|
|
1592
2300
|
const activeIssueIdentities = new Set();
|
|
1593
2301
|
const legacyUnownedAgentsByIssue = new Map();
|
|
1594
|
-
|
|
1595
|
-
if (isTerminalDispatchLifecycle(lifecycle))
|
|
1596
|
-
continue;
|
|
1597
|
-
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
1598
|
-
if (identity)
|
|
1599
|
-
activeIssueIdentities.add(identity);
|
|
1600
|
-
}
|
|
2302
|
+
const orphanedLifecycleClaimsByIssue = new Map();
|
|
1601
2303
|
for (const [, waiting] of waitingClarifications) {
|
|
1602
2304
|
const identity = githubIssueRefIdentity(waiting.issue);
|
|
1603
2305
|
if (identity)
|
|
1604
2306
|
activeIssueIdentities.add(identity);
|
|
1605
2307
|
}
|
|
2308
|
+
for (const [key, lifecycle] of lifecycles) {
|
|
2309
|
+
const identity = githubIssueRefIdentity(lifecycle.issue);
|
|
2310
|
+
if (!identity)
|
|
2311
|
+
continue;
|
|
2312
|
+
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
2313
|
+
if (!activeIssueIdentities.has(identity)) {
|
|
2314
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2315
|
+
}
|
|
2316
|
+
continue;
|
|
2317
|
+
}
|
|
2318
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2319
|
+
const hasLiveAgent = activeAgents.some((agent) => onlineAgents.has(agent.name));
|
|
2320
|
+
const exitRecoveryActive = activeAgents.some((agent) => this.#agentExitsInFlight.has(agent.name));
|
|
2321
|
+
const dispatchCallActive = this.#dispatchInFlight.has(issueKey(lifecycle.issue));
|
|
2322
|
+
// The provider's `factory:in-progress` transition happens immediately
|
|
2323
|
+
// before the durable lifecycle advances from dispatching to running.
|
|
2324
|
+
// A crash can therefore leave any nonterminal phase behind while the
|
|
2325
|
+
// external claim survives. The caller still verifies provider status,
|
|
2326
|
+
// open-PR absence, a second roster, and the lifecycle lease before it
|
|
2327
|
+
// releases this orphan-shaped row.
|
|
2328
|
+
if (!activeIssueIdentities.has(identity) &&
|
|
2329
|
+
!hasLiveAgent &&
|
|
2330
|
+
!exitRecoveryActive &&
|
|
2331
|
+
!dispatchCallActive) {
|
|
2332
|
+
orphanedLifecycleClaimsByIssue.set(identity, { key, lifecycle });
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
activeIssueIdentities.add(identity);
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
1606
2338
|
for (const agent of registry?.agents ?? []) {
|
|
1607
2339
|
if (!onlineAgents.has(agent.name) || !agent.issue)
|
|
1608
2340
|
continue;
|
|
@@ -1627,6 +2359,7 @@ export class FactoryLoop {
|
|
|
1627
2359
|
activeIssueIdentities,
|
|
1628
2360
|
onlineAgentNames: onlineAgents,
|
|
1629
2361
|
legacyUnownedAgentsByIssue,
|
|
2362
|
+
orphanedLifecycleClaimsByIssue,
|
|
1630
2363
|
};
|
|
1631
2364
|
}
|
|
1632
2365
|
catch (error) {
|
|
@@ -1638,15 +2371,19 @@ export class FactoryLoop {
|
|
|
1638
2371
|
}
|
|
1639
2372
|
}
|
|
1640
2373
|
async #reconcileOrphanedGithubInProgress(issue, context, dryRun) {
|
|
1641
|
-
if (dryRun
|
|
1642
|
-
return false;
|
|
2374
|
+
if (dryRun)
|
|
2375
|
+
return { recovered: false, reason: 'dry run does not release an in-progress claim' };
|
|
2376
|
+
if (!context)
|
|
2377
|
+
return { recovered: false, reason: 'orphan-recovery safety context is unavailable' };
|
|
2378
|
+
if (!isGithubIssue(issue))
|
|
2379
|
+
return { recovered: false, reason: 'issue is not GitHub-native' };
|
|
1643
2380
|
const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
|
|
1644
2381
|
const required = this.#config.safety.requireLabel.trim().toLowerCase();
|
|
1645
2382
|
if (!required ||
|
|
1646
2383
|
!labels.has(required) ||
|
|
1647
2384
|
!labels.has('factory:in-progress') ||
|
|
1648
2385
|
labels.has('factory:human-review'))
|
|
1649
|
-
return false;
|
|
2386
|
+
return { recovered: false, reason: 'issue is not an orphan-recovery candidate' };
|
|
1650
2387
|
const identity = githubIssueRefIdentity(issueRef(issue));
|
|
1651
2388
|
const legacyUnownedAgents = identity
|
|
1652
2389
|
? (context.legacyUnownedAgentsByIssue.get(identity) ?? [])
|
|
@@ -1657,12 +2394,12 @@ export class FactoryLoop {
|
|
|
1657
2394
|
context.activeIssueIdentities.has(identity) ||
|
|
1658
2395
|
[...context.onlineAgentNames].some((name) => githubAgentNameMatchesIssue(name, issue) && !legacyUnownedAgentNames.has(name))) {
|
|
1659
2396
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1660
|
-
return false;
|
|
2397
|
+
return { recovered: false, reason: 'active dispatch claim or live agent still owns the issue' };
|
|
1661
2398
|
}
|
|
1662
2399
|
const getProviderStatus = this.#githubWriteback.getIssueStatus;
|
|
1663
2400
|
if (!getProviderStatus) {
|
|
1664
2401
|
this.#increment('githubOrphanRecoveryStatusLookupUnavailable');
|
|
1665
|
-
return false;
|
|
2402
|
+
return { recovered: false, reason: 'provider-authoritative issue status lookup is unavailable' };
|
|
1666
2403
|
}
|
|
1667
2404
|
let providerStatus;
|
|
1668
2405
|
try {
|
|
@@ -1674,11 +2411,16 @@ export class FactoryLoop {
|
|
|
1674
2411
|
issue: issue.key,
|
|
1675
2412
|
error: describeError(error).errorMessage,
|
|
1676
2413
|
});
|
|
1677
|
-
return false;
|
|
2414
|
+
return { recovered: false, reason: 'provider-authoritative issue status could not be verified' };
|
|
1678
2415
|
}
|
|
1679
2416
|
if (!providerStatus || providerStatus === 'human-review') {
|
|
1680
2417
|
this.#increment('githubOrphanRecoveriesBlockedProviderStatus');
|
|
1681
|
-
return
|
|
2418
|
+
return {
|
|
2419
|
+
recovered: false,
|
|
2420
|
+
reason: providerStatus === 'human-review'
|
|
2421
|
+
? 'provider-authoritative issue status is human-review'
|
|
2422
|
+
: 'provider-authoritative issue status is unavailable',
|
|
2423
|
+
};
|
|
1682
2424
|
}
|
|
1683
2425
|
let openPr;
|
|
1684
2426
|
try {
|
|
@@ -1690,7 +2432,7 @@ export class FactoryLoop {
|
|
|
1690
2432
|
issue: issue.key,
|
|
1691
2433
|
error: describeError(error).errorMessage,
|
|
1692
2434
|
});
|
|
1693
|
-
return false;
|
|
2435
|
+
return { recovered: false, reason: 'matching open-PR absence could not be verified' };
|
|
1694
2436
|
}
|
|
1695
2437
|
if (openPr) {
|
|
1696
2438
|
let adopted = false;
|
|
@@ -1714,7 +2456,12 @@ export class FactoryLoop {
|
|
|
1714
2456
|
repo: openPr.repo,
|
|
1715
2457
|
prNumber: openPr.prNumber,
|
|
1716
2458
|
});
|
|
1717
|
-
return
|
|
2459
|
+
return {
|
|
2460
|
+
recovered: false,
|
|
2461
|
+
reason: adopted
|
|
2462
|
+
? 'matching open PR was adopted'
|
|
2463
|
+
: 'matching open PR still owns the issue',
|
|
2464
|
+
};
|
|
1718
2465
|
}
|
|
1719
2466
|
// A pre-durable local Factory may have left live, registry-proven workers
|
|
1720
2467
|
// without a lifecycle record. They are safe to adopt only once their open
|
|
@@ -1722,7 +2469,11 @@ export class FactoryLoop {
|
|
|
1722
2469
|
// and workers instead of redispatching duplicate agents.
|
|
1723
2470
|
if (legacyUnownedAgents.length > 0) {
|
|
1724
2471
|
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
1725
|
-
return false;
|
|
2472
|
+
return { recovered: false, reason: 'legacy registry-proven agents still own the issue' };
|
|
2473
|
+
}
|
|
2474
|
+
const lifecycleClaim = context.orphanedLifecycleClaimsByIssue.get(identity);
|
|
2475
|
+
if (lifecycleClaim && !await this.#releaseOrphanedGithubLifecycle(issue, identity, lifecycleClaim)) {
|
|
2476
|
+
return { recovered: false, reason: 'orphaned durable lifecycle claim could not be safely released' };
|
|
1726
2477
|
}
|
|
1727
2478
|
try {
|
|
1728
2479
|
if (providerStatus === 'in-progress') {
|
|
@@ -1731,14 +2482,25 @@ export class FactoryLoop {
|
|
|
1731
2482
|
// A crashed dispatch may leave its durable attempt marked in-flight even
|
|
1732
2483
|
// after every agent and lifecycle disappeared. Only clear that stale bit
|
|
1733
2484
|
// after all provider, agent, lifecycle, and open-PR safety checks pass.
|
|
1734
|
-
await this.#
|
|
2485
|
+
const attempt = await this.#state.getDispatchAttempts(this.#workspaceId, issueStateKey(issue));
|
|
2486
|
+
if (attempt?.terminal) {
|
|
2487
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issueStateKey(issue), {
|
|
2488
|
+
attempts: 0,
|
|
2489
|
+
inFlight: false,
|
|
2490
|
+
terminal: false,
|
|
2491
|
+
backoffUntilMs: 0,
|
|
2492
|
+
});
|
|
2493
|
+
}
|
|
2494
|
+
else {
|
|
2495
|
+
await this.#clearDispatchInFlight(issue);
|
|
2496
|
+
}
|
|
1735
2497
|
this.#reconciledGithubInProgress.add(identity);
|
|
1736
2498
|
this.#increment('githubOrphanedInProgressRecovered');
|
|
1737
2499
|
this.#logger.warn?.('[factory] recovered orphaned GitHub in-progress issue for redispatch', {
|
|
1738
2500
|
issue: issue.key,
|
|
1739
2501
|
path: issue.path,
|
|
1740
2502
|
});
|
|
1741
|
-
return true;
|
|
2503
|
+
return { recovered: true };
|
|
1742
2504
|
}
|
|
1743
2505
|
catch (error) {
|
|
1744
2506
|
this.#increment('githubOrphanRecoveryWritebackFailures');
|
|
@@ -1746,6 +2508,116 @@ export class FactoryLoop {
|
|
|
1746
2508
|
issue: issue.key,
|
|
1747
2509
|
error: describeError(error).errorMessage,
|
|
1748
2510
|
});
|
|
2511
|
+
return { recovered: false, reason: 'orphaned provider lifecycle status could not be cleared' };
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
async #releaseOrphanedGithubLifecycle(issue, identity, candidate) {
|
|
2515
|
+
let key = candidate.key;
|
|
2516
|
+
let lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
2517
|
+
if (!lifecycle)
|
|
2518
|
+
return true;
|
|
2519
|
+
if (lifecycle.runId !== candidate.lifecycle.runId ||
|
|
2520
|
+
githubIssueRefIdentity(lifecycle.issue) !== identity) {
|
|
2521
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because its durable lifecycle changed', {
|
|
2522
|
+
issue: issue.key,
|
|
2523
|
+
lifecycleKey: key,
|
|
2524
|
+
});
|
|
2525
|
+
return false;
|
|
2526
|
+
}
|
|
2527
|
+
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
2528
|
+
const roster = await this.#fleet.roster();
|
|
2529
|
+
const online = new Set(roster.agents.map((agent) => agent.name));
|
|
2530
|
+
if (activeAgents.some((agent) => online.has(agent.name)) ||
|
|
2531
|
+
roster.agents.some((agent) => githubAgentNameMatchesIssue(agent.name, issue))) {
|
|
2532
|
+
this.#increment('githubOrphanRecoveriesBlockedActive');
|
|
2533
|
+
return false;
|
|
2534
|
+
}
|
|
2535
|
+
let epoch;
|
|
2536
|
+
if (!isTerminalDispatchLifecycle(lifecycle)) {
|
|
2537
|
+
epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
2538
|
+
if (epoch !== undefined) {
|
|
2539
|
+
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2540
|
+
if (!renewed) {
|
|
2541
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2542
|
+
return false;
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
else {
|
|
2546
|
+
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, lifecycle, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
2547
|
+
if (!claim.acquired || !claim.lease) {
|
|
2548
|
+
this.#logger.info?.('[factory] preserved orphan-shaped GitHub claim because another publisher still owns its lease', {
|
|
2549
|
+
issue: issue.key,
|
|
2550
|
+
owner: claim.lifecycle.lease?.owner,
|
|
2551
|
+
leaseUntilMs: claim.lifecycle.lease?.leaseUntilMs,
|
|
2552
|
+
});
|
|
2553
|
+
return false;
|
|
2554
|
+
}
|
|
2555
|
+
key = claim.key ?? key;
|
|
2556
|
+
lifecycle = claim.lifecycle;
|
|
2557
|
+
epoch = claim.lease.epoch;
|
|
2558
|
+
this.#dispatchLifecycleEpochs.set(key, epoch);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
try {
|
|
2562
|
+
for (const agent of lifecycle.agents.filter((entry) => entry.releasedAtMs === undefined)) {
|
|
2563
|
+
try {
|
|
2564
|
+
await this.#fleet.release(agent.name, 'orphaned-claim');
|
|
2565
|
+
}
|
|
2566
|
+
catch (error) {
|
|
2567
|
+
// The roster was checked twice before fencing this lifecycle, so a
|
|
2568
|
+
// missing/dead worker is the expected crash-recovery shape. Do not
|
|
2569
|
+
// let a control-plane "agent not found" response make the durable
|
|
2570
|
+
// claim immortal; a failed fresh spawn remains visible to the normal
|
|
2571
|
+
// dispatch retry machinery.
|
|
2572
|
+
this.#increment('githubOrphanedLifecycleAgentReleaseFailures');
|
|
2573
|
+
this.#logger.warn?.('[factory] dead claim agent release failed; clearing fenced lifecycle anyway', {
|
|
2574
|
+
issue: issue.key,
|
|
2575
|
+
agent: agent.name,
|
|
2576
|
+
error: describeError(error).errorMessage,
|
|
2577
|
+
});
|
|
2578
|
+
}
|
|
2579
|
+
this.#fleet.markAgentTerminal?.(agent.name, 'orphaned-claim');
|
|
2580
|
+
}
|
|
2581
|
+
if (epoch !== undefined && !await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS)) {
|
|
2582
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2583
|
+
return false;
|
|
2584
|
+
}
|
|
2585
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
2586
|
+
if (retryTimer)
|
|
2587
|
+
clearTimeout(retryTimer);
|
|
2588
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
2589
|
+
const batch = await this.#batch();
|
|
2590
|
+
batch.abandon(lifecycle.issue);
|
|
2591
|
+
await this.#state.clearDispatchLifecycle(this.#workspaceId, key);
|
|
2592
|
+
await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
|
|
2593
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2594
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
2595
|
+
this.#resolveDispatchTerminalWaiters(lifecycle.issue);
|
|
2596
|
+
await this.#writeInFlightRegistry().catch((error) => {
|
|
2597
|
+
this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
|
|
2598
|
+
issue: issue.key,
|
|
2599
|
+
error: describeError(error).errorMessage,
|
|
2600
|
+
});
|
|
2601
|
+
});
|
|
2602
|
+
this.#increment('githubOrphanedLifecycleClaimsReleased');
|
|
2603
|
+
this.#logger.warn?.('[factory] released orphaned GitHub dispatch claim with no live agents', {
|
|
2604
|
+
issue: issue.key,
|
|
2605
|
+
lifecycleKey: key,
|
|
2606
|
+
runId: lifecycle.runId,
|
|
2607
|
+
agents: activeAgents.map((agent) => agent.name),
|
|
2608
|
+
});
|
|
2609
|
+
return true;
|
|
2610
|
+
}
|
|
2611
|
+
catch (error) {
|
|
2612
|
+
if (epoch !== undefined) {
|
|
2613
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch).catch(() => undefined);
|
|
2614
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
2615
|
+
}
|
|
2616
|
+
this.#logger.warn?.('[factory] failed to release orphaned GitHub dispatch claim; preserving it', {
|
|
2617
|
+
issue: issue.key,
|
|
2618
|
+
lifecycleKey: key,
|
|
2619
|
+
error: describeError(error).errorMessage,
|
|
2620
|
+
});
|
|
1749
2621
|
return false;
|
|
1750
2622
|
}
|
|
1751
2623
|
}
|
|
@@ -1904,13 +2776,60 @@ export class FactoryLoop {
|
|
|
1904
2776
|
this.#increment('githubOrphanedPullRequestsAdopted');
|
|
1905
2777
|
return true;
|
|
1906
2778
|
}
|
|
1907
|
-
|
|
1908
|
-
|
|
2779
|
+
// `cache` is opt-in and must stay limited to call sites that walk a full
|
|
2780
|
+
// repository/issue root to enumerate everything under it — the sweep this
|
|
2781
|
+
// whole feature exists to make incremental. A cached listing can lag the
|
|
2782
|
+
// live tree by up to one sweep, which is fine for enumeration but wrong
|
|
2783
|
+
// for a point lookup that needs the current state: a PR-confirmation poll
|
|
2784
|
+
// that expects a specific PR to appear, or an escalation/comment-replay
|
|
2785
|
+
// scan that must not miss a marker or reply that landed after the cache
|
|
2786
|
+
// was populated. Those callers must omit `cache` and pay for a fresh list.
|
|
2787
|
+
async #listRelayfileTree(prefix, phase, opts = {}) {
|
|
2788
|
+
if (this.#discoverySweepLeaseLost) {
|
|
2789
|
+
throw new Error('discovery sweep lease was lost; refusing another tree request');
|
|
2790
|
+
}
|
|
2791
|
+
const cached = opts.cache ? this.#cachedDiscoveryTree(prefix) : undefined;
|
|
2792
|
+
if (cached) {
|
|
2793
|
+
this.#increment('discoveryTreeCacheHits');
|
|
2794
|
+
this.#logger.debug?.('[factory] reused cached Relayfile tree', { phase, prefix, count: cached.length });
|
|
2795
|
+
return cached;
|
|
2796
|
+
}
|
|
2797
|
+
if (opts.cache)
|
|
2798
|
+
this.#increment('discoveryTreeCacheMisses');
|
|
2799
|
+
const paths = await this.#withRelayfileOperation('listTree', { phase, prefix }, () => this.#mount.listTree(prefix), {
|
|
1909
2800
|
count: (paths) => paths.length,
|
|
1910
2801
|
logFailure: true,
|
|
1911
2802
|
logStart: true,
|
|
1912
2803
|
logComplete: true,
|
|
1913
2804
|
});
|
|
2805
|
+
if (opts.cache)
|
|
2806
|
+
await this.#rememberDiscoveryTree(prefix, paths);
|
|
2807
|
+
return paths;
|
|
2808
|
+
}
|
|
2809
|
+
#cachedDiscoveryTree(prefix) {
|
|
2810
|
+
const session = this.#discoverySession;
|
|
2811
|
+
if (!session)
|
|
2812
|
+
return undefined;
|
|
2813
|
+
if (session.mode === 'full' && !session.listedPrefixes.has(prefix))
|
|
2814
|
+
return undefined;
|
|
2815
|
+
const paths = session.checkpoint.trees[prefix];
|
|
2816
|
+
return paths ? [...paths] : undefined;
|
|
2817
|
+
}
|
|
2818
|
+
async #rememberDiscoveryTree(prefix, paths) {
|
|
2819
|
+
const session = this.#discoverySession;
|
|
2820
|
+
if (!session)
|
|
2821
|
+
return;
|
|
2822
|
+
const uniquePaths = new Set();
|
|
2823
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
2824
|
+
uniquePaths.add(paths[index]);
|
|
2825
|
+
if ((index + 1) % LIVE_EVENT_DRAIN_BATCH_SIZE === 0) {
|
|
2826
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
2827
|
+
if (index + 1 < paths.length)
|
|
2828
|
+
await liveEventYield();
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
session.checkpoint.trees[prefix] = [...uniquePaths].sort();
|
|
2832
|
+
session.listedPrefixes.add(prefix);
|
|
1914
2833
|
}
|
|
1915
2834
|
async #ensureRelayfileSubRoot(prefix, phase, opts) {
|
|
1916
2835
|
return this.#withRelayfileOperation('ensureSubRoot', { phase, prefix }, () => this.#mount.ensureSubRoot(prefix, opts), {
|
|
@@ -1963,6 +2882,9 @@ export class FactoryLoop {
|
|
|
1963
2882
|
return result;
|
|
1964
2883
|
}
|
|
1965
2884
|
catch (error) {
|
|
2885
|
+
if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
|
|
2886
|
+
this.#discoveryOverloadError ??= error;
|
|
2887
|
+
}
|
|
1966
2888
|
if (opts.logFailure || waitWarnings > 0) {
|
|
1967
2889
|
this.#increment('relayfileOperationFailures');
|
|
1968
2890
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
@@ -2067,7 +2989,10 @@ export class FactoryLoop {
|
|
|
2067
2989
|
const dispatched = this.#dispatchUnlocked(decision, opts);
|
|
2068
2990
|
this.#dispatchInFlight.set(key, dispatched);
|
|
2069
2991
|
try {
|
|
2070
|
-
|
|
2992
|
+
const result = await dispatched;
|
|
2993
|
+
if (decision.issueResolution)
|
|
2994
|
+
result.issueResolution = structuredClone(decision.issueResolution);
|
|
2995
|
+
return result;
|
|
2071
2996
|
}
|
|
2072
2997
|
finally {
|
|
2073
2998
|
if (this.#dispatchInFlight.get(key) === dispatched) {
|
|
@@ -2101,7 +3026,11 @@ export class FactoryLoop {
|
|
|
2101
3026
|
this.#error(error, decision.issue);
|
|
2102
3027
|
throw error;
|
|
2103
3028
|
}
|
|
2104
|
-
|
|
3029
|
+
// This read runs before `decision` has any tracked record for
|
|
3030
|
+
// #deriveGithubApiFallbackEligibility to find (batch insertion happens
|
|
3031
|
+
// later, once scope/readiness are validated), so pass it directly
|
|
3032
|
+
// rather than registering it into the cache first.
|
|
3033
|
+
const liveIssue = await this.#readIssue(decision.issue.path, decision);
|
|
2105
3034
|
if (!liveIssue || !isInFactoryScope(liveIssue, this.#config.safety)) {
|
|
2106
3035
|
const error = new Error(`Refusing to dispatch ${decision.issue.key}: not factory-e2e scope`);
|
|
2107
3036
|
this.#error(error, decision.issue);
|
|
@@ -2178,7 +3107,10 @@ export class FactoryLoop {
|
|
|
2178
3107
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
2179
3108
|
// and concurrent issues can switch each other back to the base branch.
|
|
2180
3109
|
const isolateLocalWorktree = this.#fleet.placementLocality === 'local' && Boolean(this.#worktrees);
|
|
2181
|
-
|
|
3110
|
+
// Every live dispatch gets an issue-owned branch, including legacy local
|
|
3111
|
+
// fleets without a durable lifecycle or worktree manager. The publication
|
|
3112
|
+
// boundary uses this exact ref to reject a stale shared checkout.
|
|
3113
|
+
const lifecycleRunId = !dryRun ? randomUUID() : undefined;
|
|
2182
3114
|
if (lifecycleRunId) {
|
|
2183
3115
|
dispatchDecision = decisionWithLifecycleBranches(dispatchDecision, lifecycleRunId, {
|
|
2184
3116
|
isolateLocalWorktree,
|
|
@@ -2313,6 +3245,13 @@ export class FactoryLoop {
|
|
|
2313
3245
|
}
|
|
2314
3246
|
agents.push({ name: spawned.name, role: spec.role });
|
|
2315
3247
|
}
|
|
3248
|
+
if (!dryRun) {
|
|
3249
|
+
record.dispatchClaim = {
|
|
3250
|
+
state: 'pending',
|
|
3251
|
+
updatedAtMs: this.#clock.now(),
|
|
3252
|
+
};
|
|
3253
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
3254
|
+
}
|
|
2316
3255
|
await this.#writeInFlightRegistry();
|
|
2317
3256
|
const comment = dispatchComment(dispatchDecision, agents);
|
|
2318
3257
|
let implementingStateId;
|
|
@@ -2321,19 +3260,7 @@ export class FactoryLoop {
|
|
|
2321
3260
|
if (!issue || !this.#isIssueReady(issue)) {
|
|
2322
3261
|
throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
|
|
2323
3262
|
}
|
|
2324
|
-
|
|
2325
|
-
await this.#postIssueComment(issue, comment);
|
|
2326
|
-
}
|
|
2327
|
-
catch (error) {
|
|
2328
|
-
this.#logger.warn?.('[factory] comment writeback skipped', error);
|
|
2329
|
-
}
|
|
2330
|
-
if (isGithubIssue(issue)) {
|
|
2331
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
2332
|
-
}
|
|
2333
|
-
else {
|
|
2334
|
-
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
2335
|
-
await this.#linear.setState(issue, implementingStateId);
|
|
2336
|
-
}
|
|
3263
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
2337
3264
|
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
2338
3265
|
}
|
|
2339
3266
|
const result = {
|
|
@@ -2347,9 +3274,13 @@ export class FactoryLoop {
|
|
|
2347
3274
|
dryRun,
|
|
2348
3275
|
};
|
|
2349
3276
|
record.result = result;
|
|
2350
|
-
await this.#saveDispatchLifecycle(record, 'running')
|
|
3277
|
+
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3278
|
+
return result;
|
|
2351
3279
|
this.#increment('dispatched');
|
|
2352
3280
|
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
3281
|
+
if (!dryRun && this.#config.hooks?.onTicketDispatch) {
|
|
3282
|
+
await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
|
|
3283
|
+
}
|
|
2353
3284
|
if (!dryRun) {
|
|
2354
3285
|
await this.#ensureSlackDispatchThread(record, result);
|
|
2355
3286
|
}
|
|
@@ -2444,8 +3375,27 @@ export class FactoryLoop {
|
|
|
2444
3375
|
}
|
|
2445
3376
|
status() {
|
|
2446
3377
|
const batch = this.#batchView;
|
|
3378
|
+
const inFlightDispatches = batch?.inFlight
|
|
3379
|
+
.filter((record) => !record.dryRun)
|
|
3380
|
+
.map((record) => ({
|
|
3381
|
+
issue: { ...record.issue },
|
|
3382
|
+
agents: [...record.agents].map(([name, tracked]) => ({
|
|
3383
|
+
name,
|
|
3384
|
+
role: tracked.spec.role,
|
|
3385
|
+
...(tracked.sessionRef ? { sessionRef: tracked.sessionRef } : {}),
|
|
3386
|
+
...(tracked.spec.invocationId ? { invocationId: tracked.spec.invocationId } : {}),
|
|
3387
|
+
...(tracked.result?.node ? { node: tracked.result.node } : {}),
|
|
3388
|
+
})),
|
|
3389
|
+
claim: {
|
|
3390
|
+
...(record.dispatchClaim ?? this.#dispatchClaimStatuses.get(issueKey(record.issue)) ?? {
|
|
3391
|
+
state: 'pending',
|
|
3392
|
+
updatedAtMs: this.#clock.now(),
|
|
3393
|
+
}),
|
|
3394
|
+
},
|
|
3395
|
+
})) ?? [];
|
|
2447
3396
|
return {
|
|
2448
3397
|
inFlight: batch?.inFlight.map((record) => record.issue) ?? [],
|
|
3398
|
+
...(inFlightDispatches.length > 0 ? { inFlightDispatches } : {}),
|
|
2449
3399
|
queued: batch?.queued.map((queued) => queued.issue) ?? [],
|
|
2450
3400
|
parked: batch?.parked.map((parked) => ({
|
|
2451
3401
|
issue: parked.issue,
|
|
@@ -2456,8 +3406,27 @@ export class FactoryLoop {
|
|
|
2456
3406
|
counters: { ...this.#counters },
|
|
2457
3407
|
slackDegraded: this.#slackDegraded,
|
|
2458
3408
|
slackDegradedReason: this.#slackDegradedReason,
|
|
3409
|
+
eventListener: this.#eventListenerStatus(),
|
|
2459
3410
|
};
|
|
2460
3411
|
}
|
|
3412
|
+
#eventListenerStatus() {
|
|
3413
|
+
if (this.#startMode !== 'live') {
|
|
3414
|
+
return {
|
|
3415
|
+
state: 'not-listening',
|
|
3416
|
+
reason: this.#startMode ? `factory mode is ${this.#startMode}` : 'factory has not started',
|
|
3417
|
+
};
|
|
3418
|
+
}
|
|
3419
|
+
if (!this.#liveHeartbeatActive) {
|
|
3420
|
+
return { state: 'not-listening', reason: 'live daemon heartbeat is inactive' };
|
|
3421
|
+
}
|
|
3422
|
+
if (this.#subscription) {
|
|
3423
|
+
return { state: 'subscribed' };
|
|
3424
|
+
}
|
|
3425
|
+
if (this.#liveTransport === 'poll') {
|
|
3426
|
+
return { state: 'polling' };
|
|
3427
|
+
}
|
|
3428
|
+
return { state: 'starting' };
|
|
3429
|
+
}
|
|
2461
3430
|
on(event, listener) {
|
|
2462
3431
|
let listeners = this.#listeners.get(event);
|
|
2463
3432
|
if (!listeners) {
|
|
@@ -2577,10 +3546,11 @@ export class FactoryLoop {
|
|
|
2577
3546
|
if (claim.lifecycle.phase === 'waiting-for-human')
|
|
2578
3547
|
continue;
|
|
2579
3548
|
const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle);
|
|
3549
|
+
let liveIssue;
|
|
2580
3550
|
if (!durableRecord.dryRun &&
|
|
2581
3551
|
claim.lifecycle.phase !== 'writeback-applied' &&
|
|
2582
3552
|
claim.lifecycle.phase !== 'releasing') {
|
|
2583
|
-
|
|
3553
|
+
liveIssue = await this.#readIssue(durableRecord.issue.path);
|
|
2584
3554
|
// A babysat Linear issue already at Done may have merged while this
|
|
2585
3555
|
// process was down. Let authoritative PR restoration drive the
|
|
2586
3556
|
// normal `complete` path so merged work is not mislabeled abandoned.
|
|
@@ -2607,6 +3577,18 @@ export class FactoryLoop {
|
|
|
2607
3577
|
const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing'
|
|
2608
3578
|
? durableRecord
|
|
2609
3579
|
: batch.restore(durableRecord);
|
|
3580
|
+
if (claim.lifecycle.phase === 'running' &&
|
|
3581
|
+
this.#ticketDispatchNotificationIsPending(claim.lifecycle)) {
|
|
3582
|
+
if (!liveIssue) {
|
|
3583
|
+
this.#dispatchLifecycleEpochs.delete(claim.key ?? key);
|
|
3584
|
+
this.#scheduleDispatchLifecycleRetry(restored);
|
|
3585
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: issue is not currently readable`);
|
|
3586
|
+
}
|
|
3587
|
+
if (!restored.result) {
|
|
3588
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${restored.issue.key}: dispatch result is missing`);
|
|
3589
|
+
}
|
|
3590
|
+
await this.#notifyTicketDispatch(restored.decision, liveIssue, restored, restored.result);
|
|
3591
|
+
}
|
|
2610
3592
|
if (claim.lifecycle.phase !== 'running')
|
|
2611
3593
|
this.#scheduleDispatchLifecycleRetry(restored);
|
|
2612
3594
|
// Parking agents are cleanup-only. Hydrating them makes relay
|
|
@@ -3201,6 +4183,9 @@ export class FactoryLoop {
|
|
|
3201
4183
|
? this.#finalizeRunCost(record, lifecycleRunId)
|
|
3202
4184
|
: previous?.cost;
|
|
3203
4185
|
const lifecycle = lifecycleFromInFlightRecord(record, lifecycleRunId, phase, this.#clock.now(), primaryPullRequest, pullRequests, releaseReason ?? previous?.releaseReason, cost);
|
|
4186
|
+
if (previous?.ticketDispatchNotification) {
|
|
4187
|
+
lifecycle.ticketDispatchNotification = structuredClone(previous.ticketDispatchNotification);
|
|
4188
|
+
}
|
|
3204
4189
|
for (const agent of lifecycle.agents) {
|
|
3205
4190
|
const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
|
|
3206
4191
|
if (previousAgent?.releasedAtMs !== undefined)
|
|
@@ -3397,6 +4382,15 @@ export class FactoryLoop {
|
|
|
3397
4382
|
await this.#abandonDurableResume(record, 'source issue became terminal before lifecycle cleanup');
|
|
3398
4383
|
return;
|
|
3399
4384
|
}
|
|
4385
|
+
if (this.#ticketDispatchNotificationIsPending(lifecycle)) {
|
|
4386
|
+
if (!liveIssue) {
|
|
4387
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: issue is not currently readable`);
|
|
4388
|
+
}
|
|
4389
|
+
if (!record.result) {
|
|
4390
|
+
throw new Error(`Unable to recover ticket-dispatch notification ${record.issue.key}: dispatch result is missing`);
|
|
4391
|
+
}
|
|
4392
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
4393
|
+
}
|
|
3400
4394
|
}
|
|
3401
4395
|
if (acquiredNow && lifecycle.phase === 'running') {
|
|
3402
4396
|
const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
|
|
@@ -3524,7 +4518,11 @@ export class FactoryLoop {
|
|
|
3524
4518
|
async #resumeDurableDispatch(record) {
|
|
3525
4519
|
let liveIssue;
|
|
3526
4520
|
if (!record.dryRun) {
|
|
3527
|
-
|
|
4521
|
+
// record is already inserted into the batch by the time any phase
|
|
4522
|
+
// handler runs (BatchTracker.restore, called before this), so
|
|
4523
|
+
// #deriveGithubApiFallbackEligibility would find it there too — the
|
|
4524
|
+
// explicit hint is defense in depth, not a requirement.
|
|
4525
|
+
liveIssue = await this.#readIssue(record.issue.path, record.decision);
|
|
3528
4526
|
if (!liveIssue) {
|
|
3529
4527
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is not currently readable`);
|
|
3530
4528
|
}
|
|
@@ -3569,18 +4567,22 @@ export class FactoryLoop {
|
|
|
3569
4567
|
const spawned = await this.#spawnAgent(record, spec, record.dryRun);
|
|
3570
4568
|
agents.push({ name: spawned.name, role: spec.role });
|
|
3571
4569
|
}
|
|
4570
|
+
const comment = dispatchComment(record.decision, agents);
|
|
4571
|
+
let implementingStateId;
|
|
4572
|
+
if (!record.dryRun) {
|
|
4573
|
+
record.dispatchClaim = {
|
|
4574
|
+
state: 'pending',
|
|
4575
|
+
updatedAtMs: this.#clock.now(),
|
|
4576
|
+
};
|
|
4577
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4578
|
+
}
|
|
3572
4579
|
await this.#writeInFlightRegistry();
|
|
3573
4580
|
if (!record.dryRun) {
|
|
3574
4581
|
const issue = liveIssue ?? await this.#readIssue(record.issue.path);
|
|
3575
4582
|
if (!issue)
|
|
3576
4583
|
throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
|
|
3577
4584
|
await this.#ensureGithubAgentQuestionWatch(record, issue);
|
|
3578
|
-
|
|
3579
|
-
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
3580
|
-
}
|
|
3581
|
-
else {
|
|
3582
|
-
await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing'));
|
|
3583
|
-
}
|
|
4585
|
+
implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
|
|
3584
4586
|
}
|
|
3585
4587
|
const recoveredPreviews = uniquePreviewReferences([
|
|
3586
4588
|
...dispatchSpecs(record.decision).map((spec) => spec.preview),
|
|
@@ -3589,7 +4591,8 @@ export class FactoryLoop {
|
|
|
3589
4591
|
record.result ??= {
|
|
3590
4592
|
issue: record.issue,
|
|
3591
4593
|
agents,
|
|
3592
|
-
comments: [
|
|
4594
|
+
comments: [comment],
|
|
4595
|
+
stateId: implementingStateId,
|
|
3593
4596
|
...(recoveredPreviews.length > 0 ? { previews: recoveredPreviews } : {}),
|
|
3594
4597
|
dryRun: record.dryRun,
|
|
3595
4598
|
};
|
|
@@ -3598,7 +4601,8 @@ export class FactoryLoop {
|
|
|
3598
4601
|
}
|
|
3599
4602
|
if (!await this.#saveDispatchLifecycle(record, 'running'))
|
|
3600
4603
|
return;
|
|
3601
|
-
if (
|
|
4604
|
+
if (liveIssue) {
|
|
4605
|
+
await this.#notifyTicketDispatch(record.decision, liveIssue, record, record.result);
|
|
3602
4606
|
await this.#ensureSlackDispatchThread(record, record.result);
|
|
3603
4607
|
for (const tracked of record.agents.values()) {
|
|
3604
4608
|
const owned = tracked.spec.ownedPullRequest;
|
|
@@ -3887,6 +4891,97 @@ export class FactoryLoop {
|
|
|
3887
4891
|
}
|
|
3888
4892
|
await this.#linear.postComment(issue, body);
|
|
3889
4893
|
}
|
|
4894
|
+
async #applyDispatchClaim(record, issue, comment) {
|
|
4895
|
+
let implementingStateId;
|
|
4896
|
+
if (isGithubIssue(issue)) {
|
|
4897
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
|
|
4898
|
+
await this.#githubWriteback.setStatus(issue, 'in-progress');
|
|
4899
|
+
});
|
|
4900
|
+
const commentApplied = this.#githubWriteback.hasCommentMarker
|
|
4901
|
+
? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
|
|
4902
|
+
: undefined;
|
|
4903
|
+
await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => this.#githubWriteback.postComment(issue, comment), commentApplied);
|
|
4904
|
+
}
|
|
4905
|
+
else {
|
|
4906
|
+
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
4907
|
+
await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
|
|
4908
|
+
await this.#linear.setState(issue, implementingStateId);
|
|
4909
|
+
});
|
|
4910
|
+
await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
|
|
4911
|
+
await this.#linear.postComment(issue, comment);
|
|
4912
|
+
});
|
|
4913
|
+
}
|
|
4914
|
+
record.dispatchClaim = {
|
|
4915
|
+
state: 'verified',
|
|
4916
|
+
updatedAtMs: this.#clock.now(),
|
|
4917
|
+
};
|
|
4918
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4919
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
4920
|
+
return implementingStateId;
|
|
4921
|
+
}
|
|
4922
|
+
async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
|
|
4923
|
+
let lastError;
|
|
4924
|
+
for (let attempt = 1; attempt <= DISPATCH_WRITEBACK_MAX_ATTEMPTS; attempt += 1) {
|
|
4925
|
+
try {
|
|
4926
|
+
if (isApplied && await isApplied())
|
|
4927
|
+
return;
|
|
4928
|
+
await apply();
|
|
4929
|
+
if (isApplied && !await isApplied()) {
|
|
4930
|
+
throw new Error(`${write} returned without a provider read-back acknowledgement`);
|
|
4931
|
+
}
|
|
4932
|
+
return;
|
|
4933
|
+
}
|
|
4934
|
+
catch (error) {
|
|
4935
|
+
lastError = error;
|
|
4936
|
+
const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
|
|
4937
|
+
this.#increment('dispatchWritebackFailures');
|
|
4938
|
+
record.dispatchClaim = {
|
|
4939
|
+
state: 'degraded',
|
|
4940
|
+
write,
|
|
4941
|
+
attempts: attempt,
|
|
4942
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
4943
|
+
error: describeError(error).errorMessage,
|
|
4944
|
+
...(deadLettered ? { deadLettered: true } : {}),
|
|
4945
|
+
updatedAtMs: this.#clock.now(),
|
|
4946
|
+
};
|
|
4947
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
4948
|
+
await this.#writeDispatchClaimRegistry(record.issue);
|
|
4949
|
+
if (deadLettered) {
|
|
4950
|
+
this.#increment('dispatchWritebackDeadLetters');
|
|
4951
|
+
this.#logger.error?.('[factory] dispatch writeback dead-lettered after retries exhausted', {
|
|
4952
|
+
issue: issue.key,
|
|
4953
|
+
write,
|
|
4954
|
+
attempts: attempt,
|
|
4955
|
+
error: describeError(error).errorMessage,
|
|
4956
|
+
});
|
|
4957
|
+
throw error;
|
|
4958
|
+
}
|
|
4959
|
+
this.#increment('dispatchWritebackRetries');
|
|
4960
|
+
this.#logger.error?.('[factory] dispatch writeback failed; retrying', {
|
|
4961
|
+
issue: issue.key,
|
|
4962
|
+
write,
|
|
4963
|
+
attempt,
|
|
4964
|
+
maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
|
|
4965
|
+
retryMs: DISPATCH_WRITEBACK_RETRY_MS,
|
|
4966
|
+
error: describeError(error).errorMessage,
|
|
4967
|
+
});
|
|
4968
|
+
await this.#clock.sleep(DISPATCH_WRITEBACK_RETRY_MS);
|
|
4969
|
+
}
|
|
4970
|
+
}
|
|
4971
|
+
throw lastError;
|
|
4972
|
+
}
|
|
4973
|
+
async #writeDispatchClaimRegistry(issue) {
|
|
4974
|
+
try {
|
|
4975
|
+
await this.#writeInFlightRegistry();
|
|
4976
|
+
}
|
|
4977
|
+
catch (error) {
|
|
4978
|
+
this.#increment('dispatchClaimRegistryWriteFailures');
|
|
4979
|
+
this.#logger.error?.('[factory] failed to persist dispatch claim visibility', {
|
|
4980
|
+
issue: issue.key,
|
|
4981
|
+
error: describeError(error).errorMessage,
|
|
4982
|
+
});
|
|
4983
|
+
}
|
|
4984
|
+
}
|
|
3890
4985
|
// Probes the GitHub issue sub-root at most once and caches the verdict so
|
|
3891
4986
|
// repeated iterations skip listTree calls when the mount is absent.
|
|
3892
4987
|
async #ensureGithubIngestionReady() {
|
|
@@ -3944,7 +5039,7 @@ export class FactoryLoop {
|
|
|
3944
5039
|
const candidates = [];
|
|
3945
5040
|
let scanned = 0;
|
|
3946
5041
|
let lastProgressAtMs = startedAtMs;
|
|
3947
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading')) {
|
|
5042
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
|
|
3948
5043
|
await this.#refreshLiveHeartbeatIfDue();
|
|
3949
5044
|
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
3950
5045
|
continue;
|
|
@@ -3967,18 +5062,35 @@ export class FactoryLoop {
|
|
|
3967
5062
|
try {
|
|
3968
5063
|
const issuePaths = new Map();
|
|
3969
5064
|
for (const { owner, repo } of configuredGithubRepoParts(this.#config)) {
|
|
3970
|
-
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
3971
5065
|
const roots = githubIssueRepoRoots(owner, repo);
|
|
5066
|
+
const cachedBatches = roots.map((root) => this.#cachedDiscoveryTree(root));
|
|
5067
|
+
const allRootsCached = cachedBatches.every((paths) => paths !== undefined);
|
|
5068
|
+
const indexedPaths = allRootsCached
|
|
5069
|
+
? undefined
|
|
5070
|
+
: await this.#githubIssuePathsFromIndex(owner, repo);
|
|
3972
5071
|
// Keep the fallback roots as separate batches. Flattening a very large
|
|
3973
5072
|
// provider result is synchronous work and can starve the durable loop
|
|
3974
5073
|
// heartbeat before the bounded scan below gets a chance to yield.
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
5074
|
+
let pathBatches;
|
|
5075
|
+
if (allRootsCached) {
|
|
5076
|
+
pathBatches = cachedBatches;
|
|
5077
|
+
this.#increment('githubIssueDiscoveryCacheReposUsed');
|
|
5078
|
+
}
|
|
5079
|
+
else if (indexedPaths) {
|
|
5080
|
+
// Do not feed this into the discovery cache: the index only covers
|
|
5081
|
+
// open, labeled issues, so it is a filtered subset of the real
|
|
5082
|
+
// tree (and an empty result for whichever root form the index
|
|
5083
|
+
// doesn't use) — not something a later fresh listTree call may
|
|
5084
|
+
// treat as "the tree", including on the escalation-marker and
|
|
5085
|
+
// comment-replay call sites that share this cache.
|
|
5086
|
+
pathBatches = [indexedPaths];
|
|
3979
5087
|
this.#increment('githubIssueIndexReposUsed');
|
|
3980
5088
|
}
|
|
3981
5089
|
else {
|
|
5090
|
+
pathBatches = [];
|
|
5091
|
+
for (const root of roots) {
|
|
5092
|
+
pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
|
|
5093
|
+
}
|
|
3982
5094
|
this.#increment('githubIssueIndexFallbacks');
|
|
3983
5095
|
}
|
|
3984
5096
|
for (const paths of pathBatches) {
|
|
@@ -4023,6 +5135,8 @@ export class FactoryLoop {
|
|
|
4023
5135
|
return [...issuePaths.values()].sort();
|
|
4024
5136
|
}
|
|
4025
5137
|
catch (error) {
|
|
5138
|
+
if (relayfileOverload(error))
|
|
5139
|
+
throw error;
|
|
4026
5140
|
this.#githubIssuePathIndexReady = false;
|
|
4027
5141
|
this.#increment('githubIssueListFailures');
|
|
4028
5142
|
this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
|
|
@@ -4036,7 +5150,9 @@ export class FactoryLoop {
|
|
|
4036
5150
|
const { content } = await this.#readRelayfileFile(indexPath, 'GitHub issue index discovery');
|
|
4037
5151
|
parsed = parseJsonContent(content);
|
|
4038
5152
|
}
|
|
4039
|
-
catch {
|
|
5153
|
+
catch (error) {
|
|
5154
|
+
if (relayfileOverload(error))
|
|
5155
|
+
throw error;
|
|
4040
5156
|
return undefined;
|
|
4041
5157
|
}
|
|
4042
5158
|
if (!Array.isArray(parsed))
|
|
@@ -4123,10 +5239,67 @@ export class FactoryLoop {
|
|
|
4123
5239
|
this.#increment('githubIssueMirrorsCreated');
|
|
4124
5240
|
}
|
|
4125
5241
|
catch (error) {
|
|
5242
|
+
if (relayfileOverload(error))
|
|
5243
|
+
throw error;
|
|
4126
5244
|
this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
|
|
4127
5245
|
}
|
|
4128
5246
|
}
|
|
4129
|
-
|
|
5247
|
+
#rememberGithubApiFallbackEligible(identity) {
|
|
5248
|
+
rememberBoundedFallbackEligibility(this.#githubApiFallbackIssues, this.#githubApiFallbackIssuesEvicted, identity, GITHUB_API_FALLBACK_ISSUES_MAX, GITHUB_API_FALLBACK_ISSUES_EVICTED_MAX);
|
|
5249
|
+
}
|
|
5250
|
+
/**
|
|
5251
|
+
* `#githubApiFallbackIssues` is a cache, not a source of truth: eligibility
|
|
5252
|
+
* is derived here, not remembered by every caller. A restart (or any read
|
|
5253
|
+
* path added later that never registered anything) still resolves
|
|
5254
|
+
* correctly because this checks the actual current state instead of
|
|
5255
|
+
* requiring every entry point to have called a registration method first.
|
|
5256
|
+
*
|
|
5257
|
+
* A decision can durably outlive the original dispatch call in three
|
|
5258
|
+
* shapes, and this checks all three — a fourth found later means this
|
|
5259
|
+
* enumeration is incomplete, not that the approach is wrong:
|
|
5260
|
+
* 1. `batch.inFlight` — an active live-dispatched or durably-restored
|
|
5261
|
+
* record (`BatchTracker.restore`/`start` insert it before any phase
|
|
5262
|
+
* handler runs).
|
|
5263
|
+
* 2. `#state.listWaitingClarifications` — an issue parked awaiting a
|
|
5264
|
+
* human reply. Restoring it (`#restoreClarifications` et al.)
|
|
5265
|
+
* rebuilds an `InFlightIssue`-shaped record locally to re-arm the
|
|
5266
|
+
* watcher and does not insert it into the batch, so (1) alone misses
|
|
5267
|
+
* it.
|
|
5268
|
+
* 3. `#state.listDispatchLifecycles`, non-terminal — a durably-tracked
|
|
5269
|
+
* dispatch that is not currently in-memory at all yet (e.g. a
|
|
5270
|
+
* non-durable-fleet dispatch never reaches the batch either).
|
|
5271
|
+
*
|
|
5272
|
+
* `decisionHint` stays a pure optimization ahead of all three: the one
|
|
5273
|
+
* read that happens before its own record exists anywhere (the initial
|
|
5274
|
+
* live dispatch, validating scope before it is tracked) can skip the
|
|
5275
|
+
* scan entirely, but no read site is required to pass it — omitting it
|
|
5276
|
+
* only costs the scan, never correctness.
|
|
5277
|
+
*
|
|
5278
|
+
* Deliberately scans by GitHub identity (owner/repo/number) against each
|
|
5279
|
+
* candidate's own issue path rather than looking up the durable record by
|
|
5280
|
+
* its composite key (uuid/key/path): the real uuid is built from GitHub's
|
|
5281
|
+
* node_id/id when content is available, which a bare path cannot
|
|
5282
|
+
* reconstruct, so a keyed lookup would not reliably match.
|
|
5283
|
+
*/
|
|
5284
|
+
async #deriveGithubApiFallbackEligibility(identity, decisionHint) {
|
|
5285
|
+
if (decisionHint?.issueResolution?.source === 'github-api-fallback' &&
|
|
5286
|
+
githubIssueRefIdentity(decisionHint.issue) === identity) {
|
|
5287
|
+
return true;
|
|
5288
|
+
}
|
|
5289
|
+
const inFlight = (await this.#batch()).inFlight;
|
|
5290
|
+
if (isGithubApiFallbackEligible(inFlight, identity))
|
|
5291
|
+
return true;
|
|
5292
|
+
const [waitingClarifications, dispatchLifecycles] = await Promise.all([
|
|
5293
|
+
this.#state.listWaitingClarifications(this.#workspaceId),
|
|
5294
|
+
this.#state.listDispatchLifecycles(this.#workspaceId),
|
|
5295
|
+
]);
|
|
5296
|
+
const durable = [
|
|
5297
|
+
...githubApiFallbackCandidatesFromWaitingClarifications(waitingClarifications),
|
|
5298
|
+
...githubApiFallbackCandidatesFromDispatchLifecycles(dispatchLifecycles),
|
|
5299
|
+
];
|
|
5300
|
+
return isGithubApiFallbackEligible(durable, identity);
|
|
5301
|
+
}
|
|
5302
|
+
async #readGithubIssue(path, decisionHint) {
|
|
4130
5303
|
const preferredPath = await this.#preferredGithubIssuePath(path);
|
|
4131
5304
|
const candidatePaths = [...new Set([
|
|
4132
5305
|
...githubIssueReadCandidatePaths(preferredPath),
|
|
@@ -4150,8 +5323,55 @@ export class FactoryLoop {
|
|
|
4150
5323
|
}
|
|
4151
5324
|
catch (error) {
|
|
4152
5325
|
if (isMissingIssueFileError(error)) {
|
|
5326
|
+
const parts = githubIssuePathParts(path) ?? githubIssueDirectoryPathParts(path);
|
|
5327
|
+
const identity = parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
|
|
5328
|
+
const eligible = identity
|
|
5329
|
+
? this.#githubApiFallbackIssues.has(identity) ||
|
|
5330
|
+
await this.#deriveGithubApiFallbackEligibility(identity, decisionHint)
|
|
5331
|
+
: false;
|
|
5332
|
+
if (eligible && identity)
|
|
5333
|
+
this.#rememberGithubApiFallbackEligible(identity);
|
|
5334
|
+
if (parts && identity && eligible && this.#mount.githubRead) {
|
|
5335
|
+
const lookup = await this.#mount.githubRead.getIssue(`${parts.owner}/${parts.repo}`, parts.number);
|
|
5336
|
+
if (lookup.outcome === 'found') {
|
|
5337
|
+
const githubIssue = parseGithubIssue(lookup.issue.path, lookup.issue.content);
|
|
5338
|
+
this.#indexDependencyIssue(githubIssueAsFactoryIssue(githubIssue));
|
|
5339
|
+
this.#logger.warn?.('[factory] Relayfile projection missed GitHub issue; using GitHub API fallback', {
|
|
5340
|
+
repo: lookup.issue.repo,
|
|
5341
|
+
number: lookup.issue.number,
|
|
5342
|
+
source: 'github-api-fallback',
|
|
5343
|
+
});
|
|
5344
|
+
return githubIssue;
|
|
5345
|
+
}
|
|
5346
|
+
if (lookup.outcome === 'indeterminate') {
|
|
5347
|
+
// The provider lookup could not confirm absence (e.g. an
|
|
5348
|
+
// unauthenticated 404 against a repo it cannot prove is public).
|
|
5349
|
+
// That is not the same claim as "confirmed gone" — count and log
|
|
5350
|
+
// it separately so it stays visible instead of being folded into
|
|
5351
|
+
// the phantom-skip metric.
|
|
5352
|
+
this.#increment('githubIssueUnverifiable');
|
|
5353
|
+
this.#logger.warn?.('[factory] GitHub API fallback could not determine issue existence', {
|
|
5354
|
+
path,
|
|
5355
|
+
repo: `${parts.owner}/${parts.repo}`,
|
|
5356
|
+
number: parts.number,
|
|
5357
|
+
reason: lookup.reason,
|
|
5358
|
+
});
|
|
5359
|
+
return undefined;
|
|
5360
|
+
}
|
|
5361
|
+
}
|
|
5362
|
+
if (identity && this.#githubApiFallbackIssuesEvicted.has(identity)) {
|
|
5363
|
+
// identity was eligible for the API fallback but aged out of the
|
|
5364
|
+
// bounded set — a different claim from "never was eligible", and
|
|
5365
|
+
// one this signal must not silently collapse into a confirmed miss.
|
|
5366
|
+
this.#increment('githubIssueApiFallbackEligibilityEvicted');
|
|
5367
|
+
this.#logger.warn?.('[factory] GitHub API fallback eligibility was evicted from the bounded cache before this read', {
|
|
5368
|
+
path,
|
|
5369
|
+
identity,
|
|
5370
|
+
});
|
|
5371
|
+
return undefined;
|
|
5372
|
+
}
|
|
4153
5373
|
this.#increment('githubIssuePhantomSkipped');
|
|
4154
|
-
this.#logger.debug?.('[factory] skipped missing GitHub issue
|
|
5374
|
+
this.#logger.debug?.('[factory] skipped missing GitHub issue after projection and provider lookup', { path });
|
|
4155
5375
|
return undefined;
|
|
4156
5376
|
}
|
|
4157
5377
|
throw error;
|
|
@@ -4338,7 +5558,7 @@ export class FactoryLoop {
|
|
|
4338
5558
|
// The pass-scoped flag also lets every later blocked issue reuse the
|
|
4339
5559
|
// resulting dependency index instead of rescanning the full tree.
|
|
4340
5560
|
this.#dependencyLinearTreeLoaded = true;
|
|
4341
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery')) {
|
|
5561
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
|
|
4342
5562
|
if (isIssueFilePath(path))
|
|
4343
5563
|
await this.#readIssue(path);
|
|
4344
5564
|
}
|
|
@@ -4393,7 +5613,7 @@ export class FactoryLoop {
|
|
|
4393
5613
|
const pullRequest = await resolveIssuePrFromMount(this.#mount, this.#config, issue, {
|
|
4394
5614
|
allowLegacyGithubBranch: true,
|
|
4395
5615
|
repo,
|
|
4396
|
-
});
|
|
5616
|
+
}, (prefix) => this.#listRelayfileTree(prefix, 'dependency PR probe resolution'));
|
|
4397
5617
|
if (normalizePrState(pullRequest?.state) !== 'MERGED')
|
|
4398
5618
|
return false;
|
|
4399
5619
|
this.#terminalDependencyIdentities.add(identity);
|
|
@@ -4569,6 +5789,7 @@ export class FactoryLoop {
|
|
|
4569
5789
|
updatedAt: new Date(updatedAtMs).toISOString(),
|
|
4570
5790
|
updatedAtMs,
|
|
4571
5791
|
registryPath,
|
|
5792
|
+
eventListener: this.#eventListenerStatus(),
|
|
4572
5793
|
};
|
|
4573
5794
|
await mkdir(dirname(path), { recursive: true });
|
|
4574
5795
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
@@ -4715,14 +5936,14 @@ export class FactoryLoop {
|
|
|
4715
5936
|
}
|
|
4716
5937
|
const pathsByKey = new Map();
|
|
4717
5938
|
const canonicalPathsByKey = new Map();
|
|
4718
|
-
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery')) {
|
|
5939
|
+
for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
|
|
4719
5940
|
if (isIssueFilePath(path)) {
|
|
4720
5941
|
const key = keyFromPath(path);
|
|
4721
5942
|
canonicalPathsByKey.set(key, path);
|
|
4722
5943
|
pathsByKey.set(key, path);
|
|
4723
5944
|
}
|
|
4724
5945
|
}
|
|
4725
|
-
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery')) {
|
|
5946
|
+
for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
|
|
4726
5947
|
if (isIssueAliasFilePath(path)) {
|
|
4727
5948
|
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
4728
5949
|
if (canonicalPath) {
|
|
@@ -4736,10 +5957,10 @@ export class FactoryLoop {
|
|
|
4736
5957
|
}
|
|
4737
5958
|
return [...pathsByKey.values()].sort();
|
|
4738
5959
|
}
|
|
4739
|
-
async #readIssue(path) {
|
|
5960
|
+
async #readIssue(path, decisionHint) {
|
|
4740
5961
|
try {
|
|
4741
5962
|
if (isGithubIssueFilePath(path)) {
|
|
4742
|
-
const githubIssue = await this.#readGithubIssue(path);
|
|
5963
|
+
const githubIssue = await this.#readGithubIssue(path, decisionHint);
|
|
4743
5964
|
return githubIssue ? githubIssueAsFactoryIssue(githubIssue) : undefined;
|
|
4744
5965
|
}
|
|
4745
5966
|
// Newly-synced issues land as a change-event STUB at the primary
|
|
@@ -4763,6 +5984,8 @@ export class FactoryLoop {
|
|
|
4763
5984
|
return resolvedIssue;
|
|
4764
5985
|
}
|
|
4765
5986
|
catch (error) {
|
|
5987
|
+
if (relayfileOverload(error))
|
|
5988
|
+
throw error;
|
|
4766
5989
|
if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
|
|
4767
5990
|
this.#increment('phantomSkipped');
|
|
4768
5991
|
this.#logger.debug?.('[factory] skipped missing issue file discovered from issue tree', { path });
|
|
@@ -5006,6 +6229,7 @@ export class FactoryLoop {
|
|
|
5006
6229
|
}
|
|
5007
6230
|
}
|
|
5008
6231
|
const fleetTracked = this.#fleet.trackedAgents?.().get(agentName);
|
|
6232
|
+
const dispatchClaim = this.#dispatchClaimStatuses.get(issueKey(issue));
|
|
5009
6233
|
agents.push({
|
|
5010
6234
|
name: agentName,
|
|
5011
6235
|
role: tracked.spec.role,
|
|
@@ -5015,12 +6239,16 @@ export class FactoryLoop {
|
|
|
5015
6239
|
processes,
|
|
5016
6240
|
...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
|
|
5017
6241
|
...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
|
|
6242
|
+
...(dispatchClaim ? { dispatchClaim: { ...dispatchClaim } } : {}),
|
|
5018
6243
|
});
|
|
5019
6244
|
};
|
|
5020
6245
|
if (!empty) {
|
|
5021
6246
|
for (const record of (await this.#batch()).inFlight) {
|
|
5022
6247
|
if (record.dryRun)
|
|
5023
6248
|
continue;
|
|
6249
|
+
if (record.dispatchClaim) {
|
|
6250
|
+
this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
|
|
6251
|
+
}
|
|
5024
6252
|
for (const [agentName, tracked] of record.agents) {
|
|
5025
6253
|
await appendAgent(record.issue, agentName, tracked);
|
|
5026
6254
|
}
|
|
@@ -5556,10 +6784,25 @@ export class FactoryLoop {
|
|
|
5556
6784
|
}
|
|
5557
6785
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
5558
6786
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
6787
|
+
const expectedHeadRef = implementer.spec.branch;
|
|
6788
|
+
if (!expectedHeadRef) {
|
|
6789
|
+
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
6790
|
+
}
|
|
6791
|
+
const matchesCurrentConvention = factoryBranchBelongsToIssue(expectedHeadRef, record.issue.key);
|
|
6792
|
+
const matchesAuthorizedLegacyBranch = implementer.spec.existingPullRequestBranch === true &&
|
|
6793
|
+
/^\d+$/u.test(record.issue.key) &&
|
|
6794
|
+
expectedHeadRef.toLowerCase().startsWith(`${record.issue.key.toLowerCase()}-`);
|
|
6795
|
+
if (!matchesCurrentConvention && !matchesAuthorizedLegacyBranch) {
|
|
6796
|
+
throw new Error(`Refusing to publish ${record.issue.key}: branch ${expectedHeadRef} belongs to a different issue`);
|
|
6797
|
+
}
|
|
5559
6798
|
const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue));
|
|
5560
6799
|
const cached = this.#publishedPullRequests.get(key);
|
|
5561
|
-
if (cached)
|
|
6800
|
+
if (cached) {
|
|
6801
|
+
if (cached.headRef !== expectedHeadRef) {
|
|
6802
|
+
throw new Error(`Refusing cached PR for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${cached.headRef}`);
|
|
6803
|
+
}
|
|
5562
6804
|
return cached;
|
|
6805
|
+
}
|
|
5563
6806
|
const { identity, publisher } = this.#githubPullRequestPublisher();
|
|
5564
6807
|
const remoteBranch = implementer.result?.locality === 'remote' && implementer.spec.branch
|
|
5565
6808
|
? implementer.spec.branch
|
|
@@ -5579,11 +6822,13 @@ export class FactoryLoop {
|
|
|
5579
6822
|
: undefined;
|
|
5580
6823
|
const repo = normalizeGithubRepo(implementer.spec.repo, this.#config.repos.org ?? sourceOwner);
|
|
5581
6824
|
const durableReceipt = publishedPullRequests(durable).find((receipt) => receipt.repo.toLowerCase() === repo.toLowerCase());
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
6825
|
+
if (durableReceipt) {
|
|
6826
|
+
if (durableReceipt.headRef !== expectedHeadRef) {
|
|
6827
|
+
throw new Error(`Refusing durable PR receipt for ${record.issue.key}: expected head branch ${expectedHeadRef}, found ${durableReceipt.headRef}`);
|
|
6828
|
+
}
|
|
5585
6829
|
return durableReceipt;
|
|
5586
|
-
|
|
6830
|
+
}
|
|
6831
|
+
if (opts.reconcileExisting) {
|
|
5587
6832
|
const existing = await this.#openPullRequestByHead(repo, expectedHeadRef);
|
|
5588
6833
|
if (existing) {
|
|
5589
6834
|
this.#publishedPullRequests.set(key, existing);
|
|
@@ -5601,15 +6846,17 @@ export class FactoryLoop {
|
|
|
5601
6846
|
const result = await publisher.publishPullRequest({
|
|
5602
6847
|
repo,
|
|
5603
6848
|
...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }),
|
|
6849
|
+
expectedHeadRef,
|
|
5604
6850
|
baseRef,
|
|
5605
6851
|
title: `${issue.key}: ${issue.title}`,
|
|
5606
6852
|
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
6853
|
+
...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
|
|
5607
6854
|
});
|
|
5608
6855
|
const published = result.author
|
|
5609
6856
|
? result
|
|
5610
6857
|
: { ...result, author: identity };
|
|
5611
6858
|
if (published.repo.toLowerCase() !== repo.toLowerCase() ||
|
|
5612
|
-
published.headRef !==
|
|
6859
|
+
published.headRef !== expectedHeadRef ||
|
|
5613
6860
|
!Number.isInteger(published.number) ||
|
|
5614
6861
|
published.number <= 0 ||
|
|
5615
6862
|
!published.url) {
|
|
@@ -5710,9 +6957,11 @@ export class FactoryLoop {
|
|
|
5710
6957
|
for (const root of roots) {
|
|
5711
6958
|
let paths;
|
|
5712
6959
|
try {
|
|
5713
|
-
paths = await this.#
|
|
6960
|
+
paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
|
|
5714
6961
|
}
|
|
5715
|
-
catch {
|
|
6962
|
+
catch (error) {
|
|
6963
|
+
if (relayfileOverload(error))
|
|
6964
|
+
throw error;
|
|
5716
6965
|
continue;
|
|
5717
6966
|
}
|
|
5718
6967
|
for (const path of paths) {
|
|
@@ -5978,9 +7227,11 @@ export class FactoryLoop {
|
|
|
5978
7227
|
for (let attempt = 0; attempt < PUBLISHED_PR_CONFIRM_ATTEMPTS; attempt += 1) {
|
|
5979
7228
|
const paths = (await Promise.all(roots.map(async (root) => {
|
|
5980
7229
|
try {
|
|
5981
|
-
return await this.#
|
|
7230
|
+
return await this.#listRelayfileTree(root, 'published PR confirmation');
|
|
5982
7231
|
}
|
|
5983
|
-
catch {
|
|
7232
|
+
catch (error) {
|
|
7233
|
+
if (relayfileOverload(error))
|
|
7234
|
+
throw error;
|
|
5984
7235
|
return [];
|
|
5985
7236
|
}
|
|
5986
7237
|
}))).flat();
|
|
@@ -6258,6 +7509,20 @@ export class FactoryLoop {
|
|
|
6258
7509
|
if (!tracked.sessionRef) {
|
|
6259
7510
|
return;
|
|
6260
7511
|
}
|
|
7512
|
+
// Never resume an agent against a read-denied mirror: the mount lacks the
|
|
7513
|
+
// filesystem scope it needs, so the spawn would fail opaquely (roster PID
|
|
7514
|
+
// never resolves) and risk operating on stale integration state. Skip with
|
|
7515
|
+
// one clear message; a later attempt succeeds once the session is re-authed
|
|
7516
|
+
// and Factory restarted.
|
|
7517
|
+
if (this.#mount.isLocalMountAuthDegraded?.()) {
|
|
7518
|
+
this.#increment('resumeSkippedMountAuthDegraded');
|
|
7519
|
+
this.#logger.warn?.('[factory] tracked agent resume skipped: local mount is auth-degraded (cloud session missing relayfile fs scope); re-authenticate and restart Factory', {
|
|
7520
|
+
issue: record.issue.key,
|
|
7521
|
+
name,
|
|
7522
|
+
role: tracked.spec.role,
|
|
7523
|
+
});
|
|
7524
|
+
return;
|
|
7525
|
+
}
|
|
6261
7526
|
this.#logger.debug?.('[factory] tracked agent resume preparation started', {
|
|
6262
7527
|
issue: record.issue.key,
|
|
6263
7528
|
name,
|
|
@@ -7217,7 +8482,7 @@ export class FactoryLoop {
|
|
|
7217
8482
|
`${GITHUB_ISSUE_ROOT}/${owner}__${repo}/issues`,
|
|
7218
8483
|
]) {
|
|
7219
8484
|
try {
|
|
7220
|
-
for (const path of await this.#
|
|
8485
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub escalation marker listing')) {
|
|
7221
8486
|
const parts = githubIssueCommentPathParts(path);
|
|
7222
8487
|
if (parts &&
|
|
7223
8488
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7511,7 +8776,7 @@ export class FactoryLoop {
|
|
|
7511
8776
|
];
|
|
7512
8777
|
for (const prefix of prefixes) {
|
|
7513
8778
|
try {
|
|
7514
|
-
for (const path of await this.#
|
|
8779
|
+
for (const path of await this.#listRelayfileTree(prefix, 'GitHub issue comment replay listing')) {
|
|
7515
8780
|
const parts = githubIssueCommentPathParts(path);
|
|
7516
8781
|
if (parts &&
|
|
7517
8782
|
parts.owner.toLowerCase() === source.owner.toLowerCase() &&
|
|
@@ -7522,6 +8787,8 @@ export class FactoryLoop {
|
|
|
7522
8787
|
}
|
|
7523
8788
|
}
|
|
7524
8789
|
catch (error) {
|
|
8790
|
+
if (relayfileOverload(error))
|
|
8791
|
+
throw error;
|
|
7525
8792
|
this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
|
|
7526
8793
|
}
|
|
7527
8794
|
}
|
|
@@ -8681,7 +9948,24 @@ export class FactoryLoop {
|
|
|
8681
9948
|
const owner = await this.#babysitterOwnerFor(`${event.owner}/${event.repo}`, target.prNumber);
|
|
8682
9949
|
if (!owner) {
|
|
8683
9950
|
this.#increment('babysitterEventsIgnoredUnownedPr');
|
|
8684
|
-
|
|
9951
|
+
// "Unowned" means no babysitter was ever registered for this PR — the
|
|
9952
|
+
// downstream symptom of a missed or failed PR-open. Warn once per PR so
|
|
9953
|
+
// it is visible at default level, then fall back to debug for the
|
|
9954
|
+
// repeats; #sweepOrphanedBabysitterPrs is what actually recovers it.
|
|
9955
|
+
const identity = githubPrIdentity(`${event.owner}/${event.repo}`, target.prNumber);
|
|
9956
|
+
const detail = { ...event, prNumber: target.prNumber };
|
|
9957
|
+
// The size cap gates the warn itself, not just the bookkeeping: an
|
|
9958
|
+
// unbounded set of identities must not become an unbounded warn stream.
|
|
9959
|
+
const firstForPr = Boolean(identity) &&
|
|
9960
|
+
!this.#babysitterUnownedPrWarned.has(identity) &&
|
|
9961
|
+
this.#babysitterUnownedPrWarned.size < BABYSITTER_UNOWNED_PR_WARN_LIMIT;
|
|
9962
|
+
if (firstForPr) {
|
|
9963
|
+
this.#babysitterUnownedPrWarned.add(identity);
|
|
9964
|
+
this.#logger.warn?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
9965
|
+
}
|
|
9966
|
+
else {
|
|
9967
|
+
this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', detail);
|
|
9968
|
+
}
|
|
8685
9969
|
continue;
|
|
8686
9970
|
}
|
|
8687
9971
|
if (this.#mount.resourceSubscriptions &&
|
|
@@ -9135,19 +10419,92 @@ export class FactoryLoop {
|
|
|
9135
10419
|
// through the durable Relay lifecycle action; the orchestrator never runs `gh`. PR meta events here
|
|
9136
10420
|
// only (a) spawn the babysitter on open and (b) carry the latest open/draft/
|
|
9137
10421
|
// merged state used to guard the final transition.
|
|
10422
|
+
async #readPrSnapshotContent(path) {
|
|
10423
|
+
let lastError;
|
|
10424
|
+
for (let attempt = 1; attempt <= BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS; attempt += 1) {
|
|
10425
|
+
try {
|
|
10426
|
+
const { content } = await this.#mount.readFile(path);
|
|
10427
|
+
if (attempt > 1)
|
|
10428
|
+
this.#increment('babysitterPrSnapshotReadRetrySucceeded');
|
|
10429
|
+
return content;
|
|
10430
|
+
}
|
|
10431
|
+
catch (error) {
|
|
10432
|
+
lastError = error;
|
|
10433
|
+
if (attempt < BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS && !this.#stopping) {
|
|
10434
|
+
this.#increment('babysitterPrSnapshotReadRetries');
|
|
10435
|
+
await this.#clock.sleep(BABYSITTER_PR_SNAPSHOT_READ_BACKOFF_MS * attempt);
|
|
10436
|
+
continue;
|
|
10437
|
+
}
|
|
10438
|
+
break;
|
|
10439
|
+
}
|
|
10440
|
+
}
|
|
10441
|
+
throw lastError;
|
|
10442
|
+
}
|
|
10443
|
+
#recordBabysitterPrSnapshotFailure(path, error) {
|
|
10444
|
+
const { errorMessage } = describeError(error);
|
|
10445
|
+
const existing = this.#babysitterPrSnapshotDeadLetters.get(path);
|
|
10446
|
+
const failures = (existing?.failures ?? 0) + 1;
|
|
10447
|
+
// Re-insert last so the eviction below stays insertion-ordered by recency.
|
|
10448
|
+
this.#babysitterPrSnapshotDeadLetters.delete(path);
|
|
10449
|
+
this.#babysitterPrSnapshotDeadLetters.set(path, {
|
|
10450
|
+
failures,
|
|
10451
|
+
lastErrorMessage: errorMessage,
|
|
10452
|
+
firstFailedAtMs: existing?.firstFailedAtMs ?? this.#clock.now(),
|
|
10453
|
+
});
|
|
10454
|
+
while (this.#babysitterPrSnapshotDeadLetters.size > BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT) {
|
|
10455
|
+
const oldest = this.#babysitterPrSnapshotDeadLetters.keys().next().value;
|
|
10456
|
+
if (oldest === undefined)
|
|
10457
|
+
break;
|
|
10458
|
+
this.#babysitterPrSnapshotDeadLetters.delete(oldest);
|
|
10459
|
+
this.#increment('babysitterPrSnapshotDeadLettersEvicted');
|
|
10460
|
+
}
|
|
10461
|
+
this.#increment('babysitterPrSnapshotReadFailures');
|
|
10462
|
+
const detail = {
|
|
10463
|
+
path,
|
|
10464
|
+
error: errorMessage,
|
|
10465
|
+
failures,
|
|
10466
|
+
attemptsPerTry: BABYSITTER_PR_SNAPSHOT_READ_ATTEMPTS,
|
|
10467
|
+
deadLettered: this.#babysitterPrSnapshotDeadLetters.size,
|
|
10468
|
+
};
|
|
10469
|
+
// A PR whose meta cannot be read is a PR with no shepherd, which is not
|
|
10470
|
+
// debug-level information. Escalate once the sweep retries also fail.
|
|
10471
|
+
if (failures >= BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER) {
|
|
10472
|
+
this.#increment('babysitterPrSnapshotReadFailuresEscalated');
|
|
10473
|
+
// The sweep keeps retrying for as long as the mount stays faulted, so the
|
|
10474
|
+
// escalation logs on the transition and then only periodically. The
|
|
10475
|
+
// counters carry the true volume.
|
|
10476
|
+
const sinceEscalation = failures - BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_ESCALATE_AFTER;
|
|
10477
|
+
if (sinceEscalation % BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY === 0) {
|
|
10478
|
+
this.#logger.error?.('[factory] babysitter PR snapshot still unreadable after repeated retries', detail);
|
|
10479
|
+
}
|
|
10480
|
+
return;
|
|
10481
|
+
}
|
|
10482
|
+
this.#logger.warn?.('[factory] babysitter could not read PR snapshot', detail);
|
|
10483
|
+
}
|
|
9138
10484
|
async #handlePrChange(path) {
|
|
9139
10485
|
const parts = githubPullPathParts(path);
|
|
9140
10486
|
if (!parts) {
|
|
9141
10487
|
return;
|
|
9142
10488
|
}
|
|
9143
|
-
|
|
10489
|
+
// This read is the only gate between a PR opening and it getting a
|
|
10490
|
+
// babysitter, so it retries rather than treating one `fetch failed` as a
|
|
10491
|
+
// verdict. On exhaustion the path is dead-lettered for the reconcile sweep
|
|
10492
|
+
// (#sweepOrphanedBabysitterPrs) instead of being dropped.
|
|
10493
|
+
let content;
|
|
9144
10494
|
try {
|
|
9145
|
-
|
|
10495
|
+
content = await this.#readPrSnapshotContent(path);
|
|
9146
10496
|
}
|
|
9147
10497
|
catch (error) {
|
|
9148
|
-
this.#
|
|
10498
|
+
this.#recordBabysitterPrSnapshotFailure(path, error);
|
|
9149
10499
|
return;
|
|
9150
10500
|
}
|
|
10501
|
+
if (this.#babysitterPrSnapshotDeadLetters.delete(path)) {
|
|
10502
|
+
this.#increment('babysitterPrSnapshotDeadLettersRecovered');
|
|
10503
|
+
this.#logger.info?.('[factory] babysitter recovered a dead-lettered PR snapshot read', { path });
|
|
10504
|
+
}
|
|
10505
|
+
// parsePullSnapshot never throws; an undefined result is a structurally
|
|
10506
|
+
// mismatched payload, which a retry cannot fix.
|
|
10507
|
+
const snapshot = parsePullSnapshot(content, parts.number);
|
|
9151
10508
|
if (!snapshot) {
|
|
9152
10509
|
return;
|
|
9153
10510
|
}
|
|
@@ -9345,7 +10702,7 @@ export class FactoryLoop {
|
|
|
9345
10702
|
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
9346
10703
|
// over title/body references to avoid "related to AR-N" body false positives.
|
|
9347
10704
|
const githubSource = await this.#issueSource() === 'github';
|
|
9348
|
-
const paths = githubSource ? await this.#githubIssuePaths() : await this.#
|
|
10705
|
+
const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
|
|
9349
10706
|
for (const path of paths) {
|
|
9350
10707
|
if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
|
|
9351
10708
|
continue;
|
|
@@ -9811,10 +11168,12 @@ export class FactoryLoop {
|
|
|
9811
11168
|
const found = [];
|
|
9812
11169
|
for (const root of roots) {
|
|
9813
11170
|
try {
|
|
9814
|
-
const tree = await this.#
|
|
11171
|
+
const tree = await this.#listRelayfileTree(root, 'PR meta path discovery');
|
|
9815
11172
|
found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
|
|
9816
11173
|
}
|
|
9817
|
-
catch {
|
|
11174
|
+
catch (error) {
|
|
11175
|
+
if (relayfileOverload(error))
|
|
11176
|
+
throw error;
|
|
9818
11177
|
// try the next root
|
|
9819
11178
|
}
|
|
9820
11179
|
}
|
|
@@ -10007,6 +11366,124 @@ export class FactoryLoop {
|
|
|
10007
11366
|
}
|
|
10008
11367
|
}
|
|
10009
11368
|
}
|
|
11369
|
+
async #notifyTicketDispatch(decision, issue, record, result) {
|
|
11370
|
+
const hook = this.#config.hooks?.onTicketDispatch;
|
|
11371
|
+
if (!hook || result.dryRun)
|
|
11372
|
+
return;
|
|
11373
|
+
const agent = result.agents.find((candidate) => candidate.role === 'implementer')
|
|
11374
|
+
?? result.agents.find((candidate) => candidate.role === 'workflow')
|
|
11375
|
+
?? result.agents[0];
|
|
11376
|
+
if (!agent)
|
|
11377
|
+
return;
|
|
11378
|
+
if (!await this.#claimTicketDispatchNotification(record))
|
|
11379
|
+
return;
|
|
11380
|
+
const tracked = record.agents.get(agent.name);
|
|
11381
|
+
const payload = {
|
|
11382
|
+
eventType: 'ticket.dispatched',
|
|
11383
|
+
summary: `Ticket dispatched: ${issue.title} → ${agent.name}.`,
|
|
11384
|
+
issue: {
|
|
11385
|
+
id: issue.uuid,
|
|
11386
|
+
title: issue.title,
|
|
11387
|
+
url: dispatchIssueUrl(issue),
|
|
11388
|
+
},
|
|
11389
|
+
agent: {
|
|
11390
|
+
name: agent.name,
|
|
11391
|
+
sessionRef: tracked?.sessionRef ?? null,
|
|
11392
|
+
},
|
|
11393
|
+
sessionOwner: dispatchSessionOwner(decision) ?? null,
|
|
11394
|
+
timestamp: new Date(this.#clock.now()).toISOString(),
|
|
11395
|
+
};
|
|
11396
|
+
const text = ticketDispatchNotificationText(payload);
|
|
11397
|
+
for (const target of hook.notify) {
|
|
11398
|
+
if (target.surface === 'linear' && !target.commentOnIssue)
|
|
11399
|
+
continue;
|
|
11400
|
+
try {
|
|
11401
|
+
switch (target.surface) {
|
|
11402
|
+
case 'relay':
|
|
11403
|
+
await this.#fleet.sendMessage({
|
|
11404
|
+
to: `#${target.channel.replace(/^#/u, '')}`,
|
|
11405
|
+
text,
|
|
11406
|
+
});
|
|
11407
|
+
break;
|
|
11408
|
+
case 'slack':
|
|
11409
|
+
if (!target.channel && !target.dm) {
|
|
11410
|
+
throw new Error('Slack ticket-dispatch notification needs channel and/or dm');
|
|
11411
|
+
}
|
|
11412
|
+
await this.#ticketDispatchDelivery.slack({
|
|
11413
|
+
channel: target.channel,
|
|
11414
|
+
dm: target.dm,
|
|
11415
|
+
text,
|
|
11416
|
+
});
|
|
11417
|
+
break;
|
|
11418
|
+
case 'telegram':
|
|
11419
|
+
await this.#ticketDispatchDelivery.telegram({ chatId: target.chatId, text });
|
|
11420
|
+
break;
|
|
11421
|
+
case 'linear':
|
|
11422
|
+
if (isGithubIssue(issue)) {
|
|
11423
|
+
throw new Error('Linear ticket-dispatch comments require a Linear issue');
|
|
11424
|
+
}
|
|
11425
|
+
await this.#linear.postComment(issue, text);
|
|
11426
|
+
break;
|
|
11427
|
+
}
|
|
11428
|
+
this.#increment('ticketDispatchHookNotifications');
|
|
11429
|
+
}
|
|
11430
|
+
catch (error) {
|
|
11431
|
+
this.#increment('ticketDispatchHookFailures');
|
|
11432
|
+
this.#logger.warn?.('[factory] onTicketDispatch notification failed', {
|
|
11433
|
+
issue: issue.key,
|
|
11434
|
+
surface: target.surface,
|
|
11435
|
+
error: describeError(error).errorMessage,
|
|
11436
|
+
});
|
|
11437
|
+
}
|
|
11438
|
+
}
|
|
11439
|
+
}
|
|
11440
|
+
async #claimTicketDispatchNotification(record) {
|
|
11441
|
+
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
11442
|
+
return true;
|
|
11443
|
+
const key = issueKey(record.issue);
|
|
11444
|
+
return this.#serializeDispatchLifecyclePersistence(key, async () => {
|
|
11445
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
11446
|
+
if (epoch === undefined) {
|
|
11447
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11448
|
+
return false;
|
|
11449
|
+
}
|
|
11450
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
11451
|
+
if (!lifecycle) {
|
|
11452
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11453
|
+
return false;
|
|
11454
|
+
}
|
|
11455
|
+
const workUnitId = lifecycle.runId;
|
|
11456
|
+
if (lifecycle.ticketDispatchNotification?.workUnitId === workUnitId)
|
|
11457
|
+
return false;
|
|
11458
|
+
// Reserve the work unit before external side effects. Hook delivery is
|
|
11459
|
+
// best-effort per target, so takeover must prefer at-most-once fan-out
|
|
11460
|
+
// over replaying a notification whose provider acknowledgement was lost.
|
|
11461
|
+
// Follow-up for provider-idempotent retries: https://github.com/AgentWorkforce/factory/issues/239
|
|
11462
|
+
const claimedAtMs = this.#clock.now();
|
|
11463
|
+
const claimedLifecycle = {
|
|
11464
|
+
...lifecycle,
|
|
11465
|
+
ticketDispatchNotification: { workUnitId, claimedAtMs },
|
|
11466
|
+
updatedAtMs: claimedAtMs,
|
|
11467
|
+
};
|
|
11468
|
+
const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, claimedAtMs, claimedLifecycle);
|
|
11469
|
+
if (!saved) {
|
|
11470
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
11471
|
+
this.#increment('dispatchLifecycleFencesRejected');
|
|
11472
|
+
await this.#reportLifecycle(claimedLifecycle, 'factory.anomaly', {
|
|
11473
|
+
level: 'error',
|
|
11474
|
+
errorCode: 'fence_rejected',
|
|
11475
|
+
});
|
|
11476
|
+
this.#scheduleDispatchLifecycleRetry(record);
|
|
11477
|
+
return false;
|
|
11478
|
+
}
|
|
11479
|
+
return true;
|
|
11480
|
+
});
|
|
11481
|
+
}
|
|
11482
|
+
#ticketDispatchNotificationIsPending(lifecycle) {
|
|
11483
|
+
return Boolean(this.#config.hooks?.onTicketDispatch) &&
|
|
11484
|
+
!lifecycle.dryRun &&
|
|
11485
|
+
lifecycle.ticketDispatchNotification?.workUnitId !== lifecycle.runId;
|
|
11486
|
+
}
|
|
10010
11487
|
#error(error, issue) {
|
|
10011
11488
|
this.#increment('errors');
|
|
10012
11489
|
const details = describeError(error);
|
|
@@ -10801,9 +12278,11 @@ export class FactoryLoop {
|
|
|
10801
12278
|
async #findSlackUserIdByIdentity(identity) {
|
|
10802
12279
|
const list = async (prefix) => {
|
|
10803
12280
|
try {
|
|
10804
|
-
return await this.#
|
|
12281
|
+
return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
|
|
10805
12282
|
}
|
|
10806
|
-
catch {
|
|
12283
|
+
catch (error) {
|
|
12284
|
+
if (relayfileOverload(error))
|
|
12285
|
+
throw error;
|
|
10807
12286
|
return [];
|
|
10808
12287
|
}
|
|
10809
12288
|
};
|
|
@@ -11403,7 +12882,7 @@ export class FactoryLoop {
|
|
|
11403
12882
|
}, Math.max(1_000, Math.floor(CLARIFICATION_WAKE_LEASE_MS / 3)));
|
|
11404
12883
|
heartbeat.unref?.();
|
|
11405
12884
|
try {
|
|
11406
|
-
if (!await this.#clarificationIssueStillActive(waiting.issue)) {
|
|
12885
|
+
if (!await this.#clarificationIssueStillActive(waiting.issue, waiting.decision)) {
|
|
11407
12886
|
this.#assertClarificationWakeRunning();
|
|
11408
12887
|
await renewLease();
|
|
11409
12888
|
const completed = await this.#state.completeClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner);
|
|
@@ -11579,8 +13058,8 @@ export class FactoryLoop {
|
|
|
11579
13058
|
if (this.#stopping)
|
|
11580
13059
|
throw new ClarificationWakeStoppedError('factory is stopping');
|
|
11581
13060
|
}
|
|
11582
|
-
async #clarificationIssueStillActive(issueRef) {
|
|
11583
|
-
const issue = await this.#readIssue(issueRef.path);
|
|
13061
|
+
async #clarificationIssueStillActive(issueRef, decisionHint) {
|
|
13062
|
+
const issue = await this.#readIssue(issueRef.path, decisionHint);
|
|
11584
13063
|
if (!issue || !isInFactoryScope(issue, this.#config.safety) || !isDispatchableIssue(issue)) {
|
|
11585
13064
|
this.#logger.info?.('[factory] clarification wake cancelled because issue left factory scope', {
|
|
11586
13065
|
issue: issueRef.key,
|
|
@@ -11746,12 +13225,11 @@ export class FactoryLoop {
|
|
|
11746
13225
|
this.#emit('issue-queued', { issue: decision.issue });
|
|
11747
13226
|
}
|
|
11748
13227
|
}
|
|
11749
|
-
// Absolute path to the
|
|
11750
|
-
//
|
|
11751
|
-
//
|
|
11752
|
-
// against this root rather than a bare relative `.integrations/...`.
|
|
13228
|
+
// Absolute path to the one registered workspace mirror. Spawned agents run in
|
|
13229
|
+
// repo clone paths, so writeback instructions must name the shared mirror,
|
|
13230
|
+
// not a relative `.integrations` path or a per-repository re-home attempt.
|
|
11753
13231
|
#integrationsMountRoot() {
|
|
11754
|
-
return resolve(process.cwd(), '.integrations');
|
|
13232
|
+
return this.#mount.getLocalMountRoot?.() ?? resolve(process.cwd(), '.integrations');
|
|
11755
13233
|
}
|
|
11756
13234
|
async #slackChannelDir() {
|
|
11757
13235
|
if (!this.#config.slack) {
|
|
@@ -11781,7 +13259,7 @@ export class FactoryLoop {
|
|
|
11781
13259
|
}
|
|
11782
13260
|
let paths;
|
|
11783
13261
|
try {
|
|
11784
|
-
paths = await this.#
|
|
13262
|
+
paths = await this.#listRelayfileTree('/slack/channels', 'Slack channel resolution');
|
|
11785
13263
|
}
|
|
11786
13264
|
catch (error) {
|
|
11787
13265
|
this.#logger.warn?.('[factory] unable to resolve Slack channel from mount; using configured channel value', {
|
|
@@ -12269,8 +13747,17 @@ const pidsFromSpawnResult = (result) => {
|
|
|
12269
13747
|
}
|
|
12270
13748
|
return [...pids].sort((a, b) => a - b);
|
|
12271
13749
|
};
|
|
12272
|
-
|
|
13750
|
+
// `issueResolution.detail` and `projection.localMountDegradedReason` are
|
|
13751
|
+
// free text that can carry an absolute local filesystem path (e.g. "mount
|
|
13752
|
+
// state is missing at /Users/<name>/...") — safe in the JSON result and in
|
|
13753
|
+
// logs, but this comment is posted to a public GitHub issue. Emit only
|
|
13754
|
+
// `source`, a closed enum, rather than trying to scrub paths out of free
|
|
13755
|
+
// text; free text is not a safe thing to scrub, only a safe thing to omit.
|
|
13756
|
+
export const dispatchComment = (decision, agents) => [
|
|
12273
13757
|
`Factory dispatch for ${decision.issue.key}`,
|
|
13758
|
+
decision.issueResolution
|
|
13759
|
+
? `Issue resolution: ${decision.issueResolution.source}`
|
|
13760
|
+
: undefined,
|
|
12274
13761
|
`Implementers: ${agents.filter((agent) => agent.role === 'implementer').map((agent) => agent.name).join(', ') || 'none'}`,
|
|
12275
13762
|
decision.scope === 'workflow' ? `Workflow: ${agents.find((agent) => agent.role === 'workflow')?.name ?? 'none'}` : undefined,
|
|
12276
13763
|
`Reviewer: ${agents.find((agent) => agent.role === 'reviewer')?.name ?? 'none'}`,
|
|
@@ -12281,6 +13768,25 @@ function dispatchSpecs(decision) {
|
|
|
12281
13768
|
}
|
|
12282
13769
|
return [...decision.implementers, decision.reviewer];
|
|
12283
13770
|
}
|
|
13771
|
+
function dispatchSessionOwner(decision) {
|
|
13772
|
+
for (const spec of dispatchSpecs(decision)) {
|
|
13773
|
+
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
13774
|
+
if (sessionOwner)
|
|
13775
|
+
return sessionOwner;
|
|
13776
|
+
}
|
|
13777
|
+
return undefined;
|
|
13778
|
+
}
|
|
13779
|
+
function dispatchIssueUrl(issue) {
|
|
13780
|
+
const payload = wrappedPayload(issue.raw);
|
|
13781
|
+
const source = asRecord(payload.source);
|
|
13782
|
+
return stringValue(payload.url)
|
|
13783
|
+
?? stringValue(payload.html_url)
|
|
13784
|
+
?? stringValue(source?.url)
|
|
13785
|
+
?? issue.path;
|
|
13786
|
+
}
|
|
13787
|
+
function ticketDispatchNotificationText(payload) {
|
|
13788
|
+
return `${payload.summary}\n${JSON.stringify(payload)}`;
|
|
13789
|
+
}
|
|
12284
13790
|
function previewServiceForRepo(config, repo) {
|
|
12285
13791
|
const services = config.preview?.services;
|
|
12286
13792
|
if (!services)
|
|
@@ -12613,7 +14119,8 @@ function decisionWithLifecycleBranches(decision, runId, opts = {}) {
|
|
|
12613
14119
|
...(opts.isolateLocalWorktree && baseClonePath && branch ? { baseClonePath, clonePath } : {}),
|
|
12614
14120
|
// The same persisted lifecycle reuses this id after takeover, while a
|
|
12615
14121
|
// genuine reopen gets a new id and cannot replay an old placement ack.
|
|
12616
|
-
invocationId:
|
|
14122
|
+
invocationId: spec.invocationId ??
|
|
14123
|
+
`factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`,
|
|
12617
14124
|
};
|
|
12618
14125
|
return branch ? { ...lifecycleSpec, branch } : lifecycleSpec;
|
|
12619
14126
|
};
|
|
@@ -12788,11 +14295,51 @@ export const githubIssuePathParts = (path) => {
|
|
|
12788
14295
|
slug: match[7],
|
|
12789
14296
|
};
|
|
12790
14297
|
};
|
|
12791
|
-
const githubIssueIdentity = (owner, repo, number) => `${owner.toLowerCase()}/${repo.toLowerCase()}#${number}`;
|
|
14298
|
+
export const githubIssueIdentity = (owner, repo, number) => `${owner.toLowerCase()}/${repo.toLowerCase()}#${number}`;
|
|
12792
14299
|
const githubIssueRefIdentity = (issue) => {
|
|
12793
14300
|
const parts = githubIssuePathParts(issue.path);
|
|
12794
14301
|
return parts ? githubIssueIdentity(parts.owner, parts.repo, parts.number) : undefined;
|
|
12795
14302
|
};
|
|
14303
|
+
/**
|
|
14304
|
+
* Whether a GitHub API fallback read is eligible for `identity`, derived
|
|
14305
|
+
* from currently-tracked state rather than a separately-remembered flag:
|
|
14306
|
+
* `decisionHint` (the one read that happens before its own record is
|
|
14307
|
+
* tracked anywhere) or any `{issue, decision}` candidate — drawn from
|
|
14308
|
+
* whichever durable or in-memory store the caller has gathered — whose
|
|
14309
|
+
* decision was resolved through the fallback. Exported standalone so this
|
|
14310
|
+
* is directly testable — no caller has to have registered anything for
|
|
14311
|
+
* this to return true for a candidate that is genuinely present, which is
|
|
14312
|
+
* the property that keeps a future read call site from silently
|
|
14313
|
+
* reintroducing the restart-eligibility bug by omission.
|
|
14314
|
+
*
|
|
14315
|
+
* A decision durably outlives the in-memory batch in more than one shape
|
|
14316
|
+
* (in-flight dispatch, a parked clarification, a durable dispatch
|
|
14317
|
+
* lifecycle — see #deriveGithubApiFallbackEligibility for the full list
|
|
14318
|
+
* this class gathers); this function does not care which shape a
|
|
14319
|
+
* candidate came from, only whether one matches.
|
|
14320
|
+
*/
|
|
14321
|
+
export function isGithubApiFallbackEligible(candidates, identity, decisionHint) {
|
|
14322
|
+
if (decisionHint?.issueResolution?.source === 'github-api-fallback' &&
|
|
14323
|
+
githubIssueRefIdentity(decisionHint.issue) === identity) {
|
|
14324
|
+
return true;
|
|
14325
|
+
}
|
|
14326
|
+
return candidates.some((candidate) => candidate.decision.issueResolution?.source === 'github-api-fallback' &&
|
|
14327
|
+
githubIssueRefIdentity(candidate.issue) === identity);
|
|
14328
|
+
}
|
|
14329
|
+
/** Maps durable waiting-clarification records to isGithubApiFallbackEligible candidates. */
|
|
14330
|
+
export function githubApiFallbackCandidatesFromWaitingClarifications(waitingClarifications) {
|
|
14331
|
+
return waitingClarifications.map(([, waiting]) => ({ issue: waiting.issue, decision: waiting.decision }));
|
|
14332
|
+
}
|
|
14333
|
+
/**
|
|
14334
|
+
* Maps durable dispatch-lifecycle records to isGithubApiFallbackEligible
|
|
14335
|
+
* candidates, excluding terminal ones — a completed or abandoned dispatch's
|
|
14336
|
+
* decision is no longer a reason to trust a live read for that issue.
|
|
14337
|
+
*/
|
|
14338
|
+
export function githubApiFallbackCandidatesFromDispatchLifecycles(dispatchLifecycles) {
|
|
14339
|
+
return dispatchLifecycles
|
|
14340
|
+
.filter(([, lifecycle]) => !isTerminalDispatchLifecycle(lifecycle))
|
|
14341
|
+
.map(([, lifecycle]) => ({ issue: lifecycle.issue, decision: lifecycle.decision }));
|
|
14342
|
+
}
|
|
12796
14343
|
const githubIssuePathPreference = (path) => {
|
|
12797
14344
|
if (path.endsWith('/meta.json'))
|
|
12798
14345
|
return 0;
|
|
@@ -12980,17 +14527,19 @@ const linearIssueMirrorsGithubIssue = (issue, ghIssue) => {
|
|
|
12980
14527
|
.split(/\r?\n/u)
|
|
12981
14528
|
.some((line) => line.trim() === `${GITHUB_MIRROR_SOURCE_PREFIX}${ghIssue.url}`);
|
|
12982
14529
|
};
|
|
12983
|
-
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
14530
|
+
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix)) => {
|
|
12984
14531
|
const candidates = [];
|
|
12985
14532
|
const listErrors = [];
|
|
12986
14533
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
12987
14534
|
const paths = new Set();
|
|
12988
14535
|
for (const root of githubPullRoots(repo)) {
|
|
12989
14536
|
try {
|
|
12990
|
-
for (const path of await
|
|
14537
|
+
for (const path of await listTree(root))
|
|
12991
14538
|
paths.add(path);
|
|
12992
14539
|
}
|
|
12993
14540
|
catch (error) {
|
|
14541
|
+
if (relayfileOverload(error))
|
|
14542
|
+
throw error;
|
|
12994
14543
|
listErrors.push(error);
|
|
12995
14544
|
}
|
|
12996
14545
|
}
|
|
@@ -13257,8 +14806,7 @@ const issuePrMatchScore = (pr, issue, marker, opts = {}) => {
|
|
|
13257
14806
|
};
|
|
13258
14807
|
const hasTitlePrefix = (title, marker) => title === marker || title.startsWith(`${marker} `);
|
|
13259
14808
|
const factoryBranchMatchesIssue = (headRef, issueKey) => /^\d+$/u.test(issueKey)
|
|
13260
|
-
? headRef
|
|
13261
|
-
headRef.toLowerCase().startsWith(`factory/${issueKey.toLowerCase()}-`)
|
|
14809
|
+
? factoryBranchBelongsToIssue(headRef, issueKey)
|
|
13262
14810
|
: containsIssueKey(headRef, issueKey);
|
|
13263
14811
|
// Legacy Factory runs created GitHub-native branches as `<issue-number>-*`
|
|
13264
14812
|
// before the current `factory/<issue>-*` convention. Keep this matcher out of
|
|
@@ -13822,6 +15370,75 @@ const eventOccurredAtMs = (event) => {
|
|
|
13822
15370
|
const parsed = timestamp ? Date.parse(timestamp) : Number.NaN;
|
|
13823
15371
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
13824
15372
|
};
|
|
15373
|
+
const discoveryEventId = (event) => {
|
|
15374
|
+
const flat = asRecord(event) ?? {};
|
|
15375
|
+
const id = flat.id ?? flat.eventId ?? flat.event_id ?? flat.seq;
|
|
15376
|
+
return typeof id === 'string' || typeof id === 'number' ? String(id) : undefined;
|
|
15377
|
+
};
|
|
15378
|
+
const compareDiscoveryEvents = (left, right) => {
|
|
15379
|
+
const leftOccurredAt = eventOccurredAtMs(left);
|
|
15380
|
+
const rightOccurredAt = eventOccurredAtMs(right);
|
|
15381
|
+
if (leftOccurredAt !== undefined && rightOccurredAt !== undefined && leftOccurredAt !== rightOccurredAt) {
|
|
15382
|
+
return leftOccurredAt - rightOccurredAt;
|
|
15383
|
+
}
|
|
15384
|
+
const leftId = discoveryEventId(left) ?? '';
|
|
15385
|
+
const rightId = discoveryEventId(right) ?? '';
|
|
15386
|
+
const leftSequence = eventSequenceNumber(leftId);
|
|
15387
|
+
const rightSequence = eventSequenceNumber(rightId);
|
|
15388
|
+
if (leftSequence !== undefined && rightSequence !== undefined && leftSequence !== rightSequence) {
|
|
15389
|
+
return leftSequence - rightSequence;
|
|
15390
|
+
}
|
|
15391
|
+
return leftId.localeCompare(rightId);
|
|
15392
|
+
};
|
|
15393
|
+
const discoveryEventIsDeletion = (event) => {
|
|
15394
|
+
const flat = asRecord(event) ?? {};
|
|
15395
|
+
const summary = asRecord(flat.summary) ?? {};
|
|
15396
|
+
const type = stringValue(flat.type)?.toLowerCase();
|
|
15397
|
+
// filesystemEventToChangeEvent normalizes filesystem watcher events to
|
|
15398
|
+
// type: 'relayfile.changed' and stashes the real operation here, so a
|
|
15399
|
+
// deletion arrives as filesystemEventType: 'file.deleted' / 'dir.deleted'
|
|
15400
|
+
// with an outer type that does not itself say "deleted".
|
|
15401
|
+
const filesystemEventType = stringValue(flat.filesystemEventType)?.toLowerCase();
|
|
15402
|
+
const action = stringValue(flat.action)?.toLowerCase();
|
|
15403
|
+
const status = stringValue(summary.status)?.toLowerCase();
|
|
15404
|
+
const digest = stringValue(flat.digest)?.toLowerCase();
|
|
15405
|
+
return type === 'file.deleted' || type === 'dir.deleted' ||
|
|
15406
|
+
filesystemEventType === 'file.deleted' || filesystemEventType === 'dir.deleted' ||
|
|
15407
|
+
action === 'delete' || status === 'deleted' || digest?.startsWith('deleted:') === true;
|
|
15408
|
+
};
|
|
15409
|
+
const githubIssueIndexRepoRoots = (path) => {
|
|
15410
|
+
const slash = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15411
|
+
const flat = /^\/github\/repos\/([^/]+)__([^/]+)\/issues\/_index\.json$/u.exec(path);
|
|
15412
|
+
const parts = slash ?? flat;
|
|
15413
|
+
if (!parts)
|
|
15414
|
+
return [];
|
|
15415
|
+
const [, owner, repo] = parts;
|
|
15416
|
+
return [
|
|
15417
|
+
`/github/repos/${owner}/${repo}/issues`,
|
|
15418
|
+
`/github/repos/${owner}__${repo}/issues`,
|
|
15419
|
+
];
|
|
15420
|
+
};
|
|
15421
|
+
const relayfileOverload = (error) => {
|
|
15422
|
+
const flat = asRecord(error) ?? {};
|
|
15423
|
+
const response = asRecord(flat.response) ?? {};
|
|
15424
|
+
const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
|
|
15425
|
+
const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
|
|
15426
|
+
const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
|
|
15427
|
+
const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
|
|
15428
|
+
if (status !== 429)
|
|
15429
|
+
return undefined;
|
|
15430
|
+
const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
|
|
15431
|
+
const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
|
|
15432
|
+
const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
|
|
15433
|
+
const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
|
|
15434
|
+
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
15435
|
+
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
15436
|
+
};
|
|
15437
|
+
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
15438
|
+
const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
|
|
15439
|
+
const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
|
|
15440
|
+
return Math.max(retryAfterMs, exponentialMs);
|
|
15441
|
+
};
|
|
13825
15442
|
const eventSequenceNumber = (eventId) => {
|
|
13826
15443
|
const whole = Number(eventId);
|
|
13827
15444
|
if (Number.isFinite(whole))
|
|
@@ -13937,7 +15554,8 @@ const eventIdentity = (event) => {
|
|
|
13937
15554
|
// subscription/poll handler) crash-loops. Returns undefined for such events so
|
|
13938
15555
|
// callers skip them and keep processing the well-formed ones.
|
|
13939
15556
|
export const changeEventPath = (event) => {
|
|
13940
|
-
const
|
|
15557
|
+
const candidate = event;
|
|
15558
|
+
const path = candidate?.resource?.path ?? candidate?.path;
|
|
13941
15559
|
return typeof path === 'string' && path ? path : undefined;
|
|
13942
15560
|
};
|
|
13943
15561
|
const describeError = (error) => {
|
|
@@ -14144,6 +15762,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
14144
15762
|
agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
|
|
14145
15763
|
invocationIds: [...record.invocationIds],
|
|
14146
15764
|
result: record.result ? structuredClone(record.result) : undefined,
|
|
15765
|
+
...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
|
|
14147
15766
|
...(pullRequests.length > 0 ? { pullRequests: pullRequests.map((receipt) => ({ ...receipt })) } : {}),
|
|
14148
15767
|
...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
|
|
14149
15768
|
...(releaseReason ? { releaseReason } : {}),
|
|
@@ -14157,6 +15776,7 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
|
14157
15776
|
agents: new Map(lifecycle.agents.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)])),
|
|
14158
15777
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
14159
15778
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
15779
|
+
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
14160
15780
|
});
|
|
14161
15781
|
const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
|
|
14162
15782
|
issue: { ...lifecycle.issue },
|