@agent-relay/factory 0.1.64 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli/diagnose.d.ts +92 -0
- package/dist/cli/diagnose.d.ts.map +1 -0
- package/dist/cli/diagnose.js +381 -0
- package/dist/cli/diagnose.js.map +1 -0
- package/dist/cli/fleet.d.ts +9 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +97 -4
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +134 -2
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +60 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +16 -0
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +228 -13
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/hosted/orchestrator.d.ts.map +1 -1
- package/dist/hosted/orchestrator.js +1 -4
- package/dist/hosted/orchestrator.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/observability/error-class.d.ts +29 -0
- package/dist/observability/error-class.d.ts.map +1 -0
- package/dist/observability/error-class.js +36 -0
- package/dist/observability/error-class.js.map +1 -0
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/index.d.ts.map +1 -1
- package/dist/observability/index.js +1 -0
- package/dist/observability/index.js.map +1 -1
- package/dist/orchestrator/batch-tracker.d.ts +7 -0
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
- package/dist/orchestrator/batch-tracker.js +1 -0
- package/dist/orchestrator/batch-tracker.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +18 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1059 -82
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +1 -0
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -0
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts +82 -0
- package/dist/orchestrator/public-health.d.ts.map +1 -0
- package/dist/orchestrator/public-health.js +491 -0
- package/dist/orchestrator/public-health.js.map +1 -0
- package/dist/ports/state.d.ts +25 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/dispatch-lifecycle-slot.d.ts +45 -0
- package/dist/state/dispatch-lifecycle-slot.d.ts.map +1 -0
- package/dist/state/dispatch-lifecycle-slot.js +67 -0
- package/dist/state/dispatch-lifecycle-slot.js.map +1 -0
- package/dist/state/file-state-store.d.ts +8 -0
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +19 -19
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +8 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +18 -19
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/state/watch-state-document.js +1 -0
- package/dist/state/watch-state-document.js.map +1 -1
- package/dist/triage/schema.d.ts +14 -14
- package/dist/types.d.ts +180 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
|
-
import { FactoryConfigSchema } from '../config/schema.js';
|
|
4
|
+
import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
@@ -9,6 +9,7 @@ import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftNam
|
|
|
9
9
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
10
10
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
11
11
|
import { InMemoryStateStore } from '../state/in-memory-state-store.js';
|
|
12
|
+
import { dispatchHandedOffToBabysitters, dispatchLifecycleOccupiesSlot, dispatchPhaseOccupiesSlot, } from '../state/dispatch-lifecycle-slot.js';
|
|
12
13
|
import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue } from '../issue-key-match.js';
|
|
13
14
|
import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
|
|
14
15
|
import { isInFactoryScope } from '../safety/factory-scope.js';
|
|
@@ -29,6 +30,8 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
|
|
|
29
30
|
import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
|
|
30
31
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
31
32
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
33
|
+
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
34
|
+
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
32
35
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
33
36
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
34
37
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -139,9 +142,28 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
|
139
142
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
140
143
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
141
144
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
145
|
+
/**
|
|
146
|
+
* Ceiling on the durable capacity-wait re-arm (#303).
|
|
147
|
+
*
|
|
148
|
+
* The retry was a flat 1 Hz with no bound at all. When the batch was wedged,
|
|
149
|
+
* every queued issue re-read the shared state document once a second forever —
|
|
150
|
+
* production measured 1477 state GETs in 111 s — and the timers could not even
|
|
151
|
+
* keep up, so they coalesced into a continuous spin against the serialized
|
|
152
|
+
* store. Waiting for capacity is legitimate and must not be abandoned (a real
|
|
153
|
+
* multi-hour run holds the slot honestly), so what is bounded is the *rate*.
|
|
154
|
+
*/
|
|
155
|
+
const DISPATCH_LIFECYCLE_RETRY_MAX_MS = 30_000;
|
|
156
|
+
/** Rate limit for the capacity-wait warning once the backoff has capped. */
|
|
157
|
+
const DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS = 60_000;
|
|
142
158
|
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
143
159
|
const DISPATCH_WRITEBACK_RETRY_MS = 250;
|
|
144
160
|
const HELD_PAST_DEADLINE_RELEASE_REASON = 'held-past-deadline';
|
|
161
|
+
/**
|
|
162
|
+
* Release reason for a lifecycle that took a batch slot and never placed an
|
|
163
|
+
* agent (#303). Deliberately distinct from `held-past-deadline`: that one
|
|
164
|
+
* means a team ran and never finished, this one means no team ever existed.
|
|
165
|
+
*/
|
|
166
|
+
const AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON = 'agentless-slot-past-deadline';
|
|
145
167
|
const HELD_DEADLINE_OVERDUE_RETRY_MS = 1_000;
|
|
146
168
|
const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
147
169
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
@@ -185,6 +207,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
|
185
207
|
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
186
208
|
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
187
209
|
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
210
|
+
/** First rung of the ladder when the 429 advertises no `Retry-After`. */
|
|
211
|
+
const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
|
|
212
|
+
/**
|
|
213
|
+
* Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
|
|
214
|
+
* whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
|
|
215
|
+
* dependency into hammering it.
|
|
216
|
+
*/
|
|
217
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
|
|
218
|
+
/**
|
|
219
|
+
* Ceiling for the ladder once the dependency has told us how long to wait.
|
|
220
|
+
*
|
|
221
|
+
* relayfile sheds an overloaded workspace DO with a 429 in milliseconds
|
|
222
|
+
* carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
|
|
223
|
+
* that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
|
|
224
|
+
* dependency asked for five seconds, Factory slept for five minutes, probed
|
|
225
|
+
* for recovery once per cap-length window, and presented a transient upstream
|
|
226
|
+
* blip as a sustained outage. The ratchet still escalates — it is just bounded
|
|
227
|
+
* by roughly what was actually asked for, and never *below* it, so this is a
|
|
228
|
+
* ceiling and not a licence to retry sooner than the dependency allows.
|
|
229
|
+
*/
|
|
230
|
+
const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
|
|
231
|
+
/**
|
|
232
|
+
* How many relayfile operations one sweep may have shed before the sweep is
|
|
233
|
+
* abandoned rather than continued.
|
|
234
|
+
*
|
|
235
|
+
* Per-item overload skips that item and keeps going (#297, the same principle
|
|
236
|
+
* as #292), but skipping must not degenerate into grinding a shedding
|
|
237
|
+
* dependency through an entire backlog one 429 at a time. Past this many, the
|
|
238
|
+
* dependency is not serving this sweep at all: abort, back off, and let the
|
|
239
|
+
* ratchet do its job.
|
|
240
|
+
*/
|
|
241
|
+
const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
|
|
188
242
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
189
243
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
190
244
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -259,6 +313,20 @@ class DispatchLifecycleClaimRefusedError extends Error {
|
|
|
259
313
|
this.name = 'DispatchLifecycleClaimRefusedError';
|
|
260
314
|
}
|
|
261
315
|
}
|
|
316
|
+
/**
|
|
317
|
+
* The deadline that makes a hung sweep reachable by the existing recovery path.
|
|
318
|
+
* Its message is fully internal (one integer), so it is safe to persist into
|
|
319
|
+
* the operator-facing `readinessReconcile.lastError`.
|
|
320
|
+
*/
|
|
321
|
+
class ReadinessReconcileTimeoutError extends Error {
|
|
322
|
+
timeoutMs;
|
|
323
|
+
code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
|
|
324
|
+
constructor(timeoutMs) {
|
|
325
|
+
super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
|
|
326
|
+
this.timeoutMs = timeoutMs;
|
|
327
|
+
this.name = 'ReadinessReconcileTimeoutError';
|
|
328
|
+
}
|
|
329
|
+
}
|
|
262
330
|
const realClock = {
|
|
263
331
|
now: () => Date.now(),
|
|
264
332
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -369,7 +437,17 @@ export class FactoryLoop {
|
|
|
369
437
|
#dispatchLifecycleRetryTimers = new Map();
|
|
370
438
|
#dispatchLifecycleDrives = new Set();
|
|
371
439
|
#abandonedDispatchReasons = new Map();
|
|
372
|
-
|
|
440
|
+
/**
|
|
441
|
+
* Live batch-capacity waits, keyed by issue (#303).
|
|
442
|
+
*
|
|
443
|
+
* Replaces a `Set` of "already logged" keys. That set made the wait a
|
|
444
|
+
* one-shot log and nothing else: after the first line, an outage in which
|
|
445
|
+
* every issue was stuck behind a wedged slot was indistinguishable from an
|
|
446
|
+
* idle Factory on every operator surface. The wait now carries its own start
|
|
447
|
+
* instant and attempt count, which is what both the escalating warning and
|
|
448
|
+
* `status().dispatchCapacity` are derived from.
|
|
449
|
+
*/
|
|
450
|
+
#dispatchLifecycleCapacityWaits = new Map();
|
|
373
451
|
#dispatchLifecycleOwnershipWaitLogged = new Set();
|
|
374
452
|
#dispatchClaimStatuses = new Map();
|
|
375
453
|
#localReleaseCheckpoints = new Map();
|
|
@@ -408,12 +486,41 @@ export class FactoryLoop {
|
|
|
408
486
|
#readinessReconcileTimer;
|
|
409
487
|
#readinessReconcileInFlight;
|
|
410
488
|
#readinessReconcileIntervalMs = 60_000;
|
|
489
|
+
#readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
|
|
490
|
+
// Set for exactly as long as a sweep is running. `state` is derived from
|
|
491
|
+
// this, so an in-flight pass can no longer masquerade as the last settled one.
|
|
492
|
+
#readinessReconcileInFlightSinceMs;
|
|
493
|
+
/**
|
|
494
|
+
* The work a deadline gave up waiting on. The deadline bounds the wait, not
|
|
495
|
+
* the sweep, so this is still live: shutdown has to drain it, and `state` has
|
|
496
|
+
* to keep counting from when it actually started.
|
|
497
|
+
*
|
|
498
|
+
* Two fields rather than a collection, because every live abandoned wait
|
|
499
|
+
* converges on the same sweep. They are all `runOnce()` calls with the same
|
|
500
|
+
* `dryRun`, so whatever they are queued behind, the first one out starts the
|
|
501
|
+
* sweep and the rest coalesce onto it — they settle together. So the newest
|
|
502
|
+
* wait is a sufficient drain target, and the earliest start is the honest
|
|
503
|
+
* age. Both matter (#301 review): keeping only the newest start advanced the
|
|
504
|
+
* age by two intervals every two intervals, so at `reconcileTimeoutMs ===
|
|
505
|
+
* reconcileIntervalMs` it never reached three and `stalled` was never
|
|
506
|
+
* reported; keeping one record per wait grew without bound in exactly the
|
|
507
|
+
* never-settling case this change exists for.
|
|
508
|
+
*
|
|
509
|
+
* The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
|
|
510
|
+
* sweep is waited BEHIND rather than coalesced onto, so that handle can name
|
|
511
|
+
* an unrelated sweep, and the readiness pass would then start its own work
|
|
512
|
+
* after shutdown believed it had drained everything. The wait covers the
|
|
513
|
+
* queueing and the sweep it eventually runs, in every branch.
|
|
514
|
+
*/
|
|
515
|
+
#readinessReconcileAbandonedWait;
|
|
516
|
+
#readinessReconcileAbandonedSinceMs;
|
|
411
517
|
#readinessReconcileConsecutiveFailures = 0;
|
|
412
518
|
#readinessReconcileLastDurationMs;
|
|
413
519
|
#readinessReconcileLastStartedAtMs;
|
|
414
520
|
#readinessReconcileLastCompletedAtMs;
|
|
415
521
|
#readinessReconcileLastFailureAtMs;
|
|
416
522
|
#readinessReconcileLastError;
|
|
523
|
+
#readinessReconcileLastErrorClass;
|
|
417
524
|
#liveEventQueue = [];
|
|
418
525
|
#liveEventDrainScheduled = false;
|
|
419
526
|
#liveEventDrainActive = false;
|
|
@@ -501,7 +608,34 @@ export class FactoryLoop {
|
|
|
501
608
|
// so the loop's catch no longer runs the failure-handoff reaper for it; the
|
|
502
609
|
// pass reaps inline and must write to the same paths runLoop would.
|
|
503
610
|
#loopReapPaths;
|
|
611
|
+
/**
|
|
612
|
+
* The first 429 relayfile raised during this sweep, kept for its
|
|
613
|
+
* `Retry-After` and reason when the sweep decides how long to back off.
|
|
614
|
+
*
|
|
615
|
+
* Before #297 this doubled as a sweep-wide kill switch: any 429 from any
|
|
616
|
+
* relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
|
|
617
|
+
* away along with everything the sweep had already accomplished. It is now
|
|
618
|
+
* only evidence, never a verdict — see `#discoverySweepOverloads` for the
|
|
619
|
+
* fuse that still ends a sweep the dependency is genuinely refusing to serve.
|
|
620
|
+
*/
|
|
504
621
|
#discoveryOverloadError;
|
|
622
|
+
/** Relayfile operations this sweep has been shed on. */
|
|
623
|
+
#discoverySweepOverloads = 0;
|
|
624
|
+
/**
|
|
625
|
+
* The longest `Retry-After` any operation in this sweep advertised.
|
|
626
|
+
*
|
|
627
|
+
* `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
|
|
628
|
+
* from it alone would let the durable window expire before a later, longer
|
|
629
|
+
* advertised delay permits — breaking the very guarantee #297 is about. The
|
|
630
|
+
* backoff takes the maximum instead.
|
|
631
|
+
*/
|
|
632
|
+
#discoverySweepRetryAfterSeconds;
|
|
633
|
+
/**
|
|
634
|
+
* Whether relayfile served at least one ready work unit end to end during
|
|
635
|
+
* this sweep — its issue read, or a dispatch built on it. This is what
|
|
636
|
+
* decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
|
|
637
|
+
*/
|
|
638
|
+
#discoverySweepProgress = false;
|
|
505
639
|
#resolvedIssueSource;
|
|
506
640
|
#integrationInstructions;
|
|
507
641
|
#integrationInstructionsRefresh;
|
|
@@ -906,6 +1040,7 @@ export class FactoryLoop {
|
|
|
906
1040
|
clearTimeout(timer);
|
|
907
1041
|
this.#dispatchLifecycleRetryTimers.clear();
|
|
908
1042
|
this.#abandonedDispatchReasons.clear();
|
|
1043
|
+
this.#dispatchLifecycleCapacityWaits.clear();
|
|
909
1044
|
this.#dispatchLifecycleOwnershipWaitLogged.clear();
|
|
910
1045
|
if (this.#completionSweepTimer)
|
|
911
1046
|
clearTimeout(this.#completionSweepTimer);
|
|
@@ -917,6 +1052,14 @@ export class FactoryLoop {
|
|
|
917
1052
|
clearTimeout(this.#previewSweepTimer);
|
|
918
1053
|
this.#previewSweepTimer = undefined;
|
|
919
1054
|
await this.#readinessReconcileInFlight;
|
|
1055
|
+
// #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
|
|
1056
|
+
// can settle with its `runOnce()` still live. Shutdown releases dispatch
|
|
1057
|
+
// lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
|
|
1058
|
+
// fences a stopping sweep once something in it throws — so a sweep whose
|
|
1059
|
+
// dependency recovers cleanly would otherwise dispatch through torn-down
|
|
1060
|
+
// state. Draining here restores exactly the pre-deadline shutdown contract:
|
|
1061
|
+
// stop() outlives the sweep it started.
|
|
1062
|
+
await this.#readinessReconcileAbandonedWait;
|
|
920
1063
|
await this.#previewSweepInFlight;
|
|
921
1064
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
922
1065
|
try {
|
|
@@ -1067,6 +1210,9 @@ export class FactoryLoop {
|
|
|
1067
1210
|
const options = this.#liveOptions(overrides);
|
|
1068
1211
|
this.#liveTransport = options.transport;
|
|
1069
1212
|
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
1213
|
+
// `start()` overrides skip the schema's cross-field check, so re-apply its
|
|
1214
|
+
// floor here: a deadline under one interval would kill every pass.
|
|
1215
|
+
this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
|
|
1070
1216
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
1071
1217
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
1072
1218
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1103,10 +1249,27 @@ export class FactoryLoop {
|
|
|
1103
1249
|
this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
|
|
1104
1250
|
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
1105
1251
|
});
|
|
1252
|
+
// Review follow-up on #300 (P1, cubic). The startup backfill is a
|
|
1253
|
+
// discovery pass like any other, and it is the one most likely to hang:
|
|
1254
|
+
// #36 measured 61 minutes here while the Relayfile mirror hydrated on a
|
|
1255
|
+
// cold container. Stamping it means a wedged FIRST pass is visible as
|
|
1256
|
+
// in-flight, instead of leaving the timestamps empty and the derived
|
|
1257
|
+
// state reading `healthy` forever.
|
|
1258
|
+
//
|
|
1259
|
+
// Only the timestamps. `consecutiveFailures` and `lastError` belong to
|
|
1260
|
+
// the reconcile loop's own failure accounting, which owns the degraded
|
|
1261
|
+
// threshold and the #297 reason allowlist; a startup failure is already
|
|
1262
|
+
// counted by `liveStartupBackfillErrors` and reported through `#error`.
|
|
1263
|
+
const backfillStartedAtMs = this.#clock.now();
|
|
1264
|
+
this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
|
|
1106
1265
|
try {
|
|
1107
1266
|
await this.runOnce();
|
|
1267
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1268
|
+
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1108
1269
|
}
|
|
1109
1270
|
catch (error) {
|
|
1271
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1272
|
+
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1110
1273
|
// A startup backfill failure must not abort the daemon: log it and fall
|
|
1111
1274
|
// back to the live event stream (plus any buffered events) instead of
|
|
1112
1275
|
// leaving the factory down.
|
|
@@ -1198,6 +1361,7 @@ export class FactoryLoop {
|
|
|
1198
1361
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
1199
1362
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1200
1363
|
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1364
|
+
reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
|
|
1201
1365
|
};
|
|
1202
1366
|
}
|
|
1203
1367
|
async #currentEventCursor(limit) {
|
|
@@ -1246,19 +1410,82 @@ export class FactoryLoop {
|
|
|
1246
1410
|
}, delayMs);
|
|
1247
1411
|
this.#readinessReconcileTimer.unref?.();
|
|
1248
1412
|
}
|
|
1413
|
+
/**
|
|
1414
|
+
* Runs one sweep under a deadline (#296).
|
|
1415
|
+
*
|
|
1416
|
+
* The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
|
|
1417
|
+
* lease and abandoning it mid-flight is not safe — so expiry rejects *this*
|
|
1418
|
+
* wait and leaves the underlying pass to finish on its own. That is enough:
|
|
1419
|
+
* the rejection is what reaches the failure path, which re-arms the timer.
|
|
1420
|
+
* A later reconcile pass coalesces onto the still-running `runOnce()` and
|
|
1421
|
+
* fails on its own deadline too, so a persistent hang keeps counting up to
|
|
1422
|
+
* `degraded` instead of going quiet.
|
|
1423
|
+
*/
|
|
1424
|
+
async #runOnceWithReadinessDeadline() {
|
|
1425
|
+
const timeoutMs = this.#readinessReconcileTimeoutMs;
|
|
1426
|
+
const startedAtMs = this.#clock.now();
|
|
1427
|
+
const sweep = this.runOnce();
|
|
1428
|
+
let timer;
|
|
1429
|
+
try {
|
|
1430
|
+
return await new Promise((resolve, reject) => {
|
|
1431
|
+
timer = setTimeout(() => {
|
|
1432
|
+
this.#increment('readinessReconcileDeadlineExceeded');
|
|
1433
|
+
if (this.#readinessReconcileAbandonedSinceMs === undefined) {
|
|
1434
|
+
// `state` ages from the FIRST wait that gave up on this work, not
|
|
1435
|
+
// from whenever the latest one began.
|
|
1436
|
+
this.#readinessReconcileAbandonedSinceMs = startedAtMs;
|
|
1437
|
+
// The abandoned pass is still running against the live control
|
|
1438
|
+
// plane. Report where it lands, so an operator can tell a
|
|
1439
|
+
// dependency that recovered late from one that never answered.
|
|
1440
|
+
// Attached once, so a wedge is reported once and not per expiry.
|
|
1441
|
+
void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
|
|
1442
|
+
timeoutMs,
|
|
1443
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1444
|
+
dispatched: report.dispatched.length,
|
|
1445
|
+
}), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
|
|
1446
|
+
timeoutMs,
|
|
1447
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1448
|
+
error: describeError(error).errorMessage,
|
|
1449
|
+
})).catch(() => undefined);
|
|
1450
|
+
}
|
|
1451
|
+
// Newest wait wins as the drain target: it settles no earlier than
|
|
1452
|
+
// the ones before it, and clearing on it clears them all.
|
|
1453
|
+
const wait = sweep.catch(() => undefined).then(() => {
|
|
1454
|
+
if (this.#readinessReconcileAbandonedWait !== wait)
|
|
1455
|
+
return;
|
|
1456
|
+
this.#readinessReconcileAbandonedWait = undefined;
|
|
1457
|
+
this.#readinessReconcileAbandonedSinceMs = undefined;
|
|
1458
|
+
});
|
|
1459
|
+
this.#readinessReconcileAbandonedWait = wait;
|
|
1460
|
+
reject(new ReadinessReconcileTimeoutError(timeoutMs));
|
|
1461
|
+
}, timeoutMs);
|
|
1462
|
+
timer.unref?.();
|
|
1463
|
+
// Attaching handlers here is also what keeps a late rejection from the
|
|
1464
|
+
// abandoned pass from surfacing as an unhandled rejection.
|
|
1465
|
+
sweep.then(resolve, reject);
|
|
1466
|
+
});
|
|
1467
|
+
}
|
|
1468
|
+
finally {
|
|
1469
|
+
if (timer)
|
|
1470
|
+
clearTimeout(timer);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1249
1473
|
async #reconcileReadyIssues() {
|
|
1250
1474
|
const startedAtMs = this.#clock.now();
|
|
1251
1475
|
this.#readinessReconcileLastStartedAtMs = startedAtMs;
|
|
1476
|
+
this.#readinessReconcileInFlightSinceMs = startedAtMs;
|
|
1252
1477
|
this.#increment('readinessReconcileSweeps');
|
|
1253
1478
|
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1254
1479
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1480
|
+
timeoutMs: this.#readinessReconcileTimeoutMs,
|
|
1255
1481
|
});
|
|
1256
1482
|
try {
|
|
1257
|
-
const report = await this
|
|
1483
|
+
const report = await this.#runOnceWithReadinessDeadline();
|
|
1258
1484
|
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1259
1485
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1260
1486
|
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1261
1487
|
this.#readinessReconcileLastError = undefined;
|
|
1488
|
+
this.#readinessReconcileLastErrorClass = undefined;
|
|
1262
1489
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1263
1490
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1264
1491
|
candidates: report.pulled.length,
|
|
@@ -1267,11 +1494,28 @@ export class FactoryLoop {
|
|
|
1267
1494
|
});
|
|
1268
1495
|
}
|
|
1269
1496
|
catch (error) {
|
|
1270
|
-
|
|
1497
|
+
// #297: all four relayfile overload reason codes share one message, and
|
|
1498
|
+
// `lastError` is what an operator reads from /evidence. Without the
|
|
1499
|
+
// reason, "workspace durable object is busy" cannot be told apart from
|
|
1500
|
+
// three other conditions with three different remedies.
|
|
1501
|
+
//
|
|
1502
|
+
// Allowlisted, because this is a persisted operator-facing surface and
|
|
1503
|
+
// not just a log line: `lastError` is returned from `status()` and
|
|
1504
|
+
// written into the loop heartbeat file, so an unbounded
|
|
1505
|
+
// dependency-controlled string would land on disk.
|
|
1506
|
+
const overload = relayfileOverload(error);
|
|
1507
|
+
const errorMessage = overload
|
|
1508
|
+
? `${describeError(error).errorMessage} ` +
|
|
1509
|
+
`[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
|
|
1510
|
+
`${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
|
|
1511
|
+
: describeError(error).errorMessage;
|
|
1271
1512
|
this.#readinessReconcileConsecutiveFailures += 1;
|
|
1272
1513
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1273
1514
|
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1274
1515
|
this.#readinessReconcileLastError = errorMessage;
|
|
1516
|
+
// The class, unlike the message, is publishable: #295 puts it on the
|
|
1517
|
+
// unauthenticated health surface through the same allowlist.
|
|
1518
|
+
this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
|
|
1275
1519
|
this.#increment('readinessReconcileErrors');
|
|
1276
1520
|
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1277
1521
|
error: errorMessage,
|
|
@@ -1280,6 +1524,11 @@ export class FactoryLoop {
|
|
|
1280
1524
|
degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
1281
1525
|
});
|
|
1282
1526
|
}
|
|
1527
|
+
finally {
|
|
1528
|
+
// Cleared before the heartbeat write below, so a slow-but-successful pass
|
|
1529
|
+
// does not stamp its own tail as `stalled`.
|
|
1530
|
+
this.#readinessReconcileInFlightSinceMs = undefined;
|
|
1531
|
+
}
|
|
1283
1532
|
await this.#refreshLiveHeartbeat();
|
|
1284
1533
|
}
|
|
1285
1534
|
#scheduleLivePoll(delayMs, options) {
|
|
@@ -2008,13 +2257,29 @@ export class FactoryLoop {
|
|
|
2008
2257
|
this.#discoverySweepStartedAtMs = sweepStartedAtMs;
|
|
2009
2258
|
this.#discoverySweepLeaseLost = false;
|
|
2010
2259
|
this.#discoveryOverloadError = undefined;
|
|
2260
|
+
this.#discoverySweepOverloads = 0;
|
|
2261
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2262
|
+
this.#discoverySweepProgress = false;
|
|
2011
2263
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
2012
2264
|
let leaseReleased = false;
|
|
2013
2265
|
try {
|
|
2014
2266
|
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
2267
|
+
// #297: a 429 raised anywhere in the sweep used to latch and be rethrown
|
|
2268
|
+
// here, discarding a completed pass — every issue read, every dispatch —
|
|
2269
|
+
// because of one transient shed operation. The work this sweep did is
|
|
2270
|
+
// now kept instead, and the ratchet below records that the dependency is
|
|
2271
|
+
// shedding but still serving.
|
|
2015
2272
|
const report = await this.#performRunOnce(opts);
|
|
2016
|
-
|
|
2273
|
+
// The exception, and the reason skipping shed units cannot make a sweep
|
|
2274
|
+
// unconditionally green: a sweep that was shed AND got no work unit
|
|
2275
|
+
// through accomplished nothing. There is no progress to preserve, and
|
|
2276
|
+
// committing it would report a clean sweep over a dependency that served
|
|
2277
|
+
// none of it — leaving `readinessReconcile` healthy while Factory
|
|
2278
|
+
// dispatches nothing, which is the #292 wedge wearing the other costume.
|
|
2279
|
+
// Fail it so the ratchet escalates and readiness reflects reality.
|
|
2280
|
+
if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
|
|
2017
2281
|
throw this.#discoveryOverloadError;
|
|
2282
|
+
}
|
|
2018
2283
|
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
2019
2284
|
// Do not clear the durable lease while a renewal can still be waiting on
|
|
2020
2285
|
// the same state-file lock. A late renewal that observes the completed
|
|
@@ -2024,7 +2289,8 @@ export class FactoryLoop {
|
|
|
2024
2289
|
if (this.#discoverySweepLeaseLost) {
|
|
2025
2290
|
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
2026
2291
|
}
|
|
2027
|
-
const
|
|
2292
|
+
const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
|
|
2293
|
+
const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
|
|
2028
2294
|
leaseReleased = completed;
|
|
2029
2295
|
if (!completed)
|
|
2030
2296
|
throw new Error('discovery sweep lease was lost before completion');
|
|
@@ -2040,18 +2306,18 @@ export class FactoryLoop {
|
|
|
2040
2306
|
await this.#stopDiscoverySweepRenewal();
|
|
2041
2307
|
const overload = relayfileOverload(error);
|
|
2042
2308
|
if (overload) {
|
|
2043
|
-
const
|
|
2044
|
-
|
|
2045
|
-
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
2046
|
-
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
2047
|
-
this.#increment('discoveryOverloadBackoffs');
|
|
2309
|
+
const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
|
|
2310
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
|
|
2048
2311
|
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
2049
2312
|
status: overload.status,
|
|
2050
2313
|
reason: overload.reason,
|
|
2051
|
-
retryAfterSeconds:
|
|
2052
|
-
delayMs,
|
|
2053
|
-
backoffUntilMs,
|
|
2054
|
-
consecutiveOverloads,
|
|
2314
|
+
retryAfterSeconds: outcome.retryAfterSeconds,
|
|
2315
|
+
delayMs: outcome.delayMs,
|
|
2316
|
+
backoffUntilMs: outcome.backoffUntilMs,
|
|
2317
|
+
consecutiveOverloads: outcome.consecutiveOverloads,
|
|
2318
|
+
previousOverloads: claim.state.consecutiveOverloads,
|
|
2319
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2320
|
+
sweepProgress: this.#discoverySweepProgress,
|
|
2055
2321
|
});
|
|
2056
2322
|
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
2057
2323
|
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
@@ -2067,6 +2333,9 @@ export class FactoryLoop {
|
|
|
2067
2333
|
this.#discoverySweepEpoch = undefined;
|
|
2068
2334
|
this.#discoverySweepStartedAtMs = undefined;
|
|
2069
2335
|
this.#discoveryOverloadError = undefined;
|
|
2336
|
+
this.#discoverySweepOverloads = 0;
|
|
2337
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2338
|
+
this.#discoverySweepProgress = false;
|
|
2070
2339
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
2071
2340
|
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
2072
2341
|
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
@@ -2078,6 +2347,94 @@ export class FactoryLoop {
|
|
|
2078
2347
|
}
|
|
2079
2348
|
}
|
|
2080
2349
|
}
|
|
2350
|
+
/**
|
|
2351
|
+
* Commit the sweep, carrying any residual overload backoff into the store.
|
|
2352
|
+
*
|
|
2353
|
+
* `completeDiscoverySweepWithOverload` is optional on the port so a store
|
|
2354
|
+
* written before #297 keeps working. When one of those is injected and this
|
|
2355
|
+
* sweep HAS a residual, the backoff cannot be persisted and the next sweep
|
|
2356
|
+
* would retry immediately — so this says so, loudly and with a counter,
|
|
2357
|
+
* rather than degrading in silence. The sweep itself still commits: losing
|
|
2358
|
+
* the backoff is worse than pre-#297 behaviour only if nobody notices.
|
|
2359
|
+
*/
|
|
2360
|
+
async #commitDiscoverySweep(epoch, checkpoint, residual) {
|
|
2361
|
+
if (residual && this.#state.completeDiscoverySweepWithOverload) {
|
|
2362
|
+
return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
|
|
2363
|
+
}
|
|
2364
|
+
if (residual) {
|
|
2365
|
+
this.#increment('discoveryOverloadResidualUnsupported');
|
|
2366
|
+
this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
|
|
2367
|
+
store: this.#state.constructor?.name,
|
|
2368
|
+
consecutiveOverloads: residual.consecutiveOverloads,
|
|
2369
|
+
backoffUntilMs: residual.backoffUntilMs,
|
|
2370
|
+
delayMs: residual.delayMs,
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
|
|
2374
|
+
}
|
|
2375
|
+
/**
|
|
2376
|
+
* How the durable overload ratchet should read after this sweep, or
|
|
2377
|
+
* `undefined` if relayfile never shed anything and the ordinary reset
|
|
2378
|
+
* applies.
|
|
2379
|
+
*
|
|
2380
|
+
* #297, deliverable 3. `consecutiveOverloads` used to clear only on a
|
|
2381
|
+
* *fully* clean sweep, so under sustained mild load the ratchet essentially
|
|
2382
|
+
* never cleared: it climbed to the cap on the first bad sweep and stayed
|
|
2383
|
+
* there, probing for recovery once per cap-length window, long after the
|
|
2384
|
+
* dependency had recovered. Requiring perfection to clear a ratchet means
|
|
2385
|
+
* the ratchet does not clear.
|
|
2386
|
+
*
|
|
2387
|
+
* The signal that replaces "was this sweep perfect" is "did relayfile serve
|
|
2388
|
+
* any of this sweep's work units", because that is what the ratchet is
|
|
2389
|
+
* actually for. A shedding DO rejects ALL background traffic —
|
|
2390
|
+
* `oldest_inflight_age` does this regardless of how few requests are in
|
|
2391
|
+
* flight — so a sweep against a genuinely overloaded workspace gets not one
|
|
2392
|
+
* unit through and still escalates, all the way to the cap. A sweep that did
|
|
2393
|
+
* get units through proves the dependency is serving us, so it decays by one
|
|
2394
|
+
* rung. Decay, not reset: getting work done while being shed is not evidence
|
|
2395
|
+
* that the overload is over, only that it is survivable.
|
|
2396
|
+
*/
|
|
2397
|
+
#discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
|
|
2398
|
+
const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
|
|
2399
|
+
if (!overload)
|
|
2400
|
+
return undefined;
|
|
2401
|
+
// The longest delay anything in this sweep asked for, not just the one the
|
|
2402
|
+
// latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
|
|
2403
|
+
const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
|
|
2404
|
+
.filter((value) => value !== undefined);
|
|
2405
|
+
const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
|
|
2406
|
+
// Deliberately NOT "the sweep reached its end". A sweep in which relayfile
|
|
2407
|
+
// shed EVERY unit still runs the loop to the end, and it got nothing done;
|
|
2408
|
+
// treating that as progress would decay the ratchet in exactly the case it
|
|
2409
|
+
// exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
|
|
2410
|
+
// aborted one before it can commit, so the escalate branch here is reached
|
|
2411
|
+
// only through `sweepOutcome === 'aborted'` — but the rule is a property of
|
|
2412
|
+
// progress, not of which caller asked, and is written that way.)
|
|
2413
|
+
const consecutiveOverloads = this.#discoverySweepProgress
|
|
2414
|
+
? Math.max(0, previousOverloads - 1)
|
|
2415
|
+
: previousOverloads + 1;
|
|
2416
|
+
const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
|
|
2417
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
2418
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
2419
|
+
if (sweepOutcome === 'committed') {
|
|
2420
|
+
this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
|
|
2421
|
+
status: overload.status,
|
|
2422
|
+
reason: overload.reason,
|
|
2423
|
+
retryAfterSeconds,
|
|
2424
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2425
|
+
previousOverloads,
|
|
2426
|
+
consecutiveOverloads,
|
|
2427
|
+
delayMs,
|
|
2428
|
+
backoffUntilMs,
|
|
2429
|
+
});
|
|
2430
|
+
}
|
|
2431
|
+
return {
|
|
2432
|
+
consecutiveOverloads,
|
|
2433
|
+
backoffUntilMs,
|
|
2434
|
+
delayMs,
|
|
2435
|
+
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
2436
|
+
};
|
|
2437
|
+
}
|
|
2081
2438
|
async #performRunOnce(opts = {}) {
|
|
2082
2439
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
2083
2440
|
const startedAtMs = this.#clock.now();
|
|
@@ -2125,15 +2482,57 @@ export class FactoryLoop {
|
|
|
2125
2482
|
let readyIssueReads = 0;
|
|
2126
2483
|
const issueEntries = [];
|
|
2127
2484
|
for (const path of paths) {
|
|
2128
|
-
|
|
2485
|
+
let issue;
|
|
2486
|
+
let shed = false;
|
|
2487
|
+
try {
|
|
2488
|
+
issue = await this.#readIssue(path);
|
|
2489
|
+
}
|
|
2490
|
+
catch (error) {
|
|
2491
|
+
// #297: `#readIssue` rethrows relayfile overload and swallows every
|
|
2492
|
+
// other read fault, so this catch only ever sees the backend
|
|
2493
|
+
// shedding THIS issue's read. That is a fact about one work unit:
|
|
2494
|
+
// a sweep that pulled 40 issues and was shed on the 39th must keep
|
|
2495
|
+
// the other 39, exactly as #292 argued for dispatch failures. The
|
|
2496
|
+
// fuse is what still ends a sweep the backend is refusing.
|
|
2497
|
+
const overload = relayfileOverload(error);
|
|
2498
|
+
// Defensive: anything `#readIssue` did not swallow and is not a 429
|
|
2499
|
+
// is not ours to reclassify, and must surface as itself.
|
|
2500
|
+
if (!overload)
|
|
2501
|
+
throw error;
|
|
2502
|
+
const fuse = this.#discoveryOverloadFuseError();
|
|
2503
|
+
if (fuse)
|
|
2504
|
+
throw fuse;
|
|
2505
|
+
shed = true;
|
|
2506
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2507
|
+
this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
|
|
2508
|
+
path,
|
|
2509
|
+
status: overload.status,
|
|
2510
|
+
reason: overload.reason,
|
|
2511
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2512
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2513
|
+
});
|
|
2514
|
+
recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
|
|
2515
|
+
}
|
|
2129
2516
|
readyIssueReads += 1;
|
|
2517
|
+
// Relayfile served this work unit's read: the dependency is shedding
|
|
2518
|
+
// but not refusing, which is what decays the ratchet (#297).
|
|
2519
|
+
//
|
|
2520
|
+
// `issue` is required, not just `!shed`: `#readIssue` returns
|
|
2521
|
+
// `undefined` for a body it could not read at all, and the known
|
|
2522
|
+
// phantom condition — the tree listing issue paths whose bodies are
|
|
2523
|
+
// absent — makes every read return `undefined`. Crediting those would
|
|
2524
|
+
// decay the ratchet on a sweep that served nothing.
|
|
2525
|
+
if (!shed && issue)
|
|
2526
|
+
this.#discoverySweepProgress = true;
|
|
2130
2527
|
lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
|
|
2131
2528
|
? '[factory] GitHub ready issue read progress'
|
|
2132
2529
|
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
2133
|
-
if (
|
|
2134
|
-
|
|
2530
|
+
if (!shed) {
|
|
2531
|
+
if (issue && issueSource === 'linear') {
|
|
2532
|
+
await this.#recordCanonicalIssueState(issue);
|
|
2533
|
+
}
|
|
2534
|
+
issueEntries.push({ path, issue });
|
|
2135
2535
|
}
|
|
2136
|
-
issueEntries.push({ path, issue });
|
|
2137
2536
|
await this.#refreshLiveHeartbeatIfDue();
|
|
2138
2537
|
}
|
|
2139
2538
|
if (issueSource === 'github') {
|
|
@@ -2222,6 +2621,9 @@ export class FactoryLoop {
|
|
|
2222
2621
|
// A completed dispatch — even one that parks or escalates the issue —
|
|
2223
2622
|
// proves the pipeline still works, so the fuse below starts over.
|
|
2224
2623
|
unclassifiedFailuresSinceDispatch = 0;
|
|
2624
|
+
// ...and proves relayfile is still serving this sweep, which is what
|
|
2625
|
+
// decays the durable overload ratchet (#297).
|
|
2626
|
+
this.#discoverySweepProgress = true;
|
|
2225
2627
|
if (result.agents.length === 0 && !dryRun) {
|
|
2226
2628
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2227
2629
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
@@ -2235,13 +2637,47 @@ export class FactoryLoop {
|
|
|
2235
2637
|
}
|
|
2236
2638
|
}
|
|
2237
2639
|
catch (error) {
|
|
2640
|
+
// The failure may have left half-spawned agents behind, persisted as
|
|
2641
|
+
// failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
|
|
2642
|
+
// catch used to reap them because every such error aborted the pass;
|
|
2643
|
+
// now that most of them are skipped, the reap has to happen here.
|
|
2644
|
+
//
|
|
2645
|
+
// BEFORE the fatality check, not after (#298 review, round two): the
|
|
2646
|
+
// 429 that trips the overload fuse aborts this pass, and a direct
|
|
2647
|
+
// `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
|
|
2648
|
+
// has no runLoop catch behind it, so the unit that trips the fuse
|
|
2649
|
+
// would leak the agents it had already spawned. Reaping first covers
|
|
2650
|
+
// the abort and the skip with one call; it is idempotent, so the
|
|
2651
|
+
// runLoop catch finding nothing left to do is free.
|
|
2652
|
+
if (mayHaveSpawnedBeforeFailing(error)) {
|
|
2653
|
+
await this.#reapDispatchFailureHandoffsNow();
|
|
2654
|
+
}
|
|
2238
2655
|
// #292: issues in a pass are independent work units, so a failure
|
|
2239
2656
|
// that is about ONE unit costs that unit and nothing else. Only the
|
|
2240
2657
|
// conditions named in `#isPassFatalFailure` — the ones where
|
|
2241
2658
|
// continuing the pass is meaningless — abort the whole sweep.
|
|
2242
|
-
if (this.#isPassFatalFailure(error, dryRun))
|
|
2243
|
-
|
|
2244
|
-
|
|
2659
|
+
if (this.#isPassFatalFailure(error, dryRun)) {
|
|
2660
|
+
// The overload fuse may have been tripped by a 429 that a caller
|
|
2661
|
+
// swallowed, leaving an unrelated error in hand. The fence keys
|
|
2662
|
+
// the durable backoff off `relayfileOverload(error)`, so hand it
|
|
2663
|
+
// the 429 rather than whatever surfaced last.
|
|
2664
|
+
throw this.#discoveryOverloadFuseError() ?? error;
|
|
2665
|
+
}
|
|
2666
|
+
const overload = relayfileOverload(error);
|
|
2667
|
+
if (overload) {
|
|
2668
|
+
// #297: shedding is a state of the dependency, not a fault of this
|
|
2669
|
+
// work unit, so it stays out of `counters.errors` and gets its own
|
|
2670
|
+
// counter — the same split #293 made for undispatchable units.
|
|
2671
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2672
|
+
this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
|
|
2673
|
+
issue: issueRef(issue).key,
|
|
2674
|
+
status: overload.status,
|
|
2675
|
+
reason: overload.reason,
|
|
2676
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2677
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
else if (!isClassifiedPerItemDispatchFailure(error)) {
|
|
2245
2681
|
unclassifiedFailuresSinceDispatch += 1;
|
|
2246
2682
|
// A pass-wide fault can arrive disguised as a run of per-item
|
|
2247
2683
|
// faults. Skipping every unit would then hand back a green report
|
|
@@ -2260,11 +2696,6 @@ export class FactoryLoop {
|
|
|
2260
2696
|
error: describeError(error).errorMessage,
|
|
2261
2697
|
});
|
|
2262
2698
|
this.#error(error, issueRef(issue));
|
|
2263
|
-
// The failure may have left half-spawned agents behind. runLoop's
|
|
2264
|
-
// catch used to reap them because this error aborted the pass;
|
|
2265
|
-
// now that the pass survives, the reap has to happen here or the
|
|
2266
|
-
// agents leak until the next failed iteration.
|
|
2267
|
-
await this.#reapDispatchFailureHandoffsNow();
|
|
2268
2699
|
}
|
|
2269
2700
|
else {
|
|
2270
2701
|
// Not an error — the unit simply cannot be dispatched right now —
|
|
@@ -2313,6 +2744,21 @@ export class FactoryLoop {
|
|
|
2313
2744
|
}
|
|
2314
2745
|
}
|
|
2315
2746
|
}
|
|
2747
|
+
/**
|
|
2748
|
+
* The 429 that ended this sweep, when relayfile has shed
|
|
2749
|
+
* `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
|
|
2750
|
+
* unit would just be grinding a shedding dependency. Undefined below that.
|
|
2751
|
+
*
|
|
2752
|
+
* Returns the *latched* 429 rather than whatever error is in hand, because
|
|
2753
|
+
* `#runOnceWithDiscoveryFence` keys the durable backoff off
|
|
2754
|
+
* `relayfileOverload(error)` and a caller may have swallowed the 429 that
|
|
2755
|
+
* tripped the fuse.
|
|
2756
|
+
*/
|
|
2757
|
+
#discoveryOverloadFuseError() {
|
|
2758
|
+
if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
|
|
2759
|
+
return undefined;
|
|
2760
|
+
return this.#discoveryOverloadError;
|
|
2761
|
+
}
|
|
2316
2762
|
/**
|
|
2317
2763
|
* Whether a failure raised while processing ONE work unit must abort the
|
|
2318
2764
|
* whole readiness pass instead of skipping that unit.
|
|
@@ -2333,10 +2779,11 @@ export class FactoryLoop {
|
|
|
2333
2779
|
* one would be recorded as an ordinary per-issue skip. The run report
|
|
2334
2780
|
* would then claim a clean pass over work this process no longer has the
|
|
2335
2781
|
* right to touch.
|
|
2336
|
-
* - Relayfile
|
|
2337
|
-
*
|
|
2338
|
-
*
|
|
2339
|
-
*
|
|
2782
|
+
* - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
|
|
2783
|
+
* this sweep. A single shed operation is per-item and skippable (#297),
|
|
2784
|
+
* but past this the backend is not serving this sweep at all: grinding
|
|
2785
|
+
* through the remaining units makes it worse, and the fence needs the 429
|
|
2786
|
+
* to set the durable backoff.
|
|
2340
2787
|
* - The factory is stopping. Teardown is in progress and dispatching more
|
|
2341
2788
|
* agents now leaks them past the shutdown deadline.
|
|
2342
2789
|
* - The fleet control-plane circuit is no longer closed, **on a live pass**.
|
|
@@ -2365,7 +2812,15 @@ export class FactoryLoop {
|
|
|
2365
2812
|
#isPassFatalFailure(error, dryRun) {
|
|
2366
2813
|
// Sweep-scoped: these are about this process's right or ability to run the
|
|
2367
2814
|
// pass at all, so they hold for a dry run exactly as for a live one.
|
|
2368
|
-
if (this.#discoverySweepLeaseLost || this.#
|
|
2815
|
+
if (this.#discoverySweepLeaseLost || this.#stopping) {
|
|
2816
|
+
return true;
|
|
2817
|
+
}
|
|
2818
|
+
// #297: relayfile overload used to sit alongside those two, and it did not
|
|
2819
|
+
// belong there. A 429 on ONE work unit is a fact about that unit's read or
|
|
2820
|
+
// write, not about this process's right to run the pass — and because the
|
|
2821
|
+
// flag latched for the whole sweep, the first transient shed also made
|
|
2822
|
+
// every later unit fatal. Only sustained shedding is now pass-fatal.
|
|
2823
|
+
if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
|
|
2369
2824
|
return true;
|
|
2370
2825
|
}
|
|
2371
2826
|
// Fleet-scoped, and therefore live-only. See the doc comment above.
|
|
@@ -3231,14 +3686,40 @@ export class FactoryLoop {
|
|
|
3231
3686
|
return result;
|
|
3232
3687
|
}
|
|
3233
3688
|
catch (error) {
|
|
3234
|
-
|
|
3689
|
+
const overload = relayfileOverload(error);
|
|
3690
|
+
if (overload && this.#discoverySweepEpoch !== undefined) {
|
|
3235
3691
|
this.#discoveryOverloadError ??= error;
|
|
3692
|
+
this.#discoverySweepOverloads += 1;
|
|
3693
|
+
if (overload.retryAfterSeconds !== undefined) {
|
|
3694
|
+
this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
|
|
3695
|
+
}
|
|
3696
|
+
this.#increment('discoveryOverloadOperations');
|
|
3697
|
+
// #297: relayfile's four overload reason codes — inflight_limit,
|
|
3698
|
+
// oldest_inflight_age, router_inflight_limit, durable_object_overloaded
|
|
3699
|
+
// — all share ONE message string, and they mean four different things:
|
|
3700
|
+
// a DO-local admission cap, one stuck op poisoning every background
|
|
3701
|
+
// caller, a Worker-global isolate cap, and the Cloudflare runtime
|
|
3702
|
+
// shedding the object outright. `relayfileOverload()` has always parsed
|
|
3703
|
+
// the reason; nothing on this path ever printed it, and during the
|
|
3704
|
+
// 2026-08-20 outage that ambiguity was the single biggest obstacle to
|
|
3705
|
+
// diagnosis. Unconditional, unlike the failure warn below: a 429 is
|
|
3706
|
+
// always worth one line.
|
|
3707
|
+
this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
|
|
3708
|
+
this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
|
|
3709
|
+
...metadata,
|
|
3710
|
+
status: overload.status,
|
|
3711
|
+
reason: overload.reason,
|
|
3712
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
3713
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
3714
|
+
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3715
|
+
});
|
|
3236
3716
|
}
|
|
3237
3717
|
if (opts.logFailure || waitWarnings > 0) {
|
|
3238
3718
|
this.#increment('relayfileOperationFailures');
|
|
3239
3719
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
3240
3720
|
...metadata,
|
|
3241
3721
|
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3722
|
+
...(overload ? { status: overload.status, reason: overload.reason } : {}),
|
|
3242
3723
|
error: describeError(error).errorMessage,
|
|
3243
3724
|
});
|
|
3244
3725
|
}
|
|
@@ -3587,6 +4068,14 @@ export class FactoryLoop {
|
|
|
3587
4068
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
3588
4069
|
throw new Error(`Dispatch lifecycle ownership lost immediately before spawning ${dispatchDecision.issue.key}`);
|
|
3589
4070
|
}
|
|
4071
|
+
// That save stamped `slotHeldSinceAtMs`: the row occupies a batch slot from
|
|
4072
|
+
// here on. Arm its deadline before the first await rather than after a
|
|
4073
|
+
// placement succeeds — `#fleet.spawn` deliberately carries no mutation
|
|
4074
|
+
// timeout, so a first attempt that hangs in an otherwise idle process would
|
|
4075
|
+
// otherwise hold the slot with no timer that can ever fire. That is the
|
|
4076
|
+
// exact shape of #303, reached through the fresh dispatch path instead of
|
|
4077
|
+
// the durable one (#303 review, CodeRabbit).
|
|
4078
|
+
this.#scheduleHeldAgentDeadline(record);
|
|
3590
4079
|
if (!dryRun)
|
|
3591
4080
|
await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
|
|
3592
4081
|
const spawnedForReaperHandoff = [];
|
|
@@ -3778,6 +4267,7 @@ export class FactoryLoop {
|
|
|
3778
4267
|
slackDegradedReason: this.#slackDegradedReason,
|
|
3779
4268
|
eventListener: this.#eventListenerStatus(),
|
|
3780
4269
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
4270
|
+
dispatchCapacity: this.#dispatchCapacityStatus(),
|
|
3781
4271
|
heldAgents: batch?.inFlight.flatMap((record) => heldAgentsForRecord(record, nowMs, this.#config.dispatch.agentHoldTimeoutMs, this.#config.terminalState)) ?? [],
|
|
3782
4272
|
};
|
|
3783
4273
|
}
|
|
@@ -3801,17 +4291,66 @@ export class FactoryLoop {
|
|
|
3801
4291
|
}
|
|
3802
4292
|
#readinessReconcileStatus() {
|
|
3803
4293
|
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
3804
|
-
|
|
4294
|
+
// #296 owns the numerator here, #295/#300 own the derivation. The earliest
|
|
4295
|
+
// sweep still running — after a deadline expiry that is the abandoned one,
|
|
4296
|
+
// not the current wait, or every expiry would restart the clock and a
|
|
4297
|
+
// permanently stuck pass would read as merely `retrying`.
|
|
4298
|
+
const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
|
|
4299
|
+
const settled = this.#startMode !== 'live'
|
|
3805
4300
|
? 'not-running'
|
|
4301
|
+
// Failure-count ladder only. Precedence against `stalled` belongs to
|
|
4302
|
+
// `derivedReadinessReconcileState`, which outranks everything here: a
|
|
4303
|
+
// stall is the more specific fact, and `consecutiveFailures` ships
|
|
4304
|
+
// alongside so nothing an alarm keyed on `degraded` needs is lost.
|
|
3806
4305
|
: consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
|
|
3807
4306
|
? 'degraded'
|
|
3808
4307
|
: consecutiveFailures > 0
|
|
3809
4308
|
? 'retrying'
|
|
3810
4309
|
: 'healthy';
|
|
4310
|
+
// The counters above only move when a pass *settles*. A pass that hangs
|
|
4311
|
+
// takes neither path, so `settled` would keep reporting the last finished
|
|
4312
|
+
// pass — `healthy` — for as long as the process is stuck (#295). The
|
|
4313
|
+
// in-flight age is the only field that can express that, so derive the
|
|
4314
|
+
// state from it rather than trusting the last write.
|
|
4315
|
+
const timestamps = {
|
|
4316
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4317
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4318
|
+
...(this.#readinessReconcileLastStartedAtMs !== undefined
|
|
4319
|
+
? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
|
|
4320
|
+
: {}),
|
|
4321
|
+
...(this.#readinessReconcileLastCompletedAtMs !== undefined
|
|
4322
|
+
? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
|
|
4323
|
+
: {}),
|
|
4324
|
+
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
4325
|
+
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
4326
|
+
: {}),
|
|
4327
|
+
};
|
|
4328
|
+
const nowMs = this.#clock.now();
|
|
4329
|
+
// Defence in depth (#300 review, CodeRabbit). These derivations are new
|
|
4330
|
+
// code from another module on a path that `status()` and every heartbeat
|
|
4331
|
+
// write depend on. A throw here would take out the liveness signal the
|
|
4332
|
+
// crash reaper reads — the diagnostic causing the outage it exists to
|
|
4333
|
+
// explain — so a failure costs the derived fields and nothing else.
|
|
4334
|
+
let inFlightMs;
|
|
4335
|
+
let derived = settled;
|
|
4336
|
+
try {
|
|
4337
|
+
inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
|
|
4338
|
+
derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
|
|
4339
|
+
}
|
|
4340
|
+
catch (error) {
|
|
4341
|
+
this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
|
|
4342
|
+
error: describeError(error).errorMessage,
|
|
4343
|
+
});
|
|
4344
|
+
inFlightMs = undefined;
|
|
4345
|
+
derived = settled;
|
|
4346
|
+
}
|
|
3811
4347
|
return {
|
|
3812
|
-
state,
|
|
4348
|
+
state: derived === 'unknown' ? settled : derived,
|
|
3813
4349
|
consecutiveFailures,
|
|
3814
4350
|
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
4351
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4352
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4353
|
+
...(inFlightMs !== undefined ? { inFlightMs } : {}),
|
|
3815
4354
|
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
3816
4355
|
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
3817
4356
|
: {}),
|
|
@@ -3825,6 +4364,9 @@ export class FactoryLoop {
|
|
|
3825
4364
|
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
3826
4365
|
: {}),
|
|
3827
4366
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4367
|
+
...(this.#readinessReconcileLastErrorClass
|
|
4368
|
+
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
4369
|
+
: {}),
|
|
3828
4370
|
};
|
|
3829
4371
|
}
|
|
3830
4372
|
on(event, listener) {
|
|
@@ -4142,7 +4684,7 @@ export class FactoryLoop {
|
|
|
4142
4684
|
if (timer)
|
|
4143
4685
|
clearTimeout(timer);
|
|
4144
4686
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
4145
|
-
this.#
|
|
4687
|
+
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
4146
4688
|
clearedKeys.add(key);
|
|
4147
4689
|
this.#increment('dispatchLifecycleGithubAliasesCollapsed');
|
|
4148
4690
|
}
|
|
@@ -4200,10 +4742,67 @@ export class FactoryLoop {
|
|
|
4200
4742
|
}, DISPATCH_LIFECYCLE_RENEW_MS);
|
|
4201
4743
|
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
4202
4744
|
}
|
|
4745
|
+
/**
|
|
4746
|
+
* The wall-clock deadline that can free this record's batch slot, if any.
|
|
4747
|
+
*
|
|
4748
|
+
* Two clocks, never both. `heldSinceAtMs` is stamped by the first successful
|
|
4749
|
+
* placement and bounds a team that ran and never reached a terminal state.
|
|
4750
|
+
* A record that has no `heldSinceAtMs` never had a placement at all, and
|
|
4751
|
+
* before #303 that made it permanently unreapable: `#scheduleHeldAgentDeadline`
|
|
4752
|
+
* armed no timer and `#sweepHeldAgentDeadlines` skipped it, so a row that
|
|
4753
|
+
* reached `dispatching` and lost its process held the only batch slot
|
|
4754
|
+
* forever. Such a row is definitionally stuck — nothing but a placement can
|
|
4755
|
+
* move it, and no placement is coming — so it gets the much shorter
|
|
4756
|
+
* `agentlessHoldTimeoutMs` anchored on when it took the slot.
|
|
4757
|
+
*
|
|
4758
|
+
* The anchor is deliberately not `agents.size === 0`: `recordPlanned` writes
|
|
4759
|
+
* the spec before the spawn returns, so a process that died mid-spawn leaves
|
|
4760
|
+
* an agent entry with no result and still no placement.
|
|
4761
|
+
*/
|
|
4762
|
+
/**
|
|
4763
|
+
* Does this in-flight record hold a `batchSize` slot right now?
|
|
4764
|
+
*
|
|
4765
|
+
* The same predicate the state stores apply to the durable row, asked of the
|
|
4766
|
+
* in-memory one. Phase alone is not it: once every implementer repo has been
|
|
4767
|
+
* handed to a babysitter, admission stops counting the lifecycle, so
|
|
4768
|
+
* reporting it as an occupant would name slots that are not blocking
|
|
4769
|
+
* anything (#303 review, codex).
|
|
4770
|
+
*/
|
|
4771
|
+
#recordOccupiesSlot(record) {
|
|
4772
|
+
return dispatchPhaseOccupiesSlot(record.lifecyclePhase) && !dispatchHandedOffToBabysitters(record.decision.implementers, [...record.agents.values()].map((tracked) => ({ releasedAtMs: tracked.releasedAtMs, spec: tracked.spec })));
|
|
4773
|
+
}
|
|
4774
|
+
#holdDeadline(record) {
|
|
4775
|
+
if (record.dryRun)
|
|
4776
|
+
return undefined;
|
|
4777
|
+
if (record.heldSinceAtMs !== undefined) {
|
|
4778
|
+
const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
|
|
4779
|
+
return {
|
|
4780
|
+
kind: 'agents',
|
|
4781
|
+
sinceAtMs: record.heldSinceAtMs,
|
|
4782
|
+
timeoutMs,
|
|
4783
|
+
dueAtMs: record.heldSinceAtMs + timeoutMs,
|
|
4784
|
+
};
|
|
4785
|
+
}
|
|
4786
|
+
// Only a row that is actually holding a slot is worth reaping; a `queued`
|
|
4787
|
+
// or `waiting-for-human` row costs nothing and may wait indefinitely.
|
|
4788
|
+
if (record.slotHeldSinceAtMs === undefined || !this.#recordOccupiesSlot(record)) {
|
|
4789
|
+
return undefined;
|
|
4790
|
+
}
|
|
4791
|
+
const timeoutMs = this.#config.dispatch.agentlessHoldTimeoutMs;
|
|
4792
|
+
return {
|
|
4793
|
+
kind: 'agentless',
|
|
4794
|
+
sinceAtMs: record.slotHeldSinceAtMs,
|
|
4795
|
+
timeoutMs,
|
|
4796
|
+
dueAtMs: record.slotHeldSinceAtMs + timeoutMs,
|
|
4797
|
+
};
|
|
4798
|
+
}
|
|
4203
4799
|
#scheduleHeldAgentDeadline(record) {
|
|
4204
|
-
if (this.#stopping
|
|
4800
|
+
if (this.#stopping)
|
|
4205
4801
|
return;
|
|
4206
|
-
const
|
|
4802
|
+
const deadline = this.#holdDeadline(record);
|
|
4803
|
+
if (!deadline)
|
|
4804
|
+
return;
|
|
4805
|
+
const dueAtMs = deadline.dueAtMs;
|
|
4207
4806
|
if (this.#heldAgentDeadlineTimer &&
|
|
4208
4807
|
this.#heldAgentDeadlineDueAtMs !== undefined &&
|
|
4209
4808
|
this.#heldAgentDeadlineDueAtMs <= dueAtMs)
|
|
@@ -4243,14 +4842,16 @@ export class FactoryLoop {
|
|
|
4243
4842
|
}
|
|
4244
4843
|
async #sweepHeldAgentDeadlines() {
|
|
4245
4844
|
const nowMs = this.#clock.now();
|
|
4246
|
-
const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
|
|
4247
4845
|
for (const record of [...(await this.#batch()).inFlight]) {
|
|
4248
|
-
const
|
|
4249
|
-
if (
|
|
4250
|
-
heldSinceAtMs === undefined ||
|
|
4251
|
-
record.agents.size === 0 ||
|
|
4252
|
-
nowMs - heldSinceAtMs < timeoutMs)
|
|
4846
|
+
const deadline = this.#holdDeadline(record);
|
|
4847
|
+
if (!deadline || nowMs < deadline.dueAtMs)
|
|
4253
4848
|
continue;
|
|
4849
|
+
// The durable row wins the classification when there is one. The
|
|
4850
|
+
// in-memory record can lag a placement made in another process, and
|
|
4851
|
+
// reading the stale one would relabel a team that did run as
|
|
4852
|
+
// never-placed — which also excludes its agents from the release below,
|
|
4853
|
+
// leaking live workers (#303 review, CodeRabbit).
|
|
4854
|
+
let effective = deadline;
|
|
4254
4855
|
const key = issueKey(record.issue);
|
|
4255
4856
|
if (this.#abandonedDispatchReasons.has(key))
|
|
4256
4857
|
continue;
|
|
@@ -4264,26 +4865,42 @@ export class FactoryLoop {
|
|
|
4264
4865
|
await this.#finishDurableRelease(record, lifecycle.releaseReason);
|
|
4265
4866
|
continue;
|
|
4266
4867
|
}
|
|
4868
|
+
// Re-derive against the durable row before tearing anything down. The
|
|
4869
|
+
// in-memory record can be a beat behind a placement that just
|
|
4870
|
+
// succeeded in this process or a takeover in another, and the
|
|
4871
|
+
// never-placed deadline exists precisely to catch rows nothing is
|
|
4872
|
+
// moving — it must not be what ends a dispatch that just started
|
|
4873
|
+
// moving (#303 must-not-fire).
|
|
4874
|
+
const durable = this.#holdDeadline(inFlightRecordFromLifecycle(lifecycle));
|
|
4875
|
+
if (!durable || this.#clock.now() < durable.dueAtMs)
|
|
4876
|
+
continue;
|
|
4877
|
+
effective = durable;
|
|
4267
4878
|
if (!await this.#assertDispatchLifecycleOwner(record))
|
|
4268
4879
|
continue;
|
|
4269
4880
|
}
|
|
4270
|
-
const
|
|
4881
|
+
const agentless = effective.kind === 'agentless';
|
|
4882
|
+
const heldForMs = Math.max(0, this.#clock.now() - effective.sinceAtMs);
|
|
4271
4883
|
const details = {
|
|
4272
4884
|
issue: record.issue.key,
|
|
4273
4885
|
heldForMs,
|
|
4274
|
-
holdTimeoutMs: timeoutMs,
|
|
4886
|
+
holdTimeoutMs: effective.timeoutMs,
|
|
4275
4887
|
waitingForTerminalState: this.#config.terminalState,
|
|
4276
|
-
reason: HELD_PAST_DEADLINE_RELEASE_REASON,
|
|
4888
|
+
reason: agentless ? AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON : HELD_PAST_DEADLINE_RELEASE_REASON,
|
|
4277
4889
|
agents: [...record.agents.keys()].sort(),
|
|
4890
|
+
...(agentless ? { phase: record.lifecyclePhase } : {}),
|
|
4278
4891
|
};
|
|
4279
|
-
this.#logger.warn?.(
|
|
4280
|
-
|
|
4892
|
+
this.#logger.warn?.(agentless
|
|
4893
|
+
? '[factory] releasing a dispatch lifecycle that never placed an agent'
|
|
4894
|
+
: '[factory] releasing agents held past deadline', details);
|
|
4895
|
+
await this.#abandonStuckDispatch(record, details.reason);
|
|
4281
4896
|
const lifecycle = this.#usesDurableDispatchLifecycle()
|
|
4282
4897
|
? await this.#state.getDispatchLifecycle(this.#workspaceId, key)
|
|
4283
4898
|
: undefined;
|
|
4284
4899
|
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle)) {
|
|
4285
|
-
this.#increment('heldPastDeadlineReleases');
|
|
4286
|
-
this.#logger.warn?.(
|
|
4900
|
+
this.#increment(agentless ? 'agentlessSlotPastDeadlineReleases' : 'heldPastDeadlineReleases');
|
|
4901
|
+
this.#logger.warn?.(agentless
|
|
4902
|
+
? '[factory] released a dispatch lifecycle that never placed an agent'
|
|
4903
|
+
: '[factory] released agents held past deadline', details);
|
|
4287
4904
|
}
|
|
4288
4905
|
}
|
|
4289
4906
|
}
|
|
@@ -4654,6 +5271,14 @@ export class FactoryLoop {
|
|
|
4654
5271
|
}
|
|
4655
5272
|
async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
|
|
4656
5273
|
record.lifecyclePhase = phase;
|
|
5274
|
+
// Mirror what the store stamps, so the reaper's never-placed clock is
|
|
5275
|
+
// readable from the in-memory record between durable reads (#303). The
|
|
5276
|
+
// store still owns the authoritative value; this uses the same predicate so
|
|
5277
|
+
// the two cannot disagree.
|
|
5278
|
+
if (this.#recordOccupiesSlot(record))
|
|
5279
|
+
record.slotHeldSinceAtMs ??= this.#clock.now();
|
|
5280
|
+
else
|
|
5281
|
+
record.slotHeldSinceAtMs = undefined;
|
|
4657
5282
|
if (record.dryRun || !this.#usesDurableDispatchLifecycle())
|
|
4658
5283
|
return true;
|
|
4659
5284
|
if (isTerminalDispatchPhase(phase))
|
|
@@ -4721,6 +5346,20 @@ export class FactoryLoop {
|
|
|
4721
5346
|
if (isTerminalDispatchLifecycle(lifecycle)) {
|
|
4722
5347
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
4723
5348
|
}
|
|
5349
|
+
// Wake the capacity waiters exactly when this write gives a slot back —
|
|
5350
|
+
// occupied before, not occupied after — and never otherwise (#303
|
|
5351
|
+
// review, cubic).
|
|
5352
|
+
//
|
|
5353
|
+
// Not "when it goes terminal". `releasing` already does not occupy a
|
|
5354
|
+
// slot, so a normal completion frees it one save *before* `complete`,
|
|
5355
|
+
// and a babysitter handoff frees it without ever going terminal at all.
|
|
5356
|
+
// Keying on the terminal save alone therefore both fires for rows that
|
|
5357
|
+
// freed nothing (a `queued` row abandoned at startup) and misses the
|
|
5358
|
+
// writes that actually freed something. The occupancy transition is the
|
|
5359
|
+
// event; the phase is only a proxy for it.
|
|
5360
|
+
if (previous && dispatchLifecycleOccupiesSlot(previous) && !dispatchLifecycleOccupiesSlot(lifecycle)) {
|
|
5361
|
+
this.#resetDispatchCapacityBackoff();
|
|
5362
|
+
}
|
|
4724
5363
|
return true;
|
|
4725
5364
|
});
|
|
4726
5365
|
}
|
|
@@ -4734,7 +5373,130 @@ export class FactoryLoop {
|
|
|
4734
5373
|
resolve(phase);
|
|
4735
5374
|
this.#dispatchTerminalWaiters.delete(key);
|
|
4736
5375
|
}
|
|
4737
|
-
|
|
5376
|
+
/**
|
|
5377
|
+
* Re-arm delay for a capacity wait: 1 s doubling to a 30 s ceiling (#303).
|
|
5378
|
+
*
|
|
5379
|
+
* Only the capacity path backs off. An ownership wait is already bounded by
|
|
5380
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS`, and every other failure is a real error
|
|
5381
|
+
* whose fast retry is the recovery. A capacity wait has no bound at all —
|
|
5382
|
+
* it ends when some other lifecycle terminates, which may be hours away or,
|
|
5383
|
+
* before this fix, never.
|
|
5384
|
+
*/
|
|
5385
|
+
#capacityRetryDelayMs(attempts) {
|
|
5386
|
+
return Math.min(DISPATCH_LIFECYCLE_RETRY_MS * 2 ** Math.max(0, attempts - 1), DISPATCH_LIFECYCLE_RETRY_MAX_MS);
|
|
5387
|
+
}
|
|
5388
|
+
/** Issue keys currently holding a `batchSize` slot, for operator surfaces. */
|
|
5389
|
+
#dispatchSlotOccupants() {
|
|
5390
|
+
return (this.#batchView?.inFlight ?? [])
|
|
5391
|
+
.filter((record) => !record.dryRun && this.#recordOccupiesSlot(record))
|
|
5392
|
+
.map((record) => ({
|
|
5393
|
+
issue: record.issue.key,
|
|
5394
|
+
...(record.lifecyclePhase ? { phase: record.lifecyclePhase } : {}),
|
|
5395
|
+
agents: record.agents.size,
|
|
5396
|
+
// Specs, not workers: `recordPlanned` writes an entry before the spawn
|
|
5397
|
+
// returns, so `agents > 0` is not proof of a placement (#303 review).
|
|
5398
|
+
placedAgents: [...record.agents.values()].filter((tracked) => tracked.result !== undefined).length,
|
|
5399
|
+
...(record.heldSinceAtMs !== undefined
|
|
5400
|
+
? { heldForMs: Math.max(0, this.#clock.now() - record.heldSinceAtMs) }
|
|
5401
|
+
: {}),
|
|
5402
|
+
...(record.slotHeldSinceAtMs !== undefined
|
|
5403
|
+
? { slotHeldForMs: Math.max(0, this.#clock.now() - record.slotHeldSinceAtMs) }
|
|
5404
|
+
: {}),
|
|
5405
|
+
}))
|
|
5406
|
+
.sort((left, right) => left.issue.localeCompare(right.issue));
|
|
5407
|
+
}
|
|
5408
|
+
/**
|
|
5409
|
+
* Batch occupancy as an operator-readable fact (#303).
|
|
5410
|
+
*
|
|
5411
|
+
* Before this, a full batch was visible only as the *absence* of dispatch:
|
|
5412
|
+
* `readinessReconcile` stayed green, `consecutiveFailures` stayed 0, and the
|
|
5413
|
+
* one capacity log had fired hours earlier. Publishing occupancy is what
|
|
5414
|
+
* turns "nothing is being dispatched" into a question an operator can answer
|
|
5415
|
+
* without reading the state document.
|
|
5416
|
+
*/
|
|
5417
|
+
#dispatchCapacityStatus() {
|
|
5418
|
+
const nowMs = this.#clock.now();
|
|
5419
|
+
const occupants = this.#dispatchSlotOccupants();
|
|
5420
|
+
const waits = [...this.#dispatchLifecycleCapacityWaits.entries()];
|
|
5421
|
+
const longestWaitMs = waits.length === 0
|
|
5422
|
+
? undefined
|
|
5423
|
+
: Math.max(...waits.map(([, wait]) => Math.max(0, nowMs - wait.sinceAtMs)));
|
|
5424
|
+
return {
|
|
5425
|
+
batchSize: this.#config.batchSize,
|
|
5426
|
+
active: occupants.length,
|
|
5427
|
+
waiting: waits.length,
|
|
5428
|
+
waitWarnMs: this.#config.dispatch.capacityWaitWarnMs,
|
|
5429
|
+
agentlessHoldTimeoutMs: this.#config.dispatch.agentlessHoldTimeoutMs,
|
|
5430
|
+
...(longestWaitMs !== undefined ? { longestWaitMs } : {}),
|
|
5431
|
+
...(occupants.length > 0 ? { occupants } : {}),
|
|
5432
|
+
...(waits.length > 0
|
|
5433
|
+
? {
|
|
5434
|
+
waitingIssues: waits
|
|
5435
|
+
.sort(([, left], [, right]) => left.sinceAtMs - right.sinceAtMs)
|
|
5436
|
+
.map(([key]) => key),
|
|
5437
|
+
}
|
|
5438
|
+
: {}),
|
|
5439
|
+
};
|
|
5440
|
+
}
|
|
5441
|
+
#recordDispatchCapacityWait(record, key) {
|
|
5442
|
+
const nowMs = this.#clock.now();
|
|
5443
|
+
let wait = this.#dispatchLifecycleCapacityWaits.get(key);
|
|
5444
|
+
if (!wait) {
|
|
5445
|
+
wait = { record, sinceAtMs: nowMs, attempts: 0 };
|
|
5446
|
+
this.#dispatchLifecycleCapacityWaits.set(key, wait);
|
|
5447
|
+
this.#increment('dispatchLifecycleCapacityWaits');
|
|
5448
|
+
}
|
|
5449
|
+
wait.record = record;
|
|
5450
|
+
wait.attempts += 1;
|
|
5451
|
+
const retryMs = this.#capacityRetryDelayMs(wait.attempts);
|
|
5452
|
+
const waitedMs = Math.max(0, nowMs - wait.sinceAtMs);
|
|
5453
|
+
// Escalate on every backoff step, then once a minute after the delay
|
|
5454
|
+
// caps. The old behaviour logged once per key and went silent forever,
|
|
5455
|
+
// which is what made a 14-hour dispatch outage look like an idle Factory.
|
|
5456
|
+
const stepChanged = wait.lastLoggedRetryMs !== retryMs;
|
|
5457
|
+
const overdue = wait.lastLoggedAtMs === undefined ||
|
|
5458
|
+
nowMs - wait.lastLoggedAtMs >= DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS;
|
|
5459
|
+
if (stepChanged || overdue) {
|
|
5460
|
+
wait.lastLoggedAtMs = nowMs;
|
|
5461
|
+
wait.lastLoggedRetryMs = retryMs;
|
|
5462
|
+
this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
|
|
5463
|
+
issue: record.issue.key,
|
|
5464
|
+
retryMs,
|
|
5465
|
+
attempts: wait.attempts,
|
|
5466
|
+
waitedMs,
|
|
5467
|
+
batchSize: this.#config.batchSize,
|
|
5468
|
+
occupiedBy: this.#dispatchSlotOccupants().map((occupant) => occupant.issue),
|
|
5469
|
+
});
|
|
5470
|
+
}
|
|
5471
|
+
return retryMs;
|
|
5472
|
+
}
|
|
5473
|
+
/**
|
|
5474
|
+
* Put every capacity waiter back on the fast path, because a slot just freed.
|
|
5475
|
+
*
|
|
5476
|
+
* The backoff exists to stop a storm of retries asking a question whose
|
|
5477
|
+
* answer is not changing. When a lifecycle reaches a terminal phase the
|
|
5478
|
+
* answer *has* changed, so parking a waiter behind a 30 s timer would trade
|
|
5479
|
+
* the storm for latency — and for a slot released by another process, that
|
|
5480
|
+
* timer is the only signal this one gets (#303 review follow-up).
|
|
5481
|
+
*
|
|
5482
|
+
* The wait's `sinceAtMs` is deliberately untouched: the issue really has
|
|
5483
|
+
* been waiting that long, and the escalating warning should keep saying so.
|
|
5484
|
+
*/
|
|
5485
|
+
#resetDispatchCapacityBackoff() {
|
|
5486
|
+
if (this.#stopping || this.#dispatchLifecycleCapacityWaits.size === 0)
|
|
5487
|
+
return;
|
|
5488
|
+
for (const [key, wait] of this.#dispatchLifecycleCapacityWaits) {
|
|
5489
|
+
wait.attempts = 0;
|
|
5490
|
+
const timer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
5491
|
+
if (!timer)
|
|
5492
|
+
continue;
|
|
5493
|
+
clearTimeout(timer);
|
|
5494
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
5495
|
+
this.#scheduleDispatchLifecycleRetry(wait.record);
|
|
5496
|
+
}
|
|
5497
|
+
this.#increment('dispatchCapacityBackoffResets');
|
|
5498
|
+
}
|
|
5499
|
+
#scheduleDispatchLifecycleRetry(record, delayMs = DISPATCH_LIFECYCLE_RETRY_MS) {
|
|
4738
5500
|
const key = issueKey(record.issue);
|
|
4739
5501
|
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
4740
5502
|
return;
|
|
@@ -4742,23 +5504,17 @@ export class FactoryLoop {
|
|
|
4742
5504
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
4743
5505
|
const drive = this.#driveDispatchLifecycle(key)
|
|
4744
5506
|
.then(() => {
|
|
4745
|
-
this.#
|
|
5507
|
+
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
4746
5508
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
4747
5509
|
})
|
|
4748
5510
|
.catch((error) => {
|
|
5511
|
+
let nextDelayMs = DISPATCH_LIFECYCLE_RETRY_MS;
|
|
4749
5512
|
if (error instanceof DispatchLifecycleCapacityError) {
|
|
4750
5513
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
4751
|
-
|
|
4752
|
-
this.#dispatchLifecycleCapacityWaitLogged.add(key);
|
|
4753
|
-
this.#increment('dispatchLifecycleCapacityWaits');
|
|
4754
|
-
this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
|
|
4755
|
-
issue: record.issue.key,
|
|
4756
|
-
retryMs: DISPATCH_LIFECYCLE_RETRY_MS,
|
|
4757
|
-
});
|
|
4758
|
-
}
|
|
5514
|
+
nextDelayMs = this.#recordDispatchCapacityWait(record, key);
|
|
4759
5515
|
}
|
|
4760
5516
|
else if (error instanceof DispatchLifecycleOwnedElsewhereError) {
|
|
4761
|
-
this.#
|
|
5517
|
+
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
4762
5518
|
if (!this.#dispatchLifecycleOwnershipWaitLogged.has(key)) {
|
|
4763
5519
|
this.#dispatchLifecycleOwnershipWaitLogged.add(key);
|
|
4764
5520
|
this.#increment('dispatchLifecycleOwnershipWaits');
|
|
@@ -4772,18 +5528,18 @@ export class FactoryLoop {
|
|
|
4772
5528
|
}
|
|
4773
5529
|
}
|
|
4774
5530
|
else {
|
|
4775
|
-
this.#
|
|
5531
|
+
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
4776
5532
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
4777
5533
|
this.#logger.warn?.('[factory] durable dispatch lifecycle retry failed', {
|
|
4778
5534
|
issue: record.issue.key,
|
|
4779
5535
|
error: describeError(error).errorMessage,
|
|
4780
5536
|
});
|
|
4781
5537
|
}
|
|
4782
|
-
this.#scheduleDispatchLifecycleRetry(record);
|
|
5538
|
+
this.#scheduleDispatchLifecycleRetry(record, nextDelayMs);
|
|
4783
5539
|
})
|
|
4784
5540
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
4785
5541
|
this.#dispatchLifecycleDrives.add(drive);
|
|
4786
|
-
},
|
|
5542
|
+
}, delayMs);
|
|
4787
5543
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
4788
5544
|
}
|
|
4789
5545
|
#scheduleReleaseRetry(record, reason) {
|
|
@@ -6299,8 +7055,32 @@ export class FactoryLoop {
|
|
|
6299
7055
|
registryPath,
|
|
6300
7056
|
eventListener: this.#eventListenerStatus(),
|
|
6301
7057
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
7058
|
+
dispatchCapacity: this.#dispatchCapacityStatus(),
|
|
6302
7059
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
6303
7060
|
};
|
|
7061
|
+
// The deployed container serves `/healthz` straight out of this file and
|
|
7062
|
+
// has no redaction logic of its own, so publish the already-safe view here
|
|
7063
|
+
// rather than leaving that boundary to whoever reads the file (#295).
|
|
7064
|
+
// Derived against this daemon's clock: every duration in it is a
|
|
7065
|
+
// difference between timestamps this process wrote.
|
|
7066
|
+
//
|
|
7067
|
+
// Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
|
|
7068
|
+
// reaper and `/healthz` read to decide the daemon is alive, and several
|
|
7069
|
+
// callers of this method sit outside any try/catch. A projection failure
|
|
7070
|
+
// must cost the diagnostics block, never the heartbeat — the omitted block
|
|
7071
|
+
// is itself legible, since `factory diagnose` reports a missing one rather
|
|
7072
|
+
// than a false green.
|
|
7073
|
+
try {
|
|
7074
|
+
heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
|
|
7075
|
+
nowMs: updatedAtMs,
|
|
7076
|
+
staleMs: this.#config.loop.heartbeatStaleMs,
|
|
7077
|
+
});
|
|
7078
|
+
}
|
|
7079
|
+
catch (error) {
|
|
7080
|
+
this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
|
|
7081
|
+
error: describeError(error).errorMessage,
|
|
7082
|
+
});
|
|
7083
|
+
}
|
|
6304
7084
|
await mkdir(dirname(path), { recursive: true });
|
|
6305
7085
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
6306
7086
|
await this.#writeInFlightRegistry(registryPath, path);
|
|
@@ -6319,11 +7099,18 @@ export class FactoryLoop {
|
|
|
6319
7099
|
});
|
|
6320
7100
|
}
|
|
6321
7101
|
async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
|
|
6322
|
-
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6323
|
-
if (handoffs.length === 0) {
|
|
6324
|
-
return;
|
|
6325
|
-
}
|
|
6326
7102
|
try {
|
|
7103
|
+
// Inside the try, not before it (#298 review). Every caller reaps while
|
|
7104
|
+
// already handling a failure and then propagates that failure: the
|
|
7105
|
+
// per-item catch rethrows the dispatch error, runLoop's catch is mid
|
|
7106
|
+
// teardown. A throw from here would REPLACE the error in flight — and a
|
|
7107
|
+
// replaced 429 is no longer recognised as overload at the discovery
|
|
7108
|
+
// fence, silently dropping the advertised backoff. Reaping is
|
|
7109
|
+
// best-effort by construction; the caller's error always wins.
|
|
7110
|
+
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
7111
|
+
if (handoffs.length === 0) {
|
|
7112
|
+
return;
|
|
7113
|
+
}
|
|
6327
7114
|
const protectedPids = await this.#protectedPids();
|
|
6328
7115
|
let registryChanged = false;
|
|
6329
7116
|
const readyToClear = new Set();
|
|
@@ -6699,10 +7486,12 @@ export class FactoryLoop {
|
|
|
6699
7486
|
}
|
|
6700
7487
|
return [...handoffs.values()];
|
|
6701
7488
|
}
|
|
6702
|
-
async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed') {
|
|
7489
|
+
async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed', opts = {}) {
|
|
6703
7490
|
if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
|
|
6704
7491
|
return false;
|
|
6705
|
-
const failed = await this.#releaseAndTerminateAgents(handoffs
|
|
7492
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs
|
|
7493
|
+
.filter((handoff) => !opts.skipNeverPlacedAgents || handoff.tracked.result !== undefined)
|
|
7494
|
+
.map((handoff) => [handoff.name, handoff.tracked]), releaseReason, 'completion');
|
|
6706
7495
|
if (failed.length > 0)
|
|
6707
7496
|
return false;
|
|
6708
7497
|
try {
|
|
@@ -6878,6 +7667,17 @@ export class FactoryLoop {
|
|
|
6878
7667
|
: 'agent_spawn_failed',
|
|
6879
7668
|
});
|
|
6880
7669
|
}
|
|
7670
|
+
// The never-placed deadline can fire while this spawn is in flight — that
|
|
7671
|
+
// is the whole point of arming it before the first await, and it makes a
|
|
7672
|
+
// late `spawn` result newly reachable (#303 review, cubic). By now the
|
|
7673
|
+
// reaper may have fenced, released and terminalized the lifecycle, so this
|
|
7674
|
+
// placement belongs to nothing: recording it would attach a live worker to
|
|
7675
|
+
// a record the reaper has finished with, and nothing downstream would ever
|
|
7676
|
+
// release it. Hand it straight to teardown instead.
|
|
7677
|
+
if (!await this.#dispatchLifecycleStillOwned(record)) {
|
|
7678
|
+
await this.#releaseOrphanedLatePlacement(record, spec, result);
|
|
7679
|
+
throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
|
|
7680
|
+
}
|
|
6881
7681
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
6882
7682
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
6883
7683
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
@@ -6889,6 +7689,68 @@ export class FactoryLoop {
|
|
|
6889
7689
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
6890
7690
|
return { name: result.name };
|
|
6891
7691
|
}
|
|
7692
|
+
/**
|
|
7693
|
+
* Is this process still the owner of a lifecycle that is not already done?
|
|
7694
|
+
*
|
|
7695
|
+
* Cheap local checks first — a pending abandon reason, or a dropped epoch,
|
|
7696
|
+
* both of which the reaper sets before anything durable is re-read — then the
|
|
7697
|
+
* durable row, which is authoritative when another owner terminalized it.
|
|
7698
|
+
*/
|
|
7699
|
+
async #dispatchLifecycleStillOwned(record) {
|
|
7700
|
+
const key = issueKey(record.issue);
|
|
7701
|
+
if (this.#abandonedDispatchReasons.has(key))
|
|
7702
|
+
return false;
|
|
7703
|
+
if (!this.#usesDurableDispatchLifecycle())
|
|
7704
|
+
return true;
|
|
7705
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
7706
|
+
if (epoch === undefined)
|
|
7707
|
+
return false;
|
|
7708
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
7709
|
+
if (!lifecycle || isTerminalDispatchLifecycle(lifecycle))
|
|
7710
|
+
return false;
|
|
7711
|
+
// A live row is not the same as *our* row. Another owner can reclaim an
|
|
7712
|
+
// expired lease and leave it nonterminal, and the cached epoch here would
|
|
7713
|
+
// still say we hold it (#303 review, cubic). This mirrors exactly what
|
|
7714
|
+
// `saveDispatchLifecycle` will accept — owner, epoch and an unexpired
|
|
7715
|
+
// lease — so a placement is recorded only when the write that follows can
|
|
7716
|
+
// actually land. Otherwise the save fails and the worker leaks through the
|
|
7717
|
+
// generic ownership-lost path instead of orphan cleanup.
|
|
7718
|
+
const lease = lifecycle.lease;
|
|
7719
|
+
return lease !== undefined &&
|
|
7720
|
+
lease.owner === this.#dispatchLifecycleOwner &&
|
|
7721
|
+
lease.epoch === epoch &&
|
|
7722
|
+
lease.leaseUntilMs > this.#clock.now();
|
|
7723
|
+
}
|
|
7724
|
+
/**
|
|
7725
|
+
* Tear down a placement that landed after its lifecycle was already released.
|
|
7726
|
+
*
|
|
7727
|
+
* Deliberately not routed through `#abandonStuckDispatch`: that record is
|
|
7728
|
+
* terminal and its batch entry is gone, so there is nothing left to abandon.
|
|
7729
|
+
* The only thing that still exists is a live worker on the fleet.
|
|
7730
|
+
*/
|
|
7731
|
+
async #releaseOrphanedLatePlacement(record, spec, result) {
|
|
7732
|
+
const name = result.name ?? spec.name;
|
|
7733
|
+
this.#increment('lateSpawnPlacementsReleased');
|
|
7734
|
+
this.#logger.warn?.('[factory] releasing an agent that finished spawning after its dispatch was released', {
|
|
7735
|
+
issue: record.issue.key,
|
|
7736
|
+
agent: name,
|
|
7737
|
+
role: spec.role,
|
|
7738
|
+
});
|
|
7739
|
+
this.#fleet.markAgentTerminal?.(name, 'dispatch-released-before-placement');
|
|
7740
|
+
try {
|
|
7741
|
+
await this.#fleet.release(name, 'dispatch-released-before-placement');
|
|
7742
|
+
}
|
|
7743
|
+
catch (error) {
|
|
7744
|
+
// The reaper handoff owns anything this could not clean up; failing here
|
|
7745
|
+
// would only replace a released worker with an unreleased one.
|
|
7746
|
+
this.#increment('lateSpawnPlacementReleaseFailures');
|
|
7747
|
+
this.#logger.warn?.('[factory] failed to release a late placement; leaving it to the orphan reaper', {
|
|
7748
|
+
issue: record.issue.key,
|
|
7749
|
+
agent: name,
|
|
7750
|
+
error: describeError(error).errorMessage,
|
|
7751
|
+
});
|
|
7752
|
+
}
|
|
7753
|
+
}
|
|
6892
7754
|
async #handleAgentExit(name, reason) {
|
|
6893
7755
|
if (this.#stopping) {
|
|
6894
7756
|
return;
|
|
@@ -7943,7 +8805,14 @@ export class FactoryLoop {
|
|
|
7943
8805
|
this.#scheduleAbandonedDispatchRetry(record, reason);
|
|
7944
8806
|
return;
|
|
7945
8807
|
}
|
|
7946
|
-
|
|
8808
|
+
// A never-placed record carries specs, not workers: `recordPlanned` writes
|
|
8809
|
+
// the spec before the spawn returns, so a dispatch that died mid-spawn
|
|
8810
|
+
// leaves a name the broker never issued. Releasing one fails, which fails
|
|
8811
|
+
// the whole cleanup and re-arms the abandon retry forever — turning the
|
|
8812
|
+
// #303 reap into a second, quieter wedge. Their worktrees are still torn
|
|
8813
|
+
// down below.
|
|
8814
|
+
const neverPlaced = reason === AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON;
|
|
8815
|
+
const agents = [...record.agents].filter(([, tracked]) => !neverPlaced || tracked.result !== undefined);
|
|
7947
8816
|
for (const [agentName, tracked] of agents) {
|
|
7948
8817
|
if (!heldPastDeadline && tracked.spec.role === 'implementer')
|
|
7949
8818
|
continue;
|
|
@@ -7963,7 +8832,7 @@ export class FactoryLoop {
|
|
|
7963
8832
|
const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, agentReleaseReason, 'completion');
|
|
7964
8833
|
cleanupComplete = failed.length === 0;
|
|
7965
8834
|
}
|
|
7966
|
-
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason) && cleanupComplete;
|
|
8835
|
+
cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason, { skipNeverPlacedAgents: neverPlaced }) && cleanupComplete;
|
|
7967
8836
|
}
|
|
7968
8837
|
else if (agents.length > 0) {
|
|
7969
8838
|
const failed = await this.#releaseAndTerminateAgents(agents, agentReleaseReason, 'completion');
|
|
@@ -14941,6 +15810,15 @@ const githubIssueAuthor = (issue) => {
|
|
|
14941
15810
|
}
|
|
14942
15811
|
return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
|
|
14943
15812
|
};
|
|
15813
|
+
/**
|
|
15814
|
+
* An `IssueRef` for a path whose issue body could not be read at all — the
|
|
15815
|
+
* shape a relayfile-shed ready-issue read leaves behind (#297). The key is
|
|
15816
|
+
* what an operator needs to correlate the skip; the uuid falls back to it.
|
|
15817
|
+
*/
|
|
15818
|
+
const issueRefFromPath = (path) => {
|
|
15819
|
+
const key = keyFromPath(path);
|
|
15820
|
+
return { uuid: uuidFromPath(path) ?? key, key, path };
|
|
15821
|
+
};
|
|
14944
15822
|
const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
|
|
14945
15823
|
// Preserve the historical Linear state namespace while keeping GitHub-native
|
|
14946
15824
|
// issue numbers independent across repositories in the same workspace.
|
|
@@ -16854,10 +17732,54 @@ const relayfileOverload = (error) => {
|
|
|
16854
17732
|
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
16855
17733
|
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
16856
17734
|
};
|
|
17735
|
+
/**
|
|
17736
|
+
* relayfile's overload reason codes, allowlisted.
|
|
17737
|
+
*
|
|
17738
|
+
* `IterationReport.skipped[].reason` is serialized to stdout by `factory
|
|
17739
|
+
* run-once`, so it stays a fixed classification plus a known code — the same
|
|
17740
|
+
* public-surface rule #293 applied to error class names — rather than
|
|
17741
|
+
* whatever string the dependency happened to send.
|
|
17742
|
+
*/
|
|
17743
|
+
const RELAYFILE_OVERLOAD_REASONS = new Set([
|
|
17744
|
+
// Admission gate inside the workspace durable object.
|
|
17745
|
+
'inflight_limit',
|
|
17746
|
+
'oldest_inflight_age',
|
|
17747
|
+
'write_admission_limit',
|
|
17748
|
+
// Worker-side backpressure, per isolate rather than per workspace.
|
|
17749
|
+
'router_inflight_limit',
|
|
17750
|
+
// The Cloudflare runtime shed the object; relayfile only relabels it.
|
|
17751
|
+
'durable_object_overloaded',
|
|
17752
|
+
// relayfileOverload()'s fallback when the body carried no reason at all.
|
|
17753
|
+
'rate_limited',
|
|
17754
|
+
]);
|
|
17755
|
+
/**
|
|
17756
|
+
* Both the run-report reason and the per-reason counter key are built from
|
|
17757
|
+
* this, so an unknown code from the dependency can neither leak into stdout
|
|
17758
|
+
* nor open an unbounded counter namespace.
|
|
17759
|
+
*/
|
|
17760
|
+
const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
|
|
17761
|
+
/**
|
|
17762
|
+
* How long to wait before the next discovery sweep after relayfile shed this
|
|
17763
|
+
* one.
|
|
17764
|
+
*
|
|
17765
|
+
* The advertised `Retry-After` is authoritative in BOTH directions (#297):
|
|
17766
|
+
* it is the first rung, so we never retry sooner than the dependency allows,
|
|
17767
|
+
* and it bounds the ceiling, so we never sleep for minutes because of a
|
|
17768
|
+
* request to wait seconds. Without an advertised delay there is nothing to
|
|
17769
|
+
* respect and the original five-minute ladder governs unchanged.
|
|
17770
|
+
*/
|
|
16857
17771
|
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
16858
|
-
const
|
|
16859
|
-
|
|
16860
|
-
|
|
17772
|
+
const advertisedMs = retryAfterSeconds === undefined
|
|
17773
|
+
? undefined
|
|
17774
|
+
: Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
|
|
17775
|
+
const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
|
|
17776
|
+
// A dependency that asks for longer than the advertised ceiling still gets
|
|
17777
|
+
// what it asked for; the ceiling only stops the ladder from overshooting it.
|
|
17778
|
+
const ceilingMs = advertisedMs === undefined
|
|
17779
|
+
? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
|
|
17780
|
+
: Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
|
|
17781
|
+
const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
|
|
17782
|
+
return Math.min(ceilingMs, baseMs * (2 ** steps));
|
|
16861
17783
|
};
|
|
16862
17784
|
const eventSequenceNumber = (eventId) => {
|
|
16863
17785
|
const whole = Number(eventId);
|
|
@@ -17086,10 +18008,6 @@ const telemetryCategory = (value) => {
|
|
|
17086
18008
|
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
|
|
17087
18009
|
return normalized.slice(0, 120) || undefined;
|
|
17088
18010
|
};
|
|
17089
|
-
const telemetryErrorClass = (error) => {
|
|
17090
|
-
const name = error instanceof Error ? error.name : '';
|
|
17091
|
-
return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
|
|
17092
|
-
};
|
|
17093
18011
|
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
17094
18012
|
const retryOnTimeout = async (fn, opts) => {
|
|
17095
18013
|
let lastError;
|
|
@@ -17236,6 +18154,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
|
|
|
17236
18154
|
...(releaseReason ? { releaseReason } : {}),
|
|
17237
18155
|
...(cost ? { cost: structuredClone(cost) } : {}),
|
|
17238
18156
|
...(record.heldSinceAtMs !== undefined ? { heldSinceAtMs: record.heldSinceAtMs } : {}),
|
|
18157
|
+
...(record.slotHeldSinceAtMs !== undefined ? { slotHeldSinceAtMs: record.slotHeldSinceAtMs } : {}),
|
|
17239
18158
|
updatedAtMs,
|
|
17240
18159
|
});
|
|
17241
18160
|
const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
@@ -17252,9 +18171,14 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
|
|
|
17252
18171
|
invocationIds: new Set(lifecycle.invocationIds),
|
|
17253
18172
|
result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
|
|
17254
18173
|
...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
|
|
17255
|
-
heldSinceAtMs: lifecycle.heldSinceAtMs ?? (
|
|
18174
|
+
heldSinceAtMs: lifecycle.heldSinceAtMs ?? (
|
|
18175
|
+
// A live placement the durable row predates the `heldSinceAtMs` field for.
|
|
18176
|
+
// `tracked.result` is what distinguishes a placement from a spec that
|
|
18177
|
+
// `recordPlanned` wrote and no spawn ever answered (#303).
|
|
18178
|
+
lifecycle.agents.some((agent) => agent.releasedAtMs === undefined && agent.tracked.result !== undefined)
|
|
17256
18179
|
? lifecycle.updatedAtMs
|
|
17257
18180
|
: undefined),
|
|
18181
|
+
slotHeldSinceAtMs: lifecycle.slotHeldSinceAtMs,
|
|
17258
18182
|
lifecyclePhase: lifecycle.phase,
|
|
17259
18183
|
});
|
|
17260
18184
|
const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
|
|
@@ -17290,6 +18214,29 @@ export class LiveDispatchStateChangedError extends Error {
|
|
|
17290
18214
|
this.issueKey = issueKey;
|
|
17291
18215
|
}
|
|
17292
18216
|
}
|
|
18217
|
+
/**
|
|
18218
|
+
* A placement that finished spawning after its dispatch had been released.
|
|
18219
|
+
*
|
|
18220
|
+
* The never-placed deadline (#303) can terminalize a lifecycle while
|
|
18221
|
+
* `#fleet.spawn` is still in flight; the worker is released and the dispatch
|
|
18222
|
+
* unwinds. That is a known, named, self-healing race — the issue returns to the
|
|
18223
|
+
* queue and is re-dispatched — so it must be classified rather than counted as
|
|
18224
|
+
* an unexplained fault. It fires precisely under slow spawns, which is the
|
|
18225
|
+
* condition the deadline exists for, so a degraded fleet produces it
|
|
18226
|
+
* repeatedly; left unclassified, five in a row would trip
|
|
18227
|
+
* `UNCLASSIFIED_DISPATCH_FAILURE_LIMIT` and abort the whole readiness pass,
|
|
18228
|
+
* turning a bounded slot into a stopped sweep (#303 review, factory-lead).
|
|
18229
|
+
*/
|
|
18230
|
+
export class LatePlacementReleasedError extends Error {
|
|
18231
|
+
issueKey;
|
|
18232
|
+
agentName;
|
|
18233
|
+
constructor(issueKey, agentName) {
|
|
18234
|
+
super(`Dispatch lifecycle for ${issueKey} was released while ${agentName} was still spawning`);
|
|
18235
|
+
this.name = 'LatePlacementReleasedError';
|
|
18236
|
+
this.issueKey = issueKey;
|
|
18237
|
+
this.agentName = agentName;
|
|
18238
|
+
}
|
|
18239
|
+
}
|
|
17293
18240
|
/** Whether a thrown value is a {@link LiveDispatchStateChangedError}. */
|
|
17294
18241
|
export function isLiveDispatchStateChangedError(error) {
|
|
17295
18242
|
return error instanceof LiveDispatchStateChangedError;
|
|
@@ -17323,7 +18270,32 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
|
17323
18270
|
* skippable and are exempt from the consecutive-failure fuse.
|
|
17324
18271
|
*/
|
|
17325
18272
|
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
17326
|
-
error instanceof DispatchLifecycleClaimRefusedError
|
|
18273
|
+
error instanceof DispatchLifecycleClaimRefusedError ||
|
|
18274
|
+
// #303: the never-placed deadline released this dispatch while its spawn was
|
|
18275
|
+
// still in flight. Named, expected and self-healing — the issue goes back to
|
|
18276
|
+
// the queue — and it recurs under exactly the slow-spawn conditions the
|
|
18277
|
+
// deadline exists for, so leaving it unclassified would let a degraded fleet
|
|
18278
|
+
// trip the pass-abort fuse. Its own counters keep it visible.
|
|
18279
|
+
error instanceof LatePlacementReleasedError ||
|
|
18280
|
+
// Relayfile shedding one operation is a state of the dependency, not an
|
|
18281
|
+
// unexplained fault, and it has its own fuse — see #297 and
|
|
18282
|
+
// DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
|
|
18283
|
+
relayfileOverload(error) !== undefined;
|
|
18284
|
+
/**
|
|
18285
|
+
* Whether a per-item failure could have left half-spawned agents behind.
|
|
18286
|
+
*
|
|
18287
|
+
* The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
|
|
18288
|
+
* anything, so there is nothing to reap for them. Everything else can fail
|
|
18289
|
+
* *after* a spawn — including relayfile shedding a post-spawn read — having
|
|
18290
|
+
* persisted the agents as failure handoffs on the way out.
|
|
18291
|
+
*
|
|
18292
|
+
* Deliberately a denylist rather than an allowlist: a new failure mode that
|
|
18293
|
+
* nobody classified should default to "reap it", because the cost of a
|
|
18294
|
+
* needless reap is one no-op pass over an empty handoff list, while the cost
|
|
18295
|
+
* of a missed one is leaked agents and duplicate workers on the next retry.
|
|
18296
|
+
*/
|
|
18297
|
+
const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
|
|
18298
|
+
!(error instanceof DispatchLifecycleClaimRefusedError);
|
|
17327
18299
|
/**
|
|
17328
18300
|
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
17329
18301
|
*
|
|
@@ -17334,8 +18306,13 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
|
|
|
17334
18306
|
* `describeControlPlaneError` makes for circuit state.
|
|
17335
18307
|
*/
|
|
17336
18308
|
const perItemDispatchSkipReason = (error) => {
|
|
18309
|
+
const overload = relayfileOverload(error);
|
|
18310
|
+
if (overload)
|
|
18311
|
+
return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
|
|
17337
18312
|
if (error instanceof LiveDispatchStateChangedError)
|
|
17338
18313
|
return 'live state changed during dispatch';
|
|
18314
|
+
if (error instanceof LatePlacementReleasedError)
|
|
18315
|
+
return 'dispatch released while its agent was still spawning';
|
|
17339
18316
|
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
17340
18317
|
return error.refusal === 'terminal'
|
|
17341
18318
|
? 'dispatch lifecycle already terminal'
|