@agent-relay/factory 0.1.64 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli/diagnose.d.ts +92 -0
- package/dist/cli/diagnose.d.ts.map +1 -0
- package/dist/cli/diagnose.js +346 -0
- package/dist/cli/diagnose.js.map +1 -0
- package/dist/cli/fleet.d.ts +9 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +87 -4
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +69 -2
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +32 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +16 -0
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +228 -13
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/hosted/orchestrator.d.ts.map +1 -1
- package/dist/hosted/orchestrator.js +1 -4
- package/dist/hosted/orchestrator.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/observability/error-class.d.ts +29 -0
- package/dist/observability/error-class.d.ts.map +1 -0
- package/dist/observability/error-class.js +36 -0
- package/dist/observability/error-class.js.map +1 -0
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/index.d.ts.map +1 -1
- package/dist/observability/index.js +1 -0
- package/dist/observability/index.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +649 -46
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +1 -0
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -0
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts +82 -0
- package/dist/orchestrator/public-health.d.ts.map +1 -0
- package/dist/orchestrator/public-health.js +372 -0
- package/dist/orchestrator/public-health.js.map +1 -0
- package/dist/ports/state.d.ts +16 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +8 -0
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +12 -2
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +8 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +12 -2
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/types.d.ts +106 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
|
-
import { FactoryConfigSchema } from '../config/schema.js';
|
|
4
|
+
import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
@@ -29,6 +29,8 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
|
|
|
29
29
|
import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
|
|
30
30
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
31
31
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
32
|
+
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
33
|
+
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
32
34
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
33
35
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
34
36
|
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
@@ -185,6 +187,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
|
185
187
|
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
186
188
|
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
187
189
|
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
190
|
+
/** First rung of the ladder when the 429 advertises no `Retry-After`. */
|
|
191
|
+
const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
|
|
192
|
+
/**
|
|
193
|
+
* Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
|
|
194
|
+
* whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
|
|
195
|
+
* dependency into hammering it.
|
|
196
|
+
*/
|
|
197
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
|
|
198
|
+
/**
|
|
199
|
+
* Ceiling for the ladder once the dependency has told us how long to wait.
|
|
200
|
+
*
|
|
201
|
+
* relayfile sheds an overloaded workspace DO with a 429 in milliseconds
|
|
202
|
+
* carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
|
|
203
|
+
* that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
|
|
204
|
+
* dependency asked for five seconds, Factory slept for five minutes, probed
|
|
205
|
+
* for recovery once per cap-length window, and presented a transient upstream
|
|
206
|
+
* blip as a sustained outage. The ratchet still escalates — it is just bounded
|
|
207
|
+
* by roughly what was actually asked for, and never *below* it, so this is a
|
|
208
|
+
* ceiling and not a licence to retry sooner than the dependency allows.
|
|
209
|
+
*/
|
|
210
|
+
const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
|
|
211
|
+
/**
|
|
212
|
+
* How many relayfile operations one sweep may have shed before the sweep is
|
|
213
|
+
* abandoned rather than continued.
|
|
214
|
+
*
|
|
215
|
+
* Per-item overload skips that item and keeps going (#297, the same principle
|
|
216
|
+
* as #292), but skipping must not degenerate into grinding a shedding
|
|
217
|
+
* dependency through an entire backlog one 429 at a time. Past this many, the
|
|
218
|
+
* dependency is not serving this sweep at all: abort, back off, and let the
|
|
219
|
+
* ratchet do its job.
|
|
220
|
+
*/
|
|
221
|
+
const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
|
|
188
222
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
189
223
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
190
224
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -259,6 +293,20 @@ class DispatchLifecycleClaimRefusedError extends Error {
|
|
|
259
293
|
this.name = 'DispatchLifecycleClaimRefusedError';
|
|
260
294
|
}
|
|
261
295
|
}
|
|
296
|
+
/**
|
|
297
|
+
* The deadline that makes a hung sweep reachable by the existing recovery path.
|
|
298
|
+
* Its message is fully internal (one integer), so it is safe to persist into
|
|
299
|
+
* the operator-facing `readinessReconcile.lastError`.
|
|
300
|
+
*/
|
|
301
|
+
class ReadinessReconcileTimeoutError extends Error {
|
|
302
|
+
timeoutMs;
|
|
303
|
+
code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
|
|
304
|
+
constructor(timeoutMs) {
|
|
305
|
+
super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
|
|
306
|
+
this.timeoutMs = timeoutMs;
|
|
307
|
+
this.name = 'ReadinessReconcileTimeoutError';
|
|
308
|
+
}
|
|
309
|
+
}
|
|
262
310
|
const realClock = {
|
|
263
311
|
now: () => Date.now(),
|
|
264
312
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -408,12 +456,41 @@ export class FactoryLoop {
|
|
|
408
456
|
#readinessReconcileTimer;
|
|
409
457
|
#readinessReconcileInFlight;
|
|
410
458
|
#readinessReconcileIntervalMs = 60_000;
|
|
459
|
+
#readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
|
|
460
|
+
// Set for exactly as long as a sweep is running. `state` is derived from
|
|
461
|
+
// this, so an in-flight pass can no longer masquerade as the last settled one.
|
|
462
|
+
#readinessReconcileInFlightSinceMs;
|
|
463
|
+
/**
|
|
464
|
+
* The work a deadline gave up waiting on. The deadline bounds the wait, not
|
|
465
|
+
* the sweep, so this is still live: shutdown has to drain it, and `state` has
|
|
466
|
+
* to keep counting from when it actually started.
|
|
467
|
+
*
|
|
468
|
+
* Two fields rather than a collection, because every live abandoned wait
|
|
469
|
+
* converges on the same sweep. They are all `runOnce()` calls with the same
|
|
470
|
+
* `dryRun`, so whatever they are queued behind, the first one out starts the
|
|
471
|
+
* sweep and the rest coalesce onto it — they settle together. So the newest
|
|
472
|
+
* wait is a sufficient drain target, and the earliest start is the honest
|
|
473
|
+
* age. Both matter (#301 review): keeping only the newest start advanced the
|
|
474
|
+
* age by two intervals every two intervals, so at `reconcileTimeoutMs ===
|
|
475
|
+
* reconcileIntervalMs` it never reached three and `stalled` was never
|
|
476
|
+
* reported; keeping one record per wait grew without bound in exactly the
|
|
477
|
+
* never-settling case this change exists for.
|
|
478
|
+
*
|
|
479
|
+
* The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
|
|
480
|
+
* sweep is waited BEHIND rather than coalesced onto, so that handle can name
|
|
481
|
+
* an unrelated sweep, and the readiness pass would then start its own work
|
|
482
|
+
* after shutdown believed it had drained everything. The wait covers the
|
|
483
|
+
* queueing and the sweep it eventually runs, in every branch.
|
|
484
|
+
*/
|
|
485
|
+
#readinessReconcileAbandonedWait;
|
|
486
|
+
#readinessReconcileAbandonedSinceMs;
|
|
411
487
|
#readinessReconcileConsecutiveFailures = 0;
|
|
412
488
|
#readinessReconcileLastDurationMs;
|
|
413
489
|
#readinessReconcileLastStartedAtMs;
|
|
414
490
|
#readinessReconcileLastCompletedAtMs;
|
|
415
491
|
#readinessReconcileLastFailureAtMs;
|
|
416
492
|
#readinessReconcileLastError;
|
|
493
|
+
#readinessReconcileLastErrorClass;
|
|
417
494
|
#liveEventQueue = [];
|
|
418
495
|
#liveEventDrainScheduled = false;
|
|
419
496
|
#liveEventDrainActive = false;
|
|
@@ -501,7 +578,34 @@ export class FactoryLoop {
|
|
|
501
578
|
// so the loop's catch no longer runs the failure-handoff reaper for it; the
|
|
502
579
|
// pass reaps inline and must write to the same paths runLoop would.
|
|
503
580
|
#loopReapPaths;
|
|
581
|
+
/**
|
|
582
|
+
* The first 429 relayfile raised during this sweep, kept for its
|
|
583
|
+
* `Retry-After` and reason when the sweep decides how long to back off.
|
|
584
|
+
*
|
|
585
|
+
* Before #297 this doubled as a sweep-wide kill switch: any 429 from any
|
|
586
|
+
* relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
|
|
587
|
+
* away along with everything the sweep had already accomplished. It is now
|
|
588
|
+
* only evidence, never a verdict — see `#discoverySweepOverloads` for the
|
|
589
|
+
* fuse that still ends a sweep the dependency is genuinely refusing to serve.
|
|
590
|
+
*/
|
|
504
591
|
#discoveryOverloadError;
|
|
592
|
+
/** Relayfile operations this sweep has been shed on. */
|
|
593
|
+
#discoverySweepOverloads = 0;
|
|
594
|
+
/**
|
|
595
|
+
* The longest `Retry-After` any operation in this sweep advertised.
|
|
596
|
+
*
|
|
597
|
+
* `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
|
|
598
|
+
* from it alone would let the durable window expire before a later, longer
|
|
599
|
+
* advertised delay permits — breaking the very guarantee #297 is about. The
|
|
600
|
+
* backoff takes the maximum instead.
|
|
601
|
+
*/
|
|
602
|
+
#discoverySweepRetryAfterSeconds;
|
|
603
|
+
/**
|
|
604
|
+
* Whether relayfile served at least one ready work unit end to end during
|
|
605
|
+
* this sweep — its issue read, or a dispatch built on it. This is what
|
|
606
|
+
* decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
|
|
607
|
+
*/
|
|
608
|
+
#discoverySweepProgress = false;
|
|
505
609
|
#resolvedIssueSource;
|
|
506
610
|
#integrationInstructions;
|
|
507
611
|
#integrationInstructionsRefresh;
|
|
@@ -917,6 +1021,14 @@ export class FactoryLoop {
|
|
|
917
1021
|
clearTimeout(this.#previewSweepTimer);
|
|
918
1022
|
this.#previewSweepTimer = undefined;
|
|
919
1023
|
await this.#readinessReconcileInFlight;
|
|
1024
|
+
// #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
|
|
1025
|
+
// can settle with its `runOnce()` still live. Shutdown releases dispatch
|
|
1026
|
+
// lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
|
|
1027
|
+
// fences a stopping sweep once something in it throws — so a sweep whose
|
|
1028
|
+
// dependency recovers cleanly would otherwise dispatch through torn-down
|
|
1029
|
+
// state. Draining here restores exactly the pre-deadline shutdown contract:
|
|
1030
|
+
// stop() outlives the sweep it started.
|
|
1031
|
+
await this.#readinessReconcileAbandonedWait;
|
|
920
1032
|
await this.#previewSweepInFlight;
|
|
921
1033
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
922
1034
|
try {
|
|
@@ -1067,6 +1179,9 @@ export class FactoryLoop {
|
|
|
1067
1179
|
const options = this.#liveOptions(overrides);
|
|
1068
1180
|
this.#liveTransport = options.transport;
|
|
1069
1181
|
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
1182
|
+
// `start()` overrides skip the schema's cross-field check, so re-apply its
|
|
1183
|
+
// floor here: a deadline under one interval would kill every pass.
|
|
1184
|
+
this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
|
|
1070
1185
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
1071
1186
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
1072
1187
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1103,10 +1218,27 @@ export class FactoryLoop {
|
|
|
1103
1218
|
this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
|
|
1104
1219
|
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
1105
1220
|
});
|
|
1221
|
+
// Review follow-up on #300 (P1, cubic). The startup backfill is a
|
|
1222
|
+
// discovery pass like any other, and it is the one most likely to hang:
|
|
1223
|
+
// #36 measured 61 minutes here while the Relayfile mirror hydrated on a
|
|
1224
|
+
// cold container. Stamping it means a wedged FIRST pass is visible as
|
|
1225
|
+
// in-flight, instead of leaving the timestamps empty and the derived
|
|
1226
|
+
// state reading `healthy` forever.
|
|
1227
|
+
//
|
|
1228
|
+
// Only the timestamps. `consecutiveFailures` and `lastError` belong to
|
|
1229
|
+
// the reconcile loop's own failure accounting, which owns the degraded
|
|
1230
|
+
// threshold and the #297 reason allowlist; a startup failure is already
|
|
1231
|
+
// counted by `liveStartupBackfillErrors` and reported through `#error`.
|
|
1232
|
+
const backfillStartedAtMs = this.#clock.now();
|
|
1233
|
+
this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
|
|
1106
1234
|
try {
|
|
1107
1235
|
await this.runOnce();
|
|
1236
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1237
|
+
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1108
1238
|
}
|
|
1109
1239
|
catch (error) {
|
|
1240
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1241
|
+
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1110
1242
|
// A startup backfill failure must not abort the daemon: log it and fall
|
|
1111
1243
|
// back to the live event stream (plus any buffered events) instead of
|
|
1112
1244
|
// leaving the factory down.
|
|
@@ -1198,6 +1330,7 @@ export class FactoryLoop {
|
|
|
1198
1330
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
1199
1331
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1200
1332
|
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1333
|
+
reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
|
|
1201
1334
|
};
|
|
1202
1335
|
}
|
|
1203
1336
|
async #currentEventCursor(limit) {
|
|
@@ -1246,19 +1379,82 @@ export class FactoryLoop {
|
|
|
1246
1379
|
}, delayMs);
|
|
1247
1380
|
this.#readinessReconcileTimer.unref?.();
|
|
1248
1381
|
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Runs one sweep under a deadline (#296).
|
|
1384
|
+
*
|
|
1385
|
+
* The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
|
|
1386
|
+
* lease and abandoning it mid-flight is not safe — so expiry rejects *this*
|
|
1387
|
+
* wait and leaves the underlying pass to finish on its own. That is enough:
|
|
1388
|
+
* the rejection is what reaches the failure path, which re-arms the timer.
|
|
1389
|
+
* A later reconcile pass coalesces onto the still-running `runOnce()` and
|
|
1390
|
+
* fails on its own deadline too, so a persistent hang keeps counting up to
|
|
1391
|
+
* `degraded` instead of going quiet.
|
|
1392
|
+
*/
|
|
1393
|
+
async #runOnceWithReadinessDeadline() {
|
|
1394
|
+
const timeoutMs = this.#readinessReconcileTimeoutMs;
|
|
1395
|
+
const startedAtMs = this.#clock.now();
|
|
1396
|
+
const sweep = this.runOnce();
|
|
1397
|
+
let timer;
|
|
1398
|
+
try {
|
|
1399
|
+
return await new Promise((resolve, reject) => {
|
|
1400
|
+
timer = setTimeout(() => {
|
|
1401
|
+
this.#increment('readinessReconcileDeadlineExceeded');
|
|
1402
|
+
if (this.#readinessReconcileAbandonedSinceMs === undefined) {
|
|
1403
|
+
// `state` ages from the FIRST wait that gave up on this work, not
|
|
1404
|
+
// from whenever the latest one began.
|
|
1405
|
+
this.#readinessReconcileAbandonedSinceMs = startedAtMs;
|
|
1406
|
+
// The abandoned pass is still running against the live control
|
|
1407
|
+
// plane. Report where it lands, so an operator can tell a
|
|
1408
|
+
// dependency that recovered late from one that never answered.
|
|
1409
|
+
// Attached once, so a wedge is reported once and not per expiry.
|
|
1410
|
+
void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
|
|
1411
|
+
timeoutMs,
|
|
1412
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1413
|
+
dispatched: report.dispatched.length,
|
|
1414
|
+
}), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
|
|
1415
|
+
timeoutMs,
|
|
1416
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1417
|
+
error: describeError(error).errorMessage,
|
|
1418
|
+
})).catch(() => undefined);
|
|
1419
|
+
}
|
|
1420
|
+
// Newest wait wins as the drain target: it settles no earlier than
|
|
1421
|
+
// the ones before it, and clearing on it clears them all.
|
|
1422
|
+
const wait = sweep.catch(() => undefined).then(() => {
|
|
1423
|
+
if (this.#readinessReconcileAbandonedWait !== wait)
|
|
1424
|
+
return;
|
|
1425
|
+
this.#readinessReconcileAbandonedWait = undefined;
|
|
1426
|
+
this.#readinessReconcileAbandonedSinceMs = undefined;
|
|
1427
|
+
});
|
|
1428
|
+
this.#readinessReconcileAbandonedWait = wait;
|
|
1429
|
+
reject(new ReadinessReconcileTimeoutError(timeoutMs));
|
|
1430
|
+
}, timeoutMs);
|
|
1431
|
+
timer.unref?.();
|
|
1432
|
+
// Attaching handlers here is also what keeps a late rejection from the
|
|
1433
|
+
// abandoned pass from surfacing as an unhandled rejection.
|
|
1434
|
+
sweep.then(resolve, reject);
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
finally {
|
|
1438
|
+
if (timer)
|
|
1439
|
+
clearTimeout(timer);
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1249
1442
|
async #reconcileReadyIssues() {
|
|
1250
1443
|
const startedAtMs = this.#clock.now();
|
|
1251
1444
|
this.#readinessReconcileLastStartedAtMs = startedAtMs;
|
|
1445
|
+
this.#readinessReconcileInFlightSinceMs = startedAtMs;
|
|
1252
1446
|
this.#increment('readinessReconcileSweeps');
|
|
1253
1447
|
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1254
1448
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1449
|
+
timeoutMs: this.#readinessReconcileTimeoutMs,
|
|
1255
1450
|
});
|
|
1256
1451
|
try {
|
|
1257
|
-
const report = await this
|
|
1452
|
+
const report = await this.#runOnceWithReadinessDeadline();
|
|
1258
1453
|
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1259
1454
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1260
1455
|
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1261
1456
|
this.#readinessReconcileLastError = undefined;
|
|
1457
|
+
this.#readinessReconcileLastErrorClass = undefined;
|
|
1262
1458
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1263
1459
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1264
1460
|
candidates: report.pulled.length,
|
|
@@ -1267,11 +1463,28 @@ export class FactoryLoop {
|
|
|
1267
1463
|
});
|
|
1268
1464
|
}
|
|
1269
1465
|
catch (error) {
|
|
1270
|
-
|
|
1466
|
+
// #297: all four relayfile overload reason codes share one message, and
|
|
1467
|
+
// `lastError` is what an operator reads from /evidence. Without the
|
|
1468
|
+
// reason, "workspace durable object is busy" cannot be told apart from
|
|
1469
|
+
// three other conditions with three different remedies.
|
|
1470
|
+
//
|
|
1471
|
+
// Allowlisted, because this is a persisted operator-facing surface and
|
|
1472
|
+
// not just a log line: `lastError` is returned from `status()` and
|
|
1473
|
+
// written into the loop heartbeat file, so an unbounded
|
|
1474
|
+
// dependency-controlled string would land on disk.
|
|
1475
|
+
const overload = relayfileOverload(error);
|
|
1476
|
+
const errorMessage = overload
|
|
1477
|
+
? `${describeError(error).errorMessage} ` +
|
|
1478
|
+
`[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
|
|
1479
|
+
`${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
|
|
1480
|
+
: describeError(error).errorMessage;
|
|
1271
1481
|
this.#readinessReconcileConsecutiveFailures += 1;
|
|
1272
1482
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1273
1483
|
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1274
1484
|
this.#readinessReconcileLastError = errorMessage;
|
|
1485
|
+
// The class, unlike the message, is publishable: #295 puts it on the
|
|
1486
|
+
// unauthenticated health surface through the same allowlist.
|
|
1487
|
+
this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
|
|
1275
1488
|
this.#increment('readinessReconcileErrors');
|
|
1276
1489
|
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1277
1490
|
error: errorMessage,
|
|
@@ -1280,6 +1493,11 @@ export class FactoryLoop {
|
|
|
1280
1493
|
degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
1281
1494
|
});
|
|
1282
1495
|
}
|
|
1496
|
+
finally {
|
|
1497
|
+
// Cleared before the heartbeat write below, so a slow-but-successful pass
|
|
1498
|
+
// does not stamp its own tail as `stalled`.
|
|
1499
|
+
this.#readinessReconcileInFlightSinceMs = undefined;
|
|
1500
|
+
}
|
|
1283
1501
|
await this.#refreshLiveHeartbeat();
|
|
1284
1502
|
}
|
|
1285
1503
|
#scheduleLivePoll(delayMs, options) {
|
|
@@ -2008,13 +2226,29 @@ export class FactoryLoop {
|
|
|
2008
2226
|
this.#discoverySweepStartedAtMs = sweepStartedAtMs;
|
|
2009
2227
|
this.#discoverySweepLeaseLost = false;
|
|
2010
2228
|
this.#discoveryOverloadError = undefined;
|
|
2229
|
+
this.#discoverySweepOverloads = 0;
|
|
2230
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2231
|
+
this.#discoverySweepProgress = false;
|
|
2011
2232
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
2012
2233
|
let leaseReleased = false;
|
|
2013
2234
|
try {
|
|
2014
2235
|
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
2236
|
+
// #297: a 429 raised anywhere in the sweep used to latch and be rethrown
|
|
2237
|
+
// here, discarding a completed pass — every issue read, every dispatch —
|
|
2238
|
+
// because of one transient shed operation. The work this sweep did is
|
|
2239
|
+
// now kept instead, and the ratchet below records that the dependency is
|
|
2240
|
+
// shedding but still serving.
|
|
2015
2241
|
const report = await this.#performRunOnce(opts);
|
|
2016
|
-
|
|
2242
|
+
// The exception, and the reason skipping shed units cannot make a sweep
|
|
2243
|
+
// unconditionally green: a sweep that was shed AND got no work unit
|
|
2244
|
+
// through accomplished nothing. There is no progress to preserve, and
|
|
2245
|
+
// committing it would report a clean sweep over a dependency that served
|
|
2246
|
+
// none of it — leaving `readinessReconcile` healthy while Factory
|
|
2247
|
+
// dispatches nothing, which is the #292 wedge wearing the other costume.
|
|
2248
|
+
// Fail it so the ratchet escalates and readiness reflects reality.
|
|
2249
|
+
if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
|
|
2017
2250
|
throw this.#discoveryOverloadError;
|
|
2251
|
+
}
|
|
2018
2252
|
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
2019
2253
|
// Do not clear the durable lease while a renewal can still be waiting on
|
|
2020
2254
|
// the same state-file lock. A late renewal that observes the completed
|
|
@@ -2024,7 +2258,8 @@ export class FactoryLoop {
|
|
|
2024
2258
|
if (this.#discoverySweepLeaseLost) {
|
|
2025
2259
|
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
2026
2260
|
}
|
|
2027
|
-
const
|
|
2261
|
+
const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
|
|
2262
|
+
const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
|
|
2028
2263
|
leaseReleased = completed;
|
|
2029
2264
|
if (!completed)
|
|
2030
2265
|
throw new Error('discovery sweep lease was lost before completion');
|
|
@@ -2040,18 +2275,18 @@ export class FactoryLoop {
|
|
|
2040
2275
|
await this.#stopDiscoverySweepRenewal();
|
|
2041
2276
|
const overload = relayfileOverload(error);
|
|
2042
2277
|
if (overload) {
|
|
2043
|
-
const
|
|
2044
|
-
|
|
2045
|
-
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
2046
|
-
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
2047
|
-
this.#increment('discoveryOverloadBackoffs');
|
|
2278
|
+
const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
|
|
2279
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
|
|
2048
2280
|
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
2049
2281
|
status: overload.status,
|
|
2050
2282
|
reason: overload.reason,
|
|
2051
|
-
retryAfterSeconds:
|
|
2052
|
-
delayMs,
|
|
2053
|
-
backoffUntilMs,
|
|
2054
|
-
consecutiveOverloads,
|
|
2283
|
+
retryAfterSeconds: outcome.retryAfterSeconds,
|
|
2284
|
+
delayMs: outcome.delayMs,
|
|
2285
|
+
backoffUntilMs: outcome.backoffUntilMs,
|
|
2286
|
+
consecutiveOverloads: outcome.consecutiveOverloads,
|
|
2287
|
+
previousOverloads: claim.state.consecutiveOverloads,
|
|
2288
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2289
|
+
sweepProgress: this.#discoverySweepProgress,
|
|
2055
2290
|
});
|
|
2056
2291
|
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
2057
2292
|
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
@@ -2067,6 +2302,9 @@ export class FactoryLoop {
|
|
|
2067
2302
|
this.#discoverySweepEpoch = undefined;
|
|
2068
2303
|
this.#discoverySweepStartedAtMs = undefined;
|
|
2069
2304
|
this.#discoveryOverloadError = undefined;
|
|
2305
|
+
this.#discoverySweepOverloads = 0;
|
|
2306
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2307
|
+
this.#discoverySweepProgress = false;
|
|
2070
2308
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
2071
2309
|
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
2072
2310
|
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
@@ -2078,6 +2316,94 @@ export class FactoryLoop {
|
|
|
2078
2316
|
}
|
|
2079
2317
|
}
|
|
2080
2318
|
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Commit the sweep, carrying any residual overload backoff into the store.
|
|
2321
|
+
*
|
|
2322
|
+
* `completeDiscoverySweepWithOverload` is optional on the port so a store
|
|
2323
|
+
* written before #297 keeps working. When one of those is injected and this
|
|
2324
|
+
* sweep HAS a residual, the backoff cannot be persisted and the next sweep
|
|
2325
|
+
* would retry immediately — so this says so, loudly and with a counter,
|
|
2326
|
+
* rather than degrading in silence. The sweep itself still commits: losing
|
|
2327
|
+
* the backoff is worse than pre-#297 behaviour only if nobody notices.
|
|
2328
|
+
*/
|
|
2329
|
+
async #commitDiscoverySweep(epoch, checkpoint, residual) {
|
|
2330
|
+
if (residual && this.#state.completeDiscoverySweepWithOverload) {
|
|
2331
|
+
return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
|
|
2332
|
+
}
|
|
2333
|
+
if (residual) {
|
|
2334
|
+
this.#increment('discoveryOverloadResidualUnsupported');
|
|
2335
|
+
this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
|
|
2336
|
+
store: this.#state.constructor?.name,
|
|
2337
|
+
consecutiveOverloads: residual.consecutiveOverloads,
|
|
2338
|
+
backoffUntilMs: residual.backoffUntilMs,
|
|
2339
|
+
delayMs: residual.delayMs,
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
|
|
2343
|
+
}
|
|
2344
|
+
/**
|
|
2345
|
+
* How the durable overload ratchet should read after this sweep, or
|
|
2346
|
+
* `undefined` if relayfile never shed anything and the ordinary reset
|
|
2347
|
+
* applies.
|
|
2348
|
+
*
|
|
2349
|
+
* #297, deliverable 3. `consecutiveOverloads` used to clear only on a
|
|
2350
|
+
* *fully* clean sweep, so under sustained mild load the ratchet essentially
|
|
2351
|
+
* never cleared: it climbed to the cap on the first bad sweep and stayed
|
|
2352
|
+
* there, probing for recovery once per cap-length window, long after the
|
|
2353
|
+
* dependency had recovered. Requiring perfection to clear a ratchet means
|
|
2354
|
+
* the ratchet does not clear.
|
|
2355
|
+
*
|
|
2356
|
+
* The signal that replaces "was this sweep perfect" is "did relayfile serve
|
|
2357
|
+
* any of this sweep's work units", because that is what the ratchet is
|
|
2358
|
+
* actually for. A shedding DO rejects ALL background traffic —
|
|
2359
|
+
* `oldest_inflight_age` does this regardless of how few requests are in
|
|
2360
|
+
* flight — so a sweep against a genuinely overloaded workspace gets not one
|
|
2361
|
+
* unit through and still escalates, all the way to the cap. A sweep that did
|
|
2362
|
+
* get units through proves the dependency is serving us, so it decays by one
|
|
2363
|
+
* rung. Decay, not reset: getting work done while being shed is not evidence
|
|
2364
|
+
* that the overload is over, only that it is survivable.
|
|
2365
|
+
*/
|
|
2366
|
+
#discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
|
|
2367
|
+
const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
|
|
2368
|
+
if (!overload)
|
|
2369
|
+
return undefined;
|
|
2370
|
+
// The longest delay anything in this sweep asked for, not just the one the
|
|
2371
|
+
// latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
|
|
2372
|
+
const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
|
|
2373
|
+
.filter((value) => value !== undefined);
|
|
2374
|
+
const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
|
|
2375
|
+
// Deliberately NOT "the sweep reached its end". A sweep in which relayfile
|
|
2376
|
+
// shed EVERY unit still runs the loop to the end, and it got nothing done;
|
|
2377
|
+
// treating that as progress would decay the ratchet in exactly the case it
|
|
2378
|
+
// exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
|
|
2379
|
+
// aborted one before it can commit, so the escalate branch here is reached
|
|
2380
|
+
// only through `sweepOutcome === 'aborted'` — but the rule is a property of
|
|
2381
|
+
// progress, not of which caller asked, and is written that way.)
|
|
2382
|
+
const consecutiveOverloads = this.#discoverySweepProgress
|
|
2383
|
+
? Math.max(0, previousOverloads - 1)
|
|
2384
|
+
: previousOverloads + 1;
|
|
2385
|
+
const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
|
|
2386
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
2387
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
2388
|
+
if (sweepOutcome === 'committed') {
|
|
2389
|
+
this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
|
|
2390
|
+
status: overload.status,
|
|
2391
|
+
reason: overload.reason,
|
|
2392
|
+
retryAfterSeconds,
|
|
2393
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2394
|
+
previousOverloads,
|
|
2395
|
+
consecutiveOverloads,
|
|
2396
|
+
delayMs,
|
|
2397
|
+
backoffUntilMs,
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
return {
|
|
2401
|
+
consecutiveOverloads,
|
|
2402
|
+
backoffUntilMs,
|
|
2403
|
+
delayMs,
|
|
2404
|
+
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2081
2407
|
async #performRunOnce(opts = {}) {
|
|
2082
2408
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
2083
2409
|
const startedAtMs = this.#clock.now();
|
|
@@ -2125,15 +2451,57 @@ export class FactoryLoop {
|
|
|
2125
2451
|
let readyIssueReads = 0;
|
|
2126
2452
|
const issueEntries = [];
|
|
2127
2453
|
for (const path of paths) {
|
|
2128
|
-
|
|
2454
|
+
let issue;
|
|
2455
|
+
let shed = false;
|
|
2456
|
+
try {
|
|
2457
|
+
issue = await this.#readIssue(path);
|
|
2458
|
+
}
|
|
2459
|
+
catch (error) {
|
|
2460
|
+
// #297: `#readIssue` rethrows relayfile overload and swallows every
|
|
2461
|
+
// other read fault, so this catch only ever sees the backend
|
|
2462
|
+
// shedding THIS issue's read. That is a fact about one work unit:
|
|
2463
|
+
// a sweep that pulled 40 issues and was shed on the 39th must keep
|
|
2464
|
+
// the other 39, exactly as #292 argued for dispatch failures. The
|
|
2465
|
+
// fuse is what still ends a sweep the backend is refusing.
|
|
2466
|
+
const overload = relayfileOverload(error);
|
|
2467
|
+
// Defensive: anything `#readIssue` did not swallow and is not a 429
|
|
2468
|
+
// is not ours to reclassify, and must surface as itself.
|
|
2469
|
+
if (!overload)
|
|
2470
|
+
throw error;
|
|
2471
|
+
const fuse = this.#discoveryOverloadFuseError();
|
|
2472
|
+
if (fuse)
|
|
2473
|
+
throw fuse;
|
|
2474
|
+
shed = true;
|
|
2475
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2476
|
+
this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
|
|
2477
|
+
path,
|
|
2478
|
+
status: overload.status,
|
|
2479
|
+
reason: overload.reason,
|
|
2480
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2481
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2482
|
+
});
|
|
2483
|
+
recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
|
|
2484
|
+
}
|
|
2129
2485
|
readyIssueReads += 1;
|
|
2486
|
+
// Relayfile served this work unit's read: the dependency is shedding
|
|
2487
|
+
// but not refusing, which is what decays the ratchet (#297).
|
|
2488
|
+
//
|
|
2489
|
+
// `issue` is required, not just `!shed`: `#readIssue` returns
|
|
2490
|
+
// `undefined` for a body it could not read at all, and the known
|
|
2491
|
+
// phantom condition — the tree listing issue paths whose bodies are
|
|
2492
|
+
// absent — makes every read return `undefined`. Crediting those would
|
|
2493
|
+
// decay the ratchet on a sweep that served nothing.
|
|
2494
|
+
if (!shed && issue)
|
|
2495
|
+
this.#discoverySweepProgress = true;
|
|
2130
2496
|
lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
|
|
2131
2497
|
? '[factory] GitHub ready issue read progress'
|
|
2132
2498
|
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
2133
|
-
if (
|
|
2134
|
-
|
|
2499
|
+
if (!shed) {
|
|
2500
|
+
if (issue && issueSource === 'linear') {
|
|
2501
|
+
await this.#recordCanonicalIssueState(issue);
|
|
2502
|
+
}
|
|
2503
|
+
issueEntries.push({ path, issue });
|
|
2135
2504
|
}
|
|
2136
|
-
issueEntries.push({ path, issue });
|
|
2137
2505
|
await this.#refreshLiveHeartbeatIfDue();
|
|
2138
2506
|
}
|
|
2139
2507
|
if (issueSource === 'github') {
|
|
@@ -2222,6 +2590,9 @@ export class FactoryLoop {
|
|
|
2222
2590
|
// A completed dispatch — even one that parks or escalates the issue —
|
|
2223
2591
|
// proves the pipeline still works, so the fuse below starts over.
|
|
2224
2592
|
unclassifiedFailuresSinceDispatch = 0;
|
|
2593
|
+
// ...and proves relayfile is still serving this sweep, which is what
|
|
2594
|
+
// decays the durable overload ratchet (#297).
|
|
2595
|
+
this.#discoverySweepProgress = true;
|
|
2225
2596
|
if (result.agents.length === 0 && !dryRun) {
|
|
2226
2597
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2227
2598
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
@@ -2235,13 +2606,47 @@ export class FactoryLoop {
|
|
|
2235
2606
|
}
|
|
2236
2607
|
}
|
|
2237
2608
|
catch (error) {
|
|
2609
|
+
// The failure may have left half-spawned agents behind, persisted as
|
|
2610
|
+
// failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
|
|
2611
|
+
// catch used to reap them because every such error aborted the pass;
|
|
2612
|
+
// now that most of them are skipped, the reap has to happen here.
|
|
2613
|
+
//
|
|
2614
|
+
// BEFORE the fatality check, not after (#298 review, round two): the
|
|
2615
|
+
// 429 that trips the overload fuse aborts this pass, and a direct
|
|
2616
|
+
// `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
|
|
2617
|
+
// has no runLoop catch behind it, so the unit that trips the fuse
|
|
2618
|
+
// would leak the agents it had already spawned. Reaping first covers
|
|
2619
|
+
// the abort and the skip with one call; it is idempotent, so the
|
|
2620
|
+
// runLoop catch finding nothing left to do is free.
|
|
2621
|
+
if (mayHaveSpawnedBeforeFailing(error)) {
|
|
2622
|
+
await this.#reapDispatchFailureHandoffsNow();
|
|
2623
|
+
}
|
|
2238
2624
|
// #292: issues in a pass are independent work units, so a failure
|
|
2239
2625
|
// that is about ONE unit costs that unit and nothing else. Only the
|
|
2240
2626
|
// conditions named in `#isPassFatalFailure` — the ones where
|
|
2241
2627
|
// continuing the pass is meaningless — abort the whole sweep.
|
|
2242
|
-
if (this.#isPassFatalFailure(error, dryRun))
|
|
2243
|
-
|
|
2244
|
-
|
|
2628
|
+
if (this.#isPassFatalFailure(error, dryRun)) {
|
|
2629
|
+
// The overload fuse may have been tripped by a 429 that a caller
|
|
2630
|
+
// swallowed, leaving an unrelated error in hand. The fence keys
|
|
2631
|
+
// the durable backoff off `relayfileOverload(error)`, so hand it
|
|
2632
|
+
// the 429 rather than whatever surfaced last.
|
|
2633
|
+
throw this.#discoveryOverloadFuseError() ?? error;
|
|
2634
|
+
}
|
|
2635
|
+
const overload = relayfileOverload(error);
|
|
2636
|
+
if (overload) {
|
|
2637
|
+
// #297: shedding is a state of the dependency, not a fault of this
|
|
2638
|
+
// work unit, so it stays out of `counters.errors` and gets its own
|
|
2639
|
+
// counter — the same split #293 made for undispatchable units.
|
|
2640
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2641
|
+
this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
|
|
2642
|
+
issue: issueRef(issue).key,
|
|
2643
|
+
status: overload.status,
|
|
2644
|
+
reason: overload.reason,
|
|
2645
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2646
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2647
|
+
});
|
|
2648
|
+
}
|
|
2649
|
+
else if (!isClassifiedPerItemDispatchFailure(error)) {
|
|
2245
2650
|
unclassifiedFailuresSinceDispatch += 1;
|
|
2246
2651
|
// A pass-wide fault can arrive disguised as a run of per-item
|
|
2247
2652
|
// faults. Skipping every unit would then hand back a green report
|
|
@@ -2260,11 +2665,6 @@ export class FactoryLoop {
|
|
|
2260
2665
|
error: describeError(error).errorMessage,
|
|
2261
2666
|
});
|
|
2262
2667
|
this.#error(error, issueRef(issue));
|
|
2263
|
-
// The failure may have left half-spawned agents behind. runLoop's
|
|
2264
|
-
// catch used to reap them because this error aborted the pass;
|
|
2265
|
-
// now that the pass survives, the reap has to happen here or the
|
|
2266
|
-
// agents leak until the next failed iteration.
|
|
2267
|
-
await this.#reapDispatchFailureHandoffsNow();
|
|
2268
2668
|
}
|
|
2269
2669
|
else {
|
|
2270
2670
|
// Not an error — the unit simply cannot be dispatched right now —
|
|
@@ -2313,6 +2713,21 @@ export class FactoryLoop {
|
|
|
2313
2713
|
}
|
|
2314
2714
|
}
|
|
2315
2715
|
}
|
|
2716
|
+
/**
|
|
2717
|
+
* The 429 that ended this sweep, when relayfile has shed
|
|
2718
|
+
* `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
|
|
2719
|
+
* unit would just be grinding a shedding dependency. Undefined below that.
|
|
2720
|
+
*
|
|
2721
|
+
* Returns the *latched* 429 rather than whatever error is in hand, because
|
|
2722
|
+
* `#runOnceWithDiscoveryFence` keys the durable backoff off
|
|
2723
|
+
* `relayfileOverload(error)` and a caller may have swallowed the 429 that
|
|
2724
|
+
* tripped the fuse.
|
|
2725
|
+
*/
|
|
2726
|
+
#discoveryOverloadFuseError() {
|
|
2727
|
+
if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
|
|
2728
|
+
return undefined;
|
|
2729
|
+
return this.#discoveryOverloadError;
|
|
2730
|
+
}
|
|
2316
2731
|
/**
|
|
2317
2732
|
* Whether a failure raised while processing ONE work unit must abort the
|
|
2318
2733
|
* whole readiness pass instead of skipping that unit.
|
|
@@ -2333,10 +2748,11 @@ export class FactoryLoop {
|
|
|
2333
2748
|
* one would be recorded as an ordinary per-issue skip. The run report
|
|
2334
2749
|
* would then claim a clean pass over work this process no longer has the
|
|
2335
2750
|
* right to touch.
|
|
2336
|
-
* - Relayfile
|
|
2337
|
-
*
|
|
2338
|
-
*
|
|
2339
|
-
*
|
|
2751
|
+
* - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
|
|
2752
|
+
* this sweep. A single shed operation is per-item and skippable (#297),
|
|
2753
|
+
* but past this the backend is not serving this sweep at all: grinding
|
|
2754
|
+
* through the remaining units makes it worse, and the fence needs the 429
|
|
2755
|
+
* to set the durable backoff.
|
|
2340
2756
|
* - The factory is stopping. Teardown is in progress and dispatching more
|
|
2341
2757
|
* agents now leaks them past the shutdown deadline.
|
|
2342
2758
|
* - The fleet control-plane circuit is no longer closed, **on a live pass**.
|
|
@@ -2365,7 +2781,15 @@ export class FactoryLoop {
|
|
|
2365
2781
|
#isPassFatalFailure(error, dryRun) {
|
|
2366
2782
|
// Sweep-scoped: these are about this process's right or ability to run the
|
|
2367
2783
|
// pass at all, so they hold for a dry run exactly as for a live one.
|
|
2368
|
-
if (this.#discoverySweepLeaseLost || this.#
|
|
2784
|
+
if (this.#discoverySweepLeaseLost || this.#stopping) {
|
|
2785
|
+
return true;
|
|
2786
|
+
}
|
|
2787
|
+
// #297: relayfile overload used to sit alongside those two, and it did not
|
|
2788
|
+
// belong there. A 429 on ONE work unit is a fact about that unit's read or
|
|
2789
|
+
// write, not about this process's right to run the pass — and because the
|
|
2790
|
+
// flag latched for the whole sweep, the first transient shed also made
|
|
2791
|
+
// every later unit fatal. Only sustained shedding is now pass-fatal.
|
|
2792
|
+
if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
|
|
2369
2793
|
return true;
|
|
2370
2794
|
}
|
|
2371
2795
|
// Fleet-scoped, and therefore live-only. See the doc comment above.
|
|
@@ -3231,14 +3655,40 @@ export class FactoryLoop {
|
|
|
3231
3655
|
return result;
|
|
3232
3656
|
}
|
|
3233
3657
|
catch (error) {
|
|
3234
|
-
|
|
3658
|
+
const overload = relayfileOverload(error);
|
|
3659
|
+
if (overload && this.#discoverySweepEpoch !== undefined) {
|
|
3235
3660
|
this.#discoveryOverloadError ??= error;
|
|
3661
|
+
this.#discoverySweepOverloads += 1;
|
|
3662
|
+
if (overload.retryAfterSeconds !== undefined) {
|
|
3663
|
+
this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
|
|
3664
|
+
}
|
|
3665
|
+
this.#increment('discoveryOverloadOperations');
|
|
3666
|
+
// #297: relayfile's four overload reason codes — inflight_limit,
|
|
3667
|
+
// oldest_inflight_age, router_inflight_limit, durable_object_overloaded
|
|
3668
|
+
// — all share ONE message string, and they mean four different things:
|
|
3669
|
+
// a DO-local admission cap, one stuck op poisoning every background
|
|
3670
|
+
// caller, a Worker-global isolate cap, and the Cloudflare runtime
|
|
3671
|
+
// shedding the object outright. `relayfileOverload()` has always parsed
|
|
3672
|
+
// the reason; nothing on this path ever printed it, and during the
|
|
3673
|
+
// 2026-08-20 outage that ambiguity was the single biggest obstacle to
|
|
3674
|
+
// diagnosis. Unconditional, unlike the failure warn below: a 429 is
|
|
3675
|
+
// always worth one line.
|
|
3676
|
+
this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
|
|
3677
|
+
this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
|
|
3678
|
+
...metadata,
|
|
3679
|
+
status: overload.status,
|
|
3680
|
+
reason: overload.reason,
|
|
3681
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
3682
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
3683
|
+
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3684
|
+
});
|
|
3236
3685
|
}
|
|
3237
3686
|
if (opts.logFailure || waitWarnings > 0) {
|
|
3238
3687
|
this.#increment('relayfileOperationFailures');
|
|
3239
3688
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
3240
3689
|
...metadata,
|
|
3241
3690
|
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3691
|
+
...(overload ? { status: overload.status, reason: overload.reason } : {}),
|
|
3242
3692
|
error: describeError(error).errorMessage,
|
|
3243
3693
|
});
|
|
3244
3694
|
}
|
|
@@ -3801,17 +4251,66 @@ export class FactoryLoop {
|
|
|
3801
4251
|
}
|
|
3802
4252
|
#readinessReconcileStatus() {
|
|
3803
4253
|
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
3804
|
-
|
|
4254
|
+
// #296 owns the numerator here, #295/#300 own the derivation. The earliest
|
|
4255
|
+
// sweep still running — after a deadline expiry that is the abandoned one,
|
|
4256
|
+
// not the current wait, or every expiry would restart the clock and a
|
|
4257
|
+
// permanently stuck pass would read as merely `retrying`.
|
|
4258
|
+
const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
|
|
4259
|
+
const settled = this.#startMode !== 'live'
|
|
3805
4260
|
? 'not-running'
|
|
4261
|
+
// Failure-count ladder only. Precedence against `stalled` belongs to
|
|
4262
|
+
// `derivedReadinessReconcileState`, which outranks everything here: a
|
|
4263
|
+
// stall is the more specific fact, and `consecutiveFailures` ships
|
|
4264
|
+
// alongside so nothing an alarm keyed on `degraded` needs is lost.
|
|
3806
4265
|
: consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
|
|
3807
4266
|
? 'degraded'
|
|
3808
4267
|
: consecutiveFailures > 0
|
|
3809
4268
|
? 'retrying'
|
|
3810
4269
|
: 'healthy';
|
|
4270
|
+
// The counters above only move when a pass *settles*. A pass that hangs
|
|
4271
|
+
// takes neither path, so `settled` would keep reporting the last finished
|
|
4272
|
+
// pass — `healthy` — for as long as the process is stuck (#295). The
|
|
4273
|
+
// in-flight age is the only field that can express that, so derive the
|
|
4274
|
+
// state from it rather than trusting the last write.
|
|
4275
|
+
const timestamps = {
|
|
4276
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4277
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4278
|
+
...(this.#readinessReconcileLastStartedAtMs !== undefined
|
|
4279
|
+
? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
|
|
4280
|
+
: {}),
|
|
4281
|
+
...(this.#readinessReconcileLastCompletedAtMs !== undefined
|
|
4282
|
+
? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
|
|
4283
|
+
: {}),
|
|
4284
|
+
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
4285
|
+
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
4286
|
+
: {}),
|
|
4287
|
+
};
|
|
4288
|
+
const nowMs = this.#clock.now();
|
|
4289
|
+
// Defence in depth (#300 review, CodeRabbit). These derivations are new
|
|
4290
|
+
// code from another module on a path that `status()` and every heartbeat
|
|
4291
|
+
// write depend on. A throw here would take out the liveness signal the
|
|
4292
|
+
// crash reaper reads — the diagnostic causing the outage it exists to
|
|
4293
|
+
// explain — so a failure costs the derived fields and nothing else.
|
|
4294
|
+
let inFlightMs;
|
|
4295
|
+
let derived = settled;
|
|
4296
|
+
try {
|
|
4297
|
+
inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
|
|
4298
|
+
derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
|
|
4299
|
+
}
|
|
4300
|
+
catch (error) {
|
|
4301
|
+
this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
|
|
4302
|
+
error: describeError(error).errorMessage,
|
|
4303
|
+
});
|
|
4304
|
+
inFlightMs = undefined;
|
|
4305
|
+
derived = settled;
|
|
4306
|
+
}
|
|
3811
4307
|
return {
|
|
3812
|
-
state,
|
|
4308
|
+
state: derived === 'unknown' ? settled : derived,
|
|
3813
4309
|
consecutiveFailures,
|
|
3814
4310
|
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
4311
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4312
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4313
|
+
...(inFlightMs !== undefined ? { inFlightMs } : {}),
|
|
3815
4314
|
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
3816
4315
|
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
3817
4316
|
: {}),
|
|
@@ -3825,6 +4324,9 @@ export class FactoryLoop {
|
|
|
3825
4324
|
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
3826
4325
|
: {}),
|
|
3827
4326
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4327
|
+
...(this.#readinessReconcileLastErrorClass
|
|
4328
|
+
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
4329
|
+
: {}),
|
|
3828
4330
|
};
|
|
3829
4331
|
}
|
|
3830
4332
|
on(event, listener) {
|
|
@@ -6301,6 +6803,29 @@ export class FactoryLoop {
|
|
|
6301
6803
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
6302
6804
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
6303
6805
|
};
|
|
6806
|
+
// The deployed container serves `/healthz` straight out of this file and
|
|
6807
|
+
// has no redaction logic of its own, so publish the already-safe view here
|
|
6808
|
+
// rather than leaving that boundary to whoever reads the file (#295).
|
|
6809
|
+
// Derived against this daemon's clock: every duration in it is a
|
|
6810
|
+
// difference between timestamps this process wrote.
|
|
6811
|
+
//
|
|
6812
|
+
// Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
|
|
6813
|
+
// reaper and `/healthz` read to decide the daemon is alive, and several
|
|
6814
|
+
// callers of this method sit outside any try/catch. A projection failure
|
|
6815
|
+
// must cost the diagnostics block, never the heartbeat — the omitted block
|
|
6816
|
+
// is itself legible, since `factory diagnose` reports a missing one rather
|
|
6817
|
+
// than a false green.
|
|
6818
|
+
try {
|
|
6819
|
+
heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
|
|
6820
|
+
nowMs: updatedAtMs,
|
|
6821
|
+
staleMs: this.#config.loop.heartbeatStaleMs,
|
|
6822
|
+
});
|
|
6823
|
+
}
|
|
6824
|
+
catch (error) {
|
|
6825
|
+
this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
|
|
6826
|
+
error: describeError(error).errorMessage,
|
|
6827
|
+
});
|
|
6828
|
+
}
|
|
6304
6829
|
await mkdir(dirname(path), { recursive: true });
|
|
6305
6830
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
6306
6831
|
await this.#writeInFlightRegistry(registryPath, path);
|
|
@@ -6319,11 +6844,18 @@ export class FactoryLoop {
|
|
|
6319
6844
|
});
|
|
6320
6845
|
}
|
|
6321
6846
|
async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
|
|
6322
|
-
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6323
|
-
if (handoffs.length === 0) {
|
|
6324
|
-
return;
|
|
6325
|
-
}
|
|
6326
6847
|
try {
|
|
6848
|
+
// Inside the try, not before it (#298 review). Every caller reaps while
|
|
6849
|
+
// already handling a failure and then propagates that failure: the
|
|
6850
|
+
// per-item catch rethrows the dispatch error, runLoop's catch is mid
|
|
6851
|
+
// teardown. A throw from here would REPLACE the error in flight — and a
|
|
6852
|
+
// replaced 429 is no longer recognised as overload at the discovery
|
|
6853
|
+
// fence, silently dropping the advertised backoff. Reaping is
|
|
6854
|
+
// best-effort by construction; the caller's error always wins.
|
|
6855
|
+
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6856
|
+
if (handoffs.length === 0) {
|
|
6857
|
+
return;
|
|
6858
|
+
}
|
|
6327
6859
|
const protectedPids = await this.#protectedPids();
|
|
6328
6860
|
let registryChanged = false;
|
|
6329
6861
|
const readyToClear = new Set();
|
|
@@ -14941,6 +15473,15 @@ const githubIssueAuthor = (issue) => {
|
|
|
14941
15473
|
}
|
|
14942
15474
|
return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
|
|
14943
15475
|
};
|
|
15476
|
+
/**
|
|
15477
|
+
* An `IssueRef` for a path whose issue body could not be read at all — the
|
|
15478
|
+
* shape a relayfile-shed ready-issue read leaves behind (#297). The key is
|
|
15479
|
+
* what an operator needs to correlate the skip; the uuid falls back to it.
|
|
15480
|
+
*/
|
|
15481
|
+
const issueRefFromPath = (path) => {
|
|
15482
|
+
const key = keyFromPath(path);
|
|
15483
|
+
return { uuid: uuidFromPath(path) ?? key, key, path };
|
|
15484
|
+
};
|
|
14944
15485
|
const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
|
|
14945
15486
|
// Preserve the historical Linear state namespace while keeping GitHub-native
|
|
14946
15487
|
// issue numbers independent across repositories in the same workspace.
|
|
@@ -16854,10 +17395,54 @@ const relayfileOverload = (error) => {
|
|
|
16854
17395
|
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
16855
17396
|
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
16856
17397
|
};
|
|
17398
|
+
/**
|
|
17399
|
+
* relayfile's overload reason codes, allowlisted.
|
|
17400
|
+
*
|
|
17401
|
+
* `IterationReport.skipped[].reason` is serialized to stdout by `factory
|
|
17402
|
+
* run-once`, so it stays a fixed classification plus a known code — the same
|
|
17403
|
+
* public-surface rule #293 applied to error class names — rather than
|
|
17404
|
+
* whatever string the dependency happened to send.
|
|
17405
|
+
*/
|
|
17406
|
+
const RELAYFILE_OVERLOAD_REASONS = new Set([
|
|
17407
|
+
// Admission gate inside the workspace durable object.
|
|
17408
|
+
'inflight_limit',
|
|
17409
|
+
'oldest_inflight_age',
|
|
17410
|
+
'write_admission_limit',
|
|
17411
|
+
// Worker-side backpressure, per isolate rather than per workspace.
|
|
17412
|
+
'router_inflight_limit',
|
|
17413
|
+
// The Cloudflare runtime shed the object; relayfile only relabels it.
|
|
17414
|
+
'durable_object_overloaded',
|
|
17415
|
+
// relayfileOverload()'s fallback when the body carried no reason at all.
|
|
17416
|
+
'rate_limited',
|
|
17417
|
+
]);
|
|
17418
|
+
/**
|
|
17419
|
+
* Both the run-report reason and the per-reason counter key are built from
|
|
17420
|
+
* this, so an unknown code from the dependency can neither leak into stdout
|
|
17421
|
+
* nor open an unbounded counter namespace.
|
|
17422
|
+
*/
|
|
17423
|
+
const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
|
|
17424
|
+
/**
|
|
17425
|
+
* How long to wait before the next discovery sweep after relayfile shed this
|
|
17426
|
+
* one.
|
|
17427
|
+
*
|
|
17428
|
+
* The advertised `Retry-After` is authoritative in BOTH directions (#297):
|
|
17429
|
+
* it is the first rung, so we never retry sooner than the dependency allows,
|
|
17430
|
+
* and it bounds the ceiling, so we never sleep for minutes because of a
|
|
17431
|
+
* request to wait seconds. Without an advertised delay there is nothing to
|
|
17432
|
+
* respect and the original five-minute ladder governs unchanged.
|
|
17433
|
+
*/
|
|
16857
17434
|
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
16858
|
-
const
|
|
16859
|
-
|
|
16860
|
-
|
|
17435
|
+
const advertisedMs = retryAfterSeconds === undefined
|
|
17436
|
+
? undefined
|
|
17437
|
+
: Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
|
|
17438
|
+
const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
|
|
17439
|
+
// A dependency that asks for longer than the advertised ceiling still gets
|
|
17440
|
+
// what it asked for; the ceiling only stops the ladder from overshooting it.
|
|
17441
|
+
const ceilingMs = advertisedMs === undefined
|
|
17442
|
+
? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
|
|
17443
|
+
: Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
|
|
17444
|
+
const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
|
|
17445
|
+
return Math.min(ceilingMs, baseMs * (2 ** steps));
|
|
16861
17446
|
};
|
|
16862
17447
|
const eventSequenceNumber = (eventId) => {
|
|
16863
17448
|
const whole = Number(eventId);
|
|
@@ -17086,10 +17671,6 @@ const telemetryCategory = (value) => {
|
|
|
17086
17671
|
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
|
|
17087
17672
|
return normalized.slice(0, 120) || undefined;
|
|
17088
17673
|
};
|
|
17089
|
-
const telemetryErrorClass = (error) => {
|
|
17090
|
-
const name = error instanceof Error ? error.name : '';
|
|
17091
|
-
return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
|
|
17092
|
-
};
|
|
17093
17674
|
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
17094
17675
|
const retryOnTimeout = async (fn, opts) => {
|
|
17095
17676
|
let lastError;
|
|
@@ -17323,7 +17904,26 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
|
17323
17904
|
* skippable and are exempt from the consecutive-failure fuse.
|
|
17324
17905
|
*/
|
|
17325
17906
|
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
17326
|
-
error instanceof DispatchLifecycleClaimRefusedError
|
|
17907
|
+
error instanceof DispatchLifecycleClaimRefusedError ||
|
|
17908
|
+
// Relayfile shedding one operation is a state of the dependency, not an
|
|
17909
|
+
// unexplained fault, and it has its own fuse — see #297 and
|
|
17910
|
+
// DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
|
|
17911
|
+
relayfileOverload(error) !== undefined;
|
|
17912
|
+
/**
|
|
17913
|
+
* Whether a per-item failure could have left half-spawned agents behind.
|
|
17914
|
+
*
|
|
17915
|
+
* The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
|
|
17916
|
+
* anything, so there is nothing to reap for them. Everything else can fail
|
|
17917
|
+
* *after* a spawn — including relayfile shedding a post-spawn read — having
|
|
17918
|
+
* persisted the agents as failure handoffs on the way out.
|
|
17919
|
+
*
|
|
17920
|
+
* Deliberately a denylist rather than an allowlist: a new failure mode that
|
|
17921
|
+
* nobody classified should default to "reap it", because the cost of a
|
|
17922
|
+
* needless reap is one no-op pass over an empty handoff list, while the cost
|
|
17923
|
+
* of a missed one is leaked agents and duplicate workers on the next retry.
|
|
17924
|
+
*/
|
|
17925
|
+
const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
|
|
17926
|
+
!(error instanceof DispatchLifecycleClaimRefusedError);
|
|
17327
17927
|
/**
|
|
17328
17928
|
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
17329
17929
|
*
|
|
@@ -17334,6 +17934,9 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
|
|
|
17334
17934
|
* `describeControlPlaneError` makes for circuit state.
|
|
17335
17935
|
*/
|
|
17336
17936
|
const perItemDispatchSkipReason = (error) => {
|
|
17937
|
+
const overload = relayfileOverload(error);
|
|
17938
|
+
if (overload)
|
|
17939
|
+
return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
|
|
17337
17940
|
if (error instanceof LiveDispatchStateChangedError)
|
|
17338
17941
|
return 'live state changed during dispatch';
|
|
17339
17942
|
if (error instanceof DispatchLifecycleClaimRefusedError) {
|