@agent-relay/factory 0.1.63 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli/diagnose.d.ts +92 -0
- package/dist/cli/diagnose.d.ts.map +1 -0
- package/dist/cli/diagnose.js +346 -0
- package/dist/cli/diagnose.js.map +1 -0
- package/dist/cli/fleet.d.ts +13 -1
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +118 -11
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +69 -2
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +32 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
- package/dist/fleet/control-plane-circuit.js +13 -1
- package/dist/fleet/control-plane-circuit.js.map +1 -1
- package/dist/fleet/internal-fleet-client.d.ts +16 -0
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
- package/dist/fleet/internal-fleet-client.js +228 -13
- package/dist/fleet/internal-fleet-client.js.map +1 -1
- package/dist/hosted/orchestrator.d.ts.map +1 -1
- package/dist/hosted/orchestrator.js +1 -4
- package/dist/hosted/orchestrator.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +23 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +38 -9
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/observability/cloud-reporter.d.ts +1 -0
- package/dist/observability/cloud-reporter.d.ts.map +1 -1
- package/dist/observability/cloud-reporter.js +182 -50
- package/dist/observability/cloud-reporter.js.map +1 -1
- package/dist/observability/error-class.d.ts +29 -0
- package/dist/observability/error-class.d.ts.map +1 -0
- package/dist/observability/error-class.js +36 -0
- package/dist/observability/error-class.js.map +1 -0
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/index.d.ts.map +1 -1
- package/dist/observability/index.js +1 -0
- package/dist/observability/index.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +1461 -84
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/index.d.ts +1 -0
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +1 -0
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts +82 -0
- package/dist/orchestrator/public-health.d.ts.map +1 -0
- package/dist/orchestrator/public-health.js +372 -0
- package/dist/orchestrator/public-health.js.map +1 -0
- package/dist/ports/state.d.ts +76 -3
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/document-store.d.ts +2 -1
- package/dist/state/document-store.d.ts.map +1 -1
- package/dist/state/document-store.js.map +1 -1
- package/dist/state/file-state-store.d.ts +24 -5
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +136 -10
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +21 -2
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +99 -6
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/state/watch-state-document.d.ts.map +1 -1
- package/dist/state/watch-state-document.js +45 -3
- package/dist/state/watch-state-document.js.map +1 -1
- package/dist/trajectory.d.ts +25 -0
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/trajectory.js +51 -0
- package/dist/trajectory.js.map +1 -0
- package/dist/types.d.ts +106 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
|
-
import { FactoryConfigSchema } from '../config/schema.js';
|
|
4
|
+
import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
|
|
5
5
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
6
6
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
7
7
|
import { GithubMergeGate, closeProbePr } from '../github/index.js';
|
|
@@ -29,8 +29,11 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
|
|
|
29
29
|
import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
|
|
30
30
|
import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
31
31
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
32
|
+
import { telemetryErrorClass } from '../observability/error-class.js';
|
|
33
|
+
import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
|
|
32
34
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
33
35
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
36
|
+
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
34
37
|
import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
|
|
35
38
|
class ClarificationWakeLeaseLostError extends Error {
|
|
36
39
|
}
|
|
@@ -146,8 +149,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
|
146
149
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
147
150
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
148
151
|
const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
|
|
152
|
+
// Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
|
|
153
|
+
// cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
|
|
154
|
+
// unbounded writeFile. Sizing them past that worst case would only trade a stolen
|
|
155
|
+
// claim for a stranded one — a lease long enough to survive the slowest write is
|
|
156
|
+
// equally long enough to hold the receipt hostage to a dead holder. So these
|
|
157
|
+
// bound the *idle* claim and #withRenewedProviderLease extends them for exactly
|
|
158
|
+
// as long as the write they cover is still running.
|
|
159
|
+
const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
|
|
160
|
+
const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
|
|
161
|
+
// Renewal without a ceiling is the same defect from the other side: a heartbeat
|
|
162
|
+
// that extends the claim for as long as the write runs also extends it forever
|
|
163
|
+
// when the write never returns, and nothing else can reclaim the receipt short
|
|
164
|
+
// of a restart. So renewal is bounded past the slowest write this daemon budgets
|
|
165
|
+
// for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
|
|
166
|
+
// that the write is not slow, it is wedged: the heartbeat stops, the idle lease
|
|
167
|
+
// runs out, and the retry that owns the queued replies can take them back.
|
|
168
|
+
const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
|
|
149
169
|
const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
|
|
150
170
|
const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
|
|
171
|
+
// One pass drains the whole chain (#slackReplyRoutes holds only the newest
|
|
172
|
+
// route per key and every route awaits its predecessor). The extra passes only
|
|
173
|
+
// exist so the drain can prove quiescence rather than assume it.
|
|
174
|
+
const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
|
|
175
|
+
const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
|
|
176
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
|
|
177
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
|
|
151
178
|
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
152
179
|
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
153
180
|
const MAX_LABEL_IMPLEMENTERS = 4;
|
|
@@ -160,6 +187,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
|
|
|
160
187
|
const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
|
|
161
188
|
const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
|
|
162
189
|
const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
|
|
190
|
+
/** First rung of the ladder when the 429 advertises no `Retry-After`. */
|
|
191
|
+
const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
|
|
192
|
+
/**
|
|
193
|
+
* Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
|
|
194
|
+
* whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
|
|
195
|
+
* dependency into hammering it.
|
|
196
|
+
*/
|
|
197
|
+
const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
|
|
198
|
+
/**
|
|
199
|
+
* Ceiling for the ladder once the dependency has told us how long to wait.
|
|
200
|
+
*
|
|
201
|
+
* relayfile sheds an overloaded workspace DO with a 429 in milliseconds
|
|
202
|
+
* carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
|
|
203
|
+
* that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
|
|
204
|
+
* dependency asked for five seconds, Factory slept for five minutes, probed
|
|
205
|
+
* for recovery once per cap-length window, and presented a transient upstream
|
|
206
|
+
* blip as a sustained outage. The ratchet still escalates — it is just bounded
|
|
207
|
+
* by roughly what was actually asked for, and never *below* it, so this is a
|
|
208
|
+
* ceiling and not a licence to retry sooner than the dependency allows.
|
|
209
|
+
*/
|
|
210
|
+
const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
|
|
211
|
+
/**
|
|
212
|
+
* How many relayfile operations one sweep may have shed before the sweep is
|
|
213
|
+
* abandoned rather than continued.
|
|
214
|
+
*
|
|
215
|
+
* Per-item overload skips that item and keeps going (#297, the same principle
|
|
216
|
+
* as #292), but skipping must not degenerate into grinding a shedding
|
|
217
|
+
* dependency through an entire backlog one 429 at a time. Past this many, the
|
|
218
|
+
* dependency is not serving this sweep at all: abort, back off, and let the
|
|
219
|
+
* ratchet do its job.
|
|
220
|
+
*/
|
|
221
|
+
const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
|
|
163
222
|
const GITHUB_FACTORY_LABEL = 'factory';
|
|
164
223
|
const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
|
|
165
224
|
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
@@ -215,6 +274,39 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
215
274
|
this.leaseUntilMs = leaseUntilMs;
|
|
216
275
|
}
|
|
217
276
|
}
|
|
277
|
+
/**
|
|
278
|
+
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
279
|
+
* record is already terminal, or another publisher currently holds the lease.
|
|
280
|
+
* Both are facts about that single unit — the rest of the pass is unaffected —
|
|
281
|
+
* so the readiness loop skips it and keeps going (#292).
|
|
282
|
+
*
|
|
283
|
+
* Typed rather than left as a plain `Error` so the loop can classify it by
|
|
284
|
+
* construction instead of by matching on `Refusing to dispatch ...` text.
|
|
285
|
+
*/
|
|
286
|
+
class DispatchLifecycleClaimRefusedError extends Error {
|
|
287
|
+
issueKey;
|
|
288
|
+
refusal;
|
|
289
|
+
constructor(issueKey, refusal, message) {
|
|
290
|
+
super(message);
|
|
291
|
+
this.issueKey = issueKey;
|
|
292
|
+
this.refusal = refusal;
|
|
293
|
+
this.name = 'DispatchLifecycleClaimRefusedError';
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* The deadline that makes a hung sweep reachable by the existing recovery path.
|
|
298
|
+
* Its message is fully internal (one integer), so it is safe to persist into
|
|
299
|
+
* the operator-facing `readinessReconcile.lastError`.
|
|
300
|
+
*/
|
|
301
|
+
class ReadinessReconcileTimeoutError extends Error {
|
|
302
|
+
timeoutMs;
|
|
303
|
+
code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
|
|
304
|
+
constructor(timeoutMs) {
|
|
305
|
+
super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
|
|
306
|
+
this.timeoutMs = timeoutMs;
|
|
307
|
+
this.name = 'ReadinessReconcileTimeoutError';
|
|
308
|
+
}
|
|
309
|
+
}
|
|
218
310
|
const realClock = {
|
|
219
311
|
now: () => Date.now(),
|
|
220
312
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -266,6 +358,19 @@ export class FactoryLoop {
|
|
|
266
358
|
#dispatchInFlight = new Map();
|
|
267
359
|
#slackWatchers = new Map();
|
|
268
360
|
#slackWatcherStarts = new Map();
|
|
361
|
+
#slackTerminalWatchExpiryTimers = new Map();
|
|
362
|
+
#slackTerminalReceiptRetryTimers = new Map();
|
|
363
|
+
#terminalSlackWatchIssues = new Set();
|
|
364
|
+
/**
|
|
365
|
+
* The one in-memory record of "this work unit has an in-flight Slack side
|
|
366
|
+
* effect". Both ordinary reply routes and the writebacks the terminal fence
|
|
367
|
+
* issues on their behalf register here, because this map is what the terminal
|
|
368
|
+
* drain waits on: anything that touches Slack for a work unit without
|
|
369
|
+
* registering is invisible to the drain, and the watcher teardown that follows
|
|
370
|
+
* a successful drain then pulls that effect's retry timer out from under it.
|
|
371
|
+
*/
|
|
372
|
+
#slackReplyRoutes = new Map();
|
|
373
|
+
#slackReplyRouteDrains = new Set();
|
|
269
374
|
#slackConversationTurns;
|
|
270
375
|
#slackConversationOwner = `${process.pid}:${randomUUID()}`;
|
|
271
376
|
#githubIssueCommentWatchers = new Map();
|
|
@@ -351,12 +456,41 @@ export class FactoryLoop {
|
|
|
351
456
|
#readinessReconcileTimer;
|
|
352
457
|
#readinessReconcileInFlight;
|
|
353
458
|
#readinessReconcileIntervalMs = 60_000;
|
|
459
|
+
#readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
|
|
460
|
+
// Set for exactly as long as a sweep is running. `state` is derived from
|
|
461
|
+
// this, so an in-flight pass can no longer masquerade as the last settled one.
|
|
462
|
+
#readinessReconcileInFlightSinceMs;
|
|
463
|
+
/**
|
|
464
|
+
* The work a deadline gave up waiting on. The deadline bounds the wait, not
|
|
465
|
+
* the sweep, so this is still live: shutdown has to drain it, and `state` has
|
|
466
|
+
* to keep counting from when it actually started.
|
|
467
|
+
*
|
|
468
|
+
* Two fields rather than a collection, because every live abandoned wait
|
|
469
|
+
* converges on the same sweep. They are all `runOnce()` calls with the same
|
|
470
|
+
* `dryRun`, so whatever they are queued behind, the first one out starts the
|
|
471
|
+
* sweep and the rest coalesce onto it — they settle together. So the newest
|
|
472
|
+
* wait is a sufficient drain target, and the earliest start is the honest
|
|
473
|
+
* age. Both matter (#301 review): keeping only the newest start advanced the
|
|
474
|
+
* age by two intervals every two intervals, so at `reconcileTimeoutMs ===
|
|
475
|
+
* reconcileIntervalMs` it never reached three and `stalled` was never
|
|
476
|
+
* reported; keeping one record per wait grew without bound in exactly the
|
|
477
|
+
* never-settling case this change exists for.
|
|
478
|
+
*
|
|
479
|
+
* The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
|
|
480
|
+
* sweep is waited BEHIND rather than coalesced onto, so that handle can name
|
|
481
|
+
* an unrelated sweep, and the readiness pass would then start its own work
|
|
482
|
+
* after shutdown believed it had drained everything. The wait covers the
|
|
483
|
+
* queueing and the sweep it eventually runs, in every branch.
|
|
484
|
+
*/
|
|
485
|
+
#readinessReconcileAbandonedWait;
|
|
486
|
+
#readinessReconcileAbandonedSinceMs;
|
|
354
487
|
#readinessReconcileConsecutiveFailures = 0;
|
|
355
488
|
#readinessReconcileLastDurationMs;
|
|
356
489
|
#readinessReconcileLastStartedAtMs;
|
|
357
490
|
#readinessReconcileLastCompletedAtMs;
|
|
358
491
|
#readinessReconcileLastFailureAtMs;
|
|
359
492
|
#readinessReconcileLastError;
|
|
493
|
+
#readinessReconcileLastErrorClass;
|
|
360
494
|
#liveEventQueue = [];
|
|
361
495
|
#liveEventDrainScheduled = false;
|
|
362
496
|
#liveEventDrainActive = false;
|
|
@@ -439,7 +573,39 @@ export class FactoryLoop {
|
|
|
439
573
|
#discoverySweepRenewTimer;
|
|
440
574
|
#discoverySweepRenewalInFlight;
|
|
441
575
|
#discoverySweepLeaseLost = false;
|
|
576
|
+
// Registry/heartbeat paths the in-flight runLoop iteration would use. A
|
|
577
|
+
// per-item dispatch failure now skips instead of aborting the pass (#292),
|
|
578
|
+
// so the loop's catch no longer runs the failure-handoff reaper for it; the
|
|
579
|
+
// pass reaps inline and must write to the same paths runLoop would.
|
|
580
|
+
#loopReapPaths;
|
|
581
|
+
/**
|
|
582
|
+
* The first 429 relayfile raised during this sweep, kept for its
|
|
583
|
+
* `Retry-After` and reason when the sweep decides how long to back off.
|
|
584
|
+
*
|
|
585
|
+
* Before #297 this doubled as a sweep-wide kill switch: any 429 from any
|
|
586
|
+
* relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
|
|
587
|
+
* away along with everything the sweep had already accomplished. It is now
|
|
588
|
+
* only evidence, never a verdict — see `#discoverySweepOverloads` for the
|
|
589
|
+
* fuse that still ends a sweep the dependency is genuinely refusing to serve.
|
|
590
|
+
*/
|
|
442
591
|
#discoveryOverloadError;
|
|
592
|
+
/** Relayfile operations this sweep has been shed on. */
|
|
593
|
+
#discoverySweepOverloads = 0;
|
|
594
|
+
/**
|
|
595
|
+
* The longest `Retry-After` any operation in this sweep advertised.
|
|
596
|
+
*
|
|
597
|
+
* `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
|
|
598
|
+
* from it alone would let the durable window expire before a later, longer
|
|
599
|
+
* advertised delay permits — breaking the very guarantee #297 is about. The
|
|
600
|
+
* backoff takes the maximum instead.
|
|
601
|
+
*/
|
|
602
|
+
#discoverySweepRetryAfterSeconds;
|
|
603
|
+
/**
|
|
604
|
+
* Whether relayfile served at least one ready work unit end to end during
|
|
605
|
+
* this sweep — its issue read, or a dispatch built on it. This is what
|
|
606
|
+
* decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
|
|
607
|
+
*/
|
|
608
|
+
#discoverySweepProgress = false;
|
|
443
609
|
#resolvedIssueSource;
|
|
444
610
|
#integrationInstructions;
|
|
445
611
|
#integrationInstructionsRefresh;
|
|
@@ -855,6 +1021,14 @@ export class FactoryLoop {
|
|
|
855
1021
|
clearTimeout(this.#previewSweepTimer);
|
|
856
1022
|
this.#previewSweepTimer = undefined;
|
|
857
1023
|
await this.#readinessReconcileInFlight;
|
|
1024
|
+
// #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
|
|
1025
|
+
// can settle with its `runOnce()` still live. Shutdown releases dispatch
|
|
1026
|
+
// lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
|
|
1027
|
+
// fences a stopping sweep once something in it throws — so a sweep whose
|
|
1028
|
+
// dependency recovers cleanly would otherwise dispatch through torn-down
|
|
1029
|
+
// state. Draining here restores exactly the pre-deadline shutdown contract:
|
|
1030
|
+
// stop() outlives the sweep it started.
|
|
1031
|
+
await this.#readinessReconcileAbandonedWait;
|
|
858
1032
|
await this.#previewSweepInFlight;
|
|
859
1033
|
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
860
1034
|
try {
|
|
@@ -907,6 +1081,14 @@ export class FactoryLoop {
|
|
|
907
1081
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
908
1082
|
await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
|
|
909
1083
|
this.#slackWatchers.clear();
|
|
1084
|
+
for (const timer of this.#slackTerminalWatchExpiryTimers.values())
|
|
1085
|
+
clearTimeout(timer);
|
|
1086
|
+
this.#slackTerminalWatchExpiryTimers.clear();
|
|
1087
|
+
for (const timer of this.#slackTerminalReceiptRetryTimers.values())
|
|
1088
|
+
clearTimeout(timer);
|
|
1089
|
+
this.#slackTerminalReceiptRetryTimers.clear();
|
|
1090
|
+
this.#terminalSlackWatchIssues.clear();
|
|
1091
|
+
this.#slackReplyRouteDrains.clear();
|
|
910
1092
|
await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
|
|
911
1093
|
this.#githubIssueCommentWatchers.clear();
|
|
912
1094
|
this.#githubIssueCommentWatchStates.clear();
|
|
@@ -997,6 +1179,9 @@ export class FactoryLoop {
|
|
|
997
1179
|
const options = this.#liveOptions(overrides);
|
|
998
1180
|
this.#liveTransport = options.transport;
|
|
999
1181
|
this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
|
|
1182
|
+
// `start()` overrides skip the schema's cross-field check, so re-apply its
|
|
1183
|
+
// floor here: a deadline under one interval would kill every pass.
|
|
1184
|
+
this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
|
|
1000
1185
|
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
1001
1186
|
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
1002
1187
|
const highWatermark = await this.#currentEventHighWatermark();
|
|
@@ -1033,10 +1218,27 @@ export class FactoryLoop {
|
|
|
1033
1218
|
this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
|
|
1034
1219
|
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
1035
1220
|
});
|
|
1221
|
+
// Review follow-up on #300 (P1, cubic). The startup backfill is a
|
|
1222
|
+
// discovery pass like any other, and it is the one most likely to hang:
|
|
1223
|
+
// #36 measured 61 minutes here while the Relayfile mirror hydrated on a
|
|
1224
|
+
// cold container. Stamping it means a wedged FIRST pass is visible as
|
|
1225
|
+
// in-flight, instead of leaving the timestamps empty and the derived
|
|
1226
|
+
// state reading `healthy` forever.
|
|
1227
|
+
//
|
|
1228
|
+
// Only the timestamps. `consecutiveFailures` and `lastError` belong to
|
|
1229
|
+
// the reconcile loop's own failure accounting, which owns the degraded
|
|
1230
|
+
// threshold and the #297 reason allowlist; a startup failure is already
|
|
1231
|
+
// counted by `liveStartupBackfillErrors` and reported through `#error`.
|
|
1232
|
+
const backfillStartedAtMs = this.#clock.now();
|
|
1233
|
+
this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
|
|
1036
1234
|
try {
|
|
1037
1235
|
await this.runOnce();
|
|
1236
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1237
|
+
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1038
1238
|
}
|
|
1039
1239
|
catch (error) {
|
|
1240
|
+
this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
|
|
1241
|
+
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1040
1242
|
// A startup backfill failure must not abort the daemon: log it and fall
|
|
1041
1243
|
// back to the live event stream (plus any buffered events) instead of
|
|
1042
1244
|
// leaving the factory down.
|
|
@@ -1128,6 +1330,7 @@ export class FactoryLoop {
|
|
|
1128
1330
|
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
1129
1331
|
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
1130
1332
|
reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
|
|
1333
|
+
reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
|
|
1131
1334
|
};
|
|
1132
1335
|
}
|
|
1133
1336
|
async #currentEventCursor(limit) {
|
|
@@ -1176,19 +1379,82 @@ export class FactoryLoop {
|
|
|
1176
1379
|
}, delayMs);
|
|
1177
1380
|
this.#readinessReconcileTimer.unref?.();
|
|
1178
1381
|
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Runs one sweep under a deadline (#296).
|
|
1384
|
+
*
|
|
1385
|
+
* The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
|
|
1386
|
+
* lease and abandoning it mid-flight is not safe — so expiry rejects *this*
|
|
1387
|
+
* wait and leaves the underlying pass to finish on its own. That is enough:
|
|
1388
|
+
* the rejection is what reaches the failure path, which re-arms the timer.
|
|
1389
|
+
* A later reconcile pass coalesces onto the still-running `runOnce()` and
|
|
1390
|
+
* fails on its own deadline too, so a persistent hang keeps counting up to
|
|
1391
|
+
* `degraded` instead of going quiet.
|
|
1392
|
+
*/
|
|
1393
|
+
async #runOnceWithReadinessDeadline() {
|
|
1394
|
+
const timeoutMs = this.#readinessReconcileTimeoutMs;
|
|
1395
|
+
const startedAtMs = this.#clock.now();
|
|
1396
|
+
const sweep = this.runOnce();
|
|
1397
|
+
let timer;
|
|
1398
|
+
try {
|
|
1399
|
+
return await new Promise((resolve, reject) => {
|
|
1400
|
+
timer = setTimeout(() => {
|
|
1401
|
+
this.#increment('readinessReconcileDeadlineExceeded');
|
|
1402
|
+
if (this.#readinessReconcileAbandonedSinceMs === undefined) {
|
|
1403
|
+
// `state` ages from the FIRST wait that gave up on this work, not
|
|
1404
|
+
// from whenever the latest one began.
|
|
1405
|
+
this.#readinessReconcileAbandonedSinceMs = startedAtMs;
|
|
1406
|
+
// The abandoned pass is still running against the live control
|
|
1407
|
+
// plane. Report where it lands, so an operator can tell a
|
|
1408
|
+
// dependency that recovered late from one that never answered.
|
|
1409
|
+
// Attached once, so a wedge is reported once and not per expiry.
|
|
1410
|
+
void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
|
|
1411
|
+
timeoutMs,
|
|
1412
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1413
|
+
dispatched: report.dispatched.length,
|
|
1414
|
+
}), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
|
|
1415
|
+
timeoutMs,
|
|
1416
|
+
overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
|
|
1417
|
+
error: describeError(error).errorMessage,
|
|
1418
|
+
})).catch(() => undefined);
|
|
1419
|
+
}
|
|
1420
|
+
// Newest wait wins as the drain target: it settles no earlier than
|
|
1421
|
+
// the ones before it, and clearing on it clears them all.
|
|
1422
|
+
const wait = sweep.catch(() => undefined).then(() => {
|
|
1423
|
+
if (this.#readinessReconcileAbandonedWait !== wait)
|
|
1424
|
+
return;
|
|
1425
|
+
this.#readinessReconcileAbandonedWait = undefined;
|
|
1426
|
+
this.#readinessReconcileAbandonedSinceMs = undefined;
|
|
1427
|
+
});
|
|
1428
|
+
this.#readinessReconcileAbandonedWait = wait;
|
|
1429
|
+
reject(new ReadinessReconcileTimeoutError(timeoutMs));
|
|
1430
|
+
}, timeoutMs);
|
|
1431
|
+
timer.unref?.();
|
|
1432
|
+
// Attaching handlers here is also what keeps a late rejection from the
|
|
1433
|
+
// abandoned pass from surfacing as an unhandled rejection.
|
|
1434
|
+
sweep.then(resolve, reject);
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
finally {
|
|
1438
|
+
if (timer)
|
|
1439
|
+
clearTimeout(timer);
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1179
1442
|
async #reconcileReadyIssues() {
|
|
1180
1443
|
const startedAtMs = this.#clock.now();
|
|
1181
1444
|
this.#readinessReconcileLastStartedAtMs = startedAtMs;
|
|
1445
|
+
this.#readinessReconcileInFlightSinceMs = startedAtMs;
|
|
1182
1446
|
this.#increment('readinessReconcileSweeps');
|
|
1183
1447
|
this.#logger.info?.('[factory] periodic readiness reconciliation started', {
|
|
1184
1448
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
1449
|
+
timeoutMs: this.#readinessReconcileTimeoutMs,
|
|
1185
1450
|
});
|
|
1186
1451
|
try {
|
|
1187
|
-
const report = await this
|
|
1452
|
+
const report = await this.#runOnceWithReadinessDeadline();
|
|
1188
1453
|
this.#readinessReconcileConsecutiveFailures = 0;
|
|
1189
1454
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1190
1455
|
this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
|
|
1191
1456
|
this.#readinessReconcileLastError = undefined;
|
|
1457
|
+
this.#readinessReconcileLastErrorClass = undefined;
|
|
1192
1458
|
this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
|
|
1193
1459
|
durationMs: this.#readinessReconcileLastDurationMs,
|
|
1194
1460
|
candidates: report.pulled.length,
|
|
@@ -1197,11 +1463,28 @@ export class FactoryLoop {
|
|
|
1197
1463
|
});
|
|
1198
1464
|
}
|
|
1199
1465
|
catch (error) {
|
|
1200
|
-
|
|
1466
|
+
// #297: all four relayfile overload reason codes share one message, and
|
|
1467
|
+
// `lastError` is what an operator reads from /evidence. Without the
|
|
1468
|
+
// reason, "workspace durable object is busy" cannot be told apart from
|
|
1469
|
+
// three other conditions with three different remedies.
|
|
1470
|
+
//
|
|
1471
|
+
// Allowlisted, because this is a persisted operator-facing surface and
|
|
1472
|
+
// not just a log line: `lastError` is returned from `status()` and
|
|
1473
|
+
// written into the loop heartbeat file, so an unbounded
|
|
1474
|
+
// dependency-controlled string would land on disk.
|
|
1475
|
+
const overload = relayfileOverload(error);
|
|
1476
|
+
const errorMessage = overload
|
|
1477
|
+
? `${describeError(error).errorMessage} ` +
|
|
1478
|
+
`[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
|
|
1479
|
+
`${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
|
|
1480
|
+
: describeError(error).errorMessage;
|
|
1201
1481
|
this.#readinessReconcileConsecutiveFailures += 1;
|
|
1202
1482
|
this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
|
|
1203
1483
|
this.#readinessReconcileLastFailureAtMs = this.#clock.now();
|
|
1204
1484
|
this.#readinessReconcileLastError = errorMessage;
|
|
1485
|
+
// The class, unlike the message, is publishable: #295 puts it on the
|
|
1486
|
+
// unauthenticated health surface through the same allowlist.
|
|
1487
|
+
this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
|
|
1205
1488
|
this.#increment('readinessReconcileErrors');
|
|
1206
1489
|
this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
|
|
1207
1490
|
error: errorMessage,
|
|
@@ -1210,6 +1493,11 @@ export class FactoryLoop {
|
|
|
1210
1493
|
degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
1211
1494
|
});
|
|
1212
1495
|
}
|
|
1496
|
+
finally {
|
|
1497
|
+
// Cleared before the heartbeat write below, so a slow-but-successful pass
|
|
1498
|
+
// does not stamp its own tail as `stalled`.
|
|
1499
|
+
this.#readinessReconcileInFlightSinceMs = undefined;
|
|
1500
|
+
}
|
|
1213
1501
|
await this.#refreshLiveHeartbeat();
|
|
1214
1502
|
}
|
|
1215
1503
|
#scheduleLivePoll(delayMs, options) {
|
|
@@ -1938,13 +2226,29 @@ export class FactoryLoop {
|
|
|
1938
2226
|
this.#discoverySweepStartedAtMs = sweepStartedAtMs;
|
|
1939
2227
|
this.#discoverySweepLeaseLost = false;
|
|
1940
2228
|
this.#discoveryOverloadError = undefined;
|
|
2229
|
+
this.#discoverySweepOverloads = 0;
|
|
2230
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2231
|
+
this.#discoverySweepProgress = false;
|
|
1941
2232
|
this.#startDiscoverySweepRenewal(claim.lease.epoch);
|
|
1942
2233
|
let leaseReleased = false;
|
|
1943
2234
|
try {
|
|
1944
2235
|
this.#discoverySession = await this.#prepareDiscoverySession(claim);
|
|
2236
|
+
// #297: a 429 raised anywhere in the sweep used to latch and be rethrown
|
|
2237
|
+
// here, discarding a completed pass — every issue read, every dispatch —
|
|
2238
|
+
// because of one transient shed operation. The work this sweep did is
|
|
2239
|
+
// now kept instead, and the ratchet below records that the dependency is
|
|
2240
|
+
// shedding but still serving.
|
|
1945
2241
|
const report = await this.#performRunOnce(opts);
|
|
1946
|
-
|
|
2242
|
+
// The exception, and the reason skipping shed units cannot make a sweep
|
|
2243
|
+
// unconditionally green: a sweep that was shed AND got no work unit
|
|
2244
|
+
// through accomplished nothing. There is no progress to preserve, and
|
|
2245
|
+
// committing it would report a clean sweep over a dependency that served
|
|
2246
|
+
// none of it — leaving `readinessReconcile` healthy while Factory
|
|
2247
|
+
// dispatches nothing, which is the #292 wedge wearing the other costume.
|
|
2248
|
+
// Fail it so the ratchet escalates and readiness reflects reality.
|
|
2249
|
+
if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
|
|
1947
2250
|
throw this.#discoveryOverloadError;
|
|
2251
|
+
}
|
|
1948
2252
|
const checkpoint = await this.#finalizeDiscoveryCheckpoint();
|
|
1949
2253
|
// Do not clear the durable lease while a renewal can still be waiting on
|
|
1950
2254
|
// the same state-file lock. A late renewal that observes the completed
|
|
@@ -1954,7 +2258,8 @@ export class FactoryLoop {
|
|
|
1954
2258
|
if (this.#discoverySweepLeaseLost) {
|
|
1955
2259
|
throw new Error('discovery sweep lease was lost before checkpoint commit');
|
|
1956
2260
|
}
|
|
1957
|
-
const
|
|
2261
|
+
const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
|
|
2262
|
+
const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
|
|
1958
2263
|
leaseReleased = completed;
|
|
1959
2264
|
if (!completed)
|
|
1960
2265
|
throw new Error('discovery sweep lease was lost before completion');
|
|
@@ -1970,18 +2275,18 @@ export class FactoryLoop {
|
|
|
1970
2275
|
await this.#stopDiscoverySweepRenewal();
|
|
1971
2276
|
const overload = relayfileOverload(error);
|
|
1972
2277
|
if (overload) {
|
|
1973
|
-
const
|
|
1974
|
-
|
|
1975
|
-
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
1976
|
-
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
|
|
1977
|
-
this.#increment('discoveryOverloadBackoffs');
|
|
2278
|
+
const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
|
|
2279
|
+
leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
|
|
1978
2280
|
this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
|
|
1979
2281
|
status: overload.status,
|
|
1980
2282
|
reason: overload.reason,
|
|
1981
|
-
retryAfterSeconds:
|
|
1982
|
-
delayMs,
|
|
1983
|
-
backoffUntilMs,
|
|
1984
|
-
consecutiveOverloads,
|
|
2283
|
+
retryAfterSeconds: outcome.retryAfterSeconds,
|
|
2284
|
+
delayMs: outcome.delayMs,
|
|
2285
|
+
backoffUntilMs: outcome.backoffUntilMs,
|
|
2286
|
+
consecutiveOverloads: outcome.consecutiveOverloads,
|
|
2287
|
+
previousOverloads: claim.state.consecutiveOverloads,
|
|
2288
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2289
|
+
sweepProgress: this.#discoverySweepProgress,
|
|
1985
2290
|
});
|
|
1986
2291
|
// backoffUntilMs is already durable via deferDiscoverySweep, and the
|
|
1987
2292
|
// next runOnce() honors it at the pre-claim wait above — sleeping
|
|
@@ -1997,6 +2302,9 @@ export class FactoryLoop {
|
|
|
1997
2302
|
this.#discoverySweepEpoch = undefined;
|
|
1998
2303
|
this.#discoverySweepStartedAtMs = undefined;
|
|
1999
2304
|
this.#discoveryOverloadError = undefined;
|
|
2305
|
+
this.#discoverySweepOverloads = 0;
|
|
2306
|
+
this.#discoverySweepRetryAfterSeconds = undefined;
|
|
2307
|
+
this.#discoverySweepProgress = false;
|
|
2000
2308
|
// This sweep is over either way (committed, deferred, or lease lost) —
|
|
2001
2309
|
// a stale `true` here would otherwise make every #listRelayfileTree
|
|
2002
2310
|
// call outside a fresh claim (Slack lookups, PR confirmation, the
|
|
@@ -2008,6 +2316,94 @@ export class FactoryLoop {
|
|
|
2008
2316
|
}
|
|
2009
2317
|
}
|
|
2010
2318
|
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Commit the sweep, carrying any residual overload backoff into the store.
|
|
2321
|
+
*
|
|
2322
|
+
* `completeDiscoverySweepWithOverload` is optional on the port so a store
|
|
2323
|
+
* written before #297 keeps working. When one of those is injected and this
|
|
2324
|
+
* sweep HAS a residual, the backoff cannot be persisted and the next sweep
|
|
2325
|
+
* would retry immediately — so this says so, loudly and with a counter,
|
|
2326
|
+
* rather than degrading in silence. The sweep itself still commits: losing
|
|
2327
|
+
* the backoff is worse than pre-#297 behaviour only if nobody notices.
|
|
2328
|
+
*/
|
|
2329
|
+
async #commitDiscoverySweep(epoch, checkpoint, residual) {
|
|
2330
|
+
if (residual && this.#state.completeDiscoverySweepWithOverload) {
|
|
2331
|
+
return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
|
|
2332
|
+
}
|
|
2333
|
+
if (residual) {
|
|
2334
|
+
this.#increment('discoveryOverloadResidualUnsupported');
|
|
2335
|
+
this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
|
|
2336
|
+
store: this.#state.constructor?.name,
|
|
2337
|
+
consecutiveOverloads: residual.consecutiveOverloads,
|
|
2338
|
+
backoffUntilMs: residual.backoffUntilMs,
|
|
2339
|
+
delayMs: residual.delayMs,
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
|
|
2343
|
+
}
|
|
2344
|
+
/**
|
|
2345
|
+
* How the durable overload ratchet should read after this sweep, or
|
|
2346
|
+
* `undefined` if relayfile never shed anything and the ordinary reset
|
|
2347
|
+
* applies.
|
|
2348
|
+
*
|
|
2349
|
+
* #297, deliverable 3. `consecutiveOverloads` used to clear only on a
|
|
2350
|
+
* *fully* clean sweep, so under sustained mild load the ratchet essentially
|
|
2351
|
+
* never cleared: it climbed to the cap on the first bad sweep and stayed
|
|
2352
|
+
* there, probing for recovery once per cap-length window, long after the
|
|
2353
|
+
* dependency had recovered. Requiring perfection to clear a ratchet means
|
|
2354
|
+
* the ratchet does not clear.
|
|
2355
|
+
*
|
|
2356
|
+
* The signal that replaces "was this sweep perfect" is "did relayfile serve
|
|
2357
|
+
* any of this sweep's work units", because that is what the ratchet is
|
|
2358
|
+
* actually for. A shedding DO rejects ALL background traffic —
|
|
2359
|
+
* `oldest_inflight_age` does this regardless of how few requests are in
|
|
2360
|
+
* flight — so a sweep against a genuinely overloaded workspace gets not one
|
|
2361
|
+
* unit through and still escalates, all the way to the cap. A sweep that did
|
|
2362
|
+
* get units through proves the dependency is serving us, so it decays by one
|
|
2363
|
+
* rung. Decay, not reset: getting work done while being shed is not evidence
|
|
2364
|
+
* that the overload is over, only that it is survivable.
|
|
2365
|
+
*/
|
|
2366
|
+
#discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
|
|
2367
|
+
const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
|
|
2368
|
+
if (!overload)
|
|
2369
|
+
return undefined;
|
|
2370
|
+
// The longest delay anything in this sweep asked for, not just the one the
|
|
2371
|
+
// latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
|
|
2372
|
+
const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
|
|
2373
|
+
.filter((value) => value !== undefined);
|
|
2374
|
+
const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
|
|
2375
|
+
// Deliberately NOT "the sweep reached its end". A sweep in which relayfile
|
|
2376
|
+
// shed EVERY unit still runs the loop to the end, and it got nothing done;
|
|
2377
|
+
// treating that as progress would decay the ratchet in exactly the case it
|
|
2378
|
+
// exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
|
|
2379
|
+
// aborted one before it can commit, so the escalate branch here is reached
|
|
2380
|
+
// only through `sweepOutcome === 'aborted'` — but the rule is a property of
|
|
2381
|
+
// progress, not of which caller asked, and is written that way.)
|
|
2382
|
+
const consecutiveOverloads = this.#discoverySweepProgress
|
|
2383
|
+
? Math.max(0, previousOverloads - 1)
|
|
2384
|
+
: previousOverloads + 1;
|
|
2385
|
+
const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
|
|
2386
|
+
const backoffUntilMs = this.#clock.now() + delayMs;
|
|
2387
|
+
this.#increment('discoveryOverloadBackoffs');
|
|
2388
|
+
if (sweepOutcome === 'committed') {
|
|
2389
|
+
this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
|
|
2390
|
+
status: overload.status,
|
|
2391
|
+
reason: overload.reason,
|
|
2392
|
+
retryAfterSeconds,
|
|
2393
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2394
|
+
previousOverloads,
|
|
2395
|
+
consecutiveOverloads,
|
|
2396
|
+
delayMs,
|
|
2397
|
+
backoffUntilMs,
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
return {
|
|
2401
|
+
consecutiveOverloads,
|
|
2402
|
+
backoffUntilMs,
|
|
2403
|
+
delayMs,
|
|
2404
|
+
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2011
2407
|
async #performRunOnce(opts = {}) {
|
|
2012
2408
|
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
2013
2409
|
const startedAtMs = this.#clock.now();
|
|
@@ -2047,19 +2443,65 @@ export class FactoryLoop {
|
|
|
2047
2443
|
reason: entry.reason,
|
|
2048
2444
|
});
|
|
2049
2445
|
};
|
|
2446
|
+
// Backstop for the skip-by-default catch below: see #292. Reset only by
|
|
2447
|
+
// a completed dispatch, so the name says "since a dispatch" rather than
|
|
2448
|
+
// "consecutive" — a benign classified skip in between does not clear it.
|
|
2449
|
+
let unclassifiedFailuresSinceDispatch = 0;
|
|
2050
2450
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
2051
2451
|
let readyIssueReads = 0;
|
|
2052
2452
|
const issueEntries = [];
|
|
2053
2453
|
for (const path of paths) {
|
|
2054
|
-
|
|
2454
|
+
let issue;
|
|
2455
|
+
let shed = false;
|
|
2456
|
+
try {
|
|
2457
|
+
issue = await this.#readIssue(path);
|
|
2458
|
+
}
|
|
2459
|
+
catch (error) {
|
|
2460
|
+
// #297: `#readIssue` rethrows relayfile overload and swallows every
|
|
2461
|
+
// other read fault, so this catch only ever sees the backend
|
|
2462
|
+
// shedding THIS issue's read. That is a fact about one work unit:
|
|
2463
|
+
// a sweep that pulled 40 issues and was shed on the 39th must keep
|
|
2464
|
+
// the other 39, exactly as #292 argued for dispatch failures. The
|
|
2465
|
+
// fuse is what still ends a sweep the backend is refusing.
|
|
2466
|
+
const overload = relayfileOverload(error);
|
|
2467
|
+
// Defensive: anything `#readIssue` did not swallow and is not a 429
|
|
2468
|
+
// is not ours to reclassify, and must surface as itself.
|
|
2469
|
+
if (!overload)
|
|
2470
|
+
throw error;
|
|
2471
|
+
const fuse = this.#discoveryOverloadFuseError();
|
|
2472
|
+
if (fuse)
|
|
2473
|
+
throw fuse;
|
|
2474
|
+
shed = true;
|
|
2475
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2476
|
+
this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
|
|
2477
|
+
path,
|
|
2478
|
+
status: overload.status,
|
|
2479
|
+
reason: overload.reason,
|
|
2480
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2481
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2482
|
+
});
|
|
2483
|
+
recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
|
|
2484
|
+
}
|
|
2055
2485
|
readyIssueReads += 1;
|
|
2486
|
+
// Relayfile served this work unit's read: the dependency is shedding
|
|
2487
|
+
// but not refusing, which is what decays the ratchet (#297).
|
|
2488
|
+
//
|
|
2489
|
+
// `issue` is required, not just `!shed`: `#readIssue` returns
|
|
2490
|
+
// `undefined` for a body it could not read at all, and the known
|
|
2491
|
+
// phantom condition — the tree listing issue paths whose bodies are
|
|
2492
|
+
// absent — makes every read return `undefined`. Crediting those would
|
|
2493
|
+
// decay the ratchet on a sweep that served nothing.
|
|
2494
|
+
if (!shed && issue)
|
|
2495
|
+
this.#discoverySweepProgress = true;
|
|
2056
2496
|
lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
|
|
2057
2497
|
? '[factory] GitHub ready issue read progress'
|
|
2058
2498
|
: '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
|
|
2059
|
-
if (
|
|
2060
|
-
|
|
2499
|
+
if (!shed) {
|
|
2500
|
+
if (issue && issueSource === 'linear') {
|
|
2501
|
+
await this.#recordCanonicalIssueState(issue);
|
|
2502
|
+
}
|
|
2503
|
+
issueEntries.push({ path, issue });
|
|
2061
2504
|
}
|
|
2062
|
-
issueEntries.push({ path, issue });
|
|
2063
2505
|
await this.#refreshLiveHeartbeatIfDue();
|
|
2064
2506
|
}
|
|
2065
2507
|
if (issueSource === 'github') {
|
|
@@ -2144,19 +2586,13 @@ export class FactoryLoop {
|
|
|
2144
2586
|
}
|
|
2145
2587
|
const decision = await this.triageIssue(issue);
|
|
2146
2588
|
triaged.push(decision);
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
2155
|
-
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
2156
|
-
issue: decision.issue.key,
|
|
2157
|
-
});
|
|
2158
|
-
continue;
|
|
2159
|
-
}
|
|
2589
|
+
const result = await this.dispatch(decision, { dryRun });
|
|
2590
|
+
// A completed dispatch — even one that parks or escalates the issue —
|
|
2591
|
+
// proves the pipeline still works, so the fuse below starts over.
|
|
2592
|
+
unclassifiedFailuresSinceDispatch = 0;
|
|
2593
|
+
// ...and proves relayfile is still serving this sweep, which is what
|
|
2594
|
+
// decays the durable overload ratchet (#297).
|
|
2595
|
+
this.#discoverySweepProgress = true;
|
|
2160
2596
|
if (result.agents.length === 0 && !dryRun) {
|
|
2161
2597
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2162
2598
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
@@ -2169,6 +2605,81 @@ export class FactoryLoop {
|
|
|
2169
2605
|
dispatched.push(result);
|
|
2170
2606
|
}
|
|
2171
2607
|
}
|
|
2608
|
+
catch (error) {
|
|
2609
|
+
// The failure may have left half-spawned agents behind, persisted as
|
|
2610
|
+
// failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
|
|
2611
|
+
// catch used to reap them because every such error aborted the pass;
|
|
2612
|
+
// now that most of them are skipped, the reap has to happen here.
|
|
2613
|
+
//
|
|
2614
|
+
// BEFORE the fatality check, not after (#298 review, round two): the
|
|
2615
|
+
// 429 that trips the overload fuse aborts this pass, and a direct
|
|
2616
|
+
// `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
|
|
2617
|
+
// has no runLoop catch behind it, so the unit that trips the fuse
|
|
2618
|
+
// would leak the agents it had already spawned. Reaping first covers
|
|
2619
|
+
// the abort and the skip with one call; it is idempotent, so the
|
|
2620
|
+
// runLoop catch finding nothing left to do is free.
|
|
2621
|
+
if (mayHaveSpawnedBeforeFailing(error)) {
|
|
2622
|
+
await this.#reapDispatchFailureHandoffsNow();
|
|
2623
|
+
}
|
|
2624
|
+
// #292: issues in a pass are independent work units, so a failure
|
|
2625
|
+
// that is about ONE unit costs that unit and nothing else. Only the
|
|
2626
|
+
// conditions named in `#isPassFatalFailure` — the ones where
|
|
2627
|
+
// continuing the pass is meaningless — abort the whole sweep.
|
|
2628
|
+
if (this.#isPassFatalFailure(error, dryRun)) {
|
|
2629
|
+
// The overload fuse may have been tripped by a 429 that a caller
|
|
2630
|
+
// swallowed, leaving an unrelated error in hand. The fence keys
|
|
2631
|
+
// the durable backoff off `relayfileOverload(error)`, so hand it
|
|
2632
|
+
// the 429 rather than whatever surfaced last.
|
|
2633
|
+
throw this.#discoveryOverloadFuseError() ?? error;
|
|
2634
|
+
}
|
|
2635
|
+
const overload = relayfileOverload(error);
|
|
2636
|
+
if (overload) {
|
|
2637
|
+
// #297: shedding is a state of the dependency, not a fault of this
|
|
2638
|
+
// work unit, so it stays out of `counters.errors` and gets its own
|
|
2639
|
+
// counter — the same split #293 made for undispatchable units.
|
|
2640
|
+
this.#increment('discoveryOverloadItemsSkipped');
|
|
2641
|
+
this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
|
|
2642
|
+
issue: issueRef(issue).key,
|
|
2643
|
+
status: overload.status,
|
|
2644
|
+
reason: overload.reason,
|
|
2645
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
2646
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
2647
|
+
});
|
|
2648
|
+
}
|
|
2649
|
+
else if (!isClassifiedPerItemDispatchFailure(error)) {
|
|
2650
|
+
unclassifiedFailuresSinceDispatch += 1;
|
|
2651
|
+
// A pass-wide fault can arrive disguised as a run of per-item
|
|
2652
|
+
// faults. Skipping every unit would then hand back a green report
|
|
2653
|
+
// that dispatched nothing, which is the same silent wedge #292 is
|
|
2654
|
+
// about, wearing the opposite costume. Fail the pass loudly so
|
|
2655
|
+
// `readinessReconcile.lastError` carries the cause.
|
|
2656
|
+
if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
|
|
2657
|
+
throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
|
|
2658
|
+
}
|
|
2659
|
+
this.#increment('dispatchItemFailuresSkipped');
|
|
2660
|
+
// The raw message is operator-facing only; the run report carries
|
|
2661
|
+
// the sanitized classification from `perItemDispatchSkipReason`.
|
|
2662
|
+
this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
|
|
2663
|
+
issue: issueRef(issue).key,
|
|
2664
|
+
unclassifiedFailuresSinceDispatch,
|
|
2665
|
+
error: describeError(error).errorMessage,
|
|
2666
|
+
});
|
|
2667
|
+
this.#error(error, issueRef(issue));
|
|
2668
|
+
}
|
|
2669
|
+
else {
|
|
2670
|
+
// Not an error — the unit simply cannot be dispatched right now —
|
|
2671
|
+
// so this stays out of `counters.errors` and gets its own counter
|
|
2672
|
+
// instead, or a terminal-lifecycle backlog would be invisible to
|
|
2673
|
+
// anyone watching only `dispatchItemFailuresSkipped`.
|
|
2674
|
+
this.#increment('dispatchItemsSkippedUndispatchable');
|
|
2675
|
+
this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
|
|
2676
|
+
issue: issueRef(issue).key,
|
|
2677
|
+
error: describeError(error).errorMessage,
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
|
|
2681
|
+
continue;
|
|
2682
|
+
}
|
|
2172
2683
|
finally {
|
|
2173
2684
|
if (recoveredIdentity)
|
|
2174
2685
|
this.#reconciledGithubInProgress.delete(recoveredIdentity);
|
|
@@ -2202,6 +2713,103 @@ export class FactoryLoop {
|
|
|
2202
2713
|
}
|
|
2203
2714
|
}
|
|
2204
2715
|
}
|
|
2716
|
+
/**
|
|
2717
|
+
* The 429 that ended this sweep, when relayfile has shed
|
|
2718
|
+
* `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
|
|
2719
|
+
* unit would just be grinding a shedding dependency. Undefined below that.
|
|
2720
|
+
*
|
|
2721
|
+
* Returns the *latched* 429 rather than whatever error is in hand, because
|
|
2722
|
+
* `#runOnceWithDiscoveryFence` keys the durable backoff off
|
|
2723
|
+
* `relayfileOverload(error)` and a caller may have swallowed the 429 that
|
|
2724
|
+
* tripped the fuse.
|
|
2725
|
+
*/
|
|
2726
|
+
#discoveryOverloadFuseError() {
|
|
2727
|
+
if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
|
|
2728
|
+
return undefined;
|
|
2729
|
+
return this.#discoveryOverloadError;
|
|
2730
|
+
}
|
|
2731
|
+
/**
|
|
2732
|
+
* Whether a failure raised while processing ONE work unit must abort the
|
|
2733
|
+
* whole readiness pass instead of skipping that unit.
|
|
2734
|
+
*
|
|
2735
|
+
* The default is the opposite, and that inversion is the fix for #292.
|
|
2736
|
+
* Issues in a pass are independent work units: a failure that is *about one
|
|
2737
|
+
* unit* — its dispatch-lifecycle record, its live state, a provider fault on
|
|
2738
|
+
* its own writeback — costs that unit and nothing else. Before this, every
|
|
2739
|
+
* error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
|
|
2740
|
+
* single issue whose lifecycle record had gone terminal stopped all dispatch
|
|
2741
|
+
* indefinitely, every pass.
|
|
2742
|
+
*
|
|
2743
|
+
* A condition belongs here only when continuing the pass is meaningless or
|
|
2744
|
+
* actively harmful — when the failure is about the *pass*, not the item:
|
|
2745
|
+
*
|
|
2746
|
+
* - The discovery sweep lease is gone. Another process now owns this
|
|
2747
|
+
* workspace's sweep, so every remaining read throws the same way and each
|
|
2748
|
+
* one would be recorded as an ordinary per-issue skip. The run report
|
|
2749
|
+
* would then claim a clean pass over work this process no longer has the
|
|
2750
|
+
* right to touch.
|
|
2751
|
+
* - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
|
|
2752
|
+
* this sweep. A single shed operation is per-item and skippable (#297),
|
|
2753
|
+
* but past this the backend is not serving this sweep at all: grinding
|
|
2754
|
+
* through the remaining units makes it worse, and the fence needs the 429
|
|
2755
|
+
* to set the durable backoff.
|
|
2756
|
+
* - The factory is stopping. Teardown is in progress and dispatching more
|
|
2757
|
+
* agents now leaks them past the shutdown deadline.
|
|
2758
|
+
* - The fleet control-plane circuit is no longer closed, **on a live pass**.
|
|
2759
|
+
* Dispatch is globally paused — the same condition
|
|
2760
|
+
* `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
|
|
2761
|
+
* so it must also stop one already in flight. A dry run is exempt: it
|
|
2762
|
+
* never calls that admission gate and never spawns, so a paused control
|
|
2763
|
+
* plane is irrelevant to it rather than fatal to it. Without the
|
|
2764
|
+
* exemption, one live pass that trips the circuit would poison every
|
|
2765
|
+
* later dry run — including the boot gate's own `run-once --dry-run`
|
|
2766
|
+
* probe, turning a recoverable circuit-open condition into a failed boot.
|
|
2767
|
+
* That is a nastier version of the wedge this whole change removes.
|
|
2768
|
+
*
|
|
2769
|
+
* Deliberately NOT here: JavaScript builtin error types. Classifying
|
|
2770
|
+
* "programmer faults" such as `TypeError` as fatal is the obvious next rule
|
|
2771
|
+
* and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
|
|
2772
|
+
* failed`, which is precisely the transient per-item roster lookup that
|
|
2773
|
+
* wedged the second instance (#291). A rule keyed on builtin types would
|
|
2774
|
+
* have preserved that outage verbatim.
|
|
2775
|
+
*
|
|
2776
|
+
* Everything else — a refused lifecycle claim, a terminal lifecycle record,
|
|
2777
|
+
* a transient network fault on one issue — is per-item: record a skip and
|
|
2778
|
+
* keep going. The unclassified-failure fuse in `#performRunOnce` is the
|
|
2779
|
+
* backstop for a pass-wide fault that does not announce itself as one.
|
|
2780
|
+
*/
|
|
2781
|
+
#isPassFatalFailure(error, dryRun) {
|
|
2782
|
+
// Sweep-scoped: these are about this process's right or ability to run the
|
|
2783
|
+
// pass at all, so they hold for a dry run exactly as for a live one.
|
|
2784
|
+
if (this.#discoverySweepLeaseLost || this.#stopping) {
|
|
2785
|
+
return true;
|
|
2786
|
+
}
|
|
2787
|
+
// #297: relayfile overload used to sit alongside those two, and it did not
|
|
2788
|
+
// belong there. A 429 on ONE work unit is a fact about that unit's read or
|
|
2789
|
+
// write, not about this process's right to run the pass — and because the
|
|
2790
|
+
// flag latched for the whole sweep, the first transient shed also made
|
|
2791
|
+
// every later unit fatal. Only sustained shedding is now pass-fatal.
|
|
2792
|
+
if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
|
|
2793
|
+
return true;
|
|
2794
|
+
}
|
|
2795
|
+
// Fleet-scoped, and therefore live-only. See the doc comment above.
|
|
2796
|
+
return !dryRun && this.#isFleetControlPlaneHalted(error);
|
|
2797
|
+
}
|
|
2798
|
+
/**
|
|
2799
|
+
* Whether dispatch is globally paused by the fleet control-plane circuit.
|
|
2800
|
+
*
|
|
2801
|
+
* Two reads, because the circuit announces itself two different ways. The
|
|
2802
|
+
* state read covers `guardedMutation`, which records a mutation's own
|
|
2803
|
+
* transport failure and rethrows the *original* error rather than the
|
|
2804
|
+
* circuit-open type — converting it there would be wrong, since the mutation
|
|
2805
|
+
* may already have reached the broker and callers key spawn-failure handling
|
|
2806
|
+
* off that original error. The type check covers a rejection raised without
|
|
2807
|
+
* any state transition, such as an already-open circuit refusing admission.
|
|
2808
|
+
*/
|
|
2809
|
+
#isFleetControlPlaneHalted(error) {
|
|
2810
|
+
return this.#fleetControlPlane.status().state !== 'closed' ||
|
|
2811
|
+
wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
|
|
2812
|
+
}
|
|
2205
2813
|
#startDiscoverySweepRenewal(epoch) {
|
|
2206
2814
|
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2207
2815
|
if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
|
|
@@ -3047,14 +3655,40 @@ export class FactoryLoop {
|
|
|
3047
3655
|
return result;
|
|
3048
3656
|
}
|
|
3049
3657
|
catch (error) {
|
|
3050
|
-
|
|
3658
|
+
const overload = relayfileOverload(error);
|
|
3659
|
+
if (overload && this.#discoverySweepEpoch !== undefined) {
|
|
3051
3660
|
this.#discoveryOverloadError ??= error;
|
|
3661
|
+
this.#discoverySweepOverloads += 1;
|
|
3662
|
+
if (overload.retryAfterSeconds !== undefined) {
|
|
3663
|
+
this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
|
|
3664
|
+
}
|
|
3665
|
+
this.#increment('discoveryOverloadOperations');
|
|
3666
|
+
// #297: relayfile's four overload reason codes — inflight_limit,
|
|
3667
|
+
// oldest_inflight_age, router_inflight_limit, durable_object_overloaded
|
|
3668
|
+
// — all share ONE message string, and they mean four different things:
|
|
3669
|
+
// a DO-local admission cap, one stuck op poisoning every background
|
|
3670
|
+
// caller, a Worker-global isolate cap, and the Cloudflare runtime
|
|
3671
|
+
// shedding the object outright. `relayfileOverload()` has always parsed
|
|
3672
|
+
// the reason; nothing on this path ever printed it, and during the
|
|
3673
|
+
// 2026-08-20 outage that ambiguity was the single biggest obstacle to
|
|
3674
|
+
// diagnosis. Unconditional, unlike the failure warn below: a 429 is
|
|
3675
|
+
// always worth one line.
|
|
3676
|
+
this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
|
|
3677
|
+
this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
|
|
3678
|
+
...metadata,
|
|
3679
|
+
status: overload.status,
|
|
3680
|
+
reason: overload.reason,
|
|
3681
|
+
retryAfterSeconds: overload.retryAfterSeconds,
|
|
3682
|
+
sweepOverloads: this.#discoverySweepOverloads,
|
|
3683
|
+
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3684
|
+
});
|
|
3052
3685
|
}
|
|
3053
3686
|
if (opts.logFailure || waitWarnings > 0) {
|
|
3054
3687
|
this.#increment('relayfileOperationFailures');
|
|
3055
3688
|
this.#logger.warn?.('[factory] relayfile operation failed', {
|
|
3056
3689
|
...metadata,
|
|
3057
3690
|
elapsedMs: this.#elapsedSince(startedAtMs),
|
|
3691
|
+
...(overload ? { status: overload.status, reason: overload.reason } : {}),
|
|
3058
3692
|
error: describeError(error).errorMessage,
|
|
3059
3693
|
});
|
|
3060
3694
|
}
|
|
@@ -3085,6 +3719,7 @@ export class FactoryLoop {
|
|
|
3085
3719
|
const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
|
|
3086
3720
|
const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
|
|
3087
3721
|
const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
|
|
3722
|
+
this.#loopReapPaths = { heartbeatPath, registryPath };
|
|
3088
3723
|
const reports = [];
|
|
3089
3724
|
let consecutiveFailures = 0;
|
|
3090
3725
|
let completed = false;
|
|
@@ -3140,6 +3775,7 @@ export class FactoryLoop {
|
|
|
3140
3775
|
return reports;
|
|
3141
3776
|
}
|
|
3142
3777
|
finally {
|
|
3778
|
+
this.#loopReapPaths = undefined;
|
|
3143
3779
|
if (!completed) {
|
|
3144
3780
|
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
|
|
3145
3781
|
}
|
|
@@ -3615,17 +4251,66 @@ export class FactoryLoop {
|
|
|
3615
4251
|
}
|
|
3616
4252
|
#readinessReconcileStatus() {
|
|
3617
4253
|
const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
|
|
3618
|
-
|
|
4254
|
+
// #296 owns the numerator here, #295/#300 own the derivation. The earliest
|
|
4255
|
+
// sweep still running — after a deadline expiry that is the abandoned one,
|
|
4256
|
+
// not the current wait, or every expiry would restart the clock and a
|
|
4257
|
+
// permanently stuck pass would read as merely `retrying`.
|
|
4258
|
+
const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
|
|
4259
|
+
const settled = this.#startMode !== 'live'
|
|
3619
4260
|
? 'not-running'
|
|
4261
|
+
// Failure-count ladder only. Precedence against `stalled` belongs to
|
|
4262
|
+
// `derivedReadinessReconcileState`, which outranks everything here: a
|
|
4263
|
+
// stall is the more specific fact, and `consecutiveFailures` ships
|
|
4264
|
+
// alongside so nothing an alarm keyed on `degraded` needs is lost.
|
|
3620
4265
|
: consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
|
|
3621
4266
|
? 'degraded'
|
|
3622
4267
|
: consecutiveFailures > 0
|
|
3623
4268
|
? 'retrying'
|
|
3624
4269
|
: 'healthy';
|
|
4270
|
+
// The counters above only move when a pass *settles*. A pass that hangs
|
|
4271
|
+
// takes neither path, so `settled` would keep reporting the last finished
|
|
4272
|
+
// pass — `healthy` — for as long as the process is stuck (#295). The
|
|
4273
|
+
// in-flight age is the only field that can express that, so derive the
|
|
4274
|
+
// state from it rather than trusting the last write.
|
|
4275
|
+
const timestamps = {
|
|
4276
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4277
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4278
|
+
...(this.#readinessReconcileLastStartedAtMs !== undefined
|
|
4279
|
+
? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
|
|
4280
|
+
: {}),
|
|
4281
|
+
...(this.#readinessReconcileLastCompletedAtMs !== undefined
|
|
4282
|
+
? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
|
|
4283
|
+
: {}),
|
|
4284
|
+
...(this.#readinessReconcileLastFailureAtMs !== undefined
|
|
4285
|
+
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
4286
|
+
: {}),
|
|
4287
|
+
};
|
|
4288
|
+
const nowMs = this.#clock.now();
|
|
4289
|
+
// Defence in depth (#300 review, CodeRabbit). These derivations are new
|
|
4290
|
+
// code from another module on a path that `status()` and every heartbeat
|
|
4291
|
+
// write depend on. A throw here would take out the liveness signal the
|
|
4292
|
+
// crash reaper reads — the diagnostic causing the outage it exists to
|
|
4293
|
+
// explain — so a failure costs the derived fields and nothing else.
|
|
4294
|
+
let inFlightMs;
|
|
4295
|
+
let derived = settled;
|
|
4296
|
+
try {
|
|
4297
|
+
inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
|
|
4298
|
+
derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
|
|
4299
|
+
}
|
|
4300
|
+
catch (error) {
|
|
4301
|
+
this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
|
|
4302
|
+
error: describeError(error).errorMessage,
|
|
4303
|
+
});
|
|
4304
|
+
inFlightMs = undefined;
|
|
4305
|
+
derived = settled;
|
|
4306
|
+
}
|
|
3625
4307
|
return {
|
|
3626
|
-
state,
|
|
4308
|
+
state: derived === 'unknown' ? settled : derived,
|
|
3627
4309
|
consecutiveFailures,
|
|
3628
4310
|
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
4311
|
+
intervalMs: this.#readinessReconcileIntervalMs,
|
|
4312
|
+
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
4313
|
+
...(inFlightMs !== undefined ? { inFlightMs } : {}),
|
|
3629
4314
|
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
3630
4315
|
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
3631
4316
|
: {}),
|
|
@@ -3639,6 +4324,9 @@ export class FactoryLoop {
|
|
|
3639
4324
|
? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
|
|
3640
4325
|
: {}),
|
|
3641
4326
|
...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
|
|
4327
|
+
...(this.#readinessReconcileLastErrorClass
|
|
4328
|
+
? { lastErrorClass: this.#readinessReconcileLastErrorClass }
|
|
4329
|
+
: {}),
|
|
3642
4330
|
};
|
|
3643
4331
|
}
|
|
3644
4332
|
on(event, listener) {
|
|
@@ -4142,10 +4830,11 @@ export class FactoryLoop {
|
|
|
4142
4830
|
seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
|
|
4143
4831
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
4144
4832
|
if (!claim.acquired || !claim.lease) {
|
|
4145
|
-
const
|
|
4833
|
+
const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
|
|
4834
|
+
const reason = terminal
|
|
4146
4835
|
? 'dispatch lifecycle is already terminal'
|
|
4147
4836
|
: `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
|
|
4148
|
-
throw new
|
|
4837
|
+
throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
|
|
4149
4838
|
}
|
|
4150
4839
|
this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
|
|
4151
4840
|
this.#hydrateCostLedger(claim.lifecycle);
|
|
@@ -5008,7 +5697,7 @@ export class FactoryLoop {
|
|
|
5008
5697
|
for (const [name] of record.agents) {
|
|
5009
5698
|
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
5010
5699
|
}
|
|
5011
|
-
await this.#
|
|
5700
|
+
await this.#retireSlackWatcher(record);
|
|
5012
5701
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
5013
5702
|
await this.#writeInFlightRegistry();
|
|
5014
5703
|
this.#increment('dispatchLifecycleStaleIssuesAbandoned');
|
|
@@ -6114,6 +6803,29 @@ export class FactoryLoop {
|
|
|
6114
6803
|
readinessReconcile: this.#readinessReconcileStatus(),
|
|
6115
6804
|
fleetControlPlane: this.#fleetControlPlane.status(),
|
|
6116
6805
|
};
|
|
6806
|
+
// The deployed container serves `/healthz` straight out of this file and
|
|
6807
|
+
// has no redaction logic of its own, so publish the already-safe view here
|
|
6808
|
+
// rather than leaving that boundary to whoever reads the file (#295).
|
|
6809
|
+
// Derived against this daemon's clock: every duration in it is a
|
|
6810
|
+
// difference between timestamps this process wrote.
|
|
6811
|
+
//
|
|
6812
|
+
// Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
|
|
6813
|
+
// reaper and `/healthz` read to decide the daemon is alive, and several
|
|
6814
|
+
// callers of this method sit outside any try/catch. A projection failure
|
|
6815
|
+
// must cost the diagnostics block, never the heartbeat — the omitted block
|
|
6816
|
+
// is itself legible, since `factory diagnose` reports a missing one rather
|
|
6817
|
+
// than a false green.
|
|
6818
|
+
try {
|
|
6819
|
+
heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
|
|
6820
|
+
nowMs: updatedAtMs,
|
|
6821
|
+
staleMs: this.#config.loop.heartbeatStaleMs,
|
|
6822
|
+
});
|
|
6823
|
+
}
|
|
6824
|
+
catch (error) {
|
|
6825
|
+
this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
|
|
6826
|
+
error: describeError(error).errorMessage,
|
|
6827
|
+
});
|
|
6828
|
+
}
|
|
6117
6829
|
await mkdir(dirname(path), { recursive: true });
|
|
6118
6830
|
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
6119
6831
|
await this.#writeInFlightRegistry(registryPath, path);
|
|
@@ -6131,12 +6843,19 @@ export class FactoryLoop {
|
|
|
6131
6843
|
},
|
|
6132
6844
|
});
|
|
6133
6845
|
}
|
|
6134
|
-
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
6135
|
-
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6136
|
-
if (handoffs.length === 0) {
|
|
6137
|
-
return;
|
|
6138
|
-
}
|
|
6846
|
+
async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
|
|
6139
6847
|
try {
|
|
6848
|
+
// Inside the try, not before it (#298 review). Every caller reaps while
|
|
6849
|
+
// already handling a failure and then propagates that failure: the
|
|
6850
|
+
// per-item catch rethrows the dispatch error, runLoop's catch is mid
|
|
6851
|
+
// teardown. A throw from here would REPLACE the error in flight — and a
|
|
6852
|
+
// replaced 429 is no longer recognised as overload at the discovery
|
|
6853
|
+
// fence, silently dropping the advertised backoff. Reaping is
|
|
6854
|
+
// best-effort by construction; the caller's error always wins.
|
|
6855
|
+
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6856
|
+
if (handoffs.length === 0) {
|
|
6857
|
+
return;
|
|
6858
|
+
}
|
|
6140
6859
|
const protectedPids = await this.#protectedPids();
|
|
6141
6860
|
let registryChanged = false;
|
|
6142
6861
|
const readyToClear = new Set();
|
|
@@ -7149,6 +7868,7 @@ export class FactoryLoop {
|
|
|
7149
7868
|
}
|
|
7150
7869
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
7151
7870
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
7871
|
+
const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
|
|
7152
7872
|
const expectedHeadRef = implementer.spec.branch;
|
|
7153
7873
|
if (!expectedHeadRef) {
|
|
7154
7874
|
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
@@ -7214,7 +7934,7 @@ export class FactoryLoop {
|
|
|
7214
7934
|
expectedHeadRef,
|
|
7215
7935
|
baseRef,
|
|
7216
7936
|
title: `${issue.key}: ${issue.title}`,
|
|
7217
|
-
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
7937
|
+
body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
|
|
7218
7938
|
...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
|
|
7219
7939
|
});
|
|
7220
7940
|
const published = result.author
|
|
@@ -7803,7 +8523,7 @@ export class FactoryLoop {
|
|
|
7803
8523
|
await this.#recordDispatchTerminal(record.issue);
|
|
7804
8524
|
const next = (await this.#batch()).complete(record.issue);
|
|
7805
8525
|
await this.#drainReadyClarificationWake();
|
|
7806
|
-
await this.#
|
|
8526
|
+
await this.#retireSlackWatcher(record);
|
|
7807
8527
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
7808
8528
|
await this.#writeInFlightRegistry();
|
|
7809
8529
|
if (next) {
|
|
@@ -11763,7 +12483,7 @@ export class FactoryLoop {
|
|
|
11763
12483
|
}
|
|
11764
12484
|
if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
|
|
11765
12485
|
return;
|
|
11766
|
-
await this.#
|
|
12486
|
+
await this.#retireSlackWatcher(record);
|
|
11767
12487
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
11768
12488
|
await this.#recordDispatchTerminal(record.issue);
|
|
11769
12489
|
await this.#finishDurableRelease(record, releaseReason);
|
|
@@ -12241,6 +12961,21 @@ export class FactoryLoop {
|
|
|
12241
12961
|
return;
|
|
12242
12962
|
}
|
|
12243
12963
|
const key = issueKey(record.issue);
|
|
12964
|
+
const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
12965
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
12966
|
+
if (previousWatch?.kind === 'terminal-grace') {
|
|
12967
|
+
// A reopened work unit needs a fresh dispatch notification and a fresh
|
|
12968
|
+
// conversation. Do not let the old grace-period watcher (or its expiry
|
|
12969
|
+
// timer) capture and later tear down the new dispatch.
|
|
12970
|
+
if (!await this.#stopSlackWatcher(record.issue)) {
|
|
12971
|
+
// Fail closed. An undrained reply route still holds the retired thread
|
|
12972
|
+
// and would bind it to this dispatch, delivering a stale human reply to
|
|
12973
|
+
// fresh work. Leave the fence up; the next reconcile retries the drain.
|
|
12974
|
+
this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
|
|
12975
|
+
this.#increment('slackDispatchThreadsDeferredUndrainedReply');
|
|
12976
|
+
return;
|
|
12977
|
+
}
|
|
12978
|
+
}
|
|
12244
12979
|
const existingThread = await this.#persistedSlackThread(key);
|
|
12245
12980
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
12246
12981
|
if (existingThread || watcherStart) {
|
|
@@ -12323,9 +13058,10 @@ export class FactoryLoop {
|
|
|
12323
13058
|
if (existing) {
|
|
12324
13059
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12325
13060
|
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
13061
|
+
let rebound = false;
|
|
12326
13062
|
if (owned && sessionRef &&
|
|
12327
|
-
(agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
12328
|
-
|
|
13063
|
+
(!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
13064
|
+
rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
|
|
12329
13065
|
name: agentName,
|
|
12330
13066
|
sessionRef,
|
|
12331
13067
|
role: owned.tracked.spec.role,
|
|
@@ -12339,40 +13075,39 @@ export class FactoryLoop {
|
|
|
12339
13075
|
}
|
|
12340
13076
|
if (existing.pending.length > 0 || existing.delivery) {
|
|
12341
13077
|
const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
|
|
12342
|
-
if (!waiting)
|
|
13078
|
+
if (!waiting && (existing.agent || rebound))
|
|
12343
13079
|
this.#slackConversationTurns.schedule(conversationId);
|
|
12344
13080
|
}
|
|
12345
13081
|
return;
|
|
12346
13082
|
}
|
|
12347
13083
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12348
|
-
if (!owned || !sessionRef) {
|
|
12349
|
-
this.#increment('slackConversationSessionsSkippedMissingSession');
|
|
12350
|
-
return;
|
|
12351
|
-
}
|
|
12352
13084
|
const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
|
|
12353
13085
|
if (!channelDir)
|
|
12354
13086
|
return;
|
|
12355
|
-
const agentName = owned.tracked.result?.name ?? owned.name;
|
|
13087
|
+
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
12356
13088
|
const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
|
|
12357
13089
|
provider: 'slack',
|
|
12358
13090
|
issue: { ...record.issue },
|
|
12359
13091
|
externalId: threadId,
|
|
12360
13092
|
context: { channelDir },
|
|
12361
|
-
agent: {
|
|
12362
|
-
|
|
12363
|
-
|
|
12364
|
-
|
|
12365
|
-
|
|
12366
|
-
|
|
12367
|
-
|
|
12368
|
-
|
|
12369
|
-
|
|
13093
|
+
...(owned && sessionRef && agentName ? { agent: {
|
|
13094
|
+
name: agentName,
|
|
13095
|
+
sessionRef,
|
|
13096
|
+
role: owned.tracked.spec.role,
|
|
13097
|
+
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
13098
|
+
capability: owned.tracked.spec.capability,
|
|
13099
|
+
repo: owned.tracked.spec.repo,
|
|
13100
|
+
clonePath: owned.tracked.spec.clonePath,
|
|
13101
|
+
} } : {}),
|
|
12370
13102
|
history: [],
|
|
12371
13103
|
processedMessageIds: [],
|
|
13104
|
+
acknowledgedMessageIds: [],
|
|
13105
|
+
acknowledgementClaims: {},
|
|
12372
13106
|
pending: [],
|
|
12373
13107
|
});
|
|
12374
|
-
if (reserved)
|
|
12375
|
-
this.#increment('slackConversationSessionsOwned');
|
|
13108
|
+
if (reserved) {
|
|
13109
|
+
this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
|
|
13110
|
+
}
|
|
12376
13111
|
}
|
|
12377
13112
|
// Called right after a babysitter is spawned/reattached for an issue's PR so
|
|
12378
13113
|
// an already-owned Slack conversation session (reserved earlier by the
|
|
@@ -12400,11 +13135,15 @@ export class FactoryLoop {
|
|
|
12400
13135
|
const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
|
|
12401
13136
|
if (!claimed?.delivery) {
|
|
12402
13137
|
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
12403
|
-
if (current && (current.pending.length > 0 || current.delivery)) {
|
|
13138
|
+
if (current?.agent && (current.pending.length > 0 || current.delivery)) {
|
|
12404
13139
|
this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
|
|
12405
13140
|
}
|
|
12406
13141
|
return;
|
|
12407
13142
|
}
|
|
13143
|
+
if (!claimed.agent) {
|
|
13144
|
+
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
13145
|
+
return;
|
|
13146
|
+
}
|
|
12408
13147
|
if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
|
|
12409
13148
|
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
12410
13149
|
this.#increment('slackConversationTurnsSuppressedStaleOwner');
|
|
@@ -12488,10 +13227,13 @@ export class FactoryLoop {
|
|
|
12488
13227
|
}
|
|
12489
13228
|
}
|
|
12490
13229
|
async #recordSlackConversationResume(session, result) {
|
|
13230
|
+
const sessionAgent = session.agent;
|
|
13231
|
+
if (!sessionAgent)
|
|
13232
|
+
return;
|
|
12491
13233
|
const record = (await this.#batch()).getIssue(session.issue);
|
|
12492
13234
|
if (!record)
|
|
12493
13235
|
return;
|
|
12494
|
-
const entry = [...record.agents.entries()].find(([name, tracked]) => name ===
|
|
13236
|
+
const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
|
|
12495
13237
|
if (!entry)
|
|
12496
13238
|
return;
|
|
12497
13239
|
const [previousName, tracked] = entry;
|
|
@@ -12817,7 +13559,14 @@ export class FactoryLoop {
|
|
|
12817
13559
|
`Question: ${triageEscalationQuestion(decision, issue)}`,
|
|
12818
13560
|
].join('\n'),
|
|
12819
13561
|
});
|
|
12820
|
-
|
|
13562
|
+
const key = issueKey(decision.issue);
|
|
13563
|
+
await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
|
|
13564
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
13565
|
+
kind: 'triage',
|
|
13566
|
+
issue: { ...decision.issue },
|
|
13567
|
+
decision: structuredClone(decision),
|
|
13568
|
+
threadId: root.threadId,
|
|
13569
|
+
});
|
|
12821
13570
|
const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
|
|
12822
13571
|
this.#recordSlackWritebackSuccess('triage-escalation');
|
|
12823
13572
|
return replayedResult;
|
|
@@ -12883,6 +13632,11 @@ export class FactoryLoop {
|
|
|
12883
13632
|
if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
|
|
12884
13633
|
return;
|
|
12885
13634
|
}
|
|
13635
|
+
if (allowPreExisting &&
|
|
13636
|
+
options.replayAfterMs !== undefined &&
|
|
13637
|
+
slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
|
|
13638
|
+
return;
|
|
13639
|
+
}
|
|
12886
13640
|
const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
|
|
12887
13641
|
if (seenReplyMessages.has(replyMessageKey)) {
|
|
12888
13642
|
this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
|
|
@@ -13061,6 +13815,47 @@ export class FactoryLoop {
|
|
|
13061
13815
|
this.#slackConversationTurns.schedule(conversationId);
|
|
13062
13816
|
}
|
|
13063
13817
|
}
|
|
13818
|
+
for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
|
|
13819
|
+
if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
|
|
13820
|
+
continue;
|
|
13821
|
+
if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
|
|
13822
|
+
await this.#stopSlackWatcher(watch.issue);
|
|
13823
|
+
continue;
|
|
13824
|
+
}
|
|
13825
|
+
await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
|
|
13826
|
+
const watchRecord = escalationWatchRecord(watch.decision);
|
|
13827
|
+
if (watch.kind === 'terminal-grace') {
|
|
13828
|
+
const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
|
|
13829
|
+
if (watch.retiredAtMs !== retiredAtMs) {
|
|
13830
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
|
|
13831
|
+
}
|
|
13832
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
13833
|
+
const conversationId = slackConversationId(watch.threadId);
|
|
13834
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
13835
|
+
try {
|
|
13836
|
+
await this.#surfaceUndeliveredSlackConversation(watch.threadId);
|
|
13837
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13838
|
+
}
|
|
13839
|
+
catch (error) {
|
|
13840
|
+
// The undelivered-reply receipt needs Slack writeback, which may be
|
|
13841
|
+
// unavailable at startup. That is retryable state maintenance for this
|
|
13842
|
+
// one thread, not a reason to abandon rehydration: aborting here would
|
|
13843
|
+
// leave every remaining thread watched by nobody. Keep the queued
|
|
13844
|
+
// replies (clearing them now would drop replies nobody was told about)
|
|
13845
|
+
// and carry on re-arming.
|
|
13846
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
|
|
13847
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
13848
|
+
this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
|
|
13849
|
+
}
|
|
13850
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
|
|
13851
|
+
replayConversationReplies: true,
|
|
13852
|
+
replayAfterMs: retiredAtMs,
|
|
13853
|
+
});
|
|
13854
|
+
this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
|
|
13855
|
+
continue;
|
|
13856
|
+
}
|
|
13857
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
|
|
13858
|
+
}
|
|
13064
13859
|
await this.#sweepWaitingClarifications();
|
|
13065
13860
|
for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
|
|
13066
13861
|
if (!waiting.threadId)
|
|
@@ -13206,8 +14001,73 @@ export class FactoryLoop {
|
|
|
13206
14001
|
this.#clarificationSweepTimer = timer;
|
|
13207
14002
|
this.#clarificationSweepDueAtMs = dueAtMs;
|
|
13208
14003
|
}
|
|
14004
|
+
// The terminal fence is the only thing that makes an in-flight reply route
|
|
14005
|
+
// answer "no active agent" instead of binding the retired thread to whatever
|
|
14006
|
+
// dispatch owns this key. Routes are chained per work unit, so awaiting the
|
|
14007
|
+
// newest one drains every reply queued behind it.
|
|
14008
|
+
async #drainSlackReplyRoutes(key) {
|
|
14009
|
+
// Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
|
|
14010
|
+
// that is still inside its mount read when the drain starts registers its
|
|
14011
|
+
// route *after* the snapshot, so it would run once the fence is gone and
|
|
14012
|
+
// bind the retired thread to the next dispatch — the same escape one level
|
|
14013
|
+
// in. Bar *routing* for this key first (the bar and the registration are
|
|
14014
|
+
// both synchronous, so nothing can slip between them), then drain whatever
|
|
14015
|
+
// is already chained, then prove the set is empty before reporting success.
|
|
14016
|
+
// A barred reply still answers the human, and that writeback registers here
|
|
14017
|
+
// like any other effect, so the extra passes are what pick it up: quiescence
|
|
14018
|
+
// means every effect this work unit started has settled, not merely the ones
|
|
14019
|
+
// that existed when the drain began.
|
|
14020
|
+
const nested = this.#slackReplyRouteDrains.has(key);
|
|
14021
|
+
this.#slackReplyRouteDrains.add(key);
|
|
14022
|
+
try {
|
|
14023
|
+
for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
|
|
14024
|
+
const route = this.#slackReplyRoutes.get(key);
|
|
14025
|
+
if (!route)
|
|
14026
|
+
return true;
|
|
14027
|
+
try {
|
|
14028
|
+
await route;
|
|
14029
|
+
}
|
|
14030
|
+
catch (error) {
|
|
14031
|
+
// A route that *rejects* is not drained: the watcher replays it after
|
|
14032
|
+
// SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
|
|
14033
|
+
// dispatch. Fail closed and let the caller keep the fence up rather
|
|
14034
|
+
// than leak a stale human reply onto fresh work.
|
|
14035
|
+
this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
|
|
14036
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
14037
|
+
return false;
|
|
14038
|
+
}
|
|
14039
|
+
// The owner clears its own entry when it settles; retiring it here too
|
|
14040
|
+
// keeps the loop monotonic if that finally has not run yet.
|
|
14041
|
+
if (this.#slackReplyRoutes.get(key) === route)
|
|
14042
|
+
this.#slackReplyRoutes.delete(key);
|
|
14043
|
+
}
|
|
14044
|
+
// Not provably quiescent. Fail closed for the same reason as a rejection.
|
|
14045
|
+
this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
|
|
14046
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
14047
|
+
return false;
|
|
14048
|
+
}
|
|
14049
|
+
finally {
|
|
14050
|
+
if (!nested)
|
|
14051
|
+
this.#slackReplyRouteDrains.delete(key);
|
|
14052
|
+
}
|
|
14053
|
+
}
|
|
13209
14054
|
async #stopSlackWatcher(issue) {
|
|
13210
14055
|
const key = issueKey(issue);
|
|
14056
|
+
// Drain before clearing the fence. Clearing it first lets a reply that is
|
|
14057
|
+
// already mid-route — or one queued behind it — fall through the fence check
|
|
14058
|
+
// in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
|
|
14059
|
+
// the next dispatch of this work unit.
|
|
14060
|
+
if (!await this.#drainSlackReplyRoutes(key))
|
|
14061
|
+
return false;
|
|
14062
|
+
this.#terminalSlackWatchIssues.delete(key);
|
|
14063
|
+
const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
14064
|
+
if (expiryTimer)
|
|
14065
|
+
clearTimeout(expiryTimer);
|
|
14066
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
14067
|
+
const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
14068
|
+
if (receiptRetryTimer)
|
|
14069
|
+
clearTimeout(receiptRetryTimer);
|
|
14070
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13211
14071
|
const watcher = this.#slackWatchers.get(key);
|
|
13212
14072
|
this.#slackWatchers.delete(key);
|
|
13213
14073
|
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
@@ -13218,6 +14078,259 @@ export class FactoryLoop {
|
|
|
13218
14078
|
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13219
14079
|
}
|
|
13220
14080
|
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
14081
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
|
|
14082
|
+
return true;
|
|
14083
|
+
}
|
|
14084
|
+
async #retireSlackWatcher(record) {
|
|
14085
|
+
const key = issueKey(record.issue);
|
|
14086
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
14087
|
+
if (!threadId) {
|
|
14088
|
+
await this.#stopSlackWatcher(record.issue);
|
|
14089
|
+
return;
|
|
14090
|
+
}
|
|
14091
|
+
const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
14092
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
14093
|
+
const retiredAtMs = existingWatch?.kind === 'terminal-grace'
|
|
14094
|
+
? terminalSlackWatchRetiredAtMs(existingWatch)
|
|
14095
|
+
: this.#clock.now();
|
|
14096
|
+
const expiresAtMs = existingWatch?.kind === 'terminal-grace'
|
|
14097
|
+
? existingWatch.expiresAtMs
|
|
14098
|
+
: retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
|
|
14099
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
14100
|
+
kind: 'terminal-grace',
|
|
14101
|
+
issue: { ...record.issue },
|
|
14102
|
+
decision: structuredClone(record.decision),
|
|
14103
|
+
threadId,
|
|
14104
|
+
retiredAtMs,
|
|
14105
|
+
expiresAtMs,
|
|
14106
|
+
});
|
|
14107
|
+
// A terminal thread must never retain a resumable session for an agent that
|
|
14108
|
+
// has already exited. Keep only the exact-thread listener so a late human
|
|
14109
|
+
// reply receives the explicit no-active-agent writeback below.
|
|
14110
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
14111
|
+
await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
|
|
14112
|
+
const conversationId = slackConversationId(threadId);
|
|
14113
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
14114
|
+
try {
|
|
14115
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
14116
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
14117
|
+
}
|
|
14118
|
+
catch (error) {
|
|
14119
|
+
// The receipt fails whenever another handler holds the claim or Slack
|
|
14120
|
+
// writeback is down — neither is a reason to abort retirement. Callers
|
|
14121
|
+
// reach here having already committed the terminal phase and dropped the
|
|
14122
|
+
// pending abandon reason, so a rejection escaping would strand the
|
|
14123
|
+
// registry rewrite, the GitHub watcher stop, and the queued next dispatch
|
|
14124
|
+
// with nothing left to re-run them. Keep the queued replies and let the
|
|
14125
|
+
// retry that owns this receipt settle it inside the grace window.
|
|
14126
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
|
|
14127
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
14128
|
+
this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
|
|
14129
|
+
}
|
|
14130
|
+
if (!this.#slackWatchers.has(key) && !this.#stopping) {
|
|
14131
|
+
await this.#rearmSlackWatcher(record, threadId);
|
|
14132
|
+
}
|
|
14133
|
+
this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
|
|
14134
|
+
this.#increment('slackTerminalWatchersRetained');
|
|
14135
|
+
}
|
|
14136
|
+
// The durable half of the same record. Every caller here follows the receipt
|
|
14137
|
+
// with a state write (clearing the session), and those two cannot be one
|
|
14138
|
+
// durable step: when the state write fails, the retry that owns it must not
|
|
14139
|
+
// read "replies still queued" as "the human has not been told" and post the
|
|
14140
|
+
// notice again. So the receipt is claimed before the provider write and marked
|
|
14141
|
+
// posted after it, and a retry finds it already settled.
|
|
14142
|
+
async #surfaceUndeliveredSlackConversation(threadId) {
|
|
14143
|
+
const conversationId = slackConversationId(threadId);
|
|
14144
|
+
const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14145
|
+
const pendingCount = session
|
|
14146
|
+
? session.pending.length + (session.delivery?.messages.length ?? 0)
|
|
14147
|
+
: 0;
|
|
14148
|
+
if (pendingCount === 0)
|
|
14149
|
+
return;
|
|
14150
|
+
if (session?.terminalReceipt?.posted) {
|
|
14151
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
14152
|
+
return;
|
|
14153
|
+
}
|
|
14154
|
+
if (!this.#slack)
|
|
14155
|
+
throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
|
|
14156
|
+
const claimId = randomUUID();
|
|
14157
|
+
if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
|
|
14158
|
+
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14159
|
+
if (current?.terminalReceipt?.posted) {
|
|
14160
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
14161
|
+
return;
|
|
14162
|
+
}
|
|
14163
|
+
// Another handler is mid-write. Fail closed so the queued replies survive
|
|
14164
|
+
// for whoever settles them rather than racing a second notice onto the
|
|
14165
|
+
// same thread.
|
|
14166
|
+
throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
|
|
14167
|
+
}
|
|
14168
|
+
const noun = pendingCount === 1 ? 'reply' : 'replies';
|
|
14169
|
+
const slack = this.#slack;
|
|
14170
|
+
try {
|
|
14171
|
+
await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
|
|
14172
|
+
}
|
|
14173
|
+
catch (error) {
|
|
14174
|
+
await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
|
|
14175
|
+
throw error;
|
|
14176
|
+
}
|
|
14177
|
+
if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
|
|
14178
|
+
throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
|
|
14179
|
+
}
|
|
14180
|
+
this.#increment('slackConversationRepliesSurfacedTerminal');
|
|
14181
|
+
}
|
|
14182
|
+
// A claim only means something for as long as it outlives the work it covers.
|
|
14183
|
+
// A provider write can legitimately run past a fixed lease, at which point the
|
|
14184
|
+
// claim stops protecting the write it was taken for and a second handler can
|
|
14185
|
+
// post the same thing to the same human. Renewing on a heartbeat scopes the
|
|
14186
|
+
// lease to the work instead of to a guessed duration, and leaves the idle
|
|
14187
|
+
// timeout short enough that a holder that dies mid-write still frees it.
|
|
14188
|
+
//
|
|
14189
|
+
// The heartbeat is bounded on both ends, because a renewal loop that never
|
|
14190
|
+
// stops is a lock with no owner check: a provider write that hangs would hold
|
|
14191
|
+
// the receipt past every retry and past shutdown, and the human whose reply is
|
|
14192
|
+
// queued behind it would be told nothing until the process restarts. So it
|
|
14193
|
+
// stops at the ceiling and it stops when this daemon is stopping, and either
|
|
14194
|
+
// way it says so — from there the claim ages out on its own idle lease and
|
|
14195
|
+
// becomes reclaimable. The write may still land afterwards and duplicate the
|
|
14196
|
+
// notice; a reply nobody can ever reclaim is the worse of the two.
|
|
14197
|
+
async #withRenewedProviderLease(label, leaseMs, renew, run) {
|
|
14198
|
+
const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
|
|
14199
|
+
let renewalStopped = false;
|
|
14200
|
+
let renewalInFlight = false;
|
|
14201
|
+
const stopRenewing = (counter, reason) => {
|
|
14202
|
+
renewalStopped = true;
|
|
14203
|
+
this.#increment(counter);
|
|
14204
|
+
this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
|
|
14205
|
+
'its claim expires and the queued replies return to whoever retries them');
|
|
14206
|
+
};
|
|
14207
|
+
const heartbeat = setInterval(() => {
|
|
14208
|
+
if (renewalInFlight || renewalStopped)
|
|
14209
|
+
return;
|
|
14210
|
+
if (this.#stopping) {
|
|
14211
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
|
|
14212
|
+
return;
|
|
14213
|
+
}
|
|
14214
|
+
if (this.#clock.now() >= renewUntilMs) {
|
|
14215
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
|
|
14216
|
+
return;
|
|
14217
|
+
}
|
|
14218
|
+
renewalInFlight = true;
|
|
14219
|
+
void renew()
|
|
14220
|
+
.then((renewed) => {
|
|
14221
|
+
if (renewed)
|
|
14222
|
+
return;
|
|
14223
|
+
// Losing the lease mid-write is not recoverable from in here: the
|
|
14224
|
+
// write may already have landed. Stop renewing and let the caller's
|
|
14225
|
+
// completion check fail closed, which keeps the queued replies for
|
|
14226
|
+
// whoever holds the claim now.
|
|
14227
|
+
renewalStopped = true;
|
|
14228
|
+
this.#increment('slackProviderReceiptLeasesLost');
|
|
14229
|
+
this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
|
|
14230
|
+
})
|
|
14231
|
+
.catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
|
|
14232
|
+
error: describeError(error).errorMessage,
|
|
14233
|
+
}))
|
|
14234
|
+
.finally(() => { renewalInFlight = false; });
|
|
14235
|
+
}, Math.max(1_000, Math.floor(leaseMs / 3)));
|
|
14236
|
+
heartbeat.unref?.();
|
|
14237
|
+
try {
|
|
14238
|
+
return await run();
|
|
14239
|
+
}
|
|
14240
|
+
finally {
|
|
14241
|
+
clearInterval(heartbeat);
|
|
14242
|
+
}
|
|
14243
|
+
}
|
|
14244
|
+
// A terminal receipt that could not be written leaves the queued replies
|
|
14245
|
+
// pending with the human who wrote them told nothing. That is retryable
|
|
14246
|
+
// maintenance this daemon owns, not work to leave for the next restart: the
|
|
14247
|
+
// grace watch is the only window in which the receipt can still land on the
|
|
14248
|
+
// retired thread, so keep reattempting inside it and give up when it closes.
|
|
14249
|
+
#scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
|
|
14250
|
+
if (this.#stopping)
|
|
14251
|
+
return;
|
|
14252
|
+
const key = issueKey(issue);
|
|
14253
|
+
const existing = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
14254
|
+
if (existing)
|
|
14255
|
+
clearTimeout(existing);
|
|
14256
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
14257
|
+
const remainingMs = expiresAtMs - this.#clock.now();
|
|
14258
|
+
if (remainingMs <= 0) {
|
|
14259
|
+
this.#increment('slackTerminalWatchReceiptsAbandoned');
|
|
14260
|
+
return;
|
|
14261
|
+
}
|
|
14262
|
+
const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
|
|
14263
|
+
const timer = setTimeout(() => {
|
|
14264
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
14265
|
+
void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
|
|
14266
|
+
}, Math.max(0, Math.min(backoffMs, remainingMs)));
|
|
14267
|
+
timer.unref?.();
|
|
14268
|
+
this.#slackTerminalReceiptRetryTimers.set(key, timer);
|
|
14269
|
+
}
|
|
14270
|
+
async #retrySlackTerminalReceipt(issue, threadId, attempt) {
|
|
14271
|
+
if (this.#stopping)
|
|
14272
|
+
return;
|
|
14273
|
+
const key = issueKey(issue);
|
|
14274
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
14275
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
14276
|
+
// The grace watch is gone (expired, or the work unit reopened): the thread
|
|
14277
|
+
// this receipt would settle no longer exists, so there is nothing to say.
|
|
14278
|
+
if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
|
|
14279
|
+
return;
|
|
14280
|
+
try {
|
|
14281
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
14282
|
+
await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
|
|
14283
|
+
this.#increment('slackTerminalWatchReceiptsRecovered');
|
|
14284
|
+
}
|
|
14285
|
+
catch (error) {
|
|
14286
|
+
this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
|
|
14287
|
+
issue: issue.key,
|
|
14288
|
+
error,
|
|
14289
|
+
});
|
|
14290
|
+
this.#increment('slackTerminalWatchReceiptRetryFailures');
|
|
14291
|
+
this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
|
|
14292
|
+
}
|
|
14293
|
+
}
|
|
14294
|
+
#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
|
|
14295
|
+
if (this.#stopping)
|
|
14296
|
+
return;
|
|
14297
|
+
const key = issueKey(issue);
|
|
14298
|
+
const existing = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
14299
|
+
if (existing)
|
|
14300
|
+
clearTimeout(existing);
|
|
14301
|
+
const timer = setTimeout(() => {
|
|
14302
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
14303
|
+
void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
|
|
14304
|
+
this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
|
|
14305
|
+
issue: issue.key,
|
|
14306
|
+
error,
|
|
14307
|
+
});
|
|
14308
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
14309
|
+
});
|
|
14310
|
+
}, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
|
|
14311
|
+
timer.unref?.();
|
|
14312
|
+
this.#slackTerminalWatchExpiryTimers.set(key, timer);
|
|
14313
|
+
}
|
|
14314
|
+
async #expireSlackTerminalWatcher(issue, expiresAtMs) {
|
|
14315
|
+
const key = issueKey(issue);
|
|
14316
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
14317
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
14318
|
+
if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
|
|
14319
|
+
return;
|
|
14320
|
+
if (watch.expiresAtMs > this.#clock.now()) {
|
|
14321
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
|
|
14322
|
+
return;
|
|
14323
|
+
}
|
|
14324
|
+
// #stopSlackWatcher fails closed when an in-flight reply route will not
|
|
14325
|
+
// drain, leaving the watch and its terminal fence in place. Counting that as
|
|
14326
|
+
// an expiration retires the watch in the metrics while the real one lives
|
|
14327
|
+
// on unwatched by any expiry timer, so reschedule and count only on success.
|
|
14328
|
+
if (!await this.#stopSlackWatcher(issue)) {
|
|
14329
|
+
this.#increment('slackTerminalWatchExpiriesDeferred');
|
|
14330
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
14331
|
+
return;
|
|
14332
|
+
}
|
|
14333
|
+
this.#increment('slackTerminalWatchersExpired');
|
|
13221
14334
|
}
|
|
13222
14335
|
async #readSlackReply(path) {
|
|
13223
14336
|
try {
|
|
@@ -13282,33 +14395,147 @@ export class FactoryLoop {
|
|
|
13282
14395
|
await this.#wakeWaitingClarification(clarificationKey, claimed);
|
|
13283
14396
|
return;
|
|
13284
14397
|
}
|
|
14398
|
+
return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
|
|
14399
|
+
}
|
|
14400
|
+
async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
|
|
14401
|
+
if (this.#slackReplyRouteDrains.has(clarificationKey)) {
|
|
14402
|
+
// The terminal fence for this work unit is being drained right now.
|
|
14403
|
+
// Registering an ordinary route here would put it past the drain's
|
|
14404
|
+
// snapshot and run it once the fence is gone. Answer it the way the fence
|
|
14405
|
+
// would have — but as a tracked effect, because this writeback is still a
|
|
14406
|
+
// side effect of this work unit. Left untracked it is the same escape one
|
|
14407
|
+
// level further in: the drain reports quiescence without it, the watcher
|
|
14408
|
+
// stop clears the retry timer that owns this reply, and a slow or failed
|
|
14409
|
+
// receipt leaves the human told nothing at all.
|
|
14410
|
+
this.#increment('slackReplyRoutesFencedDuringDrain');
|
|
14411
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
|
|
14412
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
14413
|
+
return undefined;
|
|
14414
|
+
});
|
|
14415
|
+
}
|
|
14416
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
|
|
14417
|
+
}
|
|
14418
|
+
// Every Slack side effect a work unit makes on its own behalf runs through
|
|
14419
|
+
// here, so #slackReplyRoutes stays the single record the terminal drain
|
|
14420
|
+
// consults. Effects are chained per work unit: awaiting the newest one drains
|
|
14421
|
+
// everything queued behind it, and a rejection propagates to the drain, which
|
|
14422
|
+
// fails closed rather than tearing the effect's retry path down.
|
|
14423
|
+
async #trackSlackWorkUnitEffect(key, run) {
|
|
14424
|
+
const preceding = this.#slackReplyRoutes.get(key);
|
|
14425
|
+
const effect = (async () => {
|
|
14426
|
+
await preceding?.catch(() => undefined);
|
|
14427
|
+
return await run();
|
|
14428
|
+
})();
|
|
14429
|
+
this.#slackReplyRoutes.set(key, effect);
|
|
14430
|
+
try {
|
|
14431
|
+
return await effect;
|
|
14432
|
+
}
|
|
14433
|
+
finally {
|
|
14434
|
+
if (this.#slackReplyRoutes.get(key) === effect)
|
|
14435
|
+
this.#slackReplyRoutes.delete(key);
|
|
14436
|
+
}
|
|
14437
|
+
}
|
|
14438
|
+
async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
|
|
14439
|
+
if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
|
|
14440
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
14441
|
+
return;
|
|
14442
|
+
}
|
|
13285
14443
|
const conversationId = slackConversationId(reply.threadTs);
|
|
13286
|
-
|
|
14444
|
+
let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14445
|
+
let liveRecord;
|
|
14446
|
+
if (!conversation) {
|
|
14447
|
+
liveRecord = (await this.#batch()).getIssue(record.issue);
|
|
14448
|
+
if (liveRecord && !liveRecord.dryRun) {
|
|
14449
|
+
await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
|
|
14450
|
+
conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14451
|
+
}
|
|
14452
|
+
}
|
|
13287
14453
|
if (conversation && issueKey(conversation.issue) === clarificationKey) {
|
|
14454
|
+
const replyId = `${reply.threadTs}:${reply.messageTs}`;
|
|
13288
14455
|
const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
|
|
13289
|
-
id:
|
|
14456
|
+
id: replyId,
|
|
13290
14457
|
text,
|
|
13291
14458
|
receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
|
|
13292
14459
|
providerSequence: reply.messageTs,
|
|
13293
14460
|
author: reply.author,
|
|
13294
14461
|
});
|
|
13295
|
-
|
|
14462
|
+
const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14463
|
+
if (!durable || !durable.processedMessageIds.includes(replyId)) {
|
|
14464
|
+
throw new Error(`Slack reply ${replyId} was not durably queued`);
|
|
14465
|
+
}
|
|
14466
|
+
if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
14467
|
+
const acknowledgementClaimId = randomUUID();
|
|
14468
|
+
const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
|
|
14469
|
+
if (acknowledgementClaimed) {
|
|
14470
|
+
try {
|
|
14471
|
+
if (!this.#slack)
|
|
14472
|
+
throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
|
|
14473
|
+
const owner = durable.agent?.role === 'babysitter'
|
|
14474
|
+
? 'the PR babysitter'
|
|
14475
|
+
: durable.agent
|
|
14476
|
+
? 'the issue implementer'
|
|
14477
|
+
: 'an issue agent';
|
|
14478
|
+
const receipt = durable.agent
|
|
14479
|
+
? `Factory received this reply and durably queued it for ${owner}.`
|
|
14480
|
+
: 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
|
|
14481
|
+
const slack = this.#slack;
|
|
14482
|
+
// Same lease scope as the terminal receipt: this claim covers a
|
|
14483
|
+
// provider write that can outrun any fixed duration, so it is
|
|
14484
|
+
// renewed for as long as that write is actually running.
|
|
14485
|
+
await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
|
|
14486
|
+
if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
|
|
14487
|
+
throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
|
|
14488
|
+
}
|
|
14489
|
+
this.#increment('slackConversationRepliesAcknowledged');
|
|
14490
|
+
}
|
|
14491
|
+
catch (error) {
|
|
14492
|
+
await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
|
|
14493
|
+
throw error;
|
|
14494
|
+
}
|
|
14495
|
+
}
|
|
14496
|
+
else {
|
|
14497
|
+
const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
14498
|
+
if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
14499
|
+
throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
|
|
14500
|
+
}
|
|
14501
|
+
}
|
|
14502
|
+
}
|
|
14503
|
+
if (queued) {
|
|
14504
|
+
this.#increment('slackConversationRepliesQueued');
|
|
14505
|
+
}
|
|
14506
|
+
else {
|
|
13296
14507
|
this.#increment('slackConversationDuplicateRepliesSuppressed');
|
|
13297
|
-
return;
|
|
13298
14508
|
}
|
|
13299
|
-
|
|
13300
|
-
|
|
14509
|
+
const pending = durable.pending.some((message) => message.id === replyId) ||
|
|
14510
|
+
Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
|
|
14511
|
+
if (pending && durable.agent) {
|
|
14512
|
+
this.#slackConversationTurns.schedule(conversationId);
|
|
14513
|
+
}
|
|
14514
|
+
else if (pending) {
|
|
14515
|
+
this.#increment('slackConversationRepliesWaitingForOwner');
|
|
14516
|
+
}
|
|
13301
14517
|
return;
|
|
13302
14518
|
}
|
|
13303
|
-
|
|
14519
|
+
liveRecord ??= (await this.#batch()).getIssue(record.issue);
|
|
13304
14520
|
if (!liveRecord || liveRecord.dryRun) {
|
|
13305
14521
|
if (isTriageEscalationWatchRecord(record)) {
|
|
13306
14522
|
return await this.#handleTriageEscalationSlackAnswer(record, text);
|
|
13307
14523
|
}
|
|
13308
|
-
this.#
|
|
14524
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13309
14525
|
return;
|
|
13310
14526
|
}
|
|
13311
14527
|
this.#increment('slackAnswersIgnoredNoConversationSession');
|
|
14528
|
+
if (this.#slack) {
|
|
14529
|
+
await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
|
|
14530
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
14531
|
+
}
|
|
14532
|
+
}
|
|
14533
|
+
async #writeUnroutableSlackReply(threadId) {
|
|
14534
|
+
this.#increment('slackAnswersIgnoredNoInFlight');
|
|
14535
|
+
if (!this.#slack)
|
|
14536
|
+
return;
|
|
14537
|
+
await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
|
|
14538
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
13312
14539
|
}
|
|
13313
14540
|
async #wakeWaitingClarification(key, waiting) {
|
|
13314
14541
|
const existing = this.#clarificationWakeInFlight.get(key);
|
|
@@ -13680,6 +14907,7 @@ export class FactoryLoop {
|
|
|
13680
14907
|
const batch = await this.#batch();
|
|
13681
14908
|
if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
|
|
13682
14909
|
this.#increment('slackTriageAnswersIgnoredAlreadyActive');
|
|
14910
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
13683
14911
|
return;
|
|
13684
14912
|
}
|
|
13685
14913
|
if (await this.#dispatchBlockReason(record.issue)) {
|
|
@@ -13696,6 +14924,10 @@ export class FactoryLoop {
|
|
|
13696
14924
|
if (hasDispatchableRoute(decision)) {
|
|
13697
14925
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13698
14926
|
const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
|
|
14927
|
+
const active = await this.#batch();
|
|
14928
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14929
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14930
|
+
}
|
|
13699
14931
|
this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
|
|
13700
14932
|
return result;
|
|
13701
14933
|
}
|
|
@@ -13708,6 +14940,10 @@ export class FactoryLoop {
|
|
|
13708
14940
|
}
|
|
13709
14941
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13710
14942
|
const result = await this.#startOrQueueSlackClarifiedDecision(decision);
|
|
14943
|
+
const active = await this.#batch();
|
|
14944
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14945
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14946
|
+
}
|
|
13711
14947
|
this.#increment('slackTriageAnswersDispatched');
|
|
13712
14948
|
return result;
|
|
13713
14949
|
}
|
|
@@ -14237,6 +15473,15 @@ const githubIssueAuthor = (issue) => {
|
|
|
14237
15473
|
}
|
|
14238
15474
|
return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
|
|
14239
15475
|
};
|
|
15476
|
+
/**
|
|
15477
|
+
* An `IssueRef` for a path whose issue body could not be read at all — the
|
|
15478
|
+
* shape a relayfile-shed ready-issue read leaves behind (#297). The key is
|
|
15479
|
+
* what an operator needs to correlate the skip; the uuid falls back to it.
|
|
15480
|
+
*/
|
|
15481
|
+
const issueRefFromPath = (path) => {
|
|
15482
|
+
const key = keyFromPath(path);
|
|
15483
|
+
return { uuid: uuidFromPath(path) ?? key, key, path };
|
|
15484
|
+
};
|
|
14240
15485
|
const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
|
|
14241
15486
|
// Preserve the historical Linear state namespace while keeping GitHub-native
|
|
14242
15487
|
// issue numbers independent across repositories in the same workspace.
|
|
@@ -16150,10 +17395,54 @@ const relayfileOverload = (error) => {
|
|
|
16150
17395
|
stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
|
|
16151
17396
|
return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
|
|
16152
17397
|
};
|
|
17398
|
+
/**
|
|
17399
|
+
* relayfile's overload reason codes, allowlisted.
|
|
17400
|
+
*
|
|
17401
|
+
* `IterationReport.skipped[].reason` is serialized to stdout by `factory
|
|
17402
|
+
* run-once`, so it stays a fixed classification plus a known code — the same
|
|
17403
|
+
* public-surface rule #293 applied to error class names — rather than
|
|
17404
|
+
* whatever string the dependency happened to send.
|
|
17405
|
+
*/
|
|
17406
|
+
const RELAYFILE_OVERLOAD_REASONS = new Set([
|
|
17407
|
+
// Admission gate inside the workspace durable object.
|
|
17408
|
+
'inflight_limit',
|
|
17409
|
+
'oldest_inflight_age',
|
|
17410
|
+
'write_admission_limit',
|
|
17411
|
+
// Worker-side backpressure, per isolate rather than per workspace.
|
|
17412
|
+
'router_inflight_limit',
|
|
17413
|
+
// The Cloudflare runtime shed the object; relayfile only relabels it.
|
|
17414
|
+
'durable_object_overloaded',
|
|
17415
|
+
// relayfileOverload()'s fallback when the body carried no reason at all.
|
|
17416
|
+
'rate_limited',
|
|
17417
|
+
]);
|
|
17418
|
+
/**
|
|
17419
|
+
* Both the run-report reason and the per-reason counter key are built from
|
|
17420
|
+
* this, so an unknown code from the dependency can neither leak into stdout
|
|
17421
|
+
* nor open an unbounded counter namespace.
|
|
17422
|
+
*/
|
|
17423
|
+
const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
|
|
17424
|
+
/**
|
|
17425
|
+
* How long to wait before the next discovery sweep after relayfile shed this
|
|
17426
|
+
* one.
|
|
17427
|
+
*
|
|
17428
|
+
* The advertised `Retry-After` is authoritative in BOTH directions (#297):
|
|
17429
|
+
* it is the first rung, so we never retry sooner than the dependency allows,
|
|
17430
|
+
* and it bounds the ceiling, so we never sleep for minutes because of a
|
|
17431
|
+
* request to wait seconds. Without an advertised delay there is nothing to
|
|
17432
|
+
* respect and the original five-minute ladder governs unchanged.
|
|
17433
|
+
*/
|
|
16153
17434
|
const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
|
|
16154
|
-
const
|
|
16155
|
-
|
|
16156
|
-
|
|
17435
|
+
const advertisedMs = retryAfterSeconds === undefined
|
|
17436
|
+
? undefined
|
|
17437
|
+
: Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
|
|
17438
|
+
const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
|
|
17439
|
+
// A dependency that asks for longer than the advertised ceiling still gets
|
|
17440
|
+
// what it asked for; the ceiling only stops the ladder from overshooting it.
|
|
17441
|
+
const ceilingMs = advertisedMs === undefined
|
|
17442
|
+
? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
|
|
17443
|
+
: Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
|
|
17444
|
+
const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
|
|
17445
|
+
return Math.min(ceilingMs, baseMs * (2 ** steps));
|
|
16157
17446
|
};
|
|
16158
17447
|
const eventSequenceNumber = (eventId) => {
|
|
16159
17448
|
const whole = Number(eventId);
|
|
@@ -16218,8 +17507,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
16218
17507
|
}
|
|
16219
17508
|
return `${owner}/${repo}`;
|
|
16220
17509
|
};
|
|
16221
|
-
const githubPullRequestBody = (issue, preview) => [
|
|
16222
|
-
issue.description,
|
|
17510
|
+
const githubPullRequestBody = (issue, preview, sessionRef) => [
|
|
17511
|
+
stripTrajectoryPointers(issue.description),
|
|
16223
17512
|
'',
|
|
16224
17513
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
16225
17514
|
? `Fixes #${issue.key}`
|
|
@@ -16229,7 +17518,25 @@ const githubPullRequestBody = (issue, preview) => [
|
|
|
16229
17518
|
`Live preview: ${preview.url}`,
|
|
16230
17519
|
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
16231
17520
|
] : []),
|
|
17521
|
+
'',
|
|
17522
|
+
renderTrajectoryPointer({
|
|
17523
|
+
...trajectoryWorkUnitForIssue(issue),
|
|
17524
|
+
sessionRef,
|
|
17525
|
+
}),
|
|
16232
17526
|
].join('\n').trim();
|
|
17527
|
+
const trajectoryWorkUnitForIssue = (issue) => {
|
|
17528
|
+
const github = githubIssueSourceRef(issue);
|
|
17529
|
+
if (github) {
|
|
17530
|
+
return {
|
|
17531
|
+
workUnitId: `${github.owner}/${github.repo}#${github.number}`,
|
|
17532
|
+
workUnitSurface: 'github',
|
|
17533
|
+
};
|
|
17534
|
+
}
|
|
17535
|
+
if (isRealLinearIssue(issue)) {
|
|
17536
|
+
return { workUnitId: issue.key, workUnitSurface: 'linear' };
|
|
17537
|
+
}
|
|
17538
|
+
return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
|
|
17539
|
+
};
|
|
16233
17540
|
// The broker rejects re-registering a name it never released on exit
|
|
16234
17541
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
16235
17542
|
// the structured payload or the message so resume can treat it as terminal
|
|
@@ -16258,6 +17565,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
|
|
|
16258
17565
|
const seconds = Number(messageTs);
|
|
16259
17566
|
return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
|
|
16260
17567
|
};
|
|
17568
|
+
const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
|
|
17569
|
+
? watch.retiredAtMs
|
|
17570
|
+
: Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
|
|
16261
17571
|
const eventIdentity = (event) => {
|
|
16262
17572
|
const record = event;
|
|
16263
17573
|
const rawId = record.id ?? record.event_id ?? record.seq;
|
|
@@ -16361,10 +17671,6 @@ const telemetryCategory = (value) => {
|
|
|
16361
17671
|
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
|
|
16362
17672
|
return normalized.slice(0, 120) || undefined;
|
|
16363
17673
|
};
|
|
16364
|
-
const telemetryErrorClass = (error) => {
|
|
16365
|
-
const name = error instanceof Error ? error.name : '';
|
|
16366
|
-
return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
|
|
16367
|
-
};
|
|
16368
17674
|
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
16369
17675
|
const retryOnTimeout = async (fn, opts) => {
|
|
16370
17676
|
let lastError;
|
|
@@ -16569,6 +17875,77 @@ export class LiveDispatchStateChangedError extends Error {
|
|
|
16569
17875
|
export function isLiveDispatchStateChangedError(error) {
|
|
16570
17876
|
return error instanceof LiveDispatchStateChangedError;
|
|
16571
17877
|
}
|
|
17878
|
+
/** How deep to follow `cause` when classifying a wrapped failure. */
|
|
17879
|
+
const PASS_FATAL_CAUSE_DEPTH = 4;
|
|
17880
|
+
/**
|
|
17881
|
+
* Whether `error`, or anything it wraps, is an instance of `type`.
|
|
17882
|
+
* `contextualError` and the fleet control-plane guard both rethrow wrapped, so
|
|
17883
|
+
* classification has to follow the cause chain rather than trust the outermost
|
|
17884
|
+
* type.
|
|
17885
|
+
*/
|
|
17886
|
+
const wrapsErrorOfType = (error, type, depth = 0) => {
|
|
17887
|
+
if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
|
|
17888
|
+
return false;
|
|
17889
|
+
if (error instanceof type)
|
|
17890
|
+
return true;
|
|
17891
|
+
return wrapsErrorOfType(error.cause, type, depth + 1);
|
|
17892
|
+
};
|
|
17893
|
+
/**
|
|
17894
|
+
* How many *unclassified* per-item failures without an intervening successful
|
|
17895
|
+
* dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
|
|
17896
|
+
* a live-state race) never count toward it and never reset it: those
|
|
17897
|
+
* legitimately affect many units at once and are exactly the benign case #292
|
|
17898
|
+
* asks the loop to survive, so they are neither evidence of a pass-wide fault
|
|
17899
|
+
* nor evidence against one.
|
|
17900
|
+
*/
|
|
17901
|
+
const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
17902
|
+
/**
|
|
17903
|
+
* Failures the loop recognizes as belonging to one work unit. They are always
|
|
17904
|
+
* skippable and are exempt from the consecutive-failure fuse.
|
|
17905
|
+
*/
|
|
17906
|
+
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
17907
|
+
error instanceof DispatchLifecycleClaimRefusedError ||
|
|
17908
|
+
// Relayfile shedding one operation is a state of the dependency, not an
|
|
17909
|
+
// unexplained fault, and it has its own fuse — see #297 and
|
|
17910
|
+
// DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
|
|
17911
|
+
relayfileOverload(error) !== undefined;
|
|
17912
|
+
/**
|
|
17913
|
+
* Whether a per-item failure could have left half-spawned agents behind.
|
|
17914
|
+
*
|
|
17915
|
+
* The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
|
|
17916
|
+
* anything, so there is nothing to reap for them. Everything else can fail
|
|
17917
|
+
* *after* a spawn — including relayfile shedding a post-spawn read — having
|
|
17918
|
+
* persisted the agents as failure handoffs on the way out.
|
|
17919
|
+
*
|
|
17920
|
+
* Deliberately a denylist rather than an allowlist: a new failure mode that
|
|
17921
|
+
* nobody classified should default to "reap it", because the cost of a
|
|
17922
|
+
* needless reap is one no-op pass over an empty handoff list, while the cost
|
|
17923
|
+
* of a missed one is leaked agents and duplicate workers on the next retry.
|
|
17924
|
+
*/
|
|
17925
|
+
const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
|
|
17926
|
+
!(error instanceof DispatchLifecycleClaimRefusedError);
|
|
17927
|
+
/**
|
|
17928
|
+
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
17929
|
+
*
|
|
17930
|
+
* `factory run-once` serializes the whole report to stdout, so this string is
|
|
17931
|
+
* a public surface: it stays a fixed classification plus an allowlisted error
|
|
17932
|
+
* class name, never raw provider text or filesystem paths. The full message
|
|
17933
|
+
* goes to the operator log instead, the same split
|
|
17934
|
+
* `describeControlPlaneError` makes for circuit state.
|
|
17935
|
+
*/
|
|
17936
|
+
const perItemDispatchSkipReason = (error) => {
|
|
17937
|
+
const overload = relayfileOverload(error);
|
|
17938
|
+
if (overload)
|
|
17939
|
+
return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
|
|
17940
|
+
if (error instanceof LiveDispatchStateChangedError)
|
|
17941
|
+
return 'live state changed during dispatch';
|
|
17942
|
+
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
17943
|
+
return error.refusal === 'terminal'
|
|
17944
|
+
? 'dispatch lifecycle already terminal'
|
|
17945
|
+
: 'dispatch lifecycle owned by another publisher';
|
|
17946
|
+
}
|
|
17947
|
+
return `dispatch failed (${telemetryErrorClass(error)})`;
|
|
17948
|
+
};
|
|
16572
17949
|
const triageEscalationQuestion = (decision, issue) => {
|
|
16573
17950
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
16574
17951
|
const subject = issue?.title?.trim() || decision.issue.key;
|