@agent-relay/factory 0.1.77 → 0.1.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -156,6 +156,8 @@ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
156
156
  const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
157
157
  const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
158
158
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
159
+ const REMOTE_AGENT_REGISTRATION_TIMEOUT_MS = 30_000;
160
+ const REMOTE_AGENT_REGISTRATION_POLL_MS = 500;
159
161
  /**
160
162
  * Ceiling on the durable capacity-wait re-arm (#303).
161
163
  *
@@ -167,6 +169,31 @@ const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
167
169
  * multi-hour run holds the slot honestly), so what is bounded is the *rate*.
168
170
  */
169
171
  const DISPATCH_LIFECYCLE_RETRY_MAX_MS = 30_000;
172
+ /**
173
+ * How many times a work unit's RELEASE may fail before it is dead-lettered.
174
+ *
175
+ * #303 bounded the rate of the capacity-wait re-arm and deliberately left its
176
+ * count unbounded, because waiting for capacity is legitimate: the slot will
177
+ * free eventually and abandoning the wait would drop real work. Release is the
178
+ * opposite shape. It is the last step of a work unit that is already finished
179
+ * — the issue is closed, the writeback is acknowledged, the batch slot is
180
+ * gone — so a release that has failed ten times is not waiting for anything.
181
+ * It is failing, and re-arming it at 1 Hz forever buys nothing.
182
+ *
183
+ * What that costs when it happens: `#finishDurableRelease` does not throw on a
184
+ * failed release, it returns `false` and re-arms itself, so the loop runs
185
+ * through the *success* path of every `.catch()` that might otherwise have
186
+ * bounded it. Each pass calls `#terminationRoots` once per agent, and
187
+ * `#writeInFlightRegistry` calls it once per agent again — each one a process
188
+ * table scan — plus a durable read and write. Three agents is order ten scans
189
+ * and several state operations per second, indefinitely, which is the same
190
+ * serialized-store spin #303 measured at 1477 GETs in 111 s.
191
+ *
192
+ * Ten attempts at the 1 Hz floor is ~10 s of genuine retry, which covers the
193
+ * transient failures release retries exist for (a brief control-plane blip, a
194
+ * lease handover) without covering a permanent one.
195
+ */
196
+ const DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS = 10;
170
197
  /** Rate limit for the capacity-wait warning once the backoff has capped. */
171
198
  const DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS = 60_000;
172
199
  const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
@@ -324,6 +351,26 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
324
351
  this.leaseUntilMs = leaseUntilMs;
325
352
  }
326
353
  }
354
+ class FleetPlacementUnavailableError extends Error {
355
+ capability;
356
+ constructor(capability) {
357
+ super(`Refusing remote dispatch: no live fleet node advertises ${capability}`);
358
+ this.capability = capability;
359
+ this.name = 'FleetPlacementUnavailableError';
360
+ }
361
+ }
362
+ class RemoteAgentRegistrationTimeoutError extends Error {
363
+ agentName;
364
+ cleanupConfirmed;
365
+ cleanupError;
366
+ constructor(agentName, cleanupConfirmed, cleanupError) {
367
+ super(`Remote agent ${agentName} did not register with the fleet before the startup deadline`);
368
+ this.agentName = agentName;
369
+ this.cleanupConfirmed = cleanupConfirmed;
370
+ this.cleanupError = cleanupError;
371
+ this.name = 'RemoteAgentRegistrationTimeoutError';
372
+ }
373
+ }
327
374
  /**
328
375
  * The durable dispatch-lifecycle claim was refused for one work unit: its
329
376
  * record is already terminal, or another publisher currently holds the lease.
@@ -424,6 +471,8 @@ export class FactoryLoop {
424
471
  #babysitterWakeUnreachableEscalateMs;
425
472
  #babysitterWakeUnreachableRetryMs;
426
473
  #startupAgentExitDrainTimeoutMs;
474
+ #dispatchLifecycleRetryMs;
475
+ #dispatchLifecycleRenewMs;
427
476
  #state;
428
477
  #workspaceId;
429
478
  #relayflows;
@@ -500,6 +549,16 @@ export class FactoryLoop {
500
549
  #dispatchTerminalWaiters = new Map();
501
550
  #dispatchLifecycleRetryTimers = new Map();
502
551
  #dispatchLifecycleDrives = new Set();
552
+ /**
553
+ * Failed release attempts per work unit, and the units that ran out.
554
+ *
555
+ * Keyed by `dispatchLifecycleKey`, so it follows the work unit rather than
556
+ * any one agent, surface or dispatcher — the same identity rule the AR-448
557
+ * duplicate established for claims. Counted only for release re-arms; a
558
+ * capacity or ownership wait is not a failure and must not spend the budget.
559
+ */
560
+ #dispatchLifecycleReleaseAttempts = new Map();
561
+ #dispatchLifecycleReleaseAbandoned = new Set();
503
562
  #abandonedDispatchReasons = new Map();
504
563
  /**
505
564
  * Live batch-capacity waits, keyed by issue (#303).
@@ -859,6 +918,8 @@ export class FactoryLoop {
859
918
  this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
860
919
  this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
861
920
  this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
921
+ this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
922
+ this.#dispatchLifecycleRenewMs = ports.dispatchLifecycleRenewMs ?? DISPATCH_LIFECYCLE_RENEW_MS;
862
923
  this.#workspaceId = config.workspaceId ?? 'default';
863
924
  this.#relayflows = ports.relayflows;
864
925
  this.#worktrees = ports.worktrees;
@@ -2746,8 +2807,9 @@ export class FactoryLoop {
2746
2807
  }
2747
2808
  async #assertFleetControlPlaneAvailable() {
2748
2809
  try {
2749
- await this.#fleet.roster();
2810
+ const roster = await this.#fleet.roster();
2750
2811
  this.#increment('fleetControlPlaneProbeSuccesses');
2812
+ return roster;
2751
2813
  }
2752
2814
  catch (error) {
2753
2815
  const health = this.#fleetControlPlane.status();
@@ -4687,11 +4749,14 @@ export class FactoryLoop {
4687
4749
  path: progress.path,
4688
4750
  });
4689
4751
  },
4690
- // One line per repository saying why the tree walk was necessary. The walk
4691
- // is a permanent silent fallback today, and a silent permanent fallback is
4692
- // indistinguishable from a fast path that is working. This makes the day
4693
- // the index becomes usable (relayfile-adapters#271) show up in the logs
4694
- // instead of passing unnoticed.
4752
+ // One line per repository saying why the tree walk was necessary. A silent
4753
+ // fallback is indistinguishable from a fast path that is working, and
4754
+ // that is precisely how twelve minutes of sequential reads hid for so
4755
+ // long. Now that the index CAN answer the question, these reasons are the
4756
+ // only way to tell "the index answered" from "the index exists and was
4757
+ // useless" — and on a mount written before
4758
+ // `@relayfile/adapter-github@0.5.7`, `index-without-head-ref` is the
4759
+ // expected reading until a re-ingest converges the rows.
4695
4760
  onIndexFallback: (repo, reason) => {
4696
4761
  this.#increment('probePrIndexFallbacks');
4697
4762
  this.#increment(PROBE_PR_INDEX_FALLBACK_COUNTERS[reason]);
@@ -4701,6 +4766,14 @@ export class FactoryLoop {
4701
4766
  reason,
4702
4767
  });
4703
4768
  },
4769
+ onIndexHit: (repo, prNumber) => {
4770
+ this.#increment('probePrIndexHits');
4771
+ this.#logger.debug?.('[factory] PR probe resolved from the pull index', {
4772
+ issue: issue.key,
4773
+ repo,
4774
+ prNumber,
4775
+ });
4776
+ },
4704
4777
  };
4705
4778
  }
4706
4779
  #logTimedProgress(message, startedAtMs, lastLoggedAtMs, metadata) {
@@ -4971,8 +5044,10 @@ export class FactoryLoop {
4971
5044
  // consuming a dispatch attempt. The mutation proxy probes again at the
4972
5045
  // actual spawn/resume boundary so a later control-plane fault still fails
4973
5046
  // closed.
4974
- if (!dryRun)
4975
- await this.#assertFleetControlPlaneAvailable();
5047
+ const admissionRoster = !dryRun ? await this.#assertFleetControlPlaneAvailable() : undefined;
5048
+ if (admissionRoster && this.#fleet.placementLocality === 'remote') {
5049
+ dispatchDecision = decisionWithVerifiedRemotePlacements(dispatchDecision, admissionRoster);
5050
+ }
4976
5051
  const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
4977
5052
  // Local dispatches need the same deterministic branch identity as remote
4978
5053
  // ones. Without it, every worker starts in the configured shared checkout
@@ -5389,6 +5464,13 @@ export class FactoryLoop {
5389
5464
  throw error;
5390
5465
  }
5391
5466
  settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
5467
+ if ((error instanceof FleetPlacementUnavailableError ||
5468
+ (error instanceof RemoteAgentRegistrationTimeoutError && error.cleanupConfirmed)) &&
5469
+ await this.#rollbackUnregisteredRemoteDispatch(record, spawnedForReaperHandoff)) {
5470
+ this.#increment('remoteDispatchAdmissionRollbacks');
5471
+ this.#error(error, decision.issue);
5472
+ throw error;
5473
+ }
5392
5474
  // A spawn can fail after the broker accepted it but before its ack
5393
5475
  // reached Factory. Include every planned worktree agent, not only the
5394
5476
  // acknowledged spawns, so cleanup never races a name-only survivor.
@@ -5637,8 +5719,23 @@ export class FactoryLoop {
5637
5719
  consecutiveFailures,
5638
5720
  failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
5639
5721
  intervalMs: this.#readinessReconcileIntervalMs,
5722
+ // The two bounds that can actually preempt this pass, published beside
5723
+ // the cadence that cannot. Omitting them made the stanza unreadable in
5724
+ // the one situation it exists for: `intervalMs` next to a climbing
5725
+ // `inFlightMs` looks exactly like an unbounded hang, and has twice been
5726
+ // reported as one. `timeoutMs` ends the wait, `sweepBudgetMs` unwinds
5727
+ // the sweep and hands the lease back — so the second is the one that
5728
+ // answers "when does this recover".
5729
+ timeoutMs: this.#readinessReconcileTimeoutMs,
5730
+ sweepBudgetMs: this.#discoverySweepBudgetMs,
5640
5731
  ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
5641
5732
  ...(inFlightMs !== undefined ? { inFlightMs } : {}),
5733
+ // Derived here rather than left to each reader. The public projection
5734
+ // already computed it (#295/#300); the heartbeat stanza did not, so the
5735
+ // surface an operator actually opens first was the one missing it.
5736
+ ...(inFlightMs !== undefined && this.#readinessReconcileIntervalMs > 0
5737
+ ? { missedPasses: Math.floor(inFlightMs / this.#readinessReconcileIntervalMs) }
5738
+ : {}),
5642
5739
  ...(this.#readinessReconcileLastDurationMs !== undefined
5643
5740
  ? { lastDurationMs: this.#readinessReconcileLastDurationMs }
5644
5741
  : {}),
@@ -6125,7 +6222,7 @@ export class FactoryLoop {
6125
6222
  return;
6126
6223
  this.#dispatchLifecycleRenewTimer = setInterval(() => {
6127
6224
  void this.#renewDispatchLifecycles();
6128
- }, DISPATCH_LIFECYCLE_RENEW_MS);
6225
+ }, this.#dispatchLifecycleRenewMs);
6129
6226
  this.#dispatchLifecycleRenewTimer.unref?.();
6130
6227
  }
6131
6228
  /**
@@ -6322,6 +6419,18 @@ export class FactoryLoop {
6322
6419
  }
6323
6420
  async #renewDispatchLifecycles() {
6324
6421
  for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
6422
+ // The snapshot above can outlive the ownership it records: a relinquish
6423
+ // that runs while this loop is awaiting the store for an earlier key
6424
+ // leaves a stale entry here, and renewing it would re-block a key this
6425
+ // process has already handed back (#391 review, P2). Re-read the live map
6426
+ // rather than trusting the snapshot.
6427
+ //
6428
+ // This narrows the window but does not close it on its own — the delete
6429
+ // can still land after this check and before the renew resolves. Closing
6430
+ // it is `renewDispatchLifecycle`'s expiry fence, which makes a
6431
+ // relinquished lease unrenewable no matter how the two race.
6432
+ if (this.#dispatchLifecycleEpochs.get(key) !== epoch)
6433
+ continue;
6325
6434
  const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
6326
6435
  if (!renewed) {
6327
6436
  this.#dispatchLifecycleEpochs.delete(key);
@@ -6912,19 +7021,206 @@ export class FactoryLoop {
6912
7021
  }
6913
7022
  this.#increment('dispatchCapacityBackoffResets');
6914
7023
  }
6915
- #scheduleDispatchLifecycleRetry(record, delayMs = DISPATCH_LIFECYCLE_RETRY_MS) {
7024
+ /**
7025
+ * Charge one failed release against a work unit's budget.
7026
+ *
7027
+ * Returns `false` once the budget is spent, and the caller must NOT re-arm.
7028
+ * Failing closed here is deliberate and is the opposite of a dispatch gate:
7029
+ * an unrecordable dispatch claim must abort the dispatch, but an unbounded
7030
+ * release must abort the RETRY — the work is already done either way, and
7031
+ * the only thing still running is the spin.
7032
+ */
7033
+ #chargeReleaseAttempt(record, key, context) {
7034
+ if (this.#dispatchLifecycleReleaseAbandoned.has(key)) {
7035
+ // Re-entry, and the reason relinquishing the lease once is not enough.
7036
+ // `#driveDispatchLifecycle` re-claims the lease at the TOP of every
7037
+ // drive, before it has read the phase, so anything that drives an
7038
+ // already-dead-lettered key — the held-agent-deadline sweep, a registry
7039
+ // restore, a takeover — puts the epoch straight back into the renewal
7040
+ // map and re-arms the livelock this bound just escaped. Whenever the
7041
+ // budget declines a re-arm, ownership goes back too.
7042
+ this.#trackDispatchLifecycleDrive(this.#relinquishDispatchLifecycleLease(key, record.issue.key));
7043
+ return false;
7044
+ }
7045
+ const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
7046
+ this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
7047
+ if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
7048
+ return true;
7049
+ this.#dispatchLifecycleReleaseAbandoned.add(key);
7050
+ this.#dispatchLifecycleReleaseAttempts.delete(key);
7051
+ this.#increment('dispatchLifecycleReleaseAbandoned');
7052
+ const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
7053
+ // Cleanup is armed BEFORE anything that can throw (#391 review, P1). The
7054
+ // only statements above are set/map writes and a counter, none of which can
7055
+ // reject; `this.#logger.error` below is caller-supplied and can. Ordering
7056
+ // the drive first means nothing between "this unit is abandoned" and "its
7057
+ // lease is handed back" is allowed to fail in a way that skips the handback
7058
+ // — which is precisely the defect shape this whole method exists to fix.
7059
+ this.#trackDispatchLifecycleDrive(this.#releaseDeadLetteredSlot(record, key)
7060
+ .catch((error) => {
7061
+ this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
7062
+ issue: record.issue.key,
7063
+ error: describeError(error).errorMessage,
7064
+ });
7065
+ }));
7066
+ // `error`, not `warn`. Every previous layer of this failure was invisible
7067
+ // until somebody read stderr by hand; a work unit whose cleanup this
7068
+ // process has permanently given up on is exactly the event that must not
7069
+ // be inferable only from the absence of further log lines.
7070
+ this.#logger.error?.('[factory] release retries exhausted; abandoning cleanup for this work unit', {
7071
+ issue: record.issue.key,
7072
+ attempts: attempts - 1,
7073
+ maxAttempts: DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS,
7074
+ context,
7075
+ // The durable lifecycle is retained on purpose: a takeover or a restart
7076
+ // re-drives it from the persisted phase. This bounds THIS process's
7077
+ // spin, it does not declare the work unit clean.
7078
+ durableLifecycleRetained,
7079
+ });
7080
+ return false;
7081
+ }
7082
+ /** Keeps a lifecycle-side effect awaitable by `stop()` without leaking the set entry. */
7083
+ #trackDispatchLifecycleDrive(promise) {
7084
+ const drive = promise.finally(() => this.#dispatchLifecycleDrives.delete(drive));
7085
+ this.#dispatchLifecycleDrives.add(drive);
7086
+ }
7087
+ /**
7088
+ * Stop asserting durable ownership of a work unit this process will not drive
7089
+ * again.
7090
+ *
7091
+ * `#dispatchLifecycleEpochs` is not merely a cache. `#renewDispatchLifecycles`
7092
+ * walks it every `DISPATCH_LIFECYCLE_RENEW_MS` and re-stamps a full
7093
+ * `DISPATCH_LIFECYCLE_LEASE_MS` onto every key it finds, unconditionally.
7094
+ *
7095
+ * A dead-lettered release deliberately leaves its row in the non-terminal
7096
+ * `releasing` phase so a successor or a restart can re-drive the cleanup with
7097
+ * a fresh budget. That is only a recovery path if the successor can CLAIM the
7098
+ * row — and a lease renewed forever by a process that has permanently given
7099
+ * up driving it is a claim nobody can ever win. #379 bounded the retry and
7100
+ * handed back the batch slot, then kept the key locked for the life of the
7101
+ * process, which is how production reached four issues (a dispatch canary
7102
+ * among them) all logging `durable dispatch is leased by another publisher`
7103
+ * at 1 Hz for three days while the holder logged nothing but 503
7104
+ * `agent_host_unavailable`.
7105
+ *
7106
+ * The epoch is dropped BEFORE the durable release, so a renewal tick that
7107
+ * STARTS after this point finds nothing to renew. That ordering alone is not
7108
+ * sufficient and an earlier version of this comment wrongly claimed it was
7109
+ * (#391 review, P2): `#renewDispatchLifecycles` iterates a snapshot array, so
7110
+ * a tick already in flight still holds this key and would restore the lease
7111
+ * for a full term. The guarantee comes from `renewDispatchLifecycle` fencing
7112
+ * on expiry as well as owner and epoch, which makes a relinquished lease
7113
+ * unrenewable however the two race; the epoch drop and the loop's live re-read
7114
+ * narrow the window ahead of it.
7115
+ *
7116
+ * If the durable release itself fails, the dropped epoch alone still ends the
7117
+ * livelock: nothing renews the lease any more, so it expires within
7118
+ * `DISPATCH_LIFECYCLE_LEASE_MS` instead of never.
7119
+ */
7120
+ async #relinquishDispatchLifecycleLease(key, issueKey) {
7121
+ const epoch = this.#dispatchLifecycleEpochs.get(key);
7122
+ if (epoch === undefined)
7123
+ return;
7124
+ this.#dispatchLifecycleEpochs.delete(key);
7125
+ try {
7126
+ await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
7127
+ this.#increment('dispatchLifecycleLeasesRelinquished');
7128
+ }
7129
+ catch (error) {
7130
+ this.#logger.warn?.('[factory] could not relinquish the dispatch lease of an abandoned work unit', {
7131
+ issue: issueKey,
7132
+ // The lease still expires on its own now that nothing renews it.
7133
+ expiresWithinMs: DISPATCH_LIFECYCLE_LEASE_MS,
7134
+ error: describeError(error).errorMessage,
7135
+ });
7136
+ }
7137
+ }
7138
+ /**
7139
+ * Hand back the batch slot of a work unit whose release was dead-lettered.
7140
+ *
7141
+ * Without this the bound trades an unbounded 1 Hz spin for a permanently
7142
+ * leaked slot, which is not obviously the better failure (#379 review, P1):
7143
+ * a spin is loud and self-describing, whereas an in-flight record nobody will
7144
+ * ever complete silently reduces dispatch capacity until the process is
7145
+ * restarted. The slot is process-local accounting and is always freed here.
7146
+ *
7147
+ * What is NOT declared clean is the work itself. On a durable lifecycle the
7148
+ * persisted `releasing` phase is deliberately left in place, so a successor
7149
+ * or a restart re-drives the same cleanup with a fresh budget; freeing the
7150
+ * local slot does not write a terminal phase. On a local lifecycle there is
7151
+ * no durable record to retain, so the batch record is all there is and
7152
+ * completing it is what keeps capacity honest.
7153
+ *
7154
+ * The DURABLE lease has to go back with the slot. Retaining the row for a
7155
+ * successor while renewing the lease that locks the successor out is not a
7156
+ * handoff, it is a permanent block on the key — see
7157
+ * `#relinquishDispatchLifecycleLease`.
7158
+ *
7159
+ * The handback lives in a `finally` for a reason worth stating plainly
7160
+ * (#391 review, P1). The FIRST version of this fix relinquished after
7161
+ * `#writeInFlightRegistry()`, on the happy path only — so a rejecting
7162
+ * registry write would skip it and leave the abandoned key renewing its lease
7163
+ * forever, with nothing but a `warn` to show for it. That is the identical
7164
+ * shape of the bug being fixed (#379 freed the slot but not the lease, on the
7165
+ * failure path), reproduced one level up. Cleanup that only runs when the
7166
+ * rest of cleanup succeeded is not cleanup. `#batch()` and
7167
+ * `#writeInFlightRegistry()` can both reject; neither may strand the key.
7168
+ */
7169
+ async #releaseDeadLetteredSlot(record, key) {
7170
+ let next;
7171
+ try {
7172
+ const batch = await this.#batch();
7173
+ next = batch.complete(record.issue);
7174
+ this.#uncompensatedDispatchClaims.delete(key);
7175
+ await this.#writeInFlightRegistry();
7176
+ }
7177
+ finally {
7178
+ // Unconditional, and safe to put in a `finally` because
7179
+ // `#relinquishDispatchLifecycleLease` handles its own errors and cannot
7180
+ // throw — so it can never mask the failure that brought us here.
7181
+ await this.#relinquishDispatchLifecycleLease(key, record.issue.key);
7182
+ }
7183
+ // A freed slot that nothing is admitted into is only half the repair.
7184
+ if (next && !this.#stopping)
7185
+ await this.dispatch(next.decision, { dryRun: next.dryRun });
7186
+ }
7187
+ /** Clears a work unit's release budget once cleanup actually succeeds. */
7188
+ #clearReleaseAttempts(key) {
7189
+ this.#dispatchLifecycleReleaseAttempts.delete(key);
7190
+ this.#dispatchLifecycleReleaseAbandoned.delete(key);
7191
+ }
7192
+ #scheduleDispatchLifecycleRetry(record, delayMs = this.#dispatchLifecycleRetryMs, opts = {}) {
6916
7193
  const key = dispatchLifecycleKey(record.issue);
6917
7194
  if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
6918
7195
  return;
7196
+ // Only a release re-arm spends the budget. A capacity or ownership wait is
7197
+ // a legitimate wait for someone else to finish (#303) and is bounded by
7198
+ // its rate, not its count — spending the budget on one would abandon work
7199
+ // that was never failing.
7200
+ if (opts.releaseAttempt === true && !this.#chargeReleaseAttempt(record, key, 'durable-lifecycle'))
7201
+ return;
6919
7202
  const timer = setTimeout(() => {
6920
7203
  this.#dispatchLifecycleRetryTimers.delete(key);
6921
7204
  const drive = this.#driveDispatchLifecycle(key)
6922
7205
  .then(() => {
6923
7206
  this.#dispatchLifecycleCapacityWaits.delete(key);
6924
7207
  this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
7208
+ // DELIBERATELY NOT `#clearReleaseAttempts` (#379 review, P1).
7209
+ //
7210
+ // A FAILED release reaches here. `#finishDurableRelease` returns
7211
+ // `false` rather than throwing, and `#driveDispatchLifecycle`
7212
+ // discards that boolean in its `phase === 'releasing'` branch, so
7213
+ // the drive RESOLVES on a failed release and this handler runs on
7214
+ // every re-arm. Clearing the budget here zeroed it once per pass and
7215
+ // the counter could never reach the cap — the same never-fires this
7216
+ // bound exists to avoid, moved from the `.catch()` to the `.then()`.
7217
+ //
7218
+ // The budget is refunded where success is actually known:
7219
+ // `#finishDurableRelease` clears it on real per-agent progress and
7220
+ // again when the work unit completes.
6925
7221
  })
6926
7222
  .catch((error) => {
6927
- let nextDelayMs = DISPATCH_LIFECYCLE_RETRY_MS;
7223
+ let nextDelayMs = this.#dispatchLifecycleRetryMs;
6928
7224
  if (error instanceof DispatchLifecycleCapacityError) {
6929
7225
  this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
6930
7226
  nextDelayMs = this.#recordDispatchCapacityWait(record, key);
@@ -6939,7 +7235,7 @@ export class FactoryLoop {
6939
7235
  leaseRemainingMs: error.leaseUntilMs === undefined
6940
7236
  ? undefined
6941
7237
  : Math.max(0, error.leaseUntilMs - this.#clock.now()),
6942
- retryMs: DISPATCH_LIFECYCLE_RETRY_MS,
7238
+ retryMs: this.#dispatchLifecycleRetryMs,
6943
7239
  });
6944
7240
  }
6945
7241
  }
@@ -6951,6 +7247,20 @@ export class FactoryLoop {
6951
7247
  error: describeError(error).errorMessage,
6952
7248
  });
6953
7249
  }
7250
+ // Deliberately UNCHARGED (#379 review, P1). This arm re-arms for
7251
+ // dispatch, publishing and recovery failures as well as releases, and
7252
+ // charging all of them would dead-letter a work unit that was never
7253
+ // stuck in a release loop. Charging is confined to
7254
+ // `#scheduleReleaseRetry`, whose every caller is a release failure:
7255
+ // the three inside `#finishDurableRelease` and `#completeIssue`'s
7256
+ // catch once `releaseReasonForRetry` is set.
7257
+ //
7258
+ // The gap this leaves, stated plainly: a release failure that THREW
7259
+ // out of `#finishDurableRelease` instead of returning `false` would
7260
+ // reach here and re-arm unbounded. Every failure path in that method
7261
+ // returns `false` and schedules its own retry, so this is not a
7262
+ // reachable shape today — and if one appears it degrades to the
7263
+ // pre-existing unbounded behaviour rather than to a wrong dead-letter.
6954
7264
  this.#scheduleDispatchLifecycleRetry(record, nextDelayMs);
6955
7265
  })
6956
7266
  .finally(() => this.#dispatchLifecycleDrives.delete(drive));
@@ -6960,12 +7270,19 @@ export class FactoryLoop {
6960
7270
  }
6961
7271
  #scheduleReleaseRetry(record, reason) {
6962
7272
  if (this.#usesDurableDispatchLifecycle()) {
6963
- this.#scheduleDispatchLifecycleRetry(record);
7273
+ this.#scheduleDispatchLifecycleRetry(record, this.#dispatchLifecycleRetryMs, { releaseAttempt: true });
6964
7274
  return;
6965
7275
  }
6966
7276
  const key = dispatchLifecycleKey(record.issue);
6967
7277
  if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
6968
7278
  return;
7279
+ // Charged here rather than in the `.catch()` below, because the loop this
7280
+ // bounds does not go through the `.catch()`: `#finishDurableRelease`
7281
+ // returns `false` on a failed release and re-arms itself, so every re-arm
7282
+ // arrives on the resolved path. A bound on the rejection handler alone
7283
+ // would have been a fix that never fired.
7284
+ if (!this.#chargeReleaseAttempt(record, key, 'local-completion'))
7285
+ return;
6969
7286
  const timer = setTimeout(() => {
6970
7287
  this.#dispatchLifecycleRetryTimers.delete(key);
6971
7288
  const drive = this.#finishDurableRelease(record, reason)
@@ -6979,7 +7296,7 @@ export class FactoryLoop {
6979
7296
  })
6980
7297
  .finally(() => this.#dispatchLifecycleDrives.delete(drive));
6981
7298
  this.#dispatchLifecycleDrives.add(drive);
6982
- }, DISPATCH_LIFECYCLE_RETRY_MS);
7299
+ }, this.#dispatchLifecycleRetryMs);
6983
7300
  timer.unref?.();
6984
7301
  this.#dispatchLifecycleRetryTimers.set(key, timer);
6985
7302
  }
@@ -7192,6 +7509,9 @@ export class FactoryLoop {
7192
7509
  return;
7193
7510
  }
7194
7511
  if (lifecycle.phase === 'releasing') {
7512
+ // The boolean is discarded here, as it always was — and that is exactly
7513
+ // why the scheduler's success handler must not refund the release budget
7514
+ // (#379 review, P1). A failed release resolves through this line.
7195
7515
  await this.#finishDurableRelease(record, lifecycle.releaseReason);
7196
7516
  }
7197
7517
  }
@@ -7398,6 +7718,7 @@ export class FactoryLoop {
7398
7718
  .filter((agent) => agent.releasedAtMs !== undefined)
7399
7719
  .map((agent) => agent.name) ?? this.#localReleaseCheckpoints.get(releaseKey) ?? []);
7400
7720
  const failed = [];
7721
+ const releasedOnEntry = released.size;
7401
7722
  for (const agent of record.agents) {
7402
7723
  if (released.has(agent[0]))
7403
7724
  continue;
@@ -7418,6 +7739,13 @@ export class FactoryLoop {
7418
7739
  await this.#writeInFlightRegistry();
7419
7740
  if (failed.length > 0) {
7420
7741
  this.#increment('dispatchLifecycleReleaseRetries');
7742
+ // Real progress refunds the budget, so the ten attempts bound CONSECUTIVE
7743
+ // no-progress passes rather than capping a slow multi-agent release. This
7744
+ // terminates because an agent released once is checkpointed and skipped
7745
+ // on the next pass, so the remaining set strictly shrinks — a refund can
7746
+ // only be earned a finite number of times.
7747
+ if (released.size > releasedOnEntry)
7748
+ this.#clearReleaseAttempts(releaseKey);
7421
7749
  this.#scheduleReleaseRetry(record, reason);
7422
7750
  return false;
7423
7751
  }
@@ -7436,6 +7764,10 @@ export class FactoryLoop {
7436
7764
  }
7437
7765
  const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
7438
7766
  this.#localReleaseCheckpoints.delete(releaseKey);
7767
+ // Every agent is released. Nothing is left to retry, so the budget goes
7768
+ // back — including the abandoned marker, so a reopened work unit that
7769
+ // reuses this key starts with a full budget rather than a spent one.
7770
+ this.#clearReleaseAttempts(releaseKey);
7439
7771
  if (next)
7440
7772
  await this.dispatch(next.decision, { dryRun: next.dryRun });
7441
7773
  // Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
@@ -9392,13 +9724,6 @@ export class FactoryLoop {
9392
9724
  batch.recordDryRun(record, spec, invocationId);
9393
9725
  return { name: spec.name };
9394
9726
  }
9395
- // Persist intent before the remote side effect. If the owner crashes after
9396
- // the spawn ack but before recording its result, takeover retries the same
9397
- // deterministic invocation id instead of inventing a second worker.
9398
- batch.recordPlanned(record, { ...spec, invocationId });
9399
- if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
9400
- throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
9401
- }
9402
9727
  let roster;
9403
9728
  try {
9404
9729
  roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
@@ -9408,6 +9733,15 @@ export class FactoryLoop {
9408
9733
  }
9409
9734
  const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
9410
9735
  if (rosterAgent) {
9736
+ if (this.#fleet.placementLocality === 'remote') {
9737
+ const host = rosterAgent.node
9738
+ ? roster.nodes.find((node) => node.name === rosterAgent.node && node.live && node.capabilities.includes(spec.capability))
9739
+ : undefined;
9740
+ if (!host) {
9741
+ throw new FleetPlacementUnavailableError(spec.capability);
9742
+ }
9743
+ spec = { ...spec, node: host.name };
9744
+ }
9411
9745
  const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
9412
9746
  record.heldSinceAtMs ??= this.#clock.now();
9413
9747
  batch.recordSpawn(record, spec, invocationId, {
@@ -9425,6 +9759,21 @@ export class FactoryLoop {
9425
9759
  await this.#reportAgent(record, adopted, 'agent.adopted');
9426
9760
  return { name: spec.name };
9427
9761
  }
9762
+ if (this.#fleet.placementLocality === 'remote') {
9763
+ const loads = new Map();
9764
+ for (const agent of roster.agents) {
9765
+ if (agent.node)
9766
+ loads.set(agent.node, (loads.get(agent.node) ?? 0) + 1);
9767
+ }
9768
+ spec = { ...spec, node: liveFleetNodeForSpec(spec, roster, loads) };
9769
+ }
9770
+ // Persist intent before the remote side effect. If the owner crashes after
9771
+ // the spawn ack but before recording its result, takeover retries the same
9772
+ // deterministic invocation id instead of inventing a second worker.
9773
+ batch.recordPlanned(record, { ...spec, invocationId });
9774
+ if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
9775
+ throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
9776
+ }
9428
9777
  await this.#prepareAgentWorktree(record, spec);
9429
9778
  let result;
9430
9779
  try {
@@ -9464,6 +9813,26 @@ export class FactoryLoop {
9464
9813
  await this.#releaseOrphanedLatePlacement(record, spec, result);
9465
9814
  throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
9466
9815
  }
9816
+ if (this.#fleet.placementLocality === 'remote') {
9817
+ const registered = await this.#awaitRemoteAgentRegistration(result.name, spec.capability, result.node);
9818
+ if (!registered) {
9819
+ try {
9820
+ await this.#fleet.release(result.name, 'spawn-registration-timeout');
9821
+ this.#fleet.markAgentTerminal?.(result.name, 'spawn-registration-timeout');
9822
+ throw new RemoteAgentRegistrationTimeoutError(result.name, true);
9823
+ }
9824
+ catch (error) {
9825
+ if (error instanceof RemoteAgentRegistrationTimeoutError)
9826
+ throw error;
9827
+ // Cleanup is unconfirmed. Persist the placement so a successor can
9828
+ // retry the release; forgetting it would be worse than retaining a
9829
+ // nonterminal lifecycle for a worker that may still be alive.
9830
+ batch.recordSpawn(record, spec, invocationId, result);
9831
+ await this.#saveDispatchLifecycle(record, 'dispatching');
9832
+ throw new RemoteAgentRegistrationTimeoutError(result.name, false, error);
9833
+ }
9834
+ }
9835
+ }
9467
9836
  record.heldSinceAtMs ??= this.#clock.now();
9468
9837
  batch.recordSpawn(record, spec, invocationId, result);
9469
9838
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
@@ -9475,6 +9844,89 @@ export class FactoryLoop {
9475
9844
  await this.#reportAgent(record, spawned, 'agent.spawned');
9476
9845
  return { name: result.name };
9477
9846
  }
9847
+ async #awaitRemoteAgentRegistration(name, capability, expectedNode) {
9848
+ const deadlineAtMs = this.#clock.now() + REMOTE_AGENT_REGISTRATION_TIMEOUT_MS;
9849
+ do {
9850
+ try {
9851
+ if (expectedNode) {
9852
+ if (this.#fleet.isAgentRegistered) {
9853
+ if (await this.#fleet.isAgentRegistered({ name, node: expectedNode, capability }))
9854
+ return true;
9855
+ }
9856
+ else {
9857
+ const roster = await this.#fleet.roster();
9858
+ const agent = roster.agents.find((candidate) => candidate.name === name && candidate.node === expectedNode);
9859
+ const node = agent
9860
+ ? roster.nodes.find((candidate) => candidate.name === expectedNode && candidate.live && candidate.capabilities.includes(capability))
9861
+ : undefined;
9862
+ if (agent && node)
9863
+ return true;
9864
+ }
9865
+ }
9866
+ }
9867
+ catch (error) {
9868
+ this.#logger.warn?.('[factory] remote agent registration probe failed; retrying within startup bound', {
9869
+ agent: name,
9870
+ error: describeError(error).errorMessage,
9871
+ });
9872
+ }
9873
+ const remainingMs = deadlineAtMs - this.#clock.now();
9874
+ if (remainingMs <= 0)
9875
+ return false;
9876
+ await this.#clock.sleep(Math.min(REMOTE_AGENT_REGISTRATION_POLL_MS, remainingMs));
9877
+ } while (this.#clock.now() <= deadlineAtMs);
9878
+ return false;
9879
+ }
9880
+ async #rollbackUnregisteredRemoteDispatch(record, acknowledged) {
9881
+ const handoffs = this.#dispatchFailureHandoffs(record, acknowledged);
9882
+ await this.#persistDispatchFailureReaperHandoff(record, handoffs);
9883
+ const hasWorktrees = handoffs.some((handoff) => handoff.worktree);
9884
+ if (hasWorktrees) {
9885
+ if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'spawn-registration-timeout', { skipNeverPlacedAgents: true }))
9886
+ return false;
9887
+ }
9888
+ else {
9889
+ const failed = await this.#releaseAndTerminateAgents(handoffs
9890
+ .filter((handoff) => handoff.tracked.result !== undefined)
9891
+ .map((handoff) => [handoff.name, handoff.tracked]), 'spawn-registration-timeout', 'completion');
9892
+ if (failed.length > 0)
9893
+ return false;
9894
+ for (const handoff of handoffs) {
9895
+ await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
9896
+ }
9897
+ }
9898
+ try {
9899
+ await this.#teardownPreviews(record);
9900
+ }
9901
+ catch (error) {
9902
+ this.#logger.warn?.('[factory] retained unregistered spawn lifecycle after preview rollback failed', {
9903
+ issue: record.issue.key,
9904
+ error: describeError(error).errorMessage,
9905
+ });
9906
+ return false;
9907
+ }
9908
+ const key = dispatchLifecycleKey(record.issue);
9909
+ const epoch = this.#dispatchLifecycleEpochs.get(key);
9910
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
9911
+ const lease = lifecycle?.lease;
9912
+ if (epoch === undefined ||
9913
+ !lease ||
9914
+ lease.owner !== this.#dispatchLifecycleOwner ||
9915
+ lease.epoch !== epoch ||
9916
+ !await this.#state.clearClaimedDispatchLifecycle(this.#workspaceId, key, lease))
9917
+ return false;
9918
+ const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
9919
+ if (retryTimer)
9920
+ clearTimeout(retryTimer);
9921
+ this.#dispatchLifecycleRetryTimers.delete(key);
9922
+ this.#dispatchLifecycleEpochs.delete(key);
9923
+ this.#abandonedDispatchReasons.delete(key);
9924
+ const batch = await this.#batch();
9925
+ batch.abandon(record.issue);
9926
+ await this.#writeInFlightRegistry();
9927
+ this.#resetDispatchCapacityBackoff();
9928
+ return true;
9929
+ }
9478
9930
  /**
9479
9931
  * Is this process still the owner of a lifecycle that is not already done?
9480
9932
  *
@@ -18041,6 +18493,50 @@ function dispatchSpecs(decision) {
18041
18493
  }
18042
18494
  return [...decision.implementers, decision.reviewer];
18043
18495
  }
18496
+ function liveFleetNodeForSpec(spec, roster, assignedLoads) {
18497
+ const eligible = roster.nodes.filter((node) => node.live && node.capabilities.includes(spec.capability));
18498
+ if (eligible.length === 0)
18499
+ throw new FleetPlacementUnavailableError(spec.capability);
18500
+ const explicitlyRequested = spec.node && spec.node !== 'self'
18501
+ ? eligible.find((node) => node.name === spec.node)
18502
+ : undefined;
18503
+ const selected = explicitlyRequested ?? [...eligible].sort((left, right) => {
18504
+ const loadDifference = (assignedLoads.get(left.name) ?? 0) - (assignedLoads.get(right.name) ?? 0);
18505
+ return loadDifference || left.name.localeCompare(right.name);
18506
+ })[0];
18507
+ assignedLoads.set(selected.name, (assignedLoads.get(selected.name) ?? 0) + 1);
18508
+ return selected.name;
18509
+ }
18510
+ /**
18511
+ * Resolve every remote dispatch spec against one canonical roster snapshot.
18512
+ *
18513
+ * A configured node is a preference, not an entitlement: if it is offline or
18514
+ * no longer advertises the required capability, placement is re-selected from
18515
+ * the live fleet. This runs before durable lifecycle creation, so an empty
18516
+ * eligible set cannot leave a claim or consume a batch slot.
18517
+ */
18518
+ function decisionWithVerifiedRemotePlacements(decision, roster) {
18519
+ const assignedLoads = new Map();
18520
+ for (const agent of roster.agents) {
18521
+ if (agent.node)
18522
+ assignedLoads.set(agent.node, (assignedLoads.get(agent.node) ?? 0) + 1);
18523
+ }
18524
+ const place = (spec) => ({
18525
+ ...spec,
18526
+ node: liveFleetNodeForSpec(spec, roster, assignedLoads),
18527
+ });
18528
+ if (decision.scope === 'workflow') {
18529
+ return {
18530
+ ...structuredClone(decision),
18531
+ ...(decision.workflow ? { workflow: place(decision.workflow) } : {}),
18532
+ };
18533
+ }
18534
+ return {
18535
+ ...structuredClone(decision),
18536
+ implementers: decision.implementers.map(place),
18537
+ reviewer: place(decision.reviewer),
18538
+ };
18539
+ }
18044
18540
  function dispatchSessionOwner(decision) {
18045
18541
  for (const spec of dispatchSpecs(decision)) {
18046
18542
  const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
@@ -18904,28 +19400,59 @@ const PROBE_PR_INDEX_FALLBACK_COUNTERS = {
18904
19400
  'index-absent': 'probePrIndexAbsent',
18905
19401
  'index-shape-unrecognised': 'probePrIndexShapeUnrecognised',
18906
19402
  'index-without-head-ref': 'probePrIndexWithoutHeadRef',
18907
- // Not a fallback condition so much as a standing invitation: rows carry
18908
- // `headRef`, so the walk is now avoidable and this counter going non-zero is
18909
- // the cue to build the fast path.
18910
- 'index-usable': 'probePrIndexUsableButUnused',
19403
+ 'index-row-malformed': 'probePrIndexRowMalformed',
19404
+ 'index-incomplete': 'probePrIndexIncomplete',
19405
+ 'index-no-match': 'probePrIndexNoMatch',
19406
+ 'index-disagreed': 'probePrIndexDisagreed',
18911
19407
  };
18912
19408
  /**
18913
- * Classify the pull index without acting on it.
19409
+ * The lowest `issuePrMatchScore` tier the pull index can compute for itself.
18914
19410
  *
18915
- * Deliberately diagnostic only. Factory's issue-side reader
18916
- * (`#githubIssuePathsFromIndex`) uses its index as a NARROWING FILTER over
18917
- * paths it still reads individually, which is safe because it can only
18918
- * over-select. A pull index cannot be used that way today: it carries no
18919
- * `headRef`, so it cannot exclude any pull request from a branch match, and a
18920
- * title hit alone (score 20) must never be returned as the answer while an
18921
- * unread branch match (score 30) could outrank it. Until the row contract
18922
- * carries `headRef` (relayfile-adapters#271), the only honest thing to do with
18923
- * the index is say out loud why it did not help.
19411
+ * The scores are: branch match 30, issue key in title 20, explicit reference in
19412
+ * body 10. A row carries `headRef` and `title` but NOT `body`, so a row can be
19413
+ * scored exactly at the 30 and 20 tiers and is blind at 10. A pull request the
19414
+ * index scores at 20 or better therefore cannot be beaten by anything the index
19415
+ * could not see, because the best a body-only match can ever earn is 10 — which
19416
+ * makes an index hit of 20 or 30 provably the same winner the full walk would
19417
+ * have chosen. Below that threshold the index proves nothing and the walk is
19418
+ * mandatory.
19419
+ *
19420
+ * Deliberately expressed as a threshold rather than a "highest score wins"
19421
+ * shortcut: the constant is the whole safety argument, and inlining `20` at the
19422
+ * comparison would bury it.
18924
19423
  */
18925
- const classifyPullIndexFallback = async (mount, repo) => {
19424
+ const PULL_INDEX_DECISIVE_SCORE = 20;
19425
+ /**
19426
+ * Read `pulls/_index.json` and either return rows the probe may rank, or say
19427
+ * why it may not.
19428
+ *
19429
+ * All-or-nothing per repository, and the discipline is copied verbatim from
19430
+ * Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
19431
+ * ENTIRE repository if any row is legacy or malformed. One legacy row is enough
19432
+ * to poison the whole conclusion: the primary match is a branch match worth 30,
19433
+ * so a row without `headRef` cannot be ruled out as the real winner, and ranking
19434
+ * the remaining rows against each other would answer a different question from
19435
+ * the one the walk answers.
19436
+ *
19437
+ * Mixed indexes are the expected state today, not a corruption. `headRef` landed
19438
+ * in `@relayfile/adapter-github@0.5.7` and there is no pull-side backfill: the
19439
+ * incremental writers replace exactly the row they touched and pass every other
19440
+ * row through verbatim, so a webhook update hydrates one row and leaves the rest
19441
+ * legacy. Only an eager re-ingest converges a repository, because it rebuilds
19442
+ * the file from a list-pulls response in which every row carries `head.ref`.
19443
+ *
19444
+ * Row facts this relies on, from that contract: `state` is GitHub's lowercase
19445
+ * `"open"`/`"closed"` and there is no `"merged"` state — `merged: true` (with
19446
+ * `mergedAt`) is the only merged signal and `merged: false` is never written.
19447
+ * `headRef` is the bare branch name (`head.ref`), matching what
19448
+ * `readProbePrCandidate` reads off a record, NOT the `owner:branch` form of
19449
+ * `head.label`. `id` and `updated` are deliberately not required here: the
19450
+ * oldest legacy rows carry neither.
19451
+ */
19452
+ const readPullIndexForProbe = async (mount, repo) => {
18926
19453
  const [owner, name] = repo.split('/');
18927
19454
  if (!owner || !name)
18928
- return 'index-absent';
19455
+ return { reason: 'index-absent' };
18929
19456
  let parsed;
18930
19457
  try {
18931
19458
  const { content } = await mount.readFile(`${GITHUB_ISSUE_ROOT}/${owner}/${name}/pulls/_index.json`);
@@ -18936,23 +19463,159 @@ const classifyPullIndexFallback = async (mount, repo) => {
18936
19463
  // that is the same convention the tree reads below already follow.
18937
19464
  if (isPassWideRelayfileFault(error))
18938
19465
  throw error;
18939
- return 'index-absent';
19466
+ return { reason: 'index-absent' };
18940
19467
  }
18941
19468
  if (!Array.isArray(parsed))
18942
- return 'index-shape-unrecognised';
19469
+ return { reason: 'index-shape-unrecognised' };
18943
19470
  // An empty index proves nothing about the row contract, so it must not read
18944
19471
  // as usable — `[].every(...)` is vacuously true.
18945
- return parsed.length > 0 && parsed.every((entry) => typeof asRecord(entry)?.headRef === 'string')
18946
- ? 'index-usable'
18947
- : 'index-without-head-ref';
19472
+ if (parsed.length === 0)
19473
+ return { reason: 'index-without-head-ref' };
19474
+ const rows = [];
19475
+ for (const entry of parsed) {
19476
+ const row = asRecord(entry);
19477
+ if (typeof row?.headRef !== 'string')
19478
+ return { reason: 'index-without-head-ref' };
19479
+ if (!Number.isSafeInteger(row.number) || Number(row.number) <= 0 ||
19480
+ typeof row.title !== 'string' ||
19481
+ typeof row.state !== 'string' ||
19482
+ (row.merged !== undefined && typeof row.merged !== 'boolean')) {
19483
+ return { reason: 'index-row-malformed' };
19484
+ }
19485
+ rows.push({
19486
+ number: Number(row.number),
19487
+ title: row.title,
19488
+ headRef: row.headRef,
19489
+ state: row.state,
19490
+ merged: row.merged,
19491
+ });
19492
+ }
19493
+ return { rows };
18948
19494
  };
18949
- const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix), observer = {}) => {
19495
+ /**
19496
+ * Rank index rows exactly as the walk ranks records it has read.
19497
+ *
19498
+ * `body: ''` is a deliberate constant, not a placeholder for a value that
19499
+ * should have been threaded through. The index carries no body, so scoring with
19500
+ * an empty one makes `issuePrMatchScore` return only the tiers the index can
19501
+ * actually justify — 30 for a branch match, 20 for a title match, 0 otherwise —
19502
+ * and never invents the 10 it cannot see. Everything else (`requireTitleMarker`
19503
+ * gating, `allowLegacyGithubBranch`, the marker itself) is the SAME call the
19504
+ * walk makes on the same inputs, so the two paths cannot score differently.
19505
+ *
19506
+ * The tiebreak mirrors `b.score - a.score || b.prNumber - a.prNumber`: highest
19507
+ * score, then highest pull request number.
19508
+ */
19509
+ const bestPullIndexRow = (rows, issue, marker, opts) => {
19510
+ let best;
19511
+ for (const row of rows) {
19512
+ // Same derivation `readProbePrCandidate` applies to a record: a merged pull
19513
+ // request reports MERGED regardless of the raw `state` the provider wrote.
19514
+ const state = row.merged === true ? 'MERGED' : row.state;
19515
+ if (opts.openOnly && normalizePrState(state) !== 'OPEN')
19516
+ continue;
19517
+ const score = issuePrMatchScore({ title: row.title, body: '', headRef: row.headRef }, issue, marker, opts);
19518
+ if (score < PULL_INDEX_DECISIVE_SCORE)
19519
+ continue;
19520
+ if (!best || score > best.score || (score === best.score && row.number > best.number)) {
19521
+ best = { number: row.number, score };
19522
+ }
19523
+ }
19524
+ return best;
19525
+ };
19526
+ /**
19527
+ * Build the resolver's candidate shape from one pull request record.
19528
+ *
19529
+ * Shared by the index fast path and the walk so the two can never disagree on a
19530
+ * field. Differential equivalence is the whole safety property of the fast path
19531
+ * and it should hold structurally, not by two copies staying in sync.
19532
+ */
19533
+ const probePrCandidate = (repo, pr, path, score) => ({
19534
+ repo,
19535
+ prNumber: pr.number,
19536
+ draft: pr.draft,
19537
+ headRef: pr.headRef,
19538
+ headRepo: pr.headRepo,
19539
+ crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
19540
+ state: pr.state,
19541
+ url: pr.url,
19542
+ path,
19543
+ score,
19544
+ });
19545
+ /**
19546
+ * Answer "which pull request belongs to this issue?" from `pulls/_index.json`
19547
+ * instead of reading every mounted pull request record, or say why it could not.
19548
+ *
19549
+ * Reads at most two files: the index, and the one record the index names. That
19550
+ * record read is not an optimisation that was left on the table — it is load
19551
+ * bearing twice over. The index carries no `draft`, `headRepo`, `url` or mount
19552
+ * path, all of which the resolver's result type promises and callers persist as
19553
+ * `ownedPullRequest`; and re-scoring the real record is what turns "the index
19554
+ * said so" into "the record agrees", so a stale row can never decide an answer
19555
+ * on its own.
19556
+ *
19557
+ * The tree listing is NOT avoided, and that is deliberate. It costs two calls
19558
+ * per repository where the record walk cost one call per pull request — 4,000
19559
+ * of them in the live workspace, which is the cost that stalled a sweep for
19560
+ * 11m53s. Keeping it buys two things nothing else can: the exact path spelling
19561
+ * the walk would have reported (the nested layout's slug cannot be constructed,
19562
+ * only listed), and the completeness check below.
19563
+ *
19564
+ * Every condition that still walks:
19565
+ *
19566
+ * - anything `readPullIndexForProbe` refuses — absent, wrong shape, any row
19567
+ * without `headRef`, any row whose `number`/`title`/`state` is unusable;
19568
+ * - a pull request on disk with no row describing it, because an unindexed
19569
+ * record could hold the score-30 branch match;
19570
+ * - no row scoring `PULL_INDEX_DECISIVE_SCORE` or better, because a body-only
19571
+ * reference worth 10 may exist on disk and no index row can see a body;
19572
+ * - the named record failing to confirm its row — missing, different number,
19573
+ * different score, or not open when `openOnly` is set.
19574
+ */
19575
+ const resolveProbePrFromPullIndex = async (mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer) => {
19576
+ const reading = await readPullIndexForProbe(mount, repo);
19577
+ if (!reading.rows)
19578
+ return { reason: reading.reason };
19579
+ const indexed = new Set(reading.rows.map((row) => row.number));
19580
+ for (const number of pullNumbersOnDisk) {
19581
+ if (!indexed.has(number))
19582
+ return { reason: 'index-incomplete' };
19583
+ }
19584
+ const best = bestPullIndexRow(reading.rows, issue, marker, opts);
19585
+ if (!best)
19586
+ return { reason: 'index-no-match' };
19587
+ // The first spelling of this pull request in `walk` is by construction the
19588
+ // one the record loop below would have read: `walk` preserves the listing
19589
+ // order of `githubPullRoots` and has already collapsed the alias spellings,
19590
+ // keeping the first. So the fast path reports the same `path` the walk does.
19591
+ const path = walk.find((candidatePath) => githubPullPathParts(candidatePath)?.number === best.number);
19592
+ if (!path)
19593
+ return { reason: 'index-disagreed' };
19594
+ const pr = await readProbePrCandidate(mount, path);
19595
+ observer.onRead?.({ read: 1, total: 1, path });
19596
+ if (!pr)
19597
+ return { reason: 'index-disagreed' };
19598
+ if (opts.openOnly && normalizePrState(pr.state) !== 'OPEN')
19599
+ return { reason: 'index-disagreed' };
19600
+ // `readProbePrCandidate` takes the number from the PAYLOAD, falling back to
19601
+ // the path, so a record can disagree with the path that addressed it.
19602
+ if (pr.number !== best.number)
19603
+ return { reason: 'index-disagreed' };
19604
+ const score = issuePrMatchScore(pr, issue, marker, opts);
19605
+ if (score !== best.score)
19606
+ return { reason: 'index-disagreed' };
19607
+ observer.onIndexHit?.(repo, pr.number);
19608
+ return { candidate: probePrCandidate(repo, pr, path, score) };
19609
+ };
19610
+ // Exported for the differential-equivalence suite, which has to invoke the
19611
+ // index fast path and the record walk over ONE corpus and compare their two
19612
+ // answers field for field. Driving that through the loop's public surface would
19613
+ // compare a resolution to itself; the same precedent already exists in this
19614
+ // file for `githubIssuePathParts` and `keyFromPath`.
19615
+ export const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix), observer = {}) => {
18950
19616
  const candidates = [];
18951
19617
  const listErrors = [];
18952
19618
  for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
18953
- if (observer.onIndexFallback) {
18954
- observer.onIndexFallback(repo, await classifyPullIndexFallback(mount, repo));
18955
- }
18956
19619
  const paths = new Set();
18957
19620
  for (const root of githubPullRoots(repo)) {
18958
19621
  try {
@@ -18983,6 +19646,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
18983
19646
  // still read exactly as before rather than being filtered on a guess.
18984
19647
  const walk = [];
18985
19648
  const seenPulls = new Set();
19649
+ // The pull request numbers the TREE says exist, which is what makes the
19650
+ // index's completeness checkable below. Built here rather than separately
19651
+ // because `githubPullPathParts` has already been called on every path.
19652
+ const pullNumbersOnDisk = new Set();
18986
19653
  for (const path of paths) {
18987
19654
  if (!path.endsWith('.json'))
18988
19655
  continue;
@@ -18992,9 +19659,17 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
18992
19659
  if (seenPulls.has(identity))
18993
19660
  continue;
18994
19661
  seenPulls.add(identity);
19662
+ pullNumbersOnDisk.add(parts.number);
18995
19663
  }
18996
19664
  walk.push(path);
18997
19665
  }
19666
+ const marker = opts.titleMarker ?? config.safety.requireTitlePrefix;
19667
+ const fromIndex = await resolveProbePrFromPullIndex(mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer);
19668
+ if (fromIndex.candidate) {
19669
+ candidates.push(fromIndex.candidate);
19670
+ continue;
19671
+ }
19672
+ observer.onIndexFallback?.(repo, fromIndex.reason);
18998
19673
  let read = 0;
18999
19674
  for (const path of walk) {
19000
19675
  const pr = await readProbePrCandidate(mount, path);
@@ -19002,23 +19677,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
19002
19677
  observer.onRead?.({ read, total: walk.length, path });
19003
19678
  if (opts.openOnly && normalizePrState(pr?.state) !== 'OPEN')
19004
19679
  continue;
19005
- const score = pr
19006
- ? issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts)
19007
- : 0;
19680
+ const score = pr ? issuePrMatchScore(pr, issue, marker, opts) : 0;
19008
19681
  if (!pr || score <= 0)
19009
19682
  continue;
19010
- candidates.push({
19011
- repo,
19012
- prNumber: pr.number,
19013
- draft: pr.draft,
19014
- headRef: pr.headRef,
19015
- headRepo: pr.headRepo,
19016
- crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
19017
- state: pr.state,
19018
- url: pr.url,
19019
- path,
19020
- score,
19021
- });
19683
+ candidates.push(probePrCandidate(repo, pr, path, score));
19022
19684
  }
19023
19685
  }
19024
19686
  const resolved = candidates.sort((a, b) => b.score - a.score || b.prNumber - a.prNumber)[0];