@agent-relay/factory 0.1.79 → 0.1.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -156,6 +156,8 @@ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
156
156
  const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
157
157
  const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
158
158
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
159
+ const REMOTE_AGENT_REGISTRATION_TIMEOUT_MS = 30_000;
160
+ const REMOTE_AGENT_REGISTRATION_POLL_MS = 500;
159
161
  /**
160
162
  * Ceiling on the durable capacity-wait re-arm (#303).
161
163
  *
@@ -349,6 +351,26 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
349
351
  this.leaseUntilMs = leaseUntilMs;
350
352
  }
351
353
  }
354
+ class FleetPlacementUnavailableError extends Error {
355
+ capability;
356
+ constructor(capability) {
357
+ super(`Refusing remote dispatch: no live fleet node advertises ${capability}`);
358
+ this.capability = capability;
359
+ this.name = 'FleetPlacementUnavailableError';
360
+ }
361
+ }
362
+ class RemoteAgentRegistrationTimeoutError extends Error {
363
+ agentName;
364
+ cleanupConfirmed;
365
+ cleanupError;
366
+ constructor(agentName, cleanupConfirmed, cleanupError) {
367
+ super(`Remote agent ${agentName} did not register with the fleet before the startup deadline`);
368
+ this.agentName = agentName;
369
+ this.cleanupConfirmed = cleanupConfirmed;
370
+ this.cleanupError = cleanupError;
371
+ this.name = 'RemoteAgentRegistrationTimeoutError';
372
+ }
373
+ }
352
374
  /**
353
375
  * The durable dispatch-lifecycle claim was refused for one work unit: its
354
376
  * record is already terminal, or another publisher currently holds the lease.
@@ -450,6 +472,7 @@ export class FactoryLoop {
450
472
  #babysitterWakeUnreachableRetryMs;
451
473
  #startupAgentExitDrainTimeoutMs;
452
474
  #dispatchLifecycleRetryMs;
475
+ #dispatchLifecycleRenewMs;
453
476
  #state;
454
477
  #workspaceId;
455
478
  #relayflows;
@@ -896,6 +919,7 @@ export class FactoryLoop {
896
919
  this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
897
920
  this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
898
921
  this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
922
+ this.#dispatchLifecycleRenewMs = ports.dispatchLifecycleRenewMs ?? DISPATCH_LIFECYCLE_RENEW_MS;
899
923
  this.#workspaceId = config.workspaceId ?? 'default';
900
924
  this.#relayflows = ports.relayflows;
901
925
  this.#worktrees = ports.worktrees;
@@ -2783,8 +2807,9 @@ export class FactoryLoop {
2783
2807
  }
2784
2808
  async #assertFleetControlPlaneAvailable() {
2785
2809
  try {
2786
- await this.#fleet.roster();
2810
+ const roster = await this.#fleet.roster();
2787
2811
  this.#increment('fleetControlPlaneProbeSuccesses');
2812
+ return roster;
2788
2813
  }
2789
2814
  catch (error) {
2790
2815
  const health = this.#fleetControlPlane.status();
@@ -5019,8 +5044,10 @@ export class FactoryLoop {
5019
5044
  // consuming a dispatch attempt. The mutation proxy probes again at the
5020
5045
  // actual spawn/resume boundary so a later control-plane fault still fails
5021
5046
  // closed.
5022
- if (!dryRun)
5023
- await this.#assertFleetControlPlaneAvailable();
5047
+ const admissionRoster = !dryRun ? await this.#assertFleetControlPlaneAvailable() : undefined;
5048
+ if (admissionRoster && this.#fleet.placementLocality === 'remote') {
5049
+ dispatchDecision = decisionWithVerifiedRemotePlacements(dispatchDecision, admissionRoster);
5050
+ }
5024
5051
  const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
5025
5052
  // Local dispatches need the same deterministic branch identity as remote
5026
5053
  // ones. Without it, every worker starts in the configured shared checkout
@@ -5437,6 +5464,13 @@ export class FactoryLoop {
5437
5464
  throw error;
5438
5465
  }
5439
5466
  settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
5467
+ if ((error instanceof FleetPlacementUnavailableError ||
5468
+ (error instanceof RemoteAgentRegistrationTimeoutError && error.cleanupConfirmed)) &&
5469
+ await this.#rollbackUnregisteredRemoteDispatch(record, spawnedForReaperHandoff)) {
5470
+ this.#increment('remoteDispatchAdmissionRollbacks');
5471
+ this.#error(error, decision.issue);
5472
+ throw error;
5473
+ }
5440
5474
  // A spawn can fail after the broker accepted it but before its ack
5441
5475
  // reached Factory. Include every planned worktree agent, not only the
5442
5476
  // acknowledged spawns, so cleanup never races a name-only survivor.
@@ -6188,7 +6222,7 @@ export class FactoryLoop {
6188
6222
  return;
6189
6223
  this.#dispatchLifecycleRenewTimer = setInterval(() => {
6190
6224
  void this.#renewDispatchLifecycles();
6191
- }, DISPATCH_LIFECYCLE_RENEW_MS);
6225
+ }, this.#dispatchLifecycleRenewMs);
6192
6226
  this.#dispatchLifecycleRenewTimer.unref?.();
6193
6227
  }
6194
6228
  /**
@@ -6385,6 +6419,18 @@ export class FactoryLoop {
6385
6419
  }
6386
6420
  async #renewDispatchLifecycles() {
6387
6421
  for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
6422
+ // The snapshot above can outlive the ownership it records: a relinquish
6423
+ // that runs while this loop is awaiting the store for an earlier key
6424
+ // leaves a stale entry here, and renewing it would re-block a key this
6425
+ // process has already handed back (#391 review, P2). Re-read the live map
6426
+ // rather than trusting the snapshot.
6427
+ //
6428
+ // This narrows the window but does not close it on its own — the delete
6429
+ // can still land after this check and before the renew resolves. Closing
6430
+ // it is `renewDispatchLifecycle`'s expiry fence, which makes a
6431
+ // relinquished lease unrenewable no matter how the two race.
6432
+ if (this.#dispatchLifecycleEpochs.get(key) !== epoch)
6433
+ continue;
6388
6434
  const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
6389
6435
  if (!renewed) {
6390
6436
  this.#dispatchLifecycleEpochs.delete(key);
@@ -6985,8 +7031,17 @@ export class FactoryLoop {
6985
7031
  * the only thing still running is the spin.
6986
7032
  */
6987
7033
  #chargeReleaseAttempt(record, key, context) {
6988
- if (this.#dispatchLifecycleReleaseAbandoned.has(key))
7034
+ if (this.#dispatchLifecycleReleaseAbandoned.has(key)) {
7035
+ // Re-entry, and the reason relinquishing the lease once is not enough.
7036
+ // `#driveDispatchLifecycle` re-claims the lease at the TOP of every
7037
+ // drive, before it has read the phase, so anything that drives an
7038
+ // already-dead-lettered key — the held-agent-deadline sweep, a registry
7039
+ // restore, a takeover — puts the epoch straight back into the renewal
7040
+ // map and re-arms the livelock this bound just escaped. Whenever the
7041
+ // budget declines a re-arm, ownership goes back too.
7042
+ this.#trackDispatchLifecycleDrive(this.#relinquishDispatchLifecycleLease(key, record.issue.key));
6989
7043
  return false;
7044
+ }
6990
7045
  const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
6991
7046
  this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
6992
7047
  if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
@@ -6995,6 +7050,19 @@ export class FactoryLoop {
6995
7050
  this.#dispatchLifecycleReleaseAttempts.delete(key);
6996
7051
  this.#increment('dispatchLifecycleReleaseAbandoned');
6997
7052
  const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
7053
+ // Cleanup is armed BEFORE anything that can throw (#391 review, P1). The
7054
+ // only statements above are set/map writes and a counter, none of which can
7055
+ // reject; `this.#logger.error` below is caller-supplied and can. Ordering
7056
+ // the drive first means nothing between "this unit is abandoned" and "its
7057
+ // lease is handed back" is allowed to fail in a way that skips the handback
7058
+ // — which is precisely the defect shape this whole method exists to fix.
7059
+ this.#trackDispatchLifecycleDrive(this.#releaseDeadLetteredSlot(record, key)
7060
+ .catch((error) => {
7061
+ this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
7062
+ issue: record.issue.key,
7063
+ error: describeError(error).errorMessage,
7064
+ });
7065
+ }));
6998
7066
  // `error`, not `warn`. Every previous layer of this failure was invisible
6999
7067
  // until somebody read stderr by hand; a work unit whose cleanup this
7000
7068
  // process has permanently given up on is exactly the event that must not
@@ -7009,16 +7077,63 @@ export class FactoryLoop {
7009
7077
  // spin, it does not declare the work unit clean.
7010
7078
  durableLifecycleRetained,
7011
7079
  });
7012
- const drive = this.#releaseDeadLetteredSlot(record, key)
7013
- .catch((error) => {
7014
- this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
7015
- issue: record.issue.key,
7080
+ return false;
7081
+ }
7082
+ /** Keeps a lifecycle-side effect awaitable by `stop()` without leaking the set entry. */
7083
+ #trackDispatchLifecycleDrive(promise) {
7084
+ const drive = promise.finally(() => this.#dispatchLifecycleDrives.delete(drive));
7085
+ this.#dispatchLifecycleDrives.add(drive);
7086
+ }
7087
+ /**
7088
+ * Stop asserting durable ownership of a work unit this process will not drive
7089
+ * again.
7090
+ *
7091
+ * `#dispatchLifecycleEpochs` is not merely a cache. `#renewDispatchLifecycles`
7092
+ * walks it every `DISPATCH_LIFECYCLE_RENEW_MS` and re-stamps a full
7093
+ * `DISPATCH_LIFECYCLE_LEASE_MS` onto every key it finds, unconditionally.
7094
+ *
7095
+ * A dead-lettered release deliberately leaves its row in the non-terminal
7096
+ * `releasing` phase so a successor or a restart can re-drive the cleanup with
7097
+ * a fresh budget. That is only a recovery path if the successor can CLAIM the
7098
+ * row — and a lease renewed forever by a process that has permanently given
7099
+ * up driving it is a claim nobody can ever win. #379 bounded the retry and
7100
+ * handed back the batch slot, then kept the key locked for the life of the
7101
+ * process, which is how production reached four issues (a dispatch canary
7102
+ * among them) all logging `durable dispatch is leased by another publisher`
7103
+ * at 1 Hz for three days while the holder logged nothing but 503
7104
+ * `agent_host_unavailable`.
7105
+ *
7106
+ * The epoch is dropped BEFORE the durable release, so a renewal tick that
7107
+ * STARTS after this point finds nothing to renew. That ordering alone is not
7108
+ * sufficient and an earlier version of this comment wrongly claimed it was
7109
+ * (#391 review, P2): `#renewDispatchLifecycles` iterates a snapshot array, so
7110
+ * a tick already in flight still holds this key and would restore the lease
7111
+ * for a full term. The guarantee comes from `renewDispatchLifecycle` fencing
7112
+ * on expiry as well as owner and epoch, which makes a relinquished lease
7113
+ * unrenewable however the two race; the epoch drop and the loop's live re-read
7114
+ * narrow the window ahead of it.
7115
+ *
7116
+ * If the durable release itself fails, the dropped epoch alone still ends the
7117
+ * livelock: nothing renews the lease any more, so it expires within
7118
+ * `DISPATCH_LIFECYCLE_LEASE_MS` instead of never.
7119
+ */
7120
+ async #relinquishDispatchLifecycleLease(key, issueKey) {
7121
+ const epoch = this.#dispatchLifecycleEpochs.get(key);
7122
+ if (epoch === undefined)
7123
+ return;
7124
+ this.#dispatchLifecycleEpochs.delete(key);
7125
+ try {
7126
+ await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
7127
+ this.#increment('dispatchLifecycleLeasesRelinquished');
7128
+ }
7129
+ catch (error) {
7130
+ this.#logger.warn?.('[factory] could not relinquish the dispatch lease of an abandoned work unit', {
7131
+ issue: issueKey,
7132
+ // The lease still expires on its own now that nothing renews it.
7133
+ expiresWithinMs: DISPATCH_LIFECYCLE_LEASE_MS,
7016
7134
  error: describeError(error).errorMessage,
7017
7135
  });
7018
- })
7019
- .finally(() => this.#dispatchLifecycleDrives.delete(drive));
7020
- this.#dispatchLifecycleDrives.add(drive);
7021
- return false;
7136
+ }
7022
7137
  }
7023
7138
  /**
7024
7139
  * Hand back the batch slot of a work unit whose release was dead-lettered.
@@ -7035,12 +7150,36 @@ export class FactoryLoop {
7035
7150
  * local slot does not write a terminal phase. On a local lifecycle there is
7036
7151
  * no durable record to retain, so the batch record is all there is and
7037
7152
  * completing it is what keeps capacity honest.
7153
+ *
7154
+ * The DURABLE lease has to go back with the slot. Retaining the row for a
7155
+ * successor while renewing the lease that locks the successor out is not a
7156
+ * handoff, it is a permanent block on the key — see
7157
+ * `#relinquishDispatchLifecycleLease`.
7158
+ *
7159
+ * The handback lives in a `finally` for a reason worth stating plainly
7160
+ * (#391 review, P1). The FIRST version of this fix relinquished after
7161
+ * `#writeInFlightRegistry()`, on the happy path only — so a rejecting
7162
+ * registry write would skip it and leave the abandoned key renewing its lease
7163
+ * forever, with nothing but a `warn` to show for it. That is the identical
7164
+ * shape of the bug being fixed (#379 freed the slot but not the lease, on the
7165
+ * failure path), reproduced one level up. Cleanup that only runs when the
7166
+ * rest of cleanup succeeded is not cleanup. `#batch()` and
7167
+ * `#writeInFlightRegistry()` can both reject; neither may strand the key.
7038
7168
  */
7039
7169
  async #releaseDeadLetteredSlot(record, key) {
7040
- const batch = await this.#batch();
7041
- const next = batch.complete(record.issue);
7042
- this.#uncompensatedDispatchClaims.delete(key);
7043
- await this.#writeInFlightRegistry();
7170
+ let next;
7171
+ try {
7172
+ const batch = await this.#batch();
7173
+ next = batch.complete(record.issue);
7174
+ this.#uncompensatedDispatchClaims.delete(key);
7175
+ await this.#writeInFlightRegistry();
7176
+ }
7177
+ finally {
7178
+ // Unconditional, and safe to put in a `finally` because
7179
+ // `#relinquishDispatchLifecycleLease` handles its own errors and cannot
7180
+ // throw — so it can never mask the failure that brought us here.
7181
+ await this.#relinquishDispatchLifecycleLease(key, record.issue.key);
7182
+ }
7044
7183
  // A freed slot that nothing is admitted into is only half the repair.
7045
7184
  if (next && !this.#stopping)
7046
7185
  await this.dispatch(next.decision, { dryRun: next.dryRun });
@@ -9585,13 +9724,6 @@ export class FactoryLoop {
9585
9724
  batch.recordDryRun(record, spec, invocationId);
9586
9725
  return { name: spec.name };
9587
9726
  }
9588
- // Persist intent before the remote side effect. If the owner crashes after
9589
- // the spawn ack but before recording its result, takeover retries the same
9590
- // deterministic invocation id instead of inventing a second worker.
9591
- batch.recordPlanned(record, { ...spec, invocationId });
9592
- if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
9593
- throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
9594
- }
9595
9727
  let roster;
9596
9728
  try {
9597
9729
  roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
@@ -9601,6 +9733,15 @@ export class FactoryLoop {
9601
9733
  }
9602
9734
  const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
9603
9735
  if (rosterAgent) {
9736
+ if (this.#fleet.placementLocality === 'remote') {
9737
+ const host = rosterAgent.node
9738
+ ? roster.nodes.find((node) => node.name === rosterAgent.node && node.live && node.capabilities.includes(spec.capability))
9739
+ : undefined;
9740
+ if (!host) {
9741
+ throw new FleetPlacementUnavailableError(spec.capability);
9742
+ }
9743
+ spec = { ...spec, node: host.name };
9744
+ }
9604
9745
  const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
9605
9746
  record.heldSinceAtMs ??= this.#clock.now();
9606
9747
  batch.recordSpawn(record, spec, invocationId, {
@@ -9618,6 +9759,21 @@ export class FactoryLoop {
9618
9759
  await this.#reportAgent(record, adopted, 'agent.adopted');
9619
9760
  return { name: spec.name };
9620
9761
  }
9762
+ if (this.#fleet.placementLocality === 'remote') {
9763
+ const loads = new Map();
9764
+ for (const agent of roster.agents) {
9765
+ if (agent.node)
9766
+ loads.set(agent.node, (loads.get(agent.node) ?? 0) + 1);
9767
+ }
9768
+ spec = { ...spec, node: liveFleetNodeForSpec(spec, roster, loads) };
9769
+ }
9770
+ // Persist intent before the remote side effect. If the owner crashes after
9771
+ // the spawn ack but before recording its result, takeover retries the same
9772
+ // deterministic invocation id instead of inventing a second worker.
9773
+ batch.recordPlanned(record, { ...spec, invocationId });
9774
+ if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
9775
+ throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
9776
+ }
9621
9777
  await this.#prepareAgentWorktree(record, spec);
9622
9778
  let result;
9623
9779
  try {
@@ -9657,6 +9813,26 @@ export class FactoryLoop {
9657
9813
  await this.#releaseOrphanedLatePlacement(record, spec, result);
9658
9814
  throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
9659
9815
  }
9816
+ if (this.#fleet.placementLocality === 'remote') {
9817
+ const registered = await this.#awaitRemoteAgentRegistration(result.name, spec.capability, result.node);
9818
+ if (!registered) {
9819
+ try {
9820
+ await this.#fleet.release(result.name, 'spawn-registration-timeout');
9821
+ this.#fleet.markAgentTerminal?.(result.name, 'spawn-registration-timeout');
9822
+ throw new RemoteAgentRegistrationTimeoutError(result.name, true);
9823
+ }
9824
+ catch (error) {
9825
+ if (error instanceof RemoteAgentRegistrationTimeoutError)
9826
+ throw error;
9827
+ // Cleanup is unconfirmed. Persist the placement so a successor can
9828
+ // retry the release; forgetting it would be worse than retaining a
9829
+ // nonterminal lifecycle for a worker that may still be alive.
9830
+ batch.recordSpawn(record, spec, invocationId, result);
9831
+ await this.#saveDispatchLifecycle(record, 'dispatching');
9832
+ throw new RemoteAgentRegistrationTimeoutError(result.name, false, error);
9833
+ }
9834
+ }
9835
+ }
9660
9836
  record.heldSinceAtMs ??= this.#clock.now();
9661
9837
  batch.recordSpawn(record, spec, invocationId, result);
9662
9838
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
@@ -9668,6 +9844,89 @@ export class FactoryLoop {
9668
9844
  await this.#reportAgent(record, spawned, 'agent.spawned');
9669
9845
  return { name: result.name };
9670
9846
  }
9847
+ async #awaitRemoteAgentRegistration(name, capability, expectedNode) {
9848
+ const deadlineAtMs = this.#clock.now() + REMOTE_AGENT_REGISTRATION_TIMEOUT_MS;
9849
+ do {
9850
+ try {
9851
+ if (expectedNode) {
9852
+ if (this.#fleet.isAgentRegistered) {
9853
+ if (await this.#fleet.isAgentRegistered({ name, node: expectedNode, capability }))
9854
+ return true;
9855
+ }
9856
+ else {
9857
+ const roster = await this.#fleet.roster();
9858
+ const agent = roster.agents.find((candidate) => candidate.name === name && candidate.node === expectedNode);
9859
+ const node = agent
9860
+ ? roster.nodes.find((candidate) => candidate.name === expectedNode && candidate.live && candidate.capabilities.includes(capability))
9861
+ : undefined;
9862
+ if (agent && node)
9863
+ return true;
9864
+ }
9865
+ }
9866
+ }
9867
+ catch (error) {
9868
+ this.#logger.warn?.('[factory] remote agent registration probe failed; retrying within startup bound', {
9869
+ agent: name,
9870
+ error: describeError(error).errorMessage,
9871
+ });
9872
+ }
9873
+ const remainingMs = deadlineAtMs - this.#clock.now();
9874
+ if (remainingMs <= 0)
9875
+ return false;
9876
+ await this.#clock.sleep(Math.min(REMOTE_AGENT_REGISTRATION_POLL_MS, remainingMs));
9877
+ } while (this.#clock.now() <= deadlineAtMs);
9878
+ return false;
9879
+ }
9880
+ async #rollbackUnregisteredRemoteDispatch(record, acknowledged) {
9881
+ const handoffs = this.#dispatchFailureHandoffs(record, acknowledged);
9882
+ await this.#persistDispatchFailureReaperHandoff(record, handoffs);
9883
+ const hasWorktrees = handoffs.some((handoff) => handoff.worktree);
9884
+ if (hasWorktrees) {
9885
+ if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'spawn-registration-timeout', { skipNeverPlacedAgents: true }))
9886
+ return false;
9887
+ }
9888
+ else {
9889
+ const failed = await this.#releaseAndTerminateAgents(handoffs
9890
+ .filter((handoff) => handoff.tracked.result !== undefined)
9891
+ .map((handoff) => [handoff.name, handoff.tracked]), 'spawn-registration-timeout', 'completion');
9892
+ if (failed.length > 0)
9893
+ return false;
9894
+ for (const handoff of handoffs) {
9895
+ await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
9896
+ }
9897
+ }
9898
+ try {
9899
+ await this.#teardownPreviews(record);
9900
+ }
9901
+ catch (error) {
9902
+ this.#logger.warn?.('[factory] retained unregistered spawn lifecycle after preview rollback failed', {
9903
+ issue: record.issue.key,
9904
+ error: describeError(error).errorMessage,
9905
+ });
9906
+ return false;
9907
+ }
9908
+ const key = dispatchLifecycleKey(record.issue);
9909
+ const epoch = this.#dispatchLifecycleEpochs.get(key);
9910
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
9911
+ const lease = lifecycle?.lease;
9912
+ if (epoch === undefined ||
9913
+ !lease ||
9914
+ lease.owner !== this.#dispatchLifecycleOwner ||
9915
+ lease.epoch !== epoch ||
9916
+ !await this.#state.clearClaimedDispatchLifecycle(this.#workspaceId, key, lease))
9917
+ return false;
9918
+ const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
9919
+ if (retryTimer)
9920
+ clearTimeout(retryTimer);
9921
+ this.#dispatchLifecycleRetryTimers.delete(key);
9922
+ this.#dispatchLifecycleEpochs.delete(key);
9923
+ this.#abandonedDispatchReasons.delete(key);
9924
+ const batch = await this.#batch();
9925
+ batch.abandon(record.issue);
9926
+ await this.#writeInFlightRegistry();
9927
+ this.#resetDispatchCapacityBackoff();
9928
+ return true;
9929
+ }
9671
9930
  /**
9672
9931
  * Is this process still the owner of a lifecycle that is not already done?
9673
9932
  *
@@ -18234,6 +18493,50 @@ function dispatchSpecs(decision) {
18234
18493
  }
18235
18494
  return [...decision.implementers, decision.reviewer];
18236
18495
  }
18496
+ function liveFleetNodeForSpec(spec, roster, assignedLoads) {
18497
+ const eligible = roster.nodes.filter((node) => node.live && node.capabilities.includes(spec.capability));
18498
+ if (eligible.length === 0)
18499
+ throw new FleetPlacementUnavailableError(spec.capability);
18500
+ const explicitlyRequested = spec.node && spec.node !== 'self'
18501
+ ? eligible.find((node) => node.name === spec.node)
18502
+ : undefined;
18503
+ const selected = explicitlyRequested ?? [...eligible].sort((left, right) => {
18504
+ const loadDifference = (assignedLoads.get(left.name) ?? 0) - (assignedLoads.get(right.name) ?? 0);
18505
+ return loadDifference || left.name.localeCompare(right.name);
18506
+ })[0];
18507
+ assignedLoads.set(selected.name, (assignedLoads.get(selected.name) ?? 0) + 1);
18508
+ return selected.name;
18509
+ }
18510
+ /**
18511
+ * Resolve every remote dispatch spec against one canonical roster snapshot.
18512
+ *
18513
+ * A configured node is a preference, not an entitlement: if it is offline or
18514
+ * no longer advertises the required capability, placement is re-selected from
18515
+ * the live fleet. This runs before durable lifecycle creation, so an empty
18516
+ * eligible set cannot leave a claim or consume a batch slot.
18517
+ */
18518
+ function decisionWithVerifiedRemotePlacements(decision, roster) {
18519
+ const assignedLoads = new Map();
18520
+ for (const agent of roster.agents) {
18521
+ if (agent.node)
18522
+ assignedLoads.set(agent.node, (assignedLoads.get(agent.node) ?? 0) + 1);
18523
+ }
18524
+ const place = (spec) => ({
18525
+ ...spec,
18526
+ node: liveFleetNodeForSpec(spec, roster, assignedLoads),
18527
+ });
18528
+ if (decision.scope === 'workflow') {
18529
+ return {
18530
+ ...structuredClone(decision),
18531
+ ...(decision.workflow ? { workflow: place(decision.workflow) } : {}),
18532
+ };
18533
+ }
18534
+ return {
18535
+ ...structuredClone(decision),
18536
+ implementers: decision.implementers.map(place),
18537
+ reviewer: place(decision.reviewer),
18538
+ };
18539
+ }
18237
18540
  function dispatchSessionOwner(decision) {
18238
18541
  for (const spec of dispatchSpecs(decision)) {
18239
18542
  const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
@@ -19125,14 +19428,26 @@ const PULL_INDEX_DECISIVE_SCORE = 20;
19125
19428
  *
19126
19429
  * All-or-nothing per repository, and the discipline is copied verbatim from
19127
19430
  * Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
19128
- * ENTIRE repository if any row is legacy or malformed. `headRef` was added to
19129
- * the public GitHub pull index contract in `@relayfile/adapter-github@0.5.7`
19130
- * and an already-written mount only converges on re-ingest, so mixed indexes
19131
- * are the expected state, not a corruption. One legacy row is enough to poison
19132
- * the whole conclusion: the primary match is a branch match worth 30, so a row
19133
- * without `headRef` cannot be ruled out as the real winner, and ranking the
19134
- * remaining rows against each other would answer a different question from the
19135
- * one the walk answers.
19431
+ * ENTIRE repository if any row is legacy or malformed. One legacy row is enough
19432
+ * to poison the whole conclusion: the primary match is a branch match worth 30,
19433
+ * so a row without `headRef` cannot be ruled out as the real winner, and ranking
19434
+ * the remaining rows against each other would answer a different question from
19435
+ * the one the walk answers.
19436
+ *
19437
+ * Mixed indexes are the expected state today, not a corruption. `headRef` landed
19438
+ * in `@relayfile/adapter-github@0.5.7` and there is no pull-side backfill: the
19439
+ * incremental writers replace exactly the row they touched and pass every other
19440
+ * row through verbatim, so a webhook update hydrates one row and leaves the rest
19441
+ * legacy. Only an eager re-ingest converges a repository, because it rebuilds
19442
+ * the file from a list-pulls response in which every row carries `head.ref`.
19443
+ *
19444
+ * Row facts this relies on, from that contract: `state` is GitHub's lowercase
19445
+ * `"open"`/`"closed"` and there is no `"merged"` state — `merged: true` (with
19446
+ * `mergedAt`) is the only merged signal and `merged: false` is never written.
19447
+ * `headRef` is the bare branch name (`head.ref`), matching what
19448
+ * `readProbePrCandidate` reads off a record, NOT the `owner:branch` form of
19449
+ * `head.label`. `id` and `updated` are deliberately not required here: the
19450
+ * oldest legacy rows carry neither.
19136
19451
  */
19137
19452
  const readPullIndexForProbe = async (mount, repo) => {
19138
19453
  const [owner, name] = repo.split('/');