@agent-relay/factory 0.1.79 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fleet/relay-fleet-client.d.ts +5 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +7 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +18 -5
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +347 -32
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +6 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +1 -0
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +22 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +1 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +15 -1
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -156,6 +156,8 @@ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
|
|
|
156
156
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
157
157
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
158
158
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
159
|
+
const REMOTE_AGENT_REGISTRATION_TIMEOUT_MS = 30_000;
|
|
160
|
+
const REMOTE_AGENT_REGISTRATION_POLL_MS = 500;
|
|
159
161
|
/**
|
|
160
162
|
* Ceiling on the durable capacity-wait re-arm (#303).
|
|
161
163
|
*
|
|
@@ -349,6 +351,26 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
349
351
|
this.leaseUntilMs = leaseUntilMs;
|
|
350
352
|
}
|
|
351
353
|
}
|
|
354
|
+
class FleetPlacementUnavailableError extends Error {
|
|
355
|
+
capability;
|
|
356
|
+
constructor(capability) {
|
|
357
|
+
super(`Refusing remote dispatch: no live fleet node advertises ${capability}`);
|
|
358
|
+
this.capability = capability;
|
|
359
|
+
this.name = 'FleetPlacementUnavailableError';
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
class RemoteAgentRegistrationTimeoutError extends Error {
|
|
363
|
+
agentName;
|
|
364
|
+
cleanupConfirmed;
|
|
365
|
+
cleanupError;
|
|
366
|
+
constructor(agentName, cleanupConfirmed, cleanupError) {
|
|
367
|
+
super(`Remote agent ${agentName} did not register with the fleet before the startup deadline`);
|
|
368
|
+
this.agentName = agentName;
|
|
369
|
+
this.cleanupConfirmed = cleanupConfirmed;
|
|
370
|
+
this.cleanupError = cleanupError;
|
|
371
|
+
this.name = 'RemoteAgentRegistrationTimeoutError';
|
|
372
|
+
}
|
|
373
|
+
}
|
|
352
374
|
/**
|
|
353
375
|
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
354
376
|
* record is already terminal, or another publisher currently holds the lease.
|
|
@@ -450,6 +472,7 @@ export class FactoryLoop {
|
|
|
450
472
|
#babysitterWakeUnreachableRetryMs;
|
|
451
473
|
#startupAgentExitDrainTimeoutMs;
|
|
452
474
|
#dispatchLifecycleRetryMs;
|
|
475
|
+
#dispatchLifecycleRenewMs;
|
|
453
476
|
#state;
|
|
454
477
|
#workspaceId;
|
|
455
478
|
#relayflows;
|
|
@@ -896,6 +919,7 @@ export class FactoryLoop {
|
|
|
896
919
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
897
920
|
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
898
921
|
this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
|
|
922
|
+
this.#dispatchLifecycleRenewMs = ports.dispatchLifecycleRenewMs ?? DISPATCH_LIFECYCLE_RENEW_MS;
|
|
899
923
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
900
924
|
this.#relayflows = ports.relayflows;
|
|
901
925
|
this.#worktrees = ports.worktrees;
|
|
@@ -2783,8 +2807,9 @@ export class FactoryLoop {
|
|
|
2783
2807
|
}
|
|
2784
2808
|
async #assertFleetControlPlaneAvailable() {
|
|
2785
2809
|
try {
|
|
2786
|
-
await this.#fleet.roster();
|
|
2810
|
+
const roster = await this.#fleet.roster();
|
|
2787
2811
|
this.#increment('fleetControlPlaneProbeSuccesses');
|
|
2812
|
+
return roster;
|
|
2788
2813
|
}
|
|
2789
2814
|
catch (error) {
|
|
2790
2815
|
const health = this.#fleetControlPlane.status();
|
|
@@ -5019,8 +5044,10 @@ export class FactoryLoop {
|
|
|
5019
5044
|
// consuming a dispatch attempt. The mutation proxy probes again at the
|
|
5020
5045
|
// actual spawn/resume boundary so a later control-plane fault still fails
|
|
5021
5046
|
// closed.
|
|
5022
|
-
|
|
5023
|
-
|
|
5047
|
+
const admissionRoster = !dryRun ? await this.#assertFleetControlPlaneAvailable() : undefined;
|
|
5048
|
+
if (admissionRoster && this.#fleet.placementLocality === 'remote') {
|
|
5049
|
+
dispatchDecision = decisionWithVerifiedRemotePlacements(dispatchDecision, admissionRoster);
|
|
5050
|
+
}
|
|
5024
5051
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
5025
5052
|
// Local dispatches need the same deterministic branch identity as remote
|
|
5026
5053
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -5437,6 +5464,13 @@ export class FactoryLoop {
|
|
|
5437
5464
|
throw error;
|
|
5438
5465
|
}
|
|
5439
5466
|
settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
|
|
5467
|
+
if ((error instanceof FleetPlacementUnavailableError ||
|
|
5468
|
+
(error instanceof RemoteAgentRegistrationTimeoutError && error.cleanupConfirmed)) &&
|
|
5469
|
+
await this.#rollbackUnregisteredRemoteDispatch(record, spawnedForReaperHandoff)) {
|
|
5470
|
+
this.#increment('remoteDispatchAdmissionRollbacks');
|
|
5471
|
+
this.#error(error, decision.issue);
|
|
5472
|
+
throw error;
|
|
5473
|
+
}
|
|
5440
5474
|
// A spawn can fail after the broker accepted it but before its ack
|
|
5441
5475
|
// reached Factory. Include every planned worktree agent, not only the
|
|
5442
5476
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
@@ -6188,7 +6222,7 @@ export class FactoryLoop {
|
|
|
6188
6222
|
return;
|
|
6189
6223
|
this.#dispatchLifecycleRenewTimer = setInterval(() => {
|
|
6190
6224
|
void this.#renewDispatchLifecycles();
|
|
6191
|
-
},
|
|
6225
|
+
}, this.#dispatchLifecycleRenewMs);
|
|
6192
6226
|
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
6193
6227
|
}
|
|
6194
6228
|
/**
|
|
@@ -6385,6 +6419,18 @@ export class FactoryLoop {
|
|
|
6385
6419
|
}
|
|
6386
6420
|
async #renewDispatchLifecycles() {
|
|
6387
6421
|
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
6422
|
+
// The snapshot above can outlive the ownership it records: a relinquish
|
|
6423
|
+
// that runs while this loop is awaiting the store for an earlier key
|
|
6424
|
+
// leaves a stale entry here, and renewing it would re-block a key this
|
|
6425
|
+
// process has already handed back (#391 review, P2). Re-read the live map
|
|
6426
|
+
// rather than trusting the snapshot.
|
|
6427
|
+
//
|
|
6428
|
+
// This narrows the window but does not close it on its own — the delete
|
|
6429
|
+
// can still land after this check and before the renew resolves. Closing
|
|
6430
|
+
// it is `renewDispatchLifecycle`'s expiry fence, which makes a
|
|
6431
|
+
// relinquished lease unrenewable no matter how the two race.
|
|
6432
|
+
if (this.#dispatchLifecycleEpochs.get(key) !== epoch)
|
|
6433
|
+
continue;
|
|
6388
6434
|
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
6389
6435
|
if (!renewed) {
|
|
6390
6436
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
@@ -6985,8 +7031,17 @@ export class FactoryLoop {
|
|
|
6985
7031
|
* the only thing still running is the spin.
|
|
6986
7032
|
*/
|
|
6987
7033
|
#chargeReleaseAttempt(record, key, context) {
|
|
6988
|
-
if (this.#dispatchLifecycleReleaseAbandoned.has(key))
|
|
7034
|
+
if (this.#dispatchLifecycleReleaseAbandoned.has(key)) {
|
|
7035
|
+
// Re-entry, and the reason relinquishing the lease once is not enough.
|
|
7036
|
+
// `#driveDispatchLifecycle` re-claims the lease at the TOP of every
|
|
7037
|
+
// drive, before it has read the phase, so anything that drives an
|
|
7038
|
+
// already-dead-lettered key — the held-agent-deadline sweep, a registry
|
|
7039
|
+
// restore, a takeover — puts the epoch straight back into the renewal
|
|
7040
|
+
// map and re-arms the livelock this bound just escaped. Whenever the
|
|
7041
|
+
// budget declines a re-arm, ownership goes back too.
|
|
7042
|
+
this.#trackDispatchLifecycleDrive(this.#relinquishDispatchLifecycleLease(key, record.issue.key));
|
|
6989
7043
|
return false;
|
|
7044
|
+
}
|
|
6990
7045
|
const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
|
|
6991
7046
|
this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
|
|
6992
7047
|
if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
|
|
@@ -6995,6 +7050,19 @@ export class FactoryLoop {
|
|
|
6995
7050
|
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
6996
7051
|
this.#increment('dispatchLifecycleReleaseAbandoned');
|
|
6997
7052
|
const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
|
|
7053
|
+
// Cleanup is armed BEFORE anything that can throw (#391 review, P1). The
|
|
7054
|
+
// only statements above are set/map writes and a counter, none of which can
|
|
7055
|
+
// reject; `this.#logger.error` below is caller-supplied and can. Ordering
|
|
7056
|
+
// the drive first means nothing between "this unit is abandoned" and "its
|
|
7057
|
+
// lease is handed back" is allowed to fail in a way that skips the handback
|
|
7058
|
+
// — which is precisely the defect shape this whole method exists to fix.
|
|
7059
|
+
this.#trackDispatchLifecycleDrive(this.#releaseDeadLetteredSlot(record, key)
|
|
7060
|
+
.catch((error) => {
|
|
7061
|
+
this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
|
|
7062
|
+
issue: record.issue.key,
|
|
7063
|
+
error: describeError(error).errorMessage,
|
|
7064
|
+
});
|
|
7065
|
+
}));
|
|
6998
7066
|
// `error`, not `warn`. Every previous layer of this failure was invisible
|
|
6999
7067
|
// until somebody read stderr by hand; a work unit whose cleanup this
|
|
7000
7068
|
// process has permanently given up on is exactly the event that must not
|
|
@@ -7009,16 +7077,63 @@ export class FactoryLoop {
|
|
|
7009
7077
|
// spin, it does not declare the work unit clean.
|
|
7010
7078
|
durableLifecycleRetained,
|
|
7011
7079
|
});
|
|
7012
|
-
|
|
7013
|
-
|
|
7014
|
-
|
|
7015
|
-
|
|
7080
|
+
return false;
|
|
7081
|
+
}
|
|
7082
|
+
/** Keeps a lifecycle-side effect awaitable by `stop()` without leaking the set entry. */
|
|
7083
|
+
#trackDispatchLifecycleDrive(promise) {
|
|
7084
|
+
const drive = promise.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7085
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
7086
|
+
}
|
|
7087
|
+
/**
|
|
7088
|
+
* Stop asserting durable ownership of a work unit this process will not drive
|
|
7089
|
+
* again.
|
|
7090
|
+
*
|
|
7091
|
+
* `#dispatchLifecycleEpochs` is not merely a cache. `#renewDispatchLifecycles`
|
|
7092
|
+
* walks it every `DISPATCH_LIFECYCLE_RENEW_MS` and re-stamps a full
|
|
7093
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` onto every key it finds, unconditionally.
|
|
7094
|
+
*
|
|
7095
|
+
* A dead-lettered release deliberately leaves its row in the non-terminal
|
|
7096
|
+
* `releasing` phase so a successor or a restart can re-drive the cleanup with
|
|
7097
|
+
* a fresh budget. That is only a recovery path if the successor can CLAIM the
|
|
7098
|
+
* row — and a lease renewed forever by a process that has permanently given
|
|
7099
|
+
* up driving it is a claim nobody can ever win. #379 bounded the retry and
|
|
7100
|
+
* handed back the batch slot, then kept the key locked for the life of the
|
|
7101
|
+
* process, which is how production reached four issues (a dispatch canary
|
|
7102
|
+
* among them) all logging `durable dispatch is leased by another publisher`
|
|
7103
|
+
* at 1 Hz for three days while the holder logged nothing but 503
|
|
7104
|
+
* `agent_host_unavailable`.
|
|
7105
|
+
*
|
|
7106
|
+
* The epoch is dropped BEFORE the durable release, so a renewal tick that
|
|
7107
|
+
* STARTS after this point finds nothing to renew. That ordering alone is not
|
|
7108
|
+
* sufficient and an earlier version of this comment wrongly claimed it was
|
|
7109
|
+
* (#391 review, P2): `#renewDispatchLifecycles` iterates a snapshot array, so
|
|
7110
|
+
* a tick already in flight still holds this key and would restore the lease
|
|
7111
|
+
* for a full term. The guarantee comes from `renewDispatchLifecycle` fencing
|
|
7112
|
+
* on expiry as well as owner and epoch, which makes a relinquished lease
|
|
7113
|
+
* unrenewable however the two race; the epoch drop and the loop's live re-read
|
|
7114
|
+
* narrow the window ahead of it.
|
|
7115
|
+
*
|
|
7116
|
+
* If the durable release itself fails, the dropped epoch alone still ends the
|
|
7117
|
+
* livelock: nothing renews the lease any more, so it expires within
|
|
7118
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` instead of never.
|
|
7119
|
+
*/
|
|
7120
|
+
async #relinquishDispatchLifecycleLease(key, issueKey) {
|
|
7121
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
7122
|
+
if (epoch === undefined)
|
|
7123
|
+
return;
|
|
7124
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
7125
|
+
try {
|
|
7126
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
7127
|
+
this.#increment('dispatchLifecycleLeasesRelinquished');
|
|
7128
|
+
}
|
|
7129
|
+
catch (error) {
|
|
7130
|
+
this.#logger.warn?.('[factory] could not relinquish the dispatch lease of an abandoned work unit', {
|
|
7131
|
+
issue: issueKey,
|
|
7132
|
+
// The lease still expires on its own now that nothing renews it.
|
|
7133
|
+
expiresWithinMs: DISPATCH_LIFECYCLE_LEASE_MS,
|
|
7016
7134
|
error: describeError(error).errorMessage,
|
|
7017
7135
|
});
|
|
7018
|
-
}
|
|
7019
|
-
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7020
|
-
this.#dispatchLifecycleDrives.add(drive);
|
|
7021
|
-
return false;
|
|
7136
|
+
}
|
|
7022
7137
|
}
|
|
7023
7138
|
/**
|
|
7024
7139
|
* Hand back the batch slot of a work unit whose release was dead-lettered.
|
|
@@ -7035,12 +7150,36 @@ export class FactoryLoop {
|
|
|
7035
7150
|
* local slot does not write a terminal phase. On a local lifecycle there is
|
|
7036
7151
|
* no durable record to retain, so the batch record is all there is and
|
|
7037
7152
|
* completing it is what keeps capacity honest.
|
|
7153
|
+
*
|
|
7154
|
+
* The DURABLE lease has to go back with the slot. Retaining the row for a
|
|
7155
|
+
* successor while renewing the lease that locks the successor out is not a
|
|
7156
|
+
* handoff, it is a permanent block on the key — see
|
|
7157
|
+
* `#relinquishDispatchLifecycleLease`.
|
|
7158
|
+
*
|
|
7159
|
+
* The handback lives in a `finally` for a reason worth stating plainly
|
|
7160
|
+
* (#391 review, P1). The FIRST version of this fix relinquished after
|
|
7161
|
+
* `#writeInFlightRegistry()`, on the happy path only — so a rejecting
|
|
7162
|
+
* registry write would skip it and leave the abandoned key renewing its lease
|
|
7163
|
+
* forever, with nothing but a `warn` to show for it. That is the identical
|
|
7164
|
+
* shape of the bug being fixed (#379 freed the slot but not the lease, on the
|
|
7165
|
+
* failure path), reproduced one level up. Cleanup that only runs when the
|
|
7166
|
+
* rest of cleanup succeeded is not cleanup. `#batch()` and
|
|
7167
|
+
* `#writeInFlightRegistry()` can both reject; neither may strand the key.
|
|
7038
7168
|
*/
|
|
7039
7169
|
async #releaseDeadLetteredSlot(record, key) {
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7170
|
+
let next;
|
|
7171
|
+
try {
|
|
7172
|
+
const batch = await this.#batch();
|
|
7173
|
+
next = batch.complete(record.issue);
|
|
7174
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
7175
|
+
await this.#writeInFlightRegistry();
|
|
7176
|
+
}
|
|
7177
|
+
finally {
|
|
7178
|
+
// Unconditional, and safe to put in a `finally` because
|
|
7179
|
+
// `#relinquishDispatchLifecycleLease` handles its own errors and cannot
|
|
7180
|
+
// throw — so it can never mask the failure that brought us here.
|
|
7181
|
+
await this.#relinquishDispatchLifecycleLease(key, record.issue.key);
|
|
7182
|
+
}
|
|
7044
7183
|
// A freed slot that nothing is admitted into is only half the repair.
|
|
7045
7184
|
if (next && !this.#stopping)
|
|
7046
7185
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
@@ -9585,13 +9724,6 @@ export class FactoryLoop {
|
|
|
9585
9724
|
batch.recordDryRun(record, spec, invocationId);
|
|
9586
9725
|
return { name: spec.name };
|
|
9587
9726
|
}
|
|
9588
|
-
// Persist intent before the remote side effect. If the owner crashes after
|
|
9589
|
-
// the spawn ack but before recording its result, takeover retries the same
|
|
9590
|
-
// deterministic invocation id instead of inventing a second worker.
|
|
9591
|
-
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9592
|
-
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9593
|
-
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9594
|
-
}
|
|
9595
9727
|
let roster;
|
|
9596
9728
|
try {
|
|
9597
9729
|
roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
|
|
@@ -9601,6 +9733,15 @@ export class FactoryLoop {
|
|
|
9601
9733
|
}
|
|
9602
9734
|
const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
|
|
9603
9735
|
if (rosterAgent) {
|
|
9736
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9737
|
+
const host = rosterAgent.node
|
|
9738
|
+
? roster.nodes.find((node) => node.name === rosterAgent.node && node.live && node.capabilities.includes(spec.capability))
|
|
9739
|
+
: undefined;
|
|
9740
|
+
if (!host) {
|
|
9741
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
9742
|
+
}
|
|
9743
|
+
spec = { ...spec, node: host.name };
|
|
9744
|
+
}
|
|
9604
9745
|
const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
|
|
9605
9746
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9606
9747
|
batch.recordSpawn(record, spec, invocationId, {
|
|
@@ -9618,6 +9759,21 @@ export class FactoryLoop {
|
|
|
9618
9759
|
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
9619
9760
|
return { name: spec.name };
|
|
9620
9761
|
}
|
|
9762
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9763
|
+
const loads = new Map();
|
|
9764
|
+
for (const agent of roster.agents) {
|
|
9765
|
+
if (agent.node)
|
|
9766
|
+
loads.set(agent.node, (loads.get(agent.node) ?? 0) + 1);
|
|
9767
|
+
}
|
|
9768
|
+
spec = { ...spec, node: liveFleetNodeForSpec(spec, roster, loads) };
|
|
9769
|
+
}
|
|
9770
|
+
// Persist intent before the remote side effect. If the owner crashes after
|
|
9771
|
+
// the spawn ack but before recording its result, takeover retries the same
|
|
9772
|
+
// deterministic invocation id instead of inventing a second worker.
|
|
9773
|
+
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9774
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9775
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9776
|
+
}
|
|
9621
9777
|
await this.#prepareAgentWorktree(record, spec);
|
|
9622
9778
|
let result;
|
|
9623
9779
|
try {
|
|
@@ -9657,6 +9813,26 @@ export class FactoryLoop {
|
|
|
9657
9813
|
await this.#releaseOrphanedLatePlacement(record, spec, result);
|
|
9658
9814
|
throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
|
|
9659
9815
|
}
|
|
9816
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9817
|
+
const registered = await this.#awaitRemoteAgentRegistration(result.name, spec.capability, result.node);
|
|
9818
|
+
if (!registered) {
|
|
9819
|
+
try {
|
|
9820
|
+
await this.#fleet.release(result.name, 'spawn-registration-timeout');
|
|
9821
|
+
this.#fleet.markAgentTerminal?.(result.name, 'spawn-registration-timeout');
|
|
9822
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, true);
|
|
9823
|
+
}
|
|
9824
|
+
catch (error) {
|
|
9825
|
+
if (error instanceof RemoteAgentRegistrationTimeoutError)
|
|
9826
|
+
throw error;
|
|
9827
|
+
// Cleanup is unconfirmed. Persist the placement so a successor can
|
|
9828
|
+
// retry the release; forgetting it would be worse than retaining a
|
|
9829
|
+
// nonterminal lifecycle for a worker that may still be alive.
|
|
9830
|
+
batch.recordSpawn(record, spec, invocationId, result);
|
|
9831
|
+
await this.#saveDispatchLifecycle(record, 'dispatching');
|
|
9832
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, false, error);
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
}
|
|
9660
9836
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9661
9837
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
9662
9838
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
@@ -9668,6 +9844,89 @@ export class FactoryLoop {
|
|
|
9668
9844
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
9669
9845
|
return { name: result.name };
|
|
9670
9846
|
}
|
|
9847
|
+
async #awaitRemoteAgentRegistration(name, capability, expectedNode) {
|
|
9848
|
+
const deadlineAtMs = this.#clock.now() + REMOTE_AGENT_REGISTRATION_TIMEOUT_MS;
|
|
9849
|
+
do {
|
|
9850
|
+
try {
|
|
9851
|
+
if (expectedNode) {
|
|
9852
|
+
if (this.#fleet.isAgentRegistered) {
|
|
9853
|
+
if (await this.#fleet.isAgentRegistered({ name, node: expectedNode, capability }))
|
|
9854
|
+
return true;
|
|
9855
|
+
}
|
|
9856
|
+
else {
|
|
9857
|
+
const roster = await this.#fleet.roster();
|
|
9858
|
+
const agent = roster.agents.find((candidate) => candidate.name === name && candidate.node === expectedNode);
|
|
9859
|
+
const node = agent
|
|
9860
|
+
? roster.nodes.find((candidate) => candidate.name === expectedNode && candidate.live && candidate.capabilities.includes(capability))
|
|
9861
|
+
: undefined;
|
|
9862
|
+
if (agent && node)
|
|
9863
|
+
return true;
|
|
9864
|
+
}
|
|
9865
|
+
}
|
|
9866
|
+
}
|
|
9867
|
+
catch (error) {
|
|
9868
|
+
this.#logger.warn?.('[factory] remote agent registration probe failed; retrying within startup bound', {
|
|
9869
|
+
agent: name,
|
|
9870
|
+
error: describeError(error).errorMessage,
|
|
9871
|
+
});
|
|
9872
|
+
}
|
|
9873
|
+
const remainingMs = deadlineAtMs - this.#clock.now();
|
|
9874
|
+
if (remainingMs <= 0)
|
|
9875
|
+
return false;
|
|
9876
|
+
await this.#clock.sleep(Math.min(REMOTE_AGENT_REGISTRATION_POLL_MS, remainingMs));
|
|
9877
|
+
} while (this.#clock.now() <= deadlineAtMs);
|
|
9878
|
+
return false;
|
|
9879
|
+
}
|
|
9880
|
+
async #rollbackUnregisteredRemoteDispatch(record, acknowledged) {
|
|
9881
|
+
const handoffs = this.#dispatchFailureHandoffs(record, acknowledged);
|
|
9882
|
+
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
9883
|
+
const hasWorktrees = handoffs.some((handoff) => handoff.worktree);
|
|
9884
|
+
if (hasWorktrees) {
|
|
9885
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'spawn-registration-timeout', { skipNeverPlacedAgents: true }))
|
|
9886
|
+
return false;
|
|
9887
|
+
}
|
|
9888
|
+
else {
|
|
9889
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs
|
|
9890
|
+
.filter((handoff) => handoff.tracked.result !== undefined)
|
|
9891
|
+
.map((handoff) => [handoff.name, handoff.tracked]), 'spawn-registration-timeout', 'completion');
|
|
9892
|
+
if (failed.length > 0)
|
|
9893
|
+
return false;
|
|
9894
|
+
for (const handoff of handoffs) {
|
|
9895
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
9896
|
+
}
|
|
9897
|
+
}
|
|
9898
|
+
try {
|
|
9899
|
+
await this.#teardownPreviews(record);
|
|
9900
|
+
}
|
|
9901
|
+
catch (error) {
|
|
9902
|
+
this.#logger.warn?.('[factory] retained unregistered spawn lifecycle after preview rollback failed', {
|
|
9903
|
+
issue: record.issue.key,
|
|
9904
|
+
error: describeError(error).errorMessage,
|
|
9905
|
+
});
|
|
9906
|
+
return false;
|
|
9907
|
+
}
|
|
9908
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
9909
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
9910
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
9911
|
+
const lease = lifecycle?.lease;
|
|
9912
|
+
if (epoch === undefined ||
|
|
9913
|
+
!lease ||
|
|
9914
|
+
lease.owner !== this.#dispatchLifecycleOwner ||
|
|
9915
|
+
lease.epoch !== epoch ||
|
|
9916
|
+
!await this.#state.clearClaimedDispatchLifecycle(this.#workspaceId, key, lease))
|
|
9917
|
+
return false;
|
|
9918
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
9919
|
+
if (retryTimer)
|
|
9920
|
+
clearTimeout(retryTimer);
|
|
9921
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
9922
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
9923
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
9924
|
+
const batch = await this.#batch();
|
|
9925
|
+
batch.abandon(record.issue);
|
|
9926
|
+
await this.#writeInFlightRegistry();
|
|
9927
|
+
this.#resetDispatchCapacityBackoff();
|
|
9928
|
+
return true;
|
|
9929
|
+
}
|
|
9671
9930
|
/**
|
|
9672
9931
|
* Is this process still the owner of a lifecycle that is not already done?
|
|
9673
9932
|
*
|
|
@@ -18234,6 +18493,50 @@ function dispatchSpecs(decision) {
|
|
|
18234
18493
|
}
|
|
18235
18494
|
return [...decision.implementers, decision.reviewer];
|
|
18236
18495
|
}
|
|
18496
|
+
function liveFleetNodeForSpec(spec, roster, assignedLoads) {
|
|
18497
|
+
const eligible = roster.nodes.filter((node) => node.live && node.capabilities.includes(spec.capability));
|
|
18498
|
+
if (eligible.length === 0)
|
|
18499
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
18500
|
+
const explicitlyRequested = spec.node && spec.node !== 'self'
|
|
18501
|
+
? eligible.find((node) => node.name === spec.node)
|
|
18502
|
+
: undefined;
|
|
18503
|
+
const selected = explicitlyRequested ?? [...eligible].sort((left, right) => {
|
|
18504
|
+
const loadDifference = (assignedLoads.get(left.name) ?? 0) - (assignedLoads.get(right.name) ?? 0);
|
|
18505
|
+
return loadDifference || left.name.localeCompare(right.name);
|
|
18506
|
+
})[0];
|
|
18507
|
+
assignedLoads.set(selected.name, (assignedLoads.get(selected.name) ?? 0) + 1);
|
|
18508
|
+
return selected.name;
|
|
18509
|
+
}
|
|
18510
|
+
/**
|
|
18511
|
+
* Resolve every remote dispatch spec against one canonical roster snapshot.
|
|
18512
|
+
*
|
|
18513
|
+
* A configured node is a preference, not an entitlement: if it is offline or
|
|
18514
|
+
* no longer advertises the required capability, placement is re-selected from
|
|
18515
|
+
* the live fleet. This runs before durable lifecycle creation, so an empty
|
|
18516
|
+
* eligible set cannot leave a claim or consume a batch slot.
|
|
18517
|
+
*/
|
|
18518
|
+
function decisionWithVerifiedRemotePlacements(decision, roster) {
|
|
18519
|
+
const assignedLoads = new Map();
|
|
18520
|
+
for (const agent of roster.agents) {
|
|
18521
|
+
if (agent.node)
|
|
18522
|
+
assignedLoads.set(agent.node, (assignedLoads.get(agent.node) ?? 0) + 1);
|
|
18523
|
+
}
|
|
18524
|
+
const place = (spec) => ({
|
|
18525
|
+
...spec,
|
|
18526
|
+
node: liveFleetNodeForSpec(spec, roster, assignedLoads),
|
|
18527
|
+
});
|
|
18528
|
+
if (decision.scope === 'workflow') {
|
|
18529
|
+
return {
|
|
18530
|
+
...structuredClone(decision),
|
|
18531
|
+
...(decision.workflow ? { workflow: place(decision.workflow) } : {}),
|
|
18532
|
+
};
|
|
18533
|
+
}
|
|
18534
|
+
return {
|
|
18535
|
+
...structuredClone(decision),
|
|
18536
|
+
implementers: decision.implementers.map(place),
|
|
18537
|
+
reviewer: place(decision.reviewer),
|
|
18538
|
+
};
|
|
18539
|
+
}
|
|
18237
18540
|
function dispatchSessionOwner(decision) {
|
|
18238
18541
|
for (const spec of dispatchSpecs(decision)) {
|
|
18239
18542
|
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
@@ -19125,14 +19428,26 @@ const PULL_INDEX_DECISIVE_SCORE = 20;
|
|
|
19125
19428
|
*
|
|
19126
19429
|
* All-or-nothing per repository, and the discipline is copied verbatim from
|
|
19127
19430
|
* Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
|
|
19128
|
-
* ENTIRE repository if any row is legacy or malformed.
|
|
19129
|
-
* the
|
|
19130
|
-
*
|
|
19131
|
-
*
|
|
19132
|
-
* the
|
|
19133
|
-
*
|
|
19134
|
-
*
|
|
19135
|
-
*
|
|
19431
|
+
* ENTIRE repository if any row is legacy or malformed. One legacy row is enough
|
|
19432
|
+
* to poison the whole conclusion: the primary match is a branch match worth 30,
|
|
19433
|
+
* so a row without `headRef` cannot be ruled out as the real winner, and ranking
|
|
19434
|
+
* the remaining rows against each other would answer a different question from
|
|
19435
|
+
* the one the walk answers.
|
|
19436
|
+
*
|
|
19437
|
+
* Mixed indexes are the expected state today, not a corruption. `headRef` landed
|
|
19438
|
+
* in `@relayfile/adapter-github@0.5.7` and there is no pull-side backfill: the
|
|
19439
|
+
* incremental writers replace exactly the row they touched and pass every other
|
|
19440
|
+
* row through verbatim, so a webhook update hydrates one row and leaves the rest
|
|
19441
|
+
* legacy. Only an eager re-ingest converges a repository, because it rebuilds
|
|
19442
|
+
* the file from a list-pulls response in which every row carries `head.ref`.
|
|
19443
|
+
*
|
|
19444
|
+
* Row facts this relies on, from that contract: `state` is GitHub's lowercase
|
|
19445
|
+
* `"open"`/`"closed"` and there is no `"merged"` state — `merged: true` (with
|
|
19446
|
+
* `mergedAt`) is the only merged signal and `merged: false` is never written.
|
|
19447
|
+
* `headRef` is the bare branch name (`head.ref`), matching what
|
|
19448
|
+
* `readProbePrCandidate` reads off a record, NOT the `owner:branch` form of
|
|
19449
|
+
* `head.label`. `id` and `updated` are deliberately not required here: the
|
|
19450
|
+
* oldest legacy rows carry neither.
|
|
19136
19451
|
*/
|
|
19137
19452
|
const readPullIndexForProbe = async (mount, repo) => {
|
|
19138
19453
|
const [owner, name] = repo.split('/');
|