@agent-relay/factory 0.1.77 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fleet/relay-fleet-client.d.ts +5 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +7 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +78 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +723 -61
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +7 -0
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/ports/fleet.d.ts +6 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +1 -0
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +22 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +1 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +15 -1
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/types.d.ts +40 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -156,6 +156,8 @@ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
|
|
|
156
156
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
157
157
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
158
158
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
159
|
+
const REMOTE_AGENT_REGISTRATION_TIMEOUT_MS = 30_000;
|
|
160
|
+
const REMOTE_AGENT_REGISTRATION_POLL_MS = 500;
|
|
159
161
|
/**
|
|
160
162
|
* Ceiling on the durable capacity-wait re-arm (#303).
|
|
161
163
|
*
|
|
@@ -167,6 +169,31 @@ const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
|
167
169
|
* multi-hour run holds the slot honestly), so what is bounded is the *rate*.
|
|
168
170
|
*/
|
|
169
171
|
const DISPATCH_LIFECYCLE_RETRY_MAX_MS = 30_000;
|
|
172
|
+
/**
|
|
173
|
+
* How many times a work unit's RELEASE may fail before it is dead-lettered.
|
|
174
|
+
*
|
|
175
|
+
* #303 bounded the rate of the capacity-wait re-arm and deliberately left its
|
|
176
|
+
* count unbounded, because waiting for capacity is legitimate: the slot will
|
|
177
|
+
* free eventually and abandoning the wait would drop real work. Release is the
|
|
178
|
+
* opposite shape. It is the last step of a work unit that is already finished
|
|
179
|
+
* — the issue is closed, the writeback is acknowledged, the batch slot is
|
|
180
|
+
* gone — so a release that has failed ten times is not waiting for anything.
|
|
181
|
+
* It is failing, and re-arming it at 1 Hz forever buys nothing.
|
|
182
|
+
*
|
|
183
|
+
* What that costs when it happens: `#finishDurableRelease` does not throw on a
|
|
184
|
+
* failed release, it returns `false` and re-arms itself, so the loop runs
|
|
185
|
+
* through the *success* path of every `.catch()` that might otherwise have
|
|
186
|
+
* bounded it. Each pass calls `#terminationRoots` once per agent, and
|
|
187
|
+
* `#writeInFlightRegistry` calls it once per agent again — each one a process
|
|
188
|
+
* table scan — plus a durable read and write. Three agents is order ten scans
|
|
189
|
+
* and several state operations per second, indefinitely, which is the same
|
|
190
|
+
* serialized-store spin #303 measured at 1477 GETs in 111 s.
|
|
191
|
+
*
|
|
192
|
+
* Ten attempts at the 1 Hz floor is ~10 s of genuine retry, which covers the
|
|
193
|
+
* transient failures release retries exist for (a brief control-plane blip, a
|
|
194
|
+
* lease handover) without covering a permanent one.
|
|
195
|
+
*/
|
|
196
|
+
const DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS = 10;
|
|
170
197
|
/** Rate limit for the capacity-wait warning once the backoff has capped. */
|
|
171
198
|
const DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS = 60_000;
|
|
172
199
|
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
@@ -324,6 +351,26 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
324
351
|
this.leaseUntilMs = leaseUntilMs;
|
|
325
352
|
}
|
|
326
353
|
}
|
|
354
|
+
class FleetPlacementUnavailableError extends Error {
|
|
355
|
+
capability;
|
|
356
|
+
constructor(capability) {
|
|
357
|
+
super(`Refusing remote dispatch: no live fleet node advertises ${capability}`);
|
|
358
|
+
this.capability = capability;
|
|
359
|
+
this.name = 'FleetPlacementUnavailableError';
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
class RemoteAgentRegistrationTimeoutError extends Error {
|
|
363
|
+
agentName;
|
|
364
|
+
cleanupConfirmed;
|
|
365
|
+
cleanupError;
|
|
366
|
+
constructor(agentName, cleanupConfirmed, cleanupError) {
|
|
367
|
+
super(`Remote agent ${agentName} did not register with the fleet before the startup deadline`);
|
|
368
|
+
this.agentName = agentName;
|
|
369
|
+
this.cleanupConfirmed = cleanupConfirmed;
|
|
370
|
+
this.cleanupError = cleanupError;
|
|
371
|
+
this.name = 'RemoteAgentRegistrationTimeoutError';
|
|
372
|
+
}
|
|
373
|
+
}
|
|
327
374
|
/**
|
|
328
375
|
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
329
376
|
* record is already terminal, or another publisher currently holds the lease.
|
|
@@ -424,6 +471,8 @@ export class FactoryLoop {
|
|
|
424
471
|
#babysitterWakeUnreachableEscalateMs;
|
|
425
472
|
#babysitterWakeUnreachableRetryMs;
|
|
426
473
|
#startupAgentExitDrainTimeoutMs;
|
|
474
|
+
#dispatchLifecycleRetryMs;
|
|
475
|
+
#dispatchLifecycleRenewMs;
|
|
427
476
|
#state;
|
|
428
477
|
#workspaceId;
|
|
429
478
|
#relayflows;
|
|
@@ -500,6 +549,16 @@ export class FactoryLoop {
|
|
|
500
549
|
#dispatchTerminalWaiters = new Map();
|
|
501
550
|
#dispatchLifecycleRetryTimers = new Map();
|
|
502
551
|
#dispatchLifecycleDrives = new Set();
|
|
552
|
+
/**
|
|
553
|
+
* Failed release attempts per work unit, and the units that ran out.
|
|
554
|
+
*
|
|
555
|
+
* Keyed by `dispatchLifecycleKey`, so it follows the work unit rather than
|
|
556
|
+
* any one agent, surface or dispatcher — the same identity rule the AR-448
|
|
557
|
+
* duplicate established for claims. Counted only for release re-arms; a
|
|
558
|
+
* capacity or ownership wait is not a failure and must not spend the budget.
|
|
559
|
+
*/
|
|
560
|
+
#dispatchLifecycleReleaseAttempts = new Map();
|
|
561
|
+
#dispatchLifecycleReleaseAbandoned = new Set();
|
|
503
562
|
#abandonedDispatchReasons = new Map();
|
|
504
563
|
/**
|
|
505
564
|
* Live batch-capacity waits, keyed by issue (#303).
|
|
@@ -859,6 +918,8 @@ export class FactoryLoop {
|
|
|
859
918
|
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
860
919
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
861
920
|
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
921
|
+
this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
|
|
922
|
+
this.#dispatchLifecycleRenewMs = ports.dispatchLifecycleRenewMs ?? DISPATCH_LIFECYCLE_RENEW_MS;
|
|
862
923
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
863
924
|
this.#relayflows = ports.relayflows;
|
|
864
925
|
this.#worktrees = ports.worktrees;
|
|
@@ -2746,8 +2807,9 @@ export class FactoryLoop {
|
|
|
2746
2807
|
}
|
|
2747
2808
|
async #assertFleetControlPlaneAvailable() {
|
|
2748
2809
|
try {
|
|
2749
|
-
await this.#fleet.roster();
|
|
2810
|
+
const roster = await this.#fleet.roster();
|
|
2750
2811
|
this.#increment('fleetControlPlaneProbeSuccesses');
|
|
2812
|
+
return roster;
|
|
2751
2813
|
}
|
|
2752
2814
|
catch (error) {
|
|
2753
2815
|
const health = this.#fleetControlPlane.status();
|
|
@@ -4687,11 +4749,14 @@ export class FactoryLoop {
|
|
|
4687
4749
|
path: progress.path,
|
|
4688
4750
|
});
|
|
4689
4751
|
},
|
|
4690
|
-
// One line per repository saying why the tree walk was necessary.
|
|
4691
|
-
// is a
|
|
4692
|
-
//
|
|
4693
|
-
// the index
|
|
4694
|
-
//
|
|
4752
|
+
// One line per repository saying why the tree walk was necessary. A silent
|
|
4753
|
+
// fallback is indistinguishable from a fast path that is working, and
|
|
4754
|
+
// that is precisely how twelve minutes of sequential reads hid for so
|
|
4755
|
+
// long. Now that the index CAN answer the question, these reasons are the
|
|
4756
|
+
// only way to tell "the index answered" from "the index exists and was
|
|
4757
|
+
// useless" — and on a mount written before
|
|
4758
|
+
// `@relayfile/adapter-github@0.5.7`, `index-without-head-ref` is the
|
|
4759
|
+
// expected reading until a re-ingest converges the rows.
|
|
4695
4760
|
onIndexFallback: (repo, reason) => {
|
|
4696
4761
|
this.#increment('probePrIndexFallbacks');
|
|
4697
4762
|
this.#increment(PROBE_PR_INDEX_FALLBACK_COUNTERS[reason]);
|
|
@@ -4701,6 +4766,14 @@ export class FactoryLoop {
|
|
|
4701
4766
|
reason,
|
|
4702
4767
|
});
|
|
4703
4768
|
},
|
|
4769
|
+
onIndexHit: (repo, prNumber) => {
|
|
4770
|
+
this.#increment('probePrIndexHits');
|
|
4771
|
+
this.#logger.debug?.('[factory] PR probe resolved from the pull index', {
|
|
4772
|
+
issue: issue.key,
|
|
4773
|
+
repo,
|
|
4774
|
+
prNumber,
|
|
4775
|
+
});
|
|
4776
|
+
},
|
|
4704
4777
|
};
|
|
4705
4778
|
}
|
|
4706
4779
|
#logTimedProgress(message, startedAtMs, lastLoggedAtMs, metadata) {
|
|
@@ -4971,8 +5044,10 @@ export class FactoryLoop {
|
|
|
4971
5044
|
// consuming a dispatch attempt. The mutation proxy probes again at the
|
|
4972
5045
|
// actual spawn/resume boundary so a later control-plane fault still fails
|
|
4973
5046
|
// closed.
|
|
4974
|
-
|
|
4975
|
-
|
|
5047
|
+
const admissionRoster = !dryRun ? await this.#assertFleetControlPlaneAvailable() : undefined;
|
|
5048
|
+
if (admissionRoster && this.#fleet.placementLocality === 'remote') {
|
|
5049
|
+
dispatchDecision = decisionWithVerifiedRemotePlacements(dispatchDecision, admissionRoster);
|
|
5050
|
+
}
|
|
4976
5051
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
4977
5052
|
// Local dispatches need the same deterministic branch identity as remote
|
|
4978
5053
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -5389,6 +5464,13 @@ export class FactoryLoop {
|
|
|
5389
5464
|
throw error;
|
|
5390
5465
|
}
|
|
5391
5466
|
settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
|
|
5467
|
+
if ((error instanceof FleetPlacementUnavailableError ||
|
|
5468
|
+
(error instanceof RemoteAgentRegistrationTimeoutError && error.cleanupConfirmed)) &&
|
|
5469
|
+
await this.#rollbackUnregisteredRemoteDispatch(record, spawnedForReaperHandoff)) {
|
|
5470
|
+
this.#increment('remoteDispatchAdmissionRollbacks');
|
|
5471
|
+
this.#error(error, decision.issue);
|
|
5472
|
+
throw error;
|
|
5473
|
+
}
|
|
5392
5474
|
// A spawn can fail after the broker accepted it but before its ack
|
|
5393
5475
|
// reached Factory. Include every planned worktree agent, not only the
|
|
5394
5476
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
@@ -5637,8 +5719,23 @@ export class FactoryLoop {
|
|
|
5637
5719
|
consecutiveFailures,
|
|
5638
5720
|
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
5639
5721
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
5722
|
+
// The two bounds that can actually preempt this pass, published beside
|
|
5723
|
+
// the cadence that cannot. Omitting them made the stanza unreadable in
|
|
5724
|
+
// the one situation it exists for: `intervalMs` next to a climbing
|
|
5725
|
+
// `inFlightMs` looks exactly like an unbounded hang, and has twice been
|
|
5726
|
+
// reported as one. `timeoutMs` ends the wait, `sweepBudgetMs` unwinds
|
|
5727
|
+
// the sweep and hands the lease back — so the second is the one that
|
|
5728
|
+
// answers "when does this recover".
|
|
5729
|
+
timeoutMs: this.#readinessReconcileTimeoutMs,
|
|
5730
|
+
sweepBudgetMs: this.#discoverySweepBudgetMs,
|
|
5640
5731
|
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
5641
5732
|
...(inFlightMs !== undefined ? { inFlightMs } : {}),
|
|
5733
|
+
// Derived here rather than left to each reader. The public projection
|
|
5734
|
+
// already computed it (#295/#300); the heartbeat stanza did not, so the
|
|
5735
|
+
// surface an operator actually opens first was the one missing it.
|
|
5736
|
+
...(inFlightMs !== undefined && this.#readinessReconcileIntervalMs > 0
|
|
5737
|
+
? { missedPasses: Math.floor(inFlightMs / this.#readinessReconcileIntervalMs) }
|
|
5738
|
+
: {}),
|
|
5642
5739
|
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
5643
5740
|
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
5644
5741
|
: {}),
|
|
@@ -6125,7 +6222,7 @@ export class FactoryLoop {
|
|
|
6125
6222
|
return;
|
|
6126
6223
|
this.#dispatchLifecycleRenewTimer = setInterval(() => {
|
|
6127
6224
|
void this.#renewDispatchLifecycles();
|
|
6128
|
-
},
|
|
6225
|
+
}, this.#dispatchLifecycleRenewMs);
|
|
6129
6226
|
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
6130
6227
|
}
|
|
6131
6228
|
/**
|
|
@@ -6322,6 +6419,18 @@ export class FactoryLoop {
|
|
|
6322
6419
|
}
|
|
6323
6420
|
async #renewDispatchLifecycles() {
|
|
6324
6421
|
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
6422
|
+
// The snapshot above can outlive the ownership it records: a relinquish
|
|
6423
|
+
// that runs while this loop is awaiting the store for an earlier key
|
|
6424
|
+
// leaves a stale entry here, and renewing it would re-block a key this
|
|
6425
|
+
// process has already handed back (#391 review, P2). Re-read the live map
|
|
6426
|
+
// rather than trusting the snapshot.
|
|
6427
|
+
//
|
|
6428
|
+
// This narrows the window but does not close it on its own — the delete
|
|
6429
|
+
// can still land after this check and before the renew resolves. Closing
|
|
6430
|
+
// it is `renewDispatchLifecycle`'s expiry fence, which makes a
|
|
6431
|
+
// relinquished lease unrenewable no matter how the two race.
|
|
6432
|
+
if (this.#dispatchLifecycleEpochs.get(key) !== epoch)
|
|
6433
|
+
continue;
|
|
6325
6434
|
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
6326
6435
|
if (!renewed) {
|
|
6327
6436
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
@@ -6912,19 +7021,206 @@ export class FactoryLoop {
|
|
|
6912
7021
|
}
|
|
6913
7022
|
this.#increment('dispatchCapacityBackoffResets');
|
|
6914
7023
|
}
|
|
6915
|
-
|
|
7024
|
+
/**
|
|
7025
|
+
* Charge one failed release against a work unit's budget.
|
|
7026
|
+
*
|
|
7027
|
+
* Returns `false` once the budget is spent, and the caller must NOT re-arm.
|
|
7028
|
+
* Failing closed here is deliberate and is the opposite of a dispatch gate:
|
|
7029
|
+
* an unrecordable dispatch claim must abort the dispatch, but an unbounded
|
|
7030
|
+
* release must abort the RETRY — the work is already done either way, and
|
|
7031
|
+
* the only thing still running is the spin.
|
|
7032
|
+
*/
|
|
7033
|
+
#chargeReleaseAttempt(record, key, context) {
|
|
7034
|
+
if (this.#dispatchLifecycleReleaseAbandoned.has(key)) {
|
|
7035
|
+
// Re-entry, and the reason relinquishing the lease once is not enough.
|
|
7036
|
+
// `#driveDispatchLifecycle` re-claims the lease at the TOP of every
|
|
7037
|
+
// drive, before it has read the phase, so anything that drives an
|
|
7038
|
+
// already-dead-lettered key — the held-agent-deadline sweep, a registry
|
|
7039
|
+
// restore, a takeover — puts the epoch straight back into the renewal
|
|
7040
|
+
// map and re-arms the livelock this bound just escaped. Whenever the
|
|
7041
|
+
// budget declines a re-arm, ownership goes back too.
|
|
7042
|
+
this.#trackDispatchLifecycleDrive(this.#relinquishDispatchLifecycleLease(key, record.issue.key));
|
|
7043
|
+
return false;
|
|
7044
|
+
}
|
|
7045
|
+
const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
|
|
7046
|
+
this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
|
|
7047
|
+
if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
|
|
7048
|
+
return true;
|
|
7049
|
+
this.#dispatchLifecycleReleaseAbandoned.add(key);
|
|
7050
|
+
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
7051
|
+
this.#increment('dispatchLifecycleReleaseAbandoned');
|
|
7052
|
+
const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
|
|
7053
|
+
// Cleanup is armed BEFORE anything that can throw (#391 review, P1). The
|
|
7054
|
+
// only statements above are set/map writes and a counter, none of which can
|
|
7055
|
+
// reject; `this.#logger.error` below is caller-supplied and can. Ordering
|
|
7056
|
+
// the drive first means nothing between "this unit is abandoned" and "its
|
|
7057
|
+
// lease is handed back" is allowed to fail in a way that skips the handback
|
|
7058
|
+
// — which is precisely the defect shape this whole method exists to fix.
|
|
7059
|
+
this.#trackDispatchLifecycleDrive(this.#releaseDeadLetteredSlot(record, key)
|
|
7060
|
+
.catch((error) => {
|
|
7061
|
+
this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
|
|
7062
|
+
issue: record.issue.key,
|
|
7063
|
+
error: describeError(error).errorMessage,
|
|
7064
|
+
});
|
|
7065
|
+
}));
|
|
7066
|
+
// `error`, not `warn`. Every previous layer of this failure was invisible
|
|
7067
|
+
// until somebody read stderr by hand; a work unit whose cleanup this
|
|
7068
|
+
// process has permanently given up on is exactly the event that must not
|
|
7069
|
+
// be inferable only from the absence of further log lines.
|
|
7070
|
+
this.#logger.error?.('[factory] release retries exhausted; abandoning cleanup for this work unit', {
|
|
7071
|
+
issue: record.issue.key,
|
|
7072
|
+
attempts: attempts - 1,
|
|
7073
|
+
maxAttempts: DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS,
|
|
7074
|
+
context,
|
|
7075
|
+
// The durable lifecycle is retained on purpose: a takeover or a restart
|
|
7076
|
+
// re-drives it from the persisted phase. This bounds THIS process's
|
|
7077
|
+
// spin, it does not declare the work unit clean.
|
|
7078
|
+
durableLifecycleRetained,
|
|
7079
|
+
});
|
|
7080
|
+
return false;
|
|
7081
|
+
}
|
|
7082
|
+
/** Keeps a lifecycle-side effect awaitable by `stop()` without leaking the set entry. */
|
|
7083
|
+
#trackDispatchLifecycleDrive(promise) {
|
|
7084
|
+
const drive = promise.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7085
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
7086
|
+
}
|
|
7087
|
+
/**
|
|
7088
|
+
* Stop asserting durable ownership of a work unit this process will not drive
|
|
7089
|
+
* again.
|
|
7090
|
+
*
|
|
7091
|
+
* `#dispatchLifecycleEpochs` is not merely a cache. `#renewDispatchLifecycles`
|
|
7092
|
+
* walks it every `DISPATCH_LIFECYCLE_RENEW_MS` and re-stamps a full
|
|
7093
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` onto every key it finds, unconditionally.
|
|
7094
|
+
*
|
|
7095
|
+
* A dead-lettered release deliberately leaves its row in the non-terminal
|
|
7096
|
+
* `releasing` phase so a successor or a restart can re-drive the cleanup with
|
|
7097
|
+
* a fresh budget. That is only a recovery path if the successor can CLAIM the
|
|
7098
|
+
* row — and a lease renewed forever by a process that has permanently given
|
|
7099
|
+
* up driving it is a claim nobody can ever win. #379 bounded the retry and
|
|
7100
|
+
* handed back the batch slot, then kept the key locked for the life of the
|
|
7101
|
+
* process, which is how production reached four issues (a dispatch canary
|
|
7102
|
+
* among them) all logging `durable dispatch is leased by another publisher`
|
|
7103
|
+
* at 1 Hz for three days while the holder logged nothing but 503
|
|
7104
|
+
* `agent_host_unavailable`.
|
|
7105
|
+
*
|
|
7106
|
+
* The epoch is dropped BEFORE the durable release, so a renewal tick that
|
|
7107
|
+
* STARTS after this point finds nothing to renew. That ordering alone is not
|
|
7108
|
+
* sufficient and an earlier version of this comment wrongly claimed it was
|
|
7109
|
+
* (#391 review, P2): `#renewDispatchLifecycles` iterates a snapshot array, so
|
|
7110
|
+
* a tick already in flight still holds this key and would restore the lease
|
|
7111
|
+
* for a full term. The guarantee comes from `renewDispatchLifecycle` fencing
|
|
7112
|
+
* on expiry as well as owner and epoch, which makes a relinquished lease
|
|
7113
|
+
* unrenewable however the two race; the epoch drop and the loop's live re-read
|
|
7114
|
+
* narrow the window ahead of it.
|
|
7115
|
+
*
|
|
7116
|
+
* If the durable release itself fails, the dropped epoch alone still ends the
|
|
7117
|
+
* livelock: nothing renews the lease any more, so it expires within
|
|
7118
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` instead of never.
|
|
7119
|
+
*/
|
|
7120
|
+
async #relinquishDispatchLifecycleLease(key, issueKey) {
|
|
7121
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
7122
|
+
if (epoch === undefined)
|
|
7123
|
+
return;
|
|
7124
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
7125
|
+
try {
|
|
7126
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
7127
|
+
this.#increment('dispatchLifecycleLeasesRelinquished');
|
|
7128
|
+
}
|
|
7129
|
+
catch (error) {
|
|
7130
|
+
this.#logger.warn?.('[factory] could not relinquish the dispatch lease of an abandoned work unit', {
|
|
7131
|
+
issue: issueKey,
|
|
7132
|
+
// The lease still expires on its own now that nothing renews it.
|
|
7133
|
+
expiresWithinMs: DISPATCH_LIFECYCLE_LEASE_MS,
|
|
7134
|
+
error: describeError(error).errorMessage,
|
|
7135
|
+
});
|
|
7136
|
+
}
|
|
7137
|
+
}
|
|
7138
|
+
/**
|
|
7139
|
+
* Hand back the batch slot of a work unit whose release was dead-lettered.
|
|
7140
|
+
*
|
|
7141
|
+
* Without this the bound trades an unbounded 1 Hz spin for a permanently
|
|
7142
|
+
* leaked slot, which is not obviously the better failure (#379 review, P1):
|
|
7143
|
+
* a spin is loud and self-describing, whereas an in-flight record nobody will
|
|
7144
|
+
* ever complete silently reduces dispatch capacity until the process is
|
|
7145
|
+
* restarted. The slot is process-local accounting and is always freed here.
|
|
7146
|
+
*
|
|
7147
|
+
* What is NOT declared clean is the work itself. On a durable lifecycle the
|
|
7148
|
+
* persisted `releasing` phase is deliberately left in place, so a successor
|
|
7149
|
+
* or a restart re-drives the same cleanup with a fresh budget; freeing the
|
|
7150
|
+
* local slot does not write a terminal phase. On a local lifecycle there is
|
|
7151
|
+
* no durable record to retain, so the batch record is all there is and
|
|
7152
|
+
* completing it is what keeps capacity honest.
|
|
7153
|
+
*
|
|
7154
|
+
* The DURABLE lease has to go back with the slot. Retaining the row for a
|
|
7155
|
+
* successor while renewing the lease that locks the successor out is not a
|
|
7156
|
+
* handoff, it is a permanent block on the key — see
|
|
7157
|
+
* `#relinquishDispatchLifecycleLease`.
|
|
7158
|
+
*
|
|
7159
|
+
* The handback lives in a `finally` for a reason worth stating plainly
|
|
7160
|
+
* (#391 review, P1). The FIRST version of this fix relinquished after
|
|
7161
|
+
* `#writeInFlightRegistry()`, on the happy path only — so a rejecting
|
|
7162
|
+
* registry write would skip it and leave the abandoned key renewing its lease
|
|
7163
|
+
* forever, with nothing but a `warn` to show for it. That is the identical
|
|
7164
|
+
* shape of the bug being fixed (#379 freed the slot but not the lease, on the
|
|
7165
|
+
* failure path), reproduced one level up. Cleanup that only runs when the
|
|
7166
|
+
* rest of cleanup succeeded is not cleanup. `#batch()` and
|
|
7167
|
+
* `#writeInFlightRegistry()` can both reject; neither may strand the key.
|
|
7168
|
+
*/
|
|
7169
|
+
async #releaseDeadLetteredSlot(record, key) {
|
|
7170
|
+
let next;
|
|
7171
|
+
try {
|
|
7172
|
+
const batch = await this.#batch();
|
|
7173
|
+
next = batch.complete(record.issue);
|
|
7174
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
7175
|
+
await this.#writeInFlightRegistry();
|
|
7176
|
+
}
|
|
7177
|
+
finally {
|
|
7178
|
+
// Unconditional, and safe to put in a `finally` because
|
|
7179
|
+
// `#relinquishDispatchLifecycleLease` handles its own errors and cannot
|
|
7180
|
+
// throw — so it can never mask the failure that brought us here.
|
|
7181
|
+
await this.#relinquishDispatchLifecycleLease(key, record.issue.key);
|
|
7182
|
+
}
|
|
7183
|
+
// A freed slot that nothing is admitted into is only half the repair.
|
|
7184
|
+
if (next && !this.#stopping)
|
|
7185
|
+
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
7186
|
+
}
|
|
7187
|
+
/** Clears a work unit's release budget once cleanup actually succeeds. */
|
|
7188
|
+
#clearReleaseAttempts(key) {
|
|
7189
|
+
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
7190
|
+
this.#dispatchLifecycleReleaseAbandoned.delete(key);
|
|
7191
|
+
}
|
|
7192
|
+
#scheduleDispatchLifecycleRetry(record, delayMs = this.#dispatchLifecycleRetryMs, opts = {}) {
|
|
6916
7193
|
const key = dispatchLifecycleKey(record.issue);
|
|
6917
7194
|
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
6918
7195
|
return;
|
|
7196
|
+
// Only a release re-arm spends the budget. A capacity or ownership wait is
|
|
7197
|
+
// a legitimate wait for someone else to finish (#303) and is bounded by
|
|
7198
|
+
// its rate, not its count — spending the budget on one would abandon work
|
|
7199
|
+
// that was never failing.
|
|
7200
|
+
if (opts.releaseAttempt === true && !this.#chargeReleaseAttempt(record, key, 'durable-lifecycle'))
|
|
7201
|
+
return;
|
|
6919
7202
|
const timer = setTimeout(() => {
|
|
6920
7203
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
6921
7204
|
const drive = this.#driveDispatchLifecycle(key)
|
|
6922
7205
|
.then(() => {
|
|
6923
7206
|
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
6924
7207
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
7208
|
+
// DELIBERATELY NOT `#clearReleaseAttempts` (#379 review, P1).
|
|
7209
|
+
//
|
|
7210
|
+
// A FAILED release reaches here. `#finishDurableRelease` returns
|
|
7211
|
+
// `false` rather than throwing, and `#driveDispatchLifecycle`
|
|
7212
|
+
// discards that boolean in its `phase === 'releasing'` branch, so
|
|
7213
|
+
// the drive RESOLVES on a failed release and this handler runs on
|
|
7214
|
+
// every re-arm. Clearing the budget here zeroed it once per pass and
|
|
7215
|
+
// the counter could never reach the cap — the same never-fires this
|
|
7216
|
+
// bound exists to avoid, moved from the `.catch()` to the `.then()`.
|
|
7217
|
+
//
|
|
7218
|
+
// The budget is refunded where success is actually known:
|
|
7219
|
+
// `#finishDurableRelease` clears it on real per-agent progress and
|
|
7220
|
+
// again when the work unit completes.
|
|
6925
7221
|
})
|
|
6926
7222
|
.catch((error) => {
|
|
6927
|
-
let nextDelayMs =
|
|
7223
|
+
let nextDelayMs = this.#dispatchLifecycleRetryMs;
|
|
6928
7224
|
if (error instanceof DispatchLifecycleCapacityError) {
|
|
6929
7225
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
6930
7226
|
nextDelayMs = this.#recordDispatchCapacityWait(record, key);
|
|
@@ -6939,7 +7235,7 @@ export class FactoryLoop {
|
|
|
6939
7235
|
leaseRemainingMs: error.leaseUntilMs === undefined
|
|
6940
7236
|
? undefined
|
|
6941
7237
|
: Math.max(0, error.leaseUntilMs - this.#clock.now()),
|
|
6942
|
-
retryMs:
|
|
7238
|
+
retryMs: this.#dispatchLifecycleRetryMs,
|
|
6943
7239
|
});
|
|
6944
7240
|
}
|
|
6945
7241
|
}
|
|
@@ -6951,6 +7247,20 @@ export class FactoryLoop {
|
|
|
6951
7247
|
error: describeError(error).errorMessage,
|
|
6952
7248
|
});
|
|
6953
7249
|
}
|
|
7250
|
+
// Deliberately UNCHARGED (#379 review, P1). This arm re-arms for
|
|
7251
|
+
// dispatch, publishing and recovery failures as well as releases, and
|
|
7252
|
+
// charging all of them would dead-letter a work unit that was never
|
|
7253
|
+
// stuck in a release loop. Charging is confined to
|
|
7254
|
+
// `#scheduleReleaseRetry`, whose every caller is a release failure:
|
|
7255
|
+
// the three inside `#finishDurableRelease` and `#completeIssue`'s
|
|
7256
|
+
// catch once `releaseReasonForRetry` is set.
|
|
7257
|
+
//
|
|
7258
|
+
// The gap this leaves, stated plainly: a release failure that THREW
|
|
7259
|
+
// out of `#finishDurableRelease` instead of returning `false` would
|
|
7260
|
+
// reach here and re-arm unbounded. Every failure path in that method
|
|
7261
|
+
// returns `false` and schedules its own retry, so this is not a
|
|
7262
|
+
// reachable shape today — and if one appears it degrades to the
|
|
7263
|
+
// pre-existing unbounded behaviour rather than to a wrong dead-letter.
|
|
6954
7264
|
this.#scheduleDispatchLifecycleRetry(record, nextDelayMs);
|
|
6955
7265
|
})
|
|
6956
7266
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
@@ -6960,12 +7270,19 @@ export class FactoryLoop {
|
|
|
6960
7270
|
}
|
|
6961
7271
|
#scheduleReleaseRetry(record, reason) {
|
|
6962
7272
|
if (this.#usesDurableDispatchLifecycle()) {
|
|
6963
|
-
this.#scheduleDispatchLifecycleRetry(record);
|
|
7273
|
+
this.#scheduleDispatchLifecycleRetry(record, this.#dispatchLifecycleRetryMs, { releaseAttempt: true });
|
|
6964
7274
|
return;
|
|
6965
7275
|
}
|
|
6966
7276
|
const key = dispatchLifecycleKey(record.issue);
|
|
6967
7277
|
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
6968
7278
|
return;
|
|
7279
|
+
// Charged here rather than in the `.catch()` below, because the loop this
|
|
7280
|
+
// bounds does not go through the `.catch()`: `#finishDurableRelease`
|
|
7281
|
+
// returns `false` on a failed release and re-arms itself, so every re-arm
|
|
7282
|
+
// arrives on the resolved path. A bound on the rejection handler alone
|
|
7283
|
+
// would have been a fix that never fired.
|
|
7284
|
+
if (!this.#chargeReleaseAttempt(record, key, 'local-completion'))
|
|
7285
|
+
return;
|
|
6969
7286
|
const timer = setTimeout(() => {
|
|
6970
7287
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
6971
7288
|
const drive = this.#finishDurableRelease(record, reason)
|
|
@@ -6979,7 +7296,7 @@ export class FactoryLoop {
|
|
|
6979
7296
|
})
|
|
6980
7297
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
6981
7298
|
this.#dispatchLifecycleDrives.add(drive);
|
|
6982
|
-
},
|
|
7299
|
+
}, this.#dispatchLifecycleRetryMs);
|
|
6983
7300
|
timer.unref?.();
|
|
6984
7301
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
6985
7302
|
}
|
|
@@ -7192,6 +7509,9 @@ export class FactoryLoop {
|
|
|
7192
7509
|
return;
|
|
7193
7510
|
}
|
|
7194
7511
|
if (lifecycle.phase === 'releasing') {
|
|
7512
|
+
// The boolean is discarded here, as it always was — and that is exactly
|
|
7513
|
+
// why the scheduler's success handler must not refund the release budget
|
|
7514
|
+
// (#379 review, P1). A failed release resolves through this line.
|
|
7195
7515
|
await this.#finishDurableRelease(record, lifecycle.releaseReason);
|
|
7196
7516
|
}
|
|
7197
7517
|
}
|
|
@@ -7398,6 +7718,7 @@ export class FactoryLoop {
|
|
|
7398
7718
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
7399
7719
|
.map((agent) => agent.name) ?? this.#localReleaseCheckpoints.get(releaseKey) ?? []);
|
|
7400
7720
|
const failed = [];
|
|
7721
|
+
const releasedOnEntry = released.size;
|
|
7401
7722
|
for (const agent of record.agents) {
|
|
7402
7723
|
if (released.has(agent[0]))
|
|
7403
7724
|
continue;
|
|
@@ -7418,6 +7739,13 @@ export class FactoryLoop {
|
|
|
7418
7739
|
await this.#writeInFlightRegistry();
|
|
7419
7740
|
if (failed.length > 0) {
|
|
7420
7741
|
this.#increment('dispatchLifecycleReleaseRetries');
|
|
7742
|
+
// Real progress refunds the budget, so the ten attempts bound CONSECUTIVE
|
|
7743
|
+
// no-progress passes rather than capping a slow multi-agent release. This
|
|
7744
|
+
// terminates because an agent released once is checkpointed and skipped
|
|
7745
|
+
// on the next pass, so the remaining set strictly shrinks — a refund can
|
|
7746
|
+
// only be earned a finite number of times.
|
|
7747
|
+
if (released.size > releasedOnEntry)
|
|
7748
|
+
this.#clearReleaseAttempts(releaseKey);
|
|
7421
7749
|
this.#scheduleReleaseRetry(record, reason);
|
|
7422
7750
|
return false;
|
|
7423
7751
|
}
|
|
@@ -7436,6 +7764,10 @@ export class FactoryLoop {
|
|
|
7436
7764
|
}
|
|
7437
7765
|
const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
|
|
7438
7766
|
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
7767
|
+
// Every agent is released. Nothing is left to retry, so the budget goes
|
|
7768
|
+
// back — including the abandoned marker, so a reopened work unit that
|
|
7769
|
+
// reuses this key starts with a full budget rather than a spent one.
|
|
7770
|
+
this.#clearReleaseAttempts(releaseKey);
|
|
7439
7771
|
if (next)
|
|
7440
7772
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
7441
7773
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
@@ -9392,13 +9724,6 @@ export class FactoryLoop {
|
|
|
9392
9724
|
batch.recordDryRun(record, spec, invocationId);
|
|
9393
9725
|
return { name: spec.name };
|
|
9394
9726
|
}
|
|
9395
|
-
// Persist intent before the remote side effect. If the owner crashes after
|
|
9396
|
-
// the spawn ack but before recording its result, takeover retries the same
|
|
9397
|
-
// deterministic invocation id instead of inventing a second worker.
|
|
9398
|
-
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9399
|
-
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9400
|
-
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9401
|
-
}
|
|
9402
9727
|
let roster;
|
|
9403
9728
|
try {
|
|
9404
9729
|
roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
|
|
@@ -9408,6 +9733,15 @@ export class FactoryLoop {
|
|
|
9408
9733
|
}
|
|
9409
9734
|
const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
|
|
9410
9735
|
if (rosterAgent) {
|
|
9736
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9737
|
+
const host = rosterAgent.node
|
|
9738
|
+
? roster.nodes.find((node) => node.name === rosterAgent.node && node.live && node.capabilities.includes(spec.capability))
|
|
9739
|
+
: undefined;
|
|
9740
|
+
if (!host) {
|
|
9741
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
9742
|
+
}
|
|
9743
|
+
spec = { ...spec, node: host.name };
|
|
9744
|
+
}
|
|
9411
9745
|
const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
|
|
9412
9746
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9413
9747
|
batch.recordSpawn(record, spec, invocationId, {
|
|
@@ -9425,6 +9759,21 @@ export class FactoryLoop {
|
|
|
9425
9759
|
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
9426
9760
|
return { name: spec.name };
|
|
9427
9761
|
}
|
|
9762
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9763
|
+
const loads = new Map();
|
|
9764
|
+
for (const agent of roster.agents) {
|
|
9765
|
+
if (agent.node)
|
|
9766
|
+
loads.set(agent.node, (loads.get(agent.node) ?? 0) + 1);
|
|
9767
|
+
}
|
|
9768
|
+
spec = { ...spec, node: liveFleetNodeForSpec(spec, roster, loads) };
|
|
9769
|
+
}
|
|
9770
|
+
// Persist intent before the remote side effect. If the owner crashes after
|
|
9771
|
+
// the spawn ack but before recording its result, takeover retries the same
|
|
9772
|
+
// deterministic invocation id instead of inventing a second worker.
|
|
9773
|
+
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9774
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9775
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9776
|
+
}
|
|
9428
9777
|
await this.#prepareAgentWorktree(record, spec);
|
|
9429
9778
|
let result;
|
|
9430
9779
|
try {
|
|
@@ -9464,6 +9813,26 @@ export class FactoryLoop {
|
|
|
9464
9813
|
await this.#releaseOrphanedLatePlacement(record, spec, result);
|
|
9465
9814
|
throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
|
|
9466
9815
|
}
|
|
9816
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9817
|
+
const registered = await this.#awaitRemoteAgentRegistration(result.name, spec.capability, result.node);
|
|
9818
|
+
if (!registered) {
|
|
9819
|
+
try {
|
|
9820
|
+
await this.#fleet.release(result.name, 'spawn-registration-timeout');
|
|
9821
|
+
this.#fleet.markAgentTerminal?.(result.name, 'spawn-registration-timeout');
|
|
9822
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, true);
|
|
9823
|
+
}
|
|
9824
|
+
catch (error) {
|
|
9825
|
+
if (error instanceof RemoteAgentRegistrationTimeoutError)
|
|
9826
|
+
throw error;
|
|
9827
|
+
// Cleanup is unconfirmed. Persist the placement so a successor can
|
|
9828
|
+
// retry the release; forgetting it would be worse than retaining a
|
|
9829
|
+
// nonterminal lifecycle for a worker that may still be alive.
|
|
9830
|
+
batch.recordSpawn(record, spec, invocationId, result);
|
|
9831
|
+
await this.#saveDispatchLifecycle(record, 'dispatching');
|
|
9832
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, false, error);
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
}
|
|
9467
9836
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9468
9837
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
9469
9838
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
@@ -9475,6 +9844,89 @@ export class FactoryLoop {
|
|
|
9475
9844
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
9476
9845
|
return { name: result.name };
|
|
9477
9846
|
}
|
|
9847
|
+
async #awaitRemoteAgentRegistration(name, capability, expectedNode) {
|
|
9848
|
+
const deadlineAtMs = this.#clock.now() + REMOTE_AGENT_REGISTRATION_TIMEOUT_MS;
|
|
9849
|
+
do {
|
|
9850
|
+
try {
|
|
9851
|
+
if (expectedNode) {
|
|
9852
|
+
if (this.#fleet.isAgentRegistered) {
|
|
9853
|
+
if (await this.#fleet.isAgentRegistered({ name, node: expectedNode, capability }))
|
|
9854
|
+
return true;
|
|
9855
|
+
}
|
|
9856
|
+
else {
|
|
9857
|
+
const roster = await this.#fleet.roster();
|
|
9858
|
+
const agent = roster.agents.find((candidate) => candidate.name === name && candidate.node === expectedNode);
|
|
9859
|
+
const node = agent
|
|
9860
|
+
? roster.nodes.find((candidate) => candidate.name === expectedNode && candidate.live && candidate.capabilities.includes(capability))
|
|
9861
|
+
: undefined;
|
|
9862
|
+
if (agent && node)
|
|
9863
|
+
return true;
|
|
9864
|
+
}
|
|
9865
|
+
}
|
|
9866
|
+
}
|
|
9867
|
+
catch (error) {
|
|
9868
|
+
this.#logger.warn?.('[factory] remote agent registration probe failed; retrying within startup bound', {
|
|
9869
|
+
agent: name,
|
|
9870
|
+
error: describeError(error).errorMessage,
|
|
9871
|
+
});
|
|
9872
|
+
}
|
|
9873
|
+
const remainingMs = deadlineAtMs - this.#clock.now();
|
|
9874
|
+
if (remainingMs <= 0)
|
|
9875
|
+
return false;
|
|
9876
|
+
await this.#clock.sleep(Math.min(REMOTE_AGENT_REGISTRATION_POLL_MS, remainingMs));
|
|
9877
|
+
} while (this.#clock.now() <= deadlineAtMs);
|
|
9878
|
+
return false;
|
|
9879
|
+
}
|
|
9880
|
+
async #rollbackUnregisteredRemoteDispatch(record, acknowledged) {
|
|
9881
|
+
const handoffs = this.#dispatchFailureHandoffs(record, acknowledged);
|
|
9882
|
+
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
9883
|
+
const hasWorktrees = handoffs.some((handoff) => handoff.worktree);
|
|
9884
|
+
if (hasWorktrees) {
|
|
9885
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'spawn-registration-timeout', { skipNeverPlacedAgents: true }))
|
|
9886
|
+
return false;
|
|
9887
|
+
}
|
|
9888
|
+
else {
|
|
9889
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs
|
|
9890
|
+
.filter((handoff) => handoff.tracked.result !== undefined)
|
|
9891
|
+
.map((handoff) => [handoff.name, handoff.tracked]), 'spawn-registration-timeout', 'completion');
|
|
9892
|
+
if (failed.length > 0)
|
|
9893
|
+
return false;
|
|
9894
|
+
for (const handoff of handoffs) {
|
|
9895
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
9896
|
+
}
|
|
9897
|
+
}
|
|
9898
|
+
try {
|
|
9899
|
+
await this.#teardownPreviews(record);
|
|
9900
|
+
}
|
|
9901
|
+
catch (error) {
|
|
9902
|
+
this.#logger.warn?.('[factory] retained unregistered spawn lifecycle after preview rollback failed', {
|
|
9903
|
+
issue: record.issue.key,
|
|
9904
|
+
error: describeError(error).errorMessage,
|
|
9905
|
+
});
|
|
9906
|
+
return false;
|
|
9907
|
+
}
|
|
9908
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
9909
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
9910
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
9911
|
+
const lease = lifecycle?.lease;
|
|
9912
|
+
if (epoch === undefined ||
|
|
9913
|
+
!lease ||
|
|
9914
|
+
lease.owner !== this.#dispatchLifecycleOwner ||
|
|
9915
|
+
lease.epoch !== epoch ||
|
|
9916
|
+
!await this.#state.clearClaimedDispatchLifecycle(this.#workspaceId, key, lease))
|
|
9917
|
+
return false;
|
|
9918
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
9919
|
+
if (retryTimer)
|
|
9920
|
+
clearTimeout(retryTimer);
|
|
9921
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
9922
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
9923
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
9924
|
+
const batch = await this.#batch();
|
|
9925
|
+
batch.abandon(record.issue);
|
|
9926
|
+
await this.#writeInFlightRegistry();
|
|
9927
|
+
this.#resetDispatchCapacityBackoff();
|
|
9928
|
+
return true;
|
|
9929
|
+
}
|
|
9478
9930
|
/**
|
|
9479
9931
|
* Is this process still the owner of a lifecycle that is not already done?
|
|
9480
9932
|
*
|
|
@@ -18041,6 +18493,50 @@ function dispatchSpecs(decision) {
|
|
|
18041
18493
|
}
|
|
18042
18494
|
return [...decision.implementers, decision.reviewer];
|
|
18043
18495
|
}
|
|
18496
|
+
function liveFleetNodeForSpec(spec, roster, assignedLoads) {
|
|
18497
|
+
const eligible = roster.nodes.filter((node) => node.live && node.capabilities.includes(spec.capability));
|
|
18498
|
+
if (eligible.length === 0)
|
|
18499
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
18500
|
+
const explicitlyRequested = spec.node && spec.node !== 'self'
|
|
18501
|
+
? eligible.find((node) => node.name === spec.node)
|
|
18502
|
+
: undefined;
|
|
18503
|
+
const selected = explicitlyRequested ?? [...eligible].sort((left, right) => {
|
|
18504
|
+
const loadDifference = (assignedLoads.get(left.name) ?? 0) - (assignedLoads.get(right.name) ?? 0);
|
|
18505
|
+
return loadDifference || left.name.localeCompare(right.name);
|
|
18506
|
+
})[0];
|
|
18507
|
+
assignedLoads.set(selected.name, (assignedLoads.get(selected.name) ?? 0) + 1);
|
|
18508
|
+
return selected.name;
|
|
18509
|
+
}
|
|
18510
|
+
/**
|
|
18511
|
+
* Resolve every remote dispatch spec against one canonical roster snapshot.
|
|
18512
|
+
*
|
|
18513
|
+
* A configured node is a preference, not an entitlement: if it is offline or
|
|
18514
|
+
* no longer advertises the required capability, placement is re-selected from
|
|
18515
|
+
* the live fleet. This runs before durable lifecycle creation, so an empty
|
|
18516
|
+
* eligible set cannot leave a claim or consume a batch slot.
|
|
18517
|
+
*/
|
|
18518
|
+
function decisionWithVerifiedRemotePlacements(decision, roster) {
|
|
18519
|
+
const assignedLoads = new Map();
|
|
18520
|
+
for (const agent of roster.agents) {
|
|
18521
|
+
if (agent.node)
|
|
18522
|
+
assignedLoads.set(agent.node, (assignedLoads.get(agent.node) ?? 0) + 1);
|
|
18523
|
+
}
|
|
18524
|
+
const place = (spec) => ({
|
|
18525
|
+
...spec,
|
|
18526
|
+
node: liveFleetNodeForSpec(spec, roster, assignedLoads),
|
|
18527
|
+
});
|
|
18528
|
+
if (decision.scope === 'workflow') {
|
|
18529
|
+
return {
|
|
18530
|
+
...structuredClone(decision),
|
|
18531
|
+
...(decision.workflow ? { workflow: place(decision.workflow) } : {}),
|
|
18532
|
+
};
|
|
18533
|
+
}
|
|
18534
|
+
return {
|
|
18535
|
+
...structuredClone(decision),
|
|
18536
|
+
implementers: decision.implementers.map(place),
|
|
18537
|
+
reviewer: place(decision.reviewer),
|
|
18538
|
+
};
|
|
18539
|
+
}
|
|
18044
18540
|
function dispatchSessionOwner(decision) {
|
|
18045
18541
|
for (const spec of dispatchSpecs(decision)) {
|
|
18046
18542
|
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
@@ -18904,28 +19400,59 @@ const PROBE_PR_INDEX_FALLBACK_COUNTERS = {
|
|
|
18904
19400
|
'index-absent': 'probePrIndexAbsent',
|
|
18905
19401
|
'index-shape-unrecognised': 'probePrIndexShapeUnrecognised',
|
|
18906
19402
|
'index-without-head-ref': 'probePrIndexWithoutHeadRef',
|
|
18907
|
-
|
|
18908
|
-
|
|
18909
|
-
|
|
18910
|
-
'index-
|
|
19403
|
+
'index-row-malformed': 'probePrIndexRowMalformed',
|
|
19404
|
+
'index-incomplete': 'probePrIndexIncomplete',
|
|
19405
|
+
'index-no-match': 'probePrIndexNoMatch',
|
|
19406
|
+
'index-disagreed': 'probePrIndexDisagreed',
|
|
18911
19407
|
};
|
|
18912
19408
|
/**
|
|
18913
|
-
*
|
|
19409
|
+
* The lowest `issuePrMatchScore` tier the pull index can compute for itself.
|
|
18914
19410
|
*
|
|
18915
|
-
*
|
|
18916
|
-
*
|
|
18917
|
-
*
|
|
18918
|
-
*
|
|
18919
|
-
*
|
|
18920
|
-
*
|
|
18921
|
-
*
|
|
18922
|
-
*
|
|
18923
|
-
*
|
|
19411
|
+
* The scores are: branch match 30, issue key in title 20, explicit reference in
|
|
19412
|
+
* body 10. A row carries `headRef` and `title` but NOT `body`, so a row can be
|
|
19413
|
+
* scored exactly at the 30 and 20 tiers and is blind at 10. A pull request the
|
|
19414
|
+
* index scores at 20 or better therefore cannot be beaten by anything the index
|
|
19415
|
+
* could not see, because the best a body-only match can ever earn is 10 — which
|
|
19416
|
+
* makes an index hit of 20 or 30 provably the same winner the full walk would
|
|
19417
|
+
* have chosen. Below that threshold the index proves nothing and the walk is
|
|
19418
|
+
* mandatory.
|
|
19419
|
+
*
|
|
19420
|
+
* Deliberately expressed as a threshold rather than a "highest score wins"
|
|
19421
|
+
* shortcut: the constant is the whole safety argument, and inlining `20` at the
|
|
19422
|
+
* comparison would bury it.
|
|
18924
19423
|
*/
|
|
18925
|
-
const
|
|
19424
|
+
const PULL_INDEX_DECISIVE_SCORE = 20;
|
|
19425
|
+
/**
|
|
19426
|
+
* Read `pulls/_index.json` and either return rows the probe may rank, or say
|
|
19427
|
+
* why it may not.
|
|
19428
|
+
*
|
|
19429
|
+
* All-or-nothing per repository, and the discipline is copied verbatim from
|
|
19430
|
+
* Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
|
|
19431
|
+
* ENTIRE repository if any row is legacy or malformed. One legacy row is enough
|
|
19432
|
+
* to poison the whole conclusion: the primary match is a branch match worth 30,
|
|
19433
|
+
* so a row without `headRef` cannot be ruled out as the real winner, and ranking
|
|
19434
|
+
* the remaining rows against each other would answer a different question from
|
|
19435
|
+
* the one the walk answers.
|
|
19436
|
+
*
|
|
19437
|
+
* Mixed indexes are the expected state today, not a corruption. `headRef` landed
|
|
19438
|
+
* in `@relayfile/adapter-github@0.5.7` and there is no pull-side backfill: the
|
|
19439
|
+
* incremental writers replace exactly the row they touched and pass every other
|
|
19440
|
+
* row through verbatim, so a webhook update hydrates one row and leaves the rest
|
|
19441
|
+
* legacy. Only an eager re-ingest converges a repository, because it rebuilds
|
|
19442
|
+
* the file from a list-pulls response in which every row carries `head.ref`.
|
|
19443
|
+
*
|
|
19444
|
+
* Row facts this relies on, from that contract: `state` is GitHub's lowercase
|
|
19445
|
+
* `"open"`/`"closed"` and there is no `"merged"` state — `merged: true` (with
|
|
19446
|
+
* `mergedAt`) is the only merged signal and `merged: false` is never written.
|
|
19447
|
+
* `headRef` is the bare branch name (`head.ref`), matching what
|
|
19448
|
+
* `readProbePrCandidate` reads off a record, NOT the `owner:branch` form of
|
|
19449
|
+
* `head.label`. `id` and `updated` are deliberately not required here: the
|
|
19450
|
+
* oldest legacy rows carry neither.
|
|
19451
|
+
*/
|
|
19452
|
+
const readPullIndexForProbe = async (mount, repo) => {
|
|
18926
19453
|
const [owner, name] = repo.split('/');
|
|
18927
19454
|
if (!owner || !name)
|
|
18928
|
-
return 'index-absent';
|
|
19455
|
+
return { reason: 'index-absent' };
|
|
18929
19456
|
let parsed;
|
|
18930
19457
|
try {
|
|
18931
19458
|
const { content } = await mount.readFile(`${GITHUB_ISSUE_ROOT}/${owner}/${name}/pulls/_index.json`);
|
|
@@ -18936,23 +19463,159 @@ const classifyPullIndexFallback = async (mount, repo) => {
|
|
|
18936
19463
|
// that is the same convention the tree reads below already follow.
|
|
18937
19464
|
if (isPassWideRelayfileFault(error))
|
|
18938
19465
|
throw error;
|
|
18939
|
-
return 'index-absent';
|
|
19466
|
+
return { reason: 'index-absent' };
|
|
18940
19467
|
}
|
|
18941
19468
|
if (!Array.isArray(parsed))
|
|
18942
|
-
return 'index-shape-unrecognised';
|
|
19469
|
+
return { reason: 'index-shape-unrecognised' };
|
|
18943
19470
|
// An empty index proves nothing about the row contract, so it must not read
|
|
18944
19471
|
// as usable — `[].every(...)` is vacuously true.
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
|
|
19472
|
+
if (parsed.length === 0)
|
|
19473
|
+
return { reason: 'index-without-head-ref' };
|
|
19474
|
+
const rows = [];
|
|
19475
|
+
for (const entry of parsed) {
|
|
19476
|
+
const row = asRecord(entry);
|
|
19477
|
+
if (typeof row?.headRef !== 'string')
|
|
19478
|
+
return { reason: 'index-without-head-ref' };
|
|
19479
|
+
if (!Number.isSafeInteger(row.number) || Number(row.number) <= 0 ||
|
|
19480
|
+
typeof row.title !== 'string' ||
|
|
19481
|
+
typeof row.state !== 'string' ||
|
|
19482
|
+
(row.merged !== undefined && typeof row.merged !== 'boolean')) {
|
|
19483
|
+
return { reason: 'index-row-malformed' };
|
|
19484
|
+
}
|
|
19485
|
+
rows.push({
|
|
19486
|
+
number: Number(row.number),
|
|
19487
|
+
title: row.title,
|
|
19488
|
+
headRef: row.headRef,
|
|
19489
|
+
state: row.state,
|
|
19490
|
+
merged: row.merged,
|
|
19491
|
+
});
|
|
19492
|
+
}
|
|
19493
|
+
return { rows };
|
|
18948
19494
|
};
|
|
18949
|
-
|
|
19495
|
+
/**
|
|
19496
|
+
* Rank index rows exactly as the walk ranks records it has read.
|
|
19497
|
+
*
|
|
19498
|
+
* `body: ''` is a deliberate constant, not a placeholder for a value that
|
|
19499
|
+
* should have been threaded through. The index carries no body, so scoring with
|
|
19500
|
+
* an empty one makes `issuePrMatchScore` return only the tiers the index can
|
|
19501
|
+
* actually justify — 30 for a branch match, 20 for a title match, 0 otherwise —
|
|
19502
|
+
* and never invents the 10 it cannot see. Everything else (`requireTitleMarker`
|
|
19503
|
+
* gating, `allowLegacyGithubBranch`, the marker itself) is the SAME call the
|
|
19504
|
+
* walk makes on the same inputs, so the two paths cannot score differently.
|
|
19505
|
+
*
|
|
19506
|
+
* The tiebreak mirrors `b.score - a.score || b.prNumber - a.prNumber`: highest
|
|
19507
|
+
* score, then highest pull request number.
|
|
19508
|
+
*/
|
|
19509
|
+
const bestPullIndexRow = (rows, issue, marker, opts) => {
|
|
19510
|
+
let best;
|
|
19511
|
+
for (const row of rows) {
|
|
19512
|
+
// Same derivation `readProbePrCandidate` applies to a record: a merged pull
|
|
19513
|
+
// request reports MERGED regardless of the raw `state` the provider wrote.
|
|
19514
|
+
const state = row.merged === true ? 'MERGED' : row.state;
|
|
19515
|
+
if (opts.openOnly && normalizePrState(state) !== 'OPEN')
|
|
19516
|
+
continue;
|
|
19517
|
+
const score = issuePrMatchScore({ title: row.title, body: '', headRef: row.headRef }, issue, marker, opts);
|
|
19518
|
+
if (score < PULL_INDEX_DECISIVE_SCORE)
|
|
19519
|
+
continue;
|
|
19520
|
+
if (!best || score > best.score || (score === best.score && row.number > best.number)) {
|
|
19521
|
+
best = { number: row.number, score };
|
|
19522
|
+
}
|
|
19523
|
+
}
|
|
19524
|
+
return best;
|
|
19525
|
+
};
|
|
19526
|
+
/**
|
|
19527
|
+
* Build the resolver's candidate shape from one pull request record.
|
|
19528
|
+
*
|
|
19529
|
+
* Shared by the index fast path and the walk so the two can never disagree on a
|
|
19530
|
+
* field. Differential equivalence is the whole safety property of the fast path
|
|
19531
|
+
* and it should hold structurally, not by two copies staying in sync.
|
|
19532
|
+
*/
|
|
19533
|
+
const probePrCandidate = (repo, pr, path, score) => ({
|
|
19534
|
+
repo,
|
|
19535
|
+
prNumber: pr.number,
|
|
19536
|
+
draft: pr.draft,
|
|
19537
|
+
headRef: pr.headRef,
|
|
19538
|
+
headRepo: pr.headRepo,
|
|
19539
|
+
crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
|
|
19540
|
+
state: pr.state,
|
|
19541
|
+
url: pr.url,
|
|
19542
|
+
path,
|
|
19543
|
+
score,
|
|
19544
|
+
});
|
|
19545
|
+
/**
|
|
19546
|
+
* Answer "which pull request belongs to this issue?" from `pulls/_index.json`
|
|
19547
|
+
* instead of reading every mounted pull request record, or say why it could not.
|
|
19548
|
+
*
|
|
19549
|
+
* Reads at most two files: the index, and the one record the index names. That
|
|
19550
|
+
* record read is not an optimisation that was left on the table — it is load
|
|
19551
|
+
* bearing twice over. The index carries no `draft`, `headRepo`, `url` or mount
|
|
19552
|
+
* path, all of which the resolver's result type promises and callers persist as
|
|
19553
|
+
* `ownedPullRequest`; and re-scoring the real record is what turns "the index
|
|
19554
|
+
* said so" into "the record agrees", so a stale row can never decide an answer
|
|
19555
|
+
* on its own.
|
|
19556
|
+
*
|
|
19557
|
+
* The tree listing is NOT avoided, and that is deliberate. It costs two calls
|
|
19558
|
+
* per repository where the record walk cost one call per pull request — 4,000
|
|
19559
|
+
* of them in the live workspace, which is the cost that stalled a sweep for
|
|
19560
|
+
* 11m53s. Keeping it buys two things nothing else can: the exact path spelling
|
|
19561
|
+
* the walk would have reported (the nested layout's slug cannot be constructed,
|
|
19562
|
+
* only listed), and the completeness check below.
|
|
19563
|
+
*
|
|
19564
|
+
* Every condition that still walks:
|
|
19565
|
+
*
|
|
19566
|
+
* - anything `readPullIndexForProbe` refuses — absent, wrong shape, any row
|
|
19567
|
+
* without `headRef`, any row whose `number`/`title`/`state` is unusable;
|
|
19568
|
+
* - a pull request on disk with no row describing it, because an unindexed
|
|
19569
|
+
* record could hold the score-30 branch match;
|
|
19570
|
+
* - no row scoring `PULL_INDEX_DECISIVE_SCORE` or better, because a body-only
|
|
19571
|
+
* reference worth 10 may exist on disk and no index row can see a body;
|
|
19572
|
+
* - the named record failing to confirm its row — missing, different number,
|
|
19573
|
+
* different score, or not open when `openOnly` is set.
|
|
19574
|
+
*/
|
|
19575
|
+
const resolveProbePrFromPullIndex = async (mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer) => {
|
|
19576
|
+
const reading = await readPullIndexForProbe(mount, repo);
|
|
19577
|
+
if (!reading.rows)
|
|
19578
|
+
return { reason: reading.reason };
|
|
19579
|
+
const indexed = new Set(reading.rows.map((row) => row.number));
|
|
19580
|
+
for (const number of pullNumbersOnDisk) {
|
|
19581
|
+
if (!indexed.has(number))
|
|
19582
|
+
return { reason: 'index-incomplete' };
|
|
19583
|
+
}
|
|
19584
|
+
const best = bestPullIndexRow(reading.rows, issue, marker, opts);
|
|
19585
|
+
if (!best)
|
|
19586
|
+
return { reason: 'index-no-match' };
|
|
19587
|
+
// The first spelling of this pull request in `walk` is by construction the
|
|
19588
|
+
// one the record loop below would have read: `walk` preserves the listing
|
|
19589
|
+
// order of `githubPullRoots` and has already collapsed the alias spellings,
|
|
19590
|
+
// keeping the first. So the fast path reports the same `path` the walk does.
|
|
19591
|
+
const path = walk.find((candidatePath) => githubPullPathParts(candidatePath)?.number === best.number);
|
|
19592
|
+
if (!path)
|
|
19593
|
+
return { reason: 'index-disagreed' };
|
|
19594
|
+
const pr = await readProbePrCandidate(mount, path);
|
|
19595
|
+
observer.onRead?.({ read: 1, total: 1, path });
|
|
19596
|
+
if (!pr)
|
|
19597
|
+
return { reason: 'index-disagreed' };
|
|
19598
|
+
if (opts.openOnly && normalizePrState(pr.state) !== 'OPEN')
|
|
19599
|
+
return { reason: 'index-disagreed' };
|
|
19600
|
+
// `readProbePrCandidate` takes the number from the PAYLOAD, falling back to
|
|
19601
|
+
// the path, so a record can disagree with the path that addressed it.
|
|
19602
|
+
if (pr.number !== best.number)
|
|
19603
|
+
return { reason: 'index-disagreed' };
|
|
19604
|
+
const score = issuePrMatchScore(pr, issue, marker, opts);
|
|
19605
|
+
if (score !== best.score)
|
|
19606
|
+
return { reason: 'index-disagreed' };
|
|
19607
|
+
observer.onIndexHit?.(repo, pr.number);
|
|
19608
|
+
return { candidate: probePrCandidate(repo, pr, path, score) };
|
|
19609
|
+
};
|
|
19610
|
+
// Exported for the differential-equivalence suite, which has to invoke the
|
|
19611
|
+
// index fast path and the record walk over ONE corpus and compare their two
|
|
19612
|
+
// answers field for field. Driving that through the loop's public surface would
|
|
19613
|
+
// compare a resolution to itself; the same precedent already exists in this
|
|
19614
|
+
// file for `githubIssuePathParts` and `keyFromPath`.
|
|
19615
|
+
export const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix), observer = {}) => {
|
|
18950
19616
|
const candidates = [];
|
|
18951
19617
|
const listErrors = [];
|
|
18952
19618
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
18953
|
-
if (observer.onIndexFallback) {
|
|
18954
|
-
observer.onIndexFallback(repo, await classifyPullIndexFallback(mount, repo));
|
|
18955
|
-
}
|
|
18956
19619
|
const paths = new Set();
|
|
18957
19620
|
for (const root of githubPullRoots(repo)) {
|
|
18958
19621
|
try {
|
|
@@ -18983,6 +19646,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
18983
19646
|
// still read exactly as before rather than being filtered on a guess.
|
|
18984
19647
|
const walk = [];
|
|
18985
19648
|
const seenPulls = new Set();
|
|
19649
|
+
// The pull request numbers the TREE says exist, which is what makes the
|
|
19650
|
+
// index's completeness checkable below. Built here rather than separately
|
|
19651
|
+
// because `githubPullPathParts` has already been called on every path.
|
|
19652
|
+
const pullNumbersOnDisk = new Set();
|
|
18986
19653
|
for (const path of paths) {
|
|
18987
19654
|
if (!path.endsWith('.json'))
|
|
18988
19655
|
continue;
|
|
@@ -18992,9 +19659,17 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
18992
19659
|
if (seenPulls.has(identity))
|
|
18993
19660
|
continue;
|
|
18994
19661
|
seenPulls.add(identity);
|
|
19662
|
+
pullNumbersOnDisk.add(parts.number);
|
|
18995
19663
|
}
|
|
18996
19664
|
walk.push(path);
|
|
18997
19665
|
}
|
|
19666
|
+
const marker = opts.titleMarker ?? config.safety.requireTitlePrefix;
|
|
19667
|
+
const fromIndex = await resolveProbePrFromPullIndex(mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer);
|
|
19668
|
+
if (fromIndex.candidate) {
|
|
19669
|
+
candidates.push(fromIndex.candidate);
|
|
19670
|
+
continue;
|
|
19671
|
+
}
|
|
19672
|
+
observer.onIndexFallback?.(repo, fromIndex.reason);
|
|
18998
19673
|
let read = 0;
|
|
18999
19674
|
for (const path of walk) {
|
|
19000
19675
|
const pr = await readProbePrCandidate(mount, path);
|
|
@@ -19002,23 +19677,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
19002
19677
|
observer.onRead?.({ read, total: walk.length, path });
|
|
19003
19678
|
if (opts.openOnly && normalizePrState(pr?.state) !== 'OPEN')
|
|
19004
19679
|
continue;
|
|
19005
|
-
const score = pr
|
|
19006
|
-
? issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts)
|
|
19007
|
-
: 0;
|
|
19680
|
+
const score = pr ? issuePrMatchScore(pr, issue, marker, opts) : 0;
|
|
19008
19681
|
if (!pr || score <= 0)
|
|
19009
19682
|
continue;
|
|
19010
|
-
candidates.push(
|
|
19011
|
-
repo,
|
|
19012
|
-
prNumber: pr.number,
|
|
19013
|
-
draft: pr.draft,
|
|
19014
|
-
headRef: pr.headRef,
|
|
19015
|
-
headRepo: pr.headRepo,
|
|
19016
|
-
crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
|
|
19017
|
-
state: pr.state,
|
|
19018
|
-
url: pr.url,
|
|
19019
|
-
path,
|
|
19020
|
-
score,
|
|
19021
|
-
});
|
|
19683
|
+
candidates.push(probePrCandidate(repo, pr, path, score));
|
|
19022
19684
|
}
|
|
19023
19685
|
}
|
|
19024
19686
|
const resolved = candidates.sort((a, b) => b.score - a.score || b.prNumber - a.prNumber)[0];
|