@agent-relay/factory 0.1.77 → 0.1.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/orchestrator/factory.d.ts +65 -0
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +397 -50
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/orchestrator/public-health.d.ts.map +1 -1
- package/dist/orchestrator/public-health.js +7 -0
- package/dist/orchestrator/public-health.js.map +1 -1
- package/dist/types.d.ts +33 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -167,6 +167,31 @@ const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
|
167
167
|
* multi-hour run holds the slot honestly), so what is bounded is the *rate*.
|
|
168
168
|
*/
|
|
169
169
|
const DISPATCH_LIFECYCLE_RETRY_MAX_MS = 30_000;
|
|
170
|
+
/**
|
|
171
|
+
* How many times a work unit's RELEASE may fail before it is dead-lettered.
|
|
172
|
+
*
|
|
173
|
+
* #303 bounded the rate of the capacity-wait re-arm and deliberately left its
|
|
174
|
+
* count unbounded, because waiting for capacity is legitimate: the slot will
|
|
175
|
+
* free eventually and abandoning the wait would drop real work. Release is the
|
|
176
|
+
* opposite shape. It is the last step of a work unit that is already finished
|
|
177
|
+
* — the issue is closed, the writeback is acknowledged, the batch slot is
|
|
178
|
+
* gone — so a release that has failed ten times is not waiting for anything.
|
|
179
|
+
* It is failing, and re-arming it at 1 Hz forever buys nothing.
|
|
180
|
+
*
|
|
181
|
+
* What that costs when it happens: `#finishDurableRelease` does not throw on a
|
|
182
|
+
* failed release, it returns `false` and re-arms itself, so the loop runs
|
|
183
|
+
* through the *success* path of every `.catch()` that might otherwise have
|
|
184
|
+
* bounded it. Each pass calls `#terminationRoots` once per agent, and
|
|
185
|
+
* `#writeInFlightRegistry` calls it once per agent again — each one a process
|
|
186
|
+
* table scan — plus a durable read and write. Three agents is order ten scans
|
|
187
|
+
* and several state operations per second, indefinitely, which is the same
|
|
188
|
+
* serialized-store spin #303 measured at 1477 GETs in 111 s.
|
|
189
|
+
*
|
|
190
|
+
* Ten attempts at the 1 Hz floor is ~10 s of genuine retry, which covers the
|
|
191
|
+
* transient failures release retries exist for (a brief control-plane blip, a
|
|
192
|
+
* lease handover) without covering a permanent one.
|
|
193
|
+
*/
|
|
194
|
+
const DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS = 10;
|
|
170
195
|
/** Rate limit for the capacity-wait warning once the backoff has capped. */
|
|
171
196
|
const DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS = 60_000;
|
|
172
197
|
const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
|
|
@@ -424,6 +449,7 @@ export class FactoryLoop {
|
|
|
424
449
|
#babysitterWakeUnreachableEscalateMs;
|
|
425
450
|
#babysitterWakeUnreachableRetryMs;
|
|
426
451
|
#startupAgentExitDrainTimeoutMs;
|
|
452
|
+
#dispatchLifecycleRetryMs;
|
|
427
453
|
#state;
|
|
428
454
|
#workspaceId;
|
|
429
455
|
#relayflows;
|
|
@@ -500,6 +526,16 @@ export class FactoryLoop {
|
|
|
500
526
|
#dispatchTerminalWaiters = new Map();
|
|
501
527
|
#dispatchLifecycleRetryTimers = new Map();
|
|
502
528
|
#dispatchLifecycleDrives = new Set();
|
|
529
|
+
/**
|
|
530
|
+
* Failed release attempts per work unit, and the units that ran out.
|
|
531
|
+
*
|
|
532
|
+
* Keyed by `dispatchLifecycleKey`, so it follows the work unit rather than
|
|
533
|
+
* any one agent, surface or dispatcher — the same identity rule the AR-448
|
|
534
|
+
* duplicate established for claims. Counted only for release re-arms; a
|
|
535
|
+
* capacity or ownership wait is not a failure and must not spend the budget.
|
|
536
|
+
*/
|
|
537
|
+
#dispatchLifecycleReleaseAttempts = new Map();
|
|
538
|
+
#dispatchLifecycleReleaseAbandoned = new Set();
|
|
503
539
|
#abandonedDispatchReasons = new Map();
|
|
504
540
|
/**
|
|
505
541
|
* Live batch-capacity waits, keyed by issue (#303).
|
|
@@ -859,6 +895,7 @@ export class FactoryLoop {
|
|
|
859
895
|
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
860
896
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
861
897
|
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
898
|
+
this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
|
|
862
899
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
863
900
|
this.#relayflows = ports.relayflows;
|
|
864
901
|
this.#worktrees = ports.worktrees;
|
|
@@ -4687,11 +4724,14 @@ export class FactoryLoop {
|
|
|
4687
4724
|
path: progress.path,
|
|
4688
4725
|
});
|
|
4689
4726
|
},
|
|
4690
|
-
// One line per repository saying why the tree walk was necessary.
|
|
4691
|
-
// is a
|
|
4692
|
-
//
|
|
4693
|
-
// the index
|
|
4694
|
-
//
|
|
4727
|
+
// One line per repository saying why the tree walk was necessary. A silent
|
|
4728
|
+
// fallback is indistinguishable from a fast path that is working, and
|
|
4729
|
+
// that is precisely how twelve minutes of sequential reads hid for so
|
|
4730
|
+
// long. Now that the index CAN answer the question, these reasons are the
|
|
4731
|
+
// only way to tell "the index answered" from "the index exists and was
|
|
4732
|
+
// useless" — and on a mount written before
|
|
4733
|
+
// `@relayfile/adapter-github@0.5.7`, `index-without-head-ref` is the
|
|
4734
|
+
// expected reading until a re-ingest converges the rows.
|
|
4695
4735
|
onIndexFallback: (repo, reason) => {
|
|
4696
4736
|
this.#increment('probePrIndexFallbacks');
|
|
4697
4737
|
this.#increment(PROBE_PR_INDEX_FALLBACK_COUNTERS[reason]);
|
|
@@ -4701,6 +4741,14 @@ export class FactoryLoop {
|
|
|
4701
4741
|
reason,
|
|
4702
4742
|
});
|
|
4703
4743
|
},
|
|
4744
|
+
onIndexHit: (repo, prNumber) => {
|
|
4745
|
+
this.#increment('probePrIndexHits');
|
|
4746
|
+
this.#logger.debug?.('[factory] PR probe resolved from the pull index', {
|
|
4747
|
+
issue: issue.key,
|
|
4748
|
+
repo,
|
|
4749
|
+
prNumber,
|
|
4750
|
+
});
|
|
4751
|
+
},
|
|
4704
4752
|
};
|
|
4705
4753
|
}
|
|
4706
4754
|
#logTimedProgress(message, startedAtMs, lastLoggedAtMs, metadata) {
|
|
@@ -5637,8 +5685,23 @@ export class FactoryLoop {
|
|
|
5637
5685
|
consecutiveFailures,
|
|
5638
5686
|
failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
|
|
5639
5687
|
intervalMs: this.#readinessReconcileIntervalMs,
|
|
5688
|
+
// The two bounds that can actually preempt this pass, published beside
|
|
5689
|
+
// the cadence that cannot. Omitting them made the stanza unreadable in
|
|
5690
|
+
// the one situation it exists for: `intervalMs` next to a climbing
|
|
5691
|
+
// `inFlightMs` looks exactly like an unbounded hang, and has twice been
|
|
5692
|
+
// reported as one. `timeoutMs` ends the wait, `sweepBudgetMs` unwinds
|
|
5693
|
+
// the sweep and hands the lease back — so the second is the one that
|
|
5694
|
+
// answers "when does this recover".
|
|
5695
|
+
timeoutMs: this.#readinessReconcileTimeoutMs,
|
|
5696
|
+
sweepBudgetMs: this.#discoverySweepBudgetMs,
|
|
5640
5697
|
...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
|
|
5641
5698
|
...(inFlightMs !== undefined ? { inFlightMs } : {}),
|
|
5699
|
+
// Derived here rather than left to each reader. The public projection
|
|
5700
|
+
// already computed it (#295/#300); the heartbeat stanza did not, so the
|
|
5701
|
+
// surface an operator actually opens first was the one missing it.
|
|
5702
|
+
...(inFlightMs !== undefined && this.#readinessReconcileIntervalMs > 0
|
|
5703
|
+
? { missedPasses: Math.floor(inFlightMs / this.#readinessReconcileIntervalMs) }
|
|
5704
|
+
: {}),
|
|
5642
5705
|
...(this.#readinessReconcileLastDurationMs !== undefined
|
|
5643
5706
|
? { lastDurationMs: this.#readinessReconcileLastDurationMs }
|
|
5644
5707
|
: {}),
|
|
@@ -6912,19 +6975,113 @@ export class FactoryLoop {
|
|
|
6912
6975
|
}
|
|
6913
6976
|
this.#increment('dispatchCapacityBackoffResets');
|
|
6914
6977
|
}
|
|
6915
|
-
|
|
6978
|
+
/**
|
|
6979
|
+
* Charge one failed release against a work unit's budget.
|
|
6980
|
+
*
|
|
6981
|
+
* Returns `false` once the budget is spent, and the caller must NOT re-arm.
|
|
6982
|
+
* Failing closed here is deliberate and is the opposite of a dispatch gate:
|
|
6983
|
+
* an unrecordable dispatch claim must abort the dispatch, but an unbounded
|
|
6984
|
+
* release must abort the RETRY — the work is already done either way, and
|
|
6985
|
+
* the only thing still running is the spin.
|
|
6986
|
+
*/
|
|
6987
|
+
#chargeReleaseAttempt(record, key, context) {
|
|
6988
|
+
if (this.#dispatchLifecycleReleaseAbandoned.has(key))
|
|
6989
|
+
return false;
|
|
6990
|
+
const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
|
|
6991
|
+
this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
|
|
6992
|
+
if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
|
|
6993
|
+
return true;
|
|
6994
|
+
this.#dispatchLifecycleReleaseAbandoned.add(key);
|
|
6995
|
+
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
6996
|
+
this.#increment('dispatchLifecycleReleaseAbandoned');
|
|
6997
|
+
const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
|
|
6998
|
+
// `error`, not `warn`. Every previous layer of this failure was invisible
|
|
6999
|
+
// until somebody read stderr by hand; a work unit whose cleanup this
|
|
7000
|
+
// process has permanently given up on is exactly the event that must not
|
|
7001
|
+
// be inferable only from the absence of further log lines.
|
|
7002
|
+
this.#logger.error?.('[factory] release retries exhausted; abandoning cleanup for this work unit', {
|
|
7003
|
+
issue: record.issue.key,
|
|
7004
|
+
attempts: attempts - 1,
|
|
7005
|
+
maxAttempts: DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS,
|
|
7006
|
+
context,
|
|
7007
|
+
// The durable lifecycle is retained on purpose: a takeover or a restart
|
|
7008
|
+
// re-drives it from the persisted phase. This bounds THIS process's
|
|
7009
|
+
// spin, it does not declare the work unit clean.
|
|
7010
|
+
durableLifecycleRetained,
|
|
7011
|
+
});
|
|
7012
|
+
const drive = this.#releaseDeadLetteredSlot(record, key)
|
|
7013
|
+
.catch((error) => {
|
|
7014
|
+
this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
|
|
7015
|
+
issue: record.issue.key,
|
|
7016
|
+
error: describeError(error).errorMessage,
|
|
7017
|
+
});
|
|
7018
|
+
})
|
|
7019
|
+
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7020
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
7021
|
+
return false;
|
|
7022
|
+
}
|
|
7023
|
+
/**
|
|
7024
|
+
* Hand back the batch slot of a work unit whose release was dead-lettered.
|
|
7025
|
+
*
|
|
7026
|
+
* Without this the bound trades an unbounded 1 Hz spin for a permanently
|
|
7027
|
+
* leaked slot, which is not obviously the better failure (#379 review, P1):
|
|
7028
|
+
* a spin is loud and self-describing, whereas an in-flight record nobody will
|
|
7029
|
+
* ever complete silently reduces dispatch capacity until the process is
|
|
7030
|
+
* restarted. The slot is process-local accounting and is always freed here.
|
|
7031
|
+
*
|
|
7032
|
+
* What is NOT declared clean is the work itself. On a durable lifecycle the
|
|
7033
|
+
* persisted `releasing` phase is deliberately left in place, so a successor
|
|
7034
|
+
* or a restart re-drives the same cleanup with a fresh budget; freeing the
|
|
7035
|
+
* local slot does not write a terminal phase. On a local lifecycle there is
|
|
7036
|
+
* no durable record to retain, so the batch record is all there is and
|
|
7037
|
+
* completing it is what keeps capacity honest.
|
|
7038
|
+
*/
|
|
7039
|
+
async #releaseDeadLetteredSlot(record, key) {
|
|
7040
|
+
const batch = await this.#batch();
|
|
7041
|
+
const next = batch.complete(record.issue);
|
|
7042
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
7043
|
+
await this.#writeInFlightRegistry();
|
|
7044
|
+
// A freed slot that nothing is admitted into is only half the repair.
|
|
7045
|
+
if (next && !this.#stopping)
|
|
7046
|
+
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
7047
|
+
}
|
|
7048
|
+
/** Clears a work unit's release budget once cleanup actually succeeds. */
|
|
7049
|
+
#clearReleaseAttempts(key) {
|
|
7050
|
+
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
7051
|
+
this.#dispatchLifecycleReleaseAbandoned.delete(key);
|
|
7052
|
+
}
|
|
7053
|
+
#scheduleDispatchLifecycleRetry(record, delayMs = this.#dispatchLifecycleRetryMs, opts = {}) {
|
|
6916
7054
|
const key = dispatchLifecycleKey(record.issue);
|
|
6917
7055
|
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
6918
7056
|
return;
|
|
7057
|
+
// Only a release re-arm spends the budget. A capacity or ownership wait is
|
|
7058
|
+
// a legitimate wait for someone else to finish (#303) and is bounded by
|
|
7059
|
+
// its rate, not its count — spending the budget on one would abandon work
|
|
7060
|
+
// that was never failing.
|
|
7061
|
+
if (opts.releaseAttempt === true && !this.#chargeReleaseAttempt(record, key, 'durable-lifecycle'))
|
|
7062
|
+
return;
|
|
6919
7063
|
const timer = setTimeout(() => {
|
|
6920
7064
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
6921
7065
|
const drive = this.#driveDispatchLifecycle(key)
|
|
6922
7066
|
.then(() => {
|
|
6923
7067
|
this.#dispatchLifecycleCapacityWaits.delete(key);
|
|
6924
7068
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
7069
|
+
// DELIBERATELY NOT `#clearReleaseAttempts` (#379 review, P1).
|
|
7070
|
+
//
|
|
7071
|
+
// A FAILED release reaches here. `#finishDurableRelease` returns
|
|
7072
|
+
// `false` rather than throwing, and `#driveDispatchLifecycle`
|
|
7073
|
+
// discards that boolean in its `phase === 'releasing'` branch, so
|
|
7074
|
+
// the drive RESOLVES on a failed release and this handler runs on
|
|
7075
|
+
// every re-arm. Clearing the budget here zeroed it once per pass and
|
|
7076
|
+
// the counter could never reach the cap — the same never-fires this
|
|
7077
|
+
// bound exists to avoid, moved from the `.catch()` to the `.then()`.
|
|
7078
|
+
//
|
|
7079
|
+
// The budget is refunded where success is actually known:
|
|
7080
|
+
// `#finishDurableRelease` clears it on real per-agent progress and
|
|
7081
|
+
// again when the work unit completes.
|
|
6925
7082
|
})
|
|
6926
7083
|
.catch((error) => {
|
|
6927
|
-
let nextDelayMs =
|
|
7084
|
+
let nextDelayMs = this.#dispatchLifecycleRetryMs;
|
|
6928
7085
|
if (error instanceof DispatchLifecycleCapacityError) {
|
|
6929
7086
|
this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
|
|
6930
7087
|
nextDelayMs = this.#recordDispatchCapacityWait(record, key);
|
|
@@ -6939,7 +7096,7 @@ export class FactoryLoop {
|
|
|
6939
7096
|
leaseRemainingMs: error.leaseUntilMs === undefined
|
|
6940
7097
|
? undefined
|
|
6941
7098
|
: Math.max(0, error.leaseUntilMs - this.#clock.now()),
|
|
6942
|
-
retryMs:
|
|
7099
|
+
retryMs: this.#dispatchLifecycleRetryMs,
|
|
6943
7100
|
});
|
|
6944
7101
|
}
|
|
6945
7102
|
}
|
|
@@ -6951,6 +7108,20 @@ export class FactoryLoop {
|
|
|
6951
7108
|
error: describeError(error).errorMessage,
|
|
6952
7109
|
});
|
|
6953
7110
|
}
|
|
7111
|
+
// Deliberately UNCHARGED (#379 review, P1). This arm re-arms for
|
|
7112
|
+
// dispatch, publishing and recovery failures as well as releases, and
|
|
7113
|
+
// charging all of them would dead-letter a work unit that was never
|
|
7114
|
+
// stuck in a release loop. Charging is confined to
|
|
7115
|
+
// `#scheduleReleaseRetry`, whose every caller is a release failure:
|
|
7116
|
+
// the three inside `#finishDurableRelease` and `#completeIssue`'s
|
|
7117
|
+
// catch once `releaseReasonForRetry` is set.
|
|
7118
|
+
//
|
|
7119
|
+
// The gap this leaves, stated plainly: a release failure that THREW
|
|
7120
|
+
// out of `#finishDurableRelease` instead of returning `false` would
|
|
7121
|
+
// reach here and re-arm unbounded. Every failure path in that method
|
|
7122
|
+
// returns `false` and schedules its own retry, so this is not a
|
|
7123
|
+
// reachable shape today — and if one appears it degrades to the
|
|
7124
|
+
// pre-existing unbounded behaviour rather than to a wrong dead-letter.
|
|
6954
7125
|
this.#scheduleDispatchLifecycleRetry(record, nextDelayMs);
|
|
6955
7126
|
})
|
|
6956
7127
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
@@ -6960,12 +7131,19 @@ export class FactoryLoop {
|
|
|
6960
7131
|
}
|
|
6961
7132
|
#scheduleReleaseRetry(record, reason) {
|
|
6962
7133
|
if (this.#usesDurableDispatchLifecycle()) {
|
|
6963
|
-
this.#scheduleDispatchLifecycleRetry(record);
|
|
7134
|
+
this.#scheduleDispatchLifecycleRetry(record, this.#dispatchLifecycleRetryMs, { releaseAttempt: true });
|
|
6964
7135
|
return;
|
|
6965
7136
|
}
|
|
6966
7137
|
const key = dispatchLifecycleKey(record.issue);
|
|
6967
7138
|
if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
|
|
6968
7139
|
return;
|
|
7140
|
+
// Charged here rather than in the `.catch()` below, because the loop this
|
|
7141
|
+
// bounds does not go through the `.catch()`: `#finishDurableRelease`
|
|
7142
|
+
// returns `false` on a failed release and re-arms itself, so every re-arm
|
|
7143
|
+
// arrives on the resolved path. A bound on the rejection handler alone
|
|
7144
|
+
// would have been a fix that never fired.
|
|
7145
|
+
if (!this.#chargeReleaseAttempt(record, key, 'local-completion'))
|
|
7146
|
+
return;
|
|
6969
7147
|
const timer = setTimeout(() => {
|
|
6970
7148
|
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
6971
7149
|
const drive = this.#finishDurableRelease(record, reason)
|
|
@@ -6979,7 +7157,7 @@ export class FactoryLoop {
|
|
|
6979
7157
|
})
|
|
6980
7158
|
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
6981
7159
|
this.#dispatchLifecycleDrives.add(drive);
|
|
6982
|
-
},
|
|
7160
|
+
}, this.#dispatchLifecycleRetryMs);
|
|
6983
7161
|
timer.unref?.();
|
|
6984
7162
|
this.#dispatchLifecycleRetryTimers.set(key, timer);
|
|
6985
7163
|
}
|
|
@@ -7192,6 +7370,9 @@ export class FactoryLoop {
|
|
|
7192
7370
|
return;
|
|
7193
7371
|
}
|
|
7194
7372
|
if (lifecycle.phase === 'releasing') {
|
|
7373
|
+
// The boolean is discarded here, as it always was — and that is exactly
|
|
7374
|
+
// why the scheduler's success handler must not refund the release budget
|
|
7375
|
+
// (#379 review, P1). A failed release resolves through this line.
|
|
7195
7376
|
await this.#finishDurableRelease(record, lifecycle.releaseReason);
|
|
7196
7377
|
}
|
|
7197
7378
|
}
|
|
@@ -7398,6 +7579,7 @@ export class FactoryLoop {
|
|
|
7398
7579
|
.filter((agent) => agent.releasedAtMs !== undefined)
|
|
7399
7580
|
.map((agent) => agent.name) ?? this.#localReleaseCheckpoints.get(releaseKey) ?? []);
|
|
7400
7581
|
const failed = [];
|
|
7582
|
+
const releasedOnEntry = released.size;
|
|
7401
7583
|
for (const agent of record.agents) {
|
|
7402
7584
|
if (released.has(agent[0]))
|
|
7403
7585
|
continue;
|
|
@@ -7418,6 +7600,13 @@ export class FactoryLoop {
|
|
|
7418
7600
|
await this.#writeInFlightRegistry();
|
|
7419
7601
|
if (failed.length > 0) {
|
|
7420
7602
|
this.#increment('dispatchLifecycleReleaseRetries');
|
|
7603
|
+
// Real progress refunds the budget, so the ten attempts bound CONSECUTIVE
|
|
7604
|
+
// no-progress passes rather than capping a slow multi-agent release. This
|
|
7605
|
+
// terminates because an agent released once is checkpointed and skipped
|
|
7606
|
+
// on the next pass, so the remaining set strictly shrinks — a refund can
|
|
7607
|
+
// only be earned a finite number of times.
|
|
7608
|
+
if (released.size > releasedOnEntry)
|
|
7609
|
+
this.#clearReleaseAttempts(releaseKey);
|
|
7421
7610
|
this.#scheduleReleaseRetry(record, reason);
|
|
7422
7611
|
return false;
|
|
7423
7612
|
}
|
|
@@ -7436,6 +7625,10 @@ export class FactoryLoop {
|
|
|
7436
7625
|
}
|
|
7437
7626
|
const next = this.#usesDurableDispatchLifecycle() ? undefined : batch.complete(record.issue);
|
|
7438
7627
|
this.#localReleaseCheckpoints.delete(releaseKey);
|
|
7628
|
+
// Every agent is released. Nothing is left to retry, so the budget goes
|
|
7629
|
+
// back — including the abandoned marker, so a reopened work unit that
|
|
7630
|
+
// reuses this key starts with a full budget rather than a spent one.
|
|
7631
|
+
this.#clearReleaseAttempts(releaseKey);
|
|
7439
7632
|
if (next)
|
|
7440
7633
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
7441
7634
|
// Terminal lifecycle saves intentionally relinquish the owner epoch. Clear
|
|
@@ -18904,28 +19097,47 @@ const PROBE_PR_INDEX_FALLBACK_COUNTERS = {
|
|
|
18904
19097
|
'index-absent': 'probePrIndexAbsent',
|
|
18905
19098
|
'index-shape-unrecognised': 'probePrIndexShapeUnrecognised',
|
|
18906
19099
|
'index-without-head-ref': 'probePrIndexWithoutHeadRef',
|
|
18907
|
-
|
|
18908
|
-
|
|
18909
|
-
|
|
18910
|
-
'index-
|
|
19100
|
+
'index-row-malformed': 'probePrIndexRowMalformed',
|
|
19101
|
+
'index-incomplete': 'probePrIndexIncomplete',
|
|
19102
|
+
'index-no-match': 'probePrIndexNoMatch',
|
|
19103
|
+
'index-disagreed': 'probePrIndexDisagreed',
|
|
18911
19104
|
};
|
|
18912
19105
|
/**
|
|
18913
|
-
*
|
|
19106
|
+
* The lowest `issuePrMatchScore` tier the pull index can compute for itself.
|
|
18914
19107
|
*
|
|
18915
|
-
*
|
|
18916
|
-
*
|
|
18917
|
-
*
|
|
18918
|
-
*
|
|
18919
|
-
*
|
|
18920
|
-
*
|
|
18921
|
-
*
|
|
18922
|
-
*
|
|
18923
|
-
*
|
|
19108
|
+
* The scores are: branch match 30, issue key in title 20, explicit reference in
|
|
19109
|
+
* body 10. A row carries `headRef` and `title` but NOT `body`, so a row can be
|
|
19110
|
+
* scored exactly at the 30 and 20 tiers and is blind at 10. A pull request the
|
|
19111
|
+
* index scores at 20 or better therefore cannot be beaten by anything the index
|
|
19112
|
+
* could not see, because the best a body-only match can ever earn is 10 — which
|
|
19113
|
+
* makes an index hit of 20 or 30 provably the same winner the full walk would
|
|
19114
|
+
* have chosen. Below that threshold the index proves nothing and the walk is
|
|
19115
|
+
* mandatory.
|
|
19116
|
+
*
|
|
19117
|
+
* Deliberately expressed as a threshold rather than a "highest score wins"
|
|
19118
|
+
* shortcut: the constant is the whole safety argument, and inlining `20` at the
|
|
19119
|
+
* comparison would bury it.
|
|
19120
|
+
*/
|
|
19121
|
+
const PULL_INDEX_DECISIVE_SCORE = 20;
|
|
19122
|
+
/**
|
|
19123
|
+
* Read `pulls/_index.json` and either return rows the probe may rank, or say
|
|
19124
|
+
* why it may not.
|
|
19125
|
+
*
|
|
19126
|
+
* All-or-nothing per repository, and the discipline is copied verbatim from
|
|
19127
|
+
* Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
|
|
19128
|
+
* ENTIRE repository if any row is legacy or malformed. `headRef` was added to
|
|
19129
|
+
* the public GitHub pull index contract in `@relayfile/adapter-github@0.5.7`
|
|
19130
|
+
* and an already-written mount only converges on re-ingest, so mixed indexes
|
|
19131
|
+
* are the expected state, not a corruption. One legacy row is enough to poison
|
|
19132
|
+
* the whole conclusion: the primary match is a branch match worth 30, so a row
|
|
19133
|
+
* without `headRef` cannot be ruled out as the real winner, and ranking the
|
|
19134
|
+
* remaining rows against each other would answer a different question from the
|
|
19135
|
+
* one the walk answers.
|
|
18924
19136
|
*/
|
|
18925
|
-
const
|
|
19137
|
+
const readPullIndexForProbe = async (mount, repo) => {
|
|
18926
19138
|
const [owner, name] = repo.split('/');
|
|
18927
19139
|
if (!owner || !name)
|
|
18928
|
-
return 'index-absent';
|
|
19140
|
+
return { reason: 'index-absent' };
|
|
18929
19141
|
let parsed;
|
|
18930
19142
|
try {
|
|
18931
19143
|
const { content } = await mount.readFile(`${GITHUB_ISSUE_ROOT}/${owner}/${name}/pulls/_index.json`);
|
|
@@ -18936,23 +19148,159 @@ const classifyPullIndexFallback = async (mount, repo) => {
|
|
|
18936
19148
|
// that is the same convention the tree reads below already follow.
|
|
18937
19149
|
if (isPassWideRelayfileFault(error))
|
|
18938
19150
|
throw error;
|
|
18939
|
-
return 'index-absent';
|
|
19151
|
+
return { reason: 'index-absent' };
|
|
18940
19152
|
}
|
|
18941
19153
|
if (!Array.isArray(parsed))
|
|
18942
|
-
return 'index-shape-unrecognised';
|
|
19154
|
+
return { reason: 'index-shape-unrecognised' };
|
|
18943
19155
|
// An empty index proves nothing about the row contract, so it must not read
|
|
18944
19156
|
// as usable — `[].every(...)` is vacuously true.
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
|
|
19157
|
+
if (parsed.length === 0)
|
|
19158
|
+
return { reason: 'index-without-head-ref' };
|
|
19159
|
+
const rows = [];
|
|
19160
|
+
for (const entry of parsed) {
|
|
19161
|
+
const row = asRecord(entry);
|
|
19162
|
+
if (typeof row?.headRef !== 'string')
|
|
19163
|
+
return { reason: 'index-without-head-ref' };
|
|
19164
|
+
if (!Number.isSafeInteger(row.number) || Number(row.number) <= 0 ||
|
|
19165
|
+
typeof row.title !== 'string' ||
|
|
19166
|
+
typeof row.state !== 'string' ||
|
|
19167
|
+
(row.merged !== undefined && typeof row.merged !== 'boolean')) {
|
|
19168
|
+
return { reason: 'index-row-malformed' };
|
|
19169
|
+
}
|
|
19170
|
+
rows.push({
|
|
19171
|
+
number: Number(row.number),
|
|
19172
|
+
title: row.title,
|
|
19173
|
+
headRef: row.headRef,
|
|
19174
|
+
state: row.state,
|
|
19175
|
+
merged: row.merged,
|
|
19176
|
+
});
|
|
19177
|
+
}
|
|
19178
|
+
return { rows };
|
|
19179
|
+
};
|
|
19180
|
+
/**
|
|
19181
|
+
* Rank index rows exactly as the walk ranks records it has read.
|
|
19182
|
+
*
|
|
19183
|
+
* `body: ''` is a deliberate constant, not a placeholder for a value that
|
|
19184
|
+
* should have been threaded through. The index carries no body, so scoring with
|
|
19185
|
+
* an empty one makes `issuePrMatchScore` return only the tiers the index can
|
|
19186
|
+
* actually justify — 30 for a branch match, 20 for a title match, 0 otherwise —
|
|
19187
|
+
* and never invents the 10 it cannot see. Everything else (`requireTitleMarker`
|
|
19188
|
+
* gating, `allowLegacyGithubBranch`, the marker itself) is the SAME call the
|
|
19189
|
+
* walk makes on the same inputs, so the two paths cannot score differently.
|
|
19190
|
+
*
|
|
19191
|
+
* The tiebreak mirrors `b.score - a.score || b.prNumber - a.prNumber`: highest
|
|
19192
|
+
* score, then highest pull request number.
|
|
19193
|
+
*/
|
|
19194
|
+
const bestPullIndexRow = (rows, issue, marker, opts) => {
|
|
19195
|
+
let best;
|
|
19196
|
+
for (const row of rows) {
|
|
19197
|
+
// Same derivation `readProbePrCandidate` applies to a record: a merged pull
|
|
19198
|
+
// request reports MERGED regardless of the raw `state` the provider wrote.
|
|
19199
|
+
const state = row.merged === true ? 'MERGED' : row.state;
|
|
19200
|
+
if (opts.openOnly && normalizePrState(state) !== 'OPEN')
|
|
19201
|
+
continue;
|
|
19202
|
+
const score = issuePrMatchScore({ title: row.title, body: '', headRef: row.headRef }, issue, marker, opts);
|
|
19203
|
+
if (score < PULL_INDEX_DECISIVE_SCORE)
|
|
19204
|
+
continue;
|
|
19205
|
+
if (!best || score > best.score || (score === best.score && row.number > best.number)) {
|
|
19206
|
+
best = { number: row.number, score };
|
|
19207
|
+
}
|
|
19208
|
+
}
|
|
19209
|
+
return best;
|
|
19210
|
+
};
|
|
19211
|
+
/**
|
|
19212
|
+
* Build the resolver's candidate shape from one pull request record.
|
|
19213
|
+
*
|
|
19214
|
+
* Shared by the index fast path and the walk so the two can never disagree on a
|
|
19215
|
+
* field. Differential equivalence is the whole safety property of the fast path
|
|
19216
|
+
* and it should hold structurally, not by two copies staying in sync.
|
|
19217
|
+
*/
|
|
19218
|
+
const probePrCandidate = (repo, pr, path, score) => ({
|
|
19219
|
+
repo,
|
|
19220
|
+
prNumber: pr.number,
|
|
19221
|
+
draft: pr.draft,
|
|
19222
|
+
headRef: pr.headRef,
|
|
19223
|
+
headRepo: pr.headRepo,
|
|
19224
|
+
crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
|
|
19225
|
+
state: pr.state,
|
|
19226
|
+
url: pr.url,
|
|
19227
|
+
path,
|
|
19228
|
+
score,
|
|
19229
|
+
});
|
|
19230
|
+
/**
|
|
19231
|
+
* Answer "which pull request belongs to this issue?" from `pulls/_index.json`
|
|
19232
|
+
* instead of reading every mounted pull request record, or say why it could not.
|
|
19233
|
+
*
|
|
19234
|
+
* Reads at most two files: the index, and the one record the index names. That
|
|
19235
|
+
* record read is not an optimisation that was left on the table — it is load
|
|
19236
|
+
* bearing twice over. The index carries no `draft`, `headRepo`, `url` or mount
|
|
19237
|
+
* path, all of which the resolver's result type promises and callers persist as
|
|
19238
|
+
* `ownedPullRequest`; and re-scoring the real record is what turns "the index
|
|
19239
|
+
* said so" into "the record agrees", so a stale row can never decide an answer
|
|
19240
|
+
* on its own.
|
|
19241
|
+
*
|
|
19242
|
+
* The tree listing is NOT avoided, and that is deliberate. It costs two calls
|
|
19243
|
+
* per repository where the record walk cost one call per pull request — 4,000
|
|
19244
|
+
* of them in the live workspace, which is the cost that stalled a sweep for
|
|
19245
|
+
* 11m53s. Keeping it buys two things nothing else can: the exact path spelling
|
|
19246
|
+
* the walk would have reported (the nested layout's slug cannot be constructed,
|
|
19247
|
+
* only listed), and the completeness check below.
|
|
19248
|
+
*
|
|
19249
|
+
* Every condition that still walks:
|
|
19250
|
+
*
|
|
19251
|
+
* - anything `readPullIndexForProbe` refuses — absent, wrong shape, any row
|
|
19252
|
+
* without `headRef`, any row whose `number`/`title`/`state` is unusable;
|
|
19253
|
+
* - a pull request on disk with no row describing it, because an unindexed
|
|
19254
|
+
* record could hold the score-30 branch match;
|
|
19255
|
+
* - no row scoring `PULL_INDEX_DECISIVE_SCORE` or better, because a body-only
|
|
19256
|
+
* reference worth 10 may exist on disk and no index row can see a body;
|
|
19257
|
+
* - the named record failing to confirm its row — missing, different number,
|
|
19258
|
+
* different score, or not open when `openOnly` is set.
|
|
19259
|
+
*/
|
|
19260
|
+
const resolveProbePrFromPullIndex = async (mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer) => {
|
|
19261
|
+
const reading = await readPullIndexForProbe(mount, repo);
|
|
19262
|
+
if (!reading.rows)
|
|
19263
|
+
return { reason: reading.reason };
|
|
19264
|
+
const indexed = new Set(reading.rows.map((row) => row.number));
|
|
19265
|
+
for (const number of pullNumbersOnDisk) {
|
|
19266
|
+
if (!indexed.has(number))
|
|
19267
|
+
return { reason: 'index-incomplete' };
|
|
19268
|
+
}
|
|
19269
|
+
const best = bestPullIndexRow(reading.rows, issue, marker, opts);
|
|
19270
|
+
if (!best)
|
|
19271
|
+
return { reason: 'index-no-match' };
|
|
19272
|
+
// The first spelling of this pull request in `walk` is by construction the
|
|
19273
|
+
// one the record loop below would have read: `walk` preserves the listing
|
|
19274
|
+
// order of `githubPullRoots` and has already collapsed the alias spellings,
|
|
19275
|
+
// keeping the first. So the fast path reports the same `path` the walk does.
|
|
19276
|
+
const path = walk.find((candidatePath) => githubPullPathParts(candidatePath)?.number === best.number);
|
|
19277
|
+
if (!path)
|
|
19278
|
+
return { reason: 'index-disagreed' };
|
|
19279
|
+
const pr = await readProbePrCandidate(mount, path);
|
|
19280
|
+
observer.onRead?.({ read: 1, total: 1, path });
|
|
19281
|
+
if (!pr)
|
|
19282
|
+
return { reason: 'index-disagreed' };
|
|
19283
|
+
if (opts.openOnly && normalizePrState(pr.state) !== 'OPEN')
|
|
19284
|
+
return { reason: 'index-disagreed' };
|
|
19285
|
+
// `readProbePrCandidate` takes the number from the PAYLOAD, falling back to
|
|
19286
|
+
// the path, so a record can disagree with the path that addressed it.
|
|
19287
|
+
if (pr.number !== best.number)
|
|
19288
|
+
return { reason: 'index-disagreed' };
|
|
19289
|
+
const score = issuePrMatchScore(pr, issue, marker, opts);
|
|
19290
|
+
if (score !== best.score)
|
|
19291
|
+
return { reason: 'index-disagreed' };
|
|
19292
|
+
observer.onIndexHit?.(repo, pr.number);
|
|
19293
|
+
return { candidate: probePrCandidate(repo, pr, path, score) };
|
|
18948
19294
|
};
|
|
18949
|
-
|
|
19295
|
+
// Exported for the differential-equivalence suite, which has to invoke the
|
|
19296
|
+
// index fast path and the record walk over ONE corpus and compare their two
|
|
19297
|
+
// answers field for field. Driving that through the loop's public surface would
|
|
19298
|
+
// compare a resolution to itself; the same precedent already exists in this
|
|
19299
|
+
// file for `githubIssuePathParts` and `keyFromPath`.
|
|
19300
|
+
export const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree = (prefix) => mount.listTree(prefix), observer = {}) => {
|
|
18950
19301
|
const candidates = [];
|
|
18951
19302
|
const listErrors = [];
|
|
18952
19303
|
for (const repo of opts.repo ? [opts.repo] : reposFromConfig(config)) {
|
|
18953
|
-
if (observer.onIndexFallback) {
|
|
18954
|
-
observer.onIndexFallback(repo, await classifyPullIndexFallback(mount, repo));
|
|
18955
|
-
}
|
|
18956
19304
|
const paths = new Set();
|
|
18957
19305
|
for (const root of githubPullRoots(repo)) {
|
|
18958
19306
|
try {
|
|
@@ -18983,6 +19331,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
18983
19331
|
// still read exactly as before rather than being filtered on a guess.
|
|
18984
19332
|
const walk = [];
|
|
18985
19333
|
const seenPulls = new Set();
|
|
19334
|
+
// The pull request numbers the TREE says exist, which is what makes the
|
|
19335
|
+
// index's completeness checkable below. Built here rather than separately
|
|
19336
|
+
// because `githubPullPathParts` has already been called on every path.
|
|
19337
|
+
const pullNumbersOnDisk = new Set();
|
|
18986
19338
|
for (const path of paths) {
|
|
18987
19339
|
if (!path.endsWith('.json'))
|
|
18988
19340
|
continue;
|
|
@@ -18992,9 +19344,17 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
18992
19344
|
if (seenPulls.has(identity))
|
|
18993
19345
|
continue;
|
|
18994
19346
|
seenPulls.add(identity);
|
|
19347
|
+
pullNumbersOnDisk.add(parts.number);
|
|
18995
19348
|
}
|
|
18996
19349
|
walk.push(path);
|
|
18997
19350
|
}
|
|
19351
|
+
const marker = opts.titleMarker ?? config.safety.requireTitlePrefix;
|
|
19352
|
+
const fromIndex = await resolveProbePrFromPullIndex(mount, repo, issue, marker, opts, walk, pullNumbersOnDisk, observer);
|
|
19353
|
+
if (fromIndex.candidate) {
|
|
19354
|
+
candidates.push(fromIndex.candidate);
|
|
19355
|
+
continue;
|
|
19356
|
+
}
|
|
19357
|
+
observer.onIndexFallback?.(repo, fromIndex.reason);
|
|
18998
19358
|
let read = 0;
|
|
18999
19359
|
for (const path of walk) {
|
|
19000
19360
|
const pr = await readProbePrCandidate(mount, path);
|
|
@@ -19002,23 +19362,10 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
|
|
|
19002
19362
|
observer.onRead?.({ read, total: walk.length, path });
|
|
19003
19363
|
if (opts.openOnly && normalizePrState(pr?.state) !== 'OPEN')
|
|
19004
19364
|
continue;
|
|
19005
|
-
const score = pr
|
|
19006
|
-
? issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts)
|
|
19007
|
-
: 0;
|
|
19365
|
+
const score = pr ? issuePrMatchScore(pr, issue, marker, opts) : 0;
|
|
19008
19366
|
if (!pr || score <= 0)
|
|
19009
19367
|
continue;
|
|
19010
|
-
candidates.push(
|
|
19011
|
-
repo,
|
|
19012
|
-
prNumber: pr.number,
|
|
19013
|
-
draft: pr.draft,
|
|
19014
|
-
headRef: pr.headRef,
|
|
19015
|
-
headRepo: pr.headRepo,
|
|
19016
|
-
crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
|
|
19017
|
-
state: pr.state,
|
|
19018
|
-
url: pr.url,
|
|
19019
|
-
path,
|
|
19020
|
-
score,
|
|
19021
|
-
});
|
|
19368
|
+
candidates.push(probePrCandidate(repo, pr, path, score));
|
|
19022
19369
|
}
|
|
19023
19370
|
}
|
|
19024
19371
|
const resolved = candidates.sort((a, b) => b.score - a.score || b.prNumber - a.prNumber)[0];
|