@cortexkit/common-auth 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claustrum/consumer.d.ts +12 -0
- package/dist/claustrum/consumer.js +36 -0
- package/dist/claustrum/custody.d.ts +16 -4
- package/dist/claustrum/custody.js +74 -9
- package/dist/claustrum/hub.d.ts +3 -2
- package/dist/claustrum/hub.js +8 -0
- package/dist/claustrum/index.d.ts +1 -0
- package/dist/claustrum/roster.d.ts +2 -0
- package/dist/claustrum/roster.js +2 -0
- package/dist/claustrum/usage.d.ts +41 -0
- package/dist/claustrum/usage.js +145 -0
- package/dist/commands/builtins.d.ts +5 -0
- package/dist/commands/builtins.js +38 -0
- package/dist/fs/private-files.d.ts +5 -0
- package/dist/fs/private-files.js +7 -0
- package/dist/fs/refresh-file-lock.js +142 -47
- package/dist/fs/with-lock.js +2 -5
- package/dist/limits/activity.d.ts +26 -0
- package/dist/limits/activity.js +52 -0
- package/dist/limits/axes.d.ts +69 -0
- package/dist/limits/axes.js +7 -0
- package/dist/limits/bridge.d.ts +4 -0
- package/dist/limits/bridge.js +10 -0
- package/dist/limits/decide.d.ts +146 -0
- package/dist/limits/decide.js +425 -0
- package/dist/limits/duration.d.ts +8 -0
- package/dist/limits/duration.js +20 -0
- package/dist/limits/envelope-loss.d.ts +29 -0
- package/dist/limits/envelope-loss.js +150 -0
- package/dist/limits/families.d.ts +59 -0
- package/dist/limits/families.js +149 -0
- package/dist/limits/header-classifier.d.ts +9 -0
- package/dist/limits/header-classifier.js +121 -0
- package/dist/limits/hold.d.ts +13 -0
- package/dist/limits/hold.js +25 -0
- package/dist/limits/identity.d.ts +14 -0
- package/dist/limits/identity.js +21 -0
- package/dist/limits/index.d.ts +45 -0
- package/dist/limits/index.js +26 -0
- package/dist/limits/jitter.d.ts +2 -0
- package/dist/limits/jitter.js +9 -0
- package/dist/limits/labels.d.ts +3 -0
- package/dist/limits/labels.js +11 -0
- package/dist/limits/marker.d.ts +69 -0
- package/dist/limits/marker.js +107 -0
- package/dist/limits/merge-laws.d.ts +15 -0
- package/dist/limits/merge-laws.js +31 -0
- package/dist/limits/namespace.d.ts +92 -0
- package/dist/limits/namespace.js +191 -0
- package/dist/limits/persistence.d.ts +33 -0
- package/dist/limits/persistence.js +9 -0
- package/dist/limits/pi-wait.d.ts +16 -0
- package/dist/limits/pi-wait.js +51 -0
- package/dist/limits/pool-adapter.d.ts +16 -0
- package/dist/limits/pool-adapter.js +162 -0
- package/dist/limits/presets.d.ts +11 -0
- package/dist/limits/presets.js +240 -0
- package/dist/limits/registry.d.ts +61 -0
- package/dist/limits/registry.js +478 -0
- package/dist/limits/request-history.d.ts +79 -0
- package/dist/limits/request-history.js +120 -0
- package/dist/limits/reset.d.ts +21 -0
- package/dist/limits/reset.js +44 -0
- package/dist/limits/routing-hold.d.ts +16 -0
- package/dist/limits/routing-hold.js +23 -0
- package/dist/limits/scope.d.ts +12 -0
- package/dist/limits/scope.js +10 -0
- package/dist/limits/signal.d.ts +41 -0
- package/dist/limits/signal.js +1 -0
- package/dist/limits/terminal.d.ts +52 -0
- package/dist/limits/terminal.js +133 -0
- package/dist/limits/unconfirmed-retain-counter.d.ts +13 -0
- package/dist/limits/unconfirmed-retain-counter.js +41 -0
- package/dist/limits/vault-adapter.d.ts +16 -0
- package/dist/limits/vault-adapter.js +123 -0
- package/dist/limits/vault.d.ts +1 -0
- package/dist/limits/vault.js +1 -0
- package/dist/limits/wake.d.ts +21 -0
- package/dist/limits/wake.js +47 -0
- package/dist/limits/wording.d.ts +25 -0
- package/dist/limits/wording.js +114 -0
- package/dist/opencode2/index.d.ts +1 -1
- package/dist/opencode2/install.d.ts +2 -2
- package/dist/opencode2/install.js +161 -18
- package/dist/opencode2/types.d.ts +95 -9
- package/dist/routing/admission.d.ts +7 -2
- package/dist/routing/admission.js +17 -7
- package/dist/routing-accessor/handle.d.ts +1 -0
- package/dist/routing-accessor/handle.js +10 -0
- package/dist/routing-accessor/index.d.ts +13 -3
- package/dist/routing-accessor/index.js +11 -10
- package/dist/rpc/port-file.d.ts +12 -3
- package/dist/rpc/port-file.js +46 -4
- package/dist/store/attribution.js +3 -2
- package/dist/store/errors.d.ts +1 -1
- package/dist/store/hooks.d.ts +5 -1
- package/dist/store/hooks.js +20 -5
- package/dist/store/index.d.ts +1 -0
- package/dist/store/limits-namespace.d.ts +7 -0
- package/dist/store/limits-namespace.js +34 -0
- package/dist/store/pool.d.ts +10 -1
- package/dist/store/pool.js +2 -0
- package/dist/store/provider-state.js +5 -5
- package/dist/store/refresh.js +1 -1
- package/dist/store/schema.js +9 -4
- package/dist/store/settings.d.ts +2 -0
- package/dist/store/settings.js +11 -4
- package/package.json +10 -2
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
import { MAX_SAME_ACCOUNT_HOLD_MS } from './hold.js';
|
|
2
|
+
import { holdDelayMs } from './jitter.js';
|
|
3
|
+
import { isRequestExcluded, REQUEST_EXCLUSION_EFFECTS, reduceRequestHistory, sameAccount, } from './request-history.js';
|
|
4
|
+
import { terminalRefusalDecision, terminalWaitDisabled } from './terminal.js';
|
|
5
|
+
import { liftedWindowsForWake, waitCheckpoint } from './wake.js';
|
|
6
|
+
function finiteFuture(value, now) {
|
|
7
|
+
return value !== undefined && Number.isFinite(value) && value > now;
|
|
8
|
+
}
|
|
9
|
+
function continuationRetry(facts, axes) {
|
|
10
|
+
return axes.execution === 'engine' && facts.phase === 'after-limit'
|
|
11
|
+
? 'veto'
|
|
12
|
+
: 'leave';
|
|
13
|
+
}
|
|
14
|
+
function refusal(facts, axes, history, reason, metadata) {
|
|
15
|
+
const terminal = metadata && terminalRefusalDecision(facts, axes, history, metadata);
|
|
16
|
+
if (terminal)
|
|
17
|
+
return terminal;
|
|
18
|
+
const policy = axes.refusals[reason][facts.host];
|
|
19
|
+
return {
|
|
20
|
+
action: 'refuse',
|
|
21
|
+
reason,
|
|
22
|
+
family: metadata?.family ?? policy.family,
|
|
23
|
+
messageKey: metadata?.messageKey ?? policy.messageKey,
|
|
24
|
+
retryable: metadata?.retryable ?? policy.retryable,
|
|
25
|
+
hostRetry: metadata?.hostRetry ?? policy.hostRetry,
|
|
26
|
+
history,
|
|
27
|
+
...(metadata?.retryAfterMs === undefined
|
|
28
|
+
? {}
|
|
29
|
+
: { retryAfterMs: metadata.retryAfterMs }),
|
|
30
|
+
...(metadata?.resetAt === undefined ? {} : { resetAt: metadata.resetAt }),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function hold(facts, axes, history, current, windows) {
|
|
34
|
+
const bound = facts.hold;
|
|
35
|
+
if (axes.shortLimit.holdBound === 'disabled' ||
|
|
36
|
+
(facts.host === 'opencode2' &&
|
|
37
|
+
(facts.hostPhase !== 'retry' ||
|
|
38
|
+
facts.remainingRetries === undefined ||
|
|
39
|
+
!(facts.remainingRetries >= 1))) ||
|
|
40
|
+
!current?.admitted ||
|
|
41
|
+
!bound ||
|
|
42
|
+
bound.maxWaitMs <= 0 ||
|
|
43
|
+
Number.isNaN(bound.maxWaitMs) ||
|
|
44
|
+
!Number.isFinite(bound.cacheExpiresAt) ||
|
|
45
|
+
(bound.maxDelayMs !== undefined &&
|
|
46
|
+
(!Number.isFinite(bound.maxDelayMs) || bound.maxDelayMs < 0)) ||
|
|
47
|
+
!windows.length ||
|
|
48
|
+
windows.some((window) => !finiteFuture(window.resetAt, facts.now)))
|
|
49
|
+
return undefined;
|
|
50
|
+
const resetAt = Math.max(...windows.map((window) => window.resetAt));
|
|
51
|
+
const delayMs = axes.shortLimit.jitter === 'session-sha256'
|
|
52
|
+
? holdDelayMs(facts.sessionId, resetAt - facts.now)
|
|
53
|
+
: resetAt - facts.now;
|
|
54
|
+
const target = facts.now + delayMs;
|
|
55
|
+
if (!Number.isFinite(target) ||
|
|
56
|
+
target > Math.min(bound.cacheExpiresAt, facts.now + bound.maxWaitMs) ||
|
|
57
|
+
(facts.host === 'opencode2' && delayMs > MAX_SAME_ACCOUNT_HOLD_MS) ||
|
|
58
|
+
(bound.maxDelayMs !== undefined && delayMs > bound.maxDelayMs))
|
|
59
|
+
return undefined;
|
|
60
|
+
return {
|
|
61
|
+
action: 'hold',
|
|
62
|
+
account: current.account,
|
|
63
|
+
delayMs,
|
|
64
|
+
retryAfterMs: delayMs,
|
|
65
|
+
target,
|
|
66
|
+
checkpoint: waitCheckpoint(current.account, facts.now, windows),
|
|
67
|
+
reason: 'confirmed-limit',
|
|
68
|
+
retryable: true,
|
|
69
|
+
hostRetry: facts.phase === 'pre-send' ? 'leave' : continuationRetry(facts, axes),
|
|
70
|
+
history,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
function poolWait(facts, axes, history, candidates) {
|
|
74
|
+
const policy = axes.poolWaiting;
|
|
75
|
+
if (policy.kind === 'off' || !candidates.length)
|
|
76
|
+
return undefined;
|
|
77
|
+
history = reduceRequestHistory(history, {
|
|
78
|
+
type: 'blocked',
|
|
79
|
+
clockNow: facts.clockNow,
|
|
80
|
+
});
|
|
81
|
+
const times = candidates
|
|
82
|
+
.map((candidate) => {
|
|
83
|
+
const blockers = candidate.blockers.filter((blocker) => !(blocker.kind === 'soft' &&
|
|
84
|
+
candidates.length === 1 &&
|
|
85
|
+
policy.singleEligibleSoftExemption));
|
|
86
|
+
if (blockers.some((blocker) => !finiteFuture(blocker.until, facts.now)))
|
|
87
|
+
return undefined;
|
|
88
|
+
return blockers.length
|
|
89
|
+
? {
|
|
90
|
+
candidate,
|
|
91
|
+
time: Math.max(...blockers.map((blocker) => blocker.until)),
|
|
92
|
+
}
|
|
93
|
+
: undefined;
|
|
94
|
+
})
|
|
95
|
+
.filter((entry) => entry !== undefined);
|
|
96
|
+
const earliest = times.reduce((best, entry) => (!best || entry.time < best.time ? entry : best), undefined);
|
|
97
|
+
const proposed = earliest
|
|
98
|
+
? earliest.time - facts.now
|
|
99
|
+
: candidates.some((candidate) => candidate.blockers.some((blocker) => blocker.kind !== 'soft'))
|
|
100
|
+
? policy.hardFallbackMs
|
|
101
|
+
: undefined;
|
|
102
|
+
const budget = facts.poolWaitBudgetMs ?? policy.budgetMs;
|
|
103
|
+
const elapsed = facts.clockNow - (history.blockedSince ?? facts.clockNow);
|
|
104
|
+
if (budget > 0 &&
|
|
105
|
+
(elapsed >= budget || (proposed !== undefined && proposed > budget)))
|
|
106
|
+
return refusal(facts, axes, history, 'quota', facts.terminal);
|
|
107
|
+
const remaining = policy.requestDeadlineMs - (facts.clockNow - history.requestStartedAt);
|
|
108
|
+
if (remaining <= 0)
|
|
109
|
+
return refusal(facts, axes, history, 'incomplete');
|
|
110
|
+
if (proposed === undefined)
|
|
111
|
+
return refusal(facts, axes, history, 'quota', facts.terminal);
|
|
112
|
+
const delayMs = Math.min(proposed, remaining, budget > 0 ? budget - elapsed : Number.POSITIVE_INFINITY);
|
|
113
|
+
const waitedAccount = earliest?.candidate ?? candidates[0];
|
|
114
|
+
if (!waitedAccount)
|
|
115
|
+
return undefined;
|
|
116
|
+
return {
|
|
117
|
+
action: 'wait-then-reselect',
|
|
118
|
+
delayMs,
|
|
119
|
+
target: facts.now + delayMs,
|
|
120
|
+
checkpoint: waitCheckpoint(waitedAccount.account, facts.now, waitedAccount.confirmedBlockers ?? []),
|
|
121
|
+
reason: 'pool-blocked',
|
|
122
|
+
retryable: true,
|
|
123
|
+
hostRetry: continuationRetry(facts, axes),
|
|
124
|
+
history,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
/** The caller supplies clocks, admission, delivery ownership and parsed evidence. */
|
|
128
|
+
export function decideAfterLimit(facts, axes) {
|
|
129
|
+
let history = facts.history;
|
|
130
|
+
const pass = (reason, retryable = facts.phase === 'pre-send'
|
|
131
|
+
? true
|
|
132
|
+
: (facts.failureRetryable ?? true), hostRetry = 'leave') => ({
|
|
133
|
+
action: 'pass',
|
|
134
|
+
reason,
|
|
135
|
+
retryable,
|
|
136
|
+
hostRetry,
|
|
137
|
+
history,
|
|
138
|
+
});
|
|
139
|
+
if (facts.phase === 'after-limit' &&
|
|
140
|
+
facts.host === 'opencode2' &&
|
|
141
|
+
(!facts.delivery.owned || !facts.delivery.attributable))
|
|
142
|
+
return pass('not-owned');
|
|
143
|
+
const status = facts.status ?? facts.signal?.status;
|
|
144
|
+
if (facts.phase === 'after-limit' && facts.attempted)
|
|
145
|
+
history = reduceRequestHistory(history, {
|
|
146
|
+
type: 'response',
|
|
147
|
+
account: facts.attempted,
|
|
148
|
+
scope: facts.requestScope,
|
|
149
|
+
status,
|
|
150
|
+
kind: axes.terminalStatusHistory?.paymentBalance &&
|
|
151
|
+
facts.signalApplicable !== false &&
|
|
152
|
+
facts.signal &&
|
|
153
|
+
sameAccount(facts.signal.account, facts.attempted)
|
|
154
|
+
? facts.signal.kind
|
|
155
|
+
: undefined,
|
|
156
|
+
});
|
|
157
|
+
if (facts.phase === 'after-limit' && !facts.delivery.replayable)
|
|
158
|
+
return pass('unsafe-output', false, 'veto');
|
|
159
|
+
const admitted = facts.candidates.filter((candidate) => candidate.admitted);
|
|
160
|
+
const candidates = admitted.filter((candidate) => !isRequestExcluded(history, candidate.account, facts.requestScope));
|
|
161
|
+
const current = candidates.find((candidate) => facts.attempted && sameAccount(candidate.account, facts.attempted));
|
|
162
|
+
const signal = facts.signalApplicable !== false &&
|
|
163
|
+
facts.signal &&
|
|
164
|
+
facts.attempted &&
|
|
165
|
+
sameAccount(facts.signal.account, facts.attempted)
|
|
166
|
+
? facts.signal
|
|
167
|
+
: undefined;
|
|
168
|
+
if (axes.requestExclusion === 'on' &&
|
|
169
|
+
signal?.kind === 'model-unavailable' &&
|
|
170
|
+
current) {
|
|
171
|
+
return {
|
|
172
|
+
action: 'exclude-for-request',
|
|
173
|
+
account: current.account,
|
|
174
|
+
scope: facts.requestScope,
|
|
175
|
+
effects: REQUEST_EXCLUSION_EFFECTS,
|
|
176
|
+
reason: 'model-unavailable',
|
|
177
|
+
retryable: true,
|
|
178
|
+
hostRetry: continuationRetry(facts, axes),
|
|
179
|
+
history,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
if (admitted.length && !candidates.length)
|
|
183
|
+
return refusal(facts, axes, history, 'model-unavailable');
|
|
184
|
+
if (facts.host === 'opencode2' &&
|
|
185
|
+
facts.hostPhase === 'retry' &&
|
|
186
|
+
axes.execution === 'host' &&
|
|
187
|
+
facts.remainingRetries !== undefined &&
|
|
188
|
+
facts.remainingRetries < 1) {
|
|
189
|
+
return facts.terminal &&
|
|
190
|
+
(facts.terminal.retryable === false ||
|
|
191
|
+
terminalWaitDisabled(facts.terminal, axes))
|
|
192
|
+
? refusal(facts, axes, history, facts.terminal.reason, facts.terminal)
|
|
193
|
+
: pass('host-retries-exhausted');
|
|
194
|
+
}
|
|
195
|
+
if (facts.wake && facts.phase === 'pre-send') {
|
|
196
|
+
const wake = facts.wake;
|
|
197
|
+
const held = candidates.find((candidate) => sameAccount(candidate.account, wake.checkpoint.accountKey));
|
|
198
|
+
const windows = held?.confirmedBlockers ??
|
|
199
|
+
(held && signal && sameAccount(signal.account, held.account)
|
|
200
|
+
? signal.windows
|
|
201
|
+
: []);
|
|
202
|
+
const liftedWindows = held
|
|
203
|
+
? liftedWindowsForWake(wake, held.account, windows, facts.now)
|
|
204
|
+
: [];
|
|
205
|
+
const bound = facts.hold;
|
|
206
|
+
const cacheValid = wake.kind === 'terminal' ||
|
|
207
|
+
(bound !== undefined &&
|
|
208
|
+
Number.isFinite(bound.cacheExpiresAt) &&
|
|
209
|
+
facts.now <= bound.cacheExpiresAt &&
|
|
210
|
+
wake.target <= bound.cacheExpiresAt &&
|
|
211
|
+
bound.maxWaitMs > 0 &&
|
|
212
|
+
wake.target - wake.checkpoint.decidedAt <= bound.maxWaitMs);
|
|
213
|
+
if (held &&
|
|
214
|
+
Number.isFinite(wake.target) &&
|
|
215
|
+
wake.target >= wake.checkpoint.decidedAt &&
|
|
216
|
+
facts.now >= wake.target &&
|
|
217
|
+
cacheValid &&
|
|
218
|
+
held.blockers.every((blocker) => blocker.until !== undefined &&
|
|
219
|
+
Number.isFinite(blocker.until) &&
|
|
220
|
+
blocker.until <= facts.now) &&
|
|
221
|
+
(held.ready || windows.some((window) => window.confirmed)) &&
|
|
222
|
+
windows
|
|
223
|
+
.filter((window) => window.confirmed)
|
|
224
|
+
.every((window) => liftedWindows.includes(window.window))) {
|
|
225
|
+
return {
|
|
226
|
+
action: 'retain',
|
|
227
|
+
account: held.account,
|
|
228
|
+
liftedWindows,
|
|
229
|
+
reason: 'wait-elapsed',
|
|
230
|
+
retryable: true,
|
|
231
|
+
hostRetry: 'leave',
|
|
232
|
+
history: reduceRequestHistory(history, { type: 'selected' }),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (facts.credentialRecovery && current) {
|
|
237
|
+
return {
|
|
238
|
+
action: 'retain',
|
|
239
|
+
account: current.account,
|
|
240
|
+
reason: 'credential-recovery',
|
|
241
|
+
...(facts.credentialRecovery.retryMs === undefined
|
|
242
|
+
? {}
|
|
243
|
+
: { retryMs: facts.credentialRecovery.retryMs }),
|
|
244
|
+
retryable: true,
|
|
245
|
+
hostRetry: continuationRetry(facts, axes),
|
|
246
|
+
history,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
// Supplying the count opts into the turn-scoped rule across all hosts;
|
|
250
|
+
// callers without it keep their existing host-specific unknown-limit policy.
|
|
251
|
+
const retainLimit = axes.unconfirmedRetainLimit;
|
|
252
|
+
const retainCount = facts.unconfirmedRetainCount;
|
|
253
|
+
const counted = status === 429 && retainLimit !== undefined && retainCount !== undefined;
|
|
254
|
+
const advance = axes.advance[facts.host];
|
|
255
|
+
if (facts.phase === 'after-limit' &&
|
|
256
|
+
axes.terminalStatusHistory?.paymentBalance &&
|
|
257
|
+
(signal?.kind === 'payment-cap' ||
|
|
258
|
+
(status === 402 && signal?.kind !== 'payment-balance')))
|
|
259
|
+
return pass('unhandled-failure');
|
|
260
|
+
if (facts.phase === 'after-limit' &&
|
|
261
|
+
!signal &&
|
|
262
|
+
!counted &&
|
|
263
|
+
!facts.terminal &&
|
|
264
|
+
(advance.attributedLimitOnly ||
|
|
265
|
+
status === undefined ||
|
|
266
|
+
!advance.statuses.includes(status)))
|
|
267
|
+
return pass('unhandled-failure');
|
|
268
|
+
const selected = candidates.find((candidate) => facts.selection &&
|
|
269
|
+
sameAccount(candidate.account, facts.selection.account) &&
|
|
270
|
+
candidate.placementEligible);
|
|
271
|
+
const selectedReady = selected &&
|
|
272
|
+
(selected.ready ||
|
|
273
|
+
(axes.poolWaiting.kind === 'earliest' &&
|
|
274
|
+
axes.poolWaiting.singleEligibleSoftExemption &&
|
|
275
|
+
candidates.length === 1 &&
|
|
276
|
+
selected.blockers.length > 0 &&
|
|
277
|
+
selected.blockers.every((blocker) => blocker.kind === 'soft')));
|
|
278
|
+
const alternative = selectedReady &&
|
|
279
|
+
(!facts.attempted || !sameAccount(selected.account, facts.attempted))
|
|
280
|
+
? selected
|
|
281
|
+
: undefined;
|
|
282
|
+
const reroute = (candidate, reason = 'ready-alternative') => ({
|
|
283
|
+
action: 'reroute',
|
|
284
|
+
account: candidate.account,
|
|
285
|
+
pin: facts.selection?.pin ?? 'preserve',
|
|
286
|
+
reason,
|
|
287
|
+
retryable: true,
|
|
288
|
+
hostRetry: continuationRetry(facts, axes),
|
|
289
|
+
history: reduceRequestHistory(history, { type: 'selected' }),
|
|
290
|
+
});
|
|
291
|
+
// Before the first send there is no attempted account or response signal;
|
|
292
|
+
// the retained selection can still have confirmed blockers worth waiting for.
|
|
293
|
+
const holdCandidate = facts.phase === 'pre-send'
|
|
294
|
+
? (current ??
|
|
295
|
+
candidates.find((candidate) => facts.selection &&
|
|
296
|
+
sameAccount(candidate.account, facts.selection.account)))
|
|
297
|
+
: current;
|
|
298
|
+
const windows = (holdCandidate?.confirmedBlockers ??
|
|
299
|
+
(facts.phase === 'after-limit' ? signal?.windows : undefined) ??
|
|
300
|
+
[]).filter((window) => window.confirmed);
|
|
301
|
+
if (windows.length) {
|
|
302
|
+
if (axes.shortLimit.preference === 'ready-alternative-first' && alternative)
|
|
303
|
+
return reroute(alternative);
|
|
304
|
+
const held = hold(facts, axes, history, holdCandidate, windows);
|
|
305
|
+
if (held)
|
|
306
|
+
return held;
|
|
307
|
+
}
|
|
308
|
+
const unknown = axes.unknown[facts.host];
|
|
309
|
+
const unconfirmed = facts.phase === 'after-limit' &&
|
|
310
|
+
!windows.length &&
|
|
311
|
+
(status === 429 ||
|
|
312
|
+
signal?.kind === 'rate-limit' ||
|
|
313
|
+
signal?.kind === 'quota-exhausted' ||
|
|
314
|
+
signal?.kind === 'capacity');
|
|
315
|
+
const retainLimitReached = counted && retainCount >= retainLimit;
|
|
316
|
+
if (unconfirmed &&
|
|
317
|
+
current &&
|
|
318
|
+
(unknown.affinity === 'always' || facts.affinity === 'sticky')) {
|
|
319
|
+
if (counted && retainLimitReached) {
|
|
320
|
+
if (alternative)
|
|
321
|
+
return reroute(alternative, 'unconfirmed-retry-limit');
|
|
322
|
+
// With no ready replacement, use the ordinary exhausted-pool outcome.
|
|
323
|
+
}
|
|
324
|
+
else if (counted || unknown.action === 'retain') {
|
|
325
|
+
const hints = [
|
|
326
|
+
...(facts.retryHintsMs ?? []),
|
|
327
|
+
signal?.retryAfterMs,
|
|
328
|
+
].filter((hint) => hint !== undefined && Number.isFinite(hint) && (counted || hint >= 0));
|
|
329
|
+
return {
|
|
330
|
+
action: 'retain',
|
|
331
|
+
account: current.account,
|
|
332
|
+
reason: 'unconfirmed-limit',
|
|
333
|
+
...(counted
|
|
334
|
+
? {
|
|
335
|
+
retryMs: Math.min(60_000, ...hints.map((hint) => Math.max(0, hint))),
|
|
336
|
+
incrementUnconfirmedRetain: true,
|
|
337
|
+
}
|
|
338
|
+
: unknown.hint === 'minimum'
|
|
339
|
+
? { retryMs: Math.min(unknown.defaultMs, ...hints) }
|
|
340
|
+
: {}),
|
|
341
|
+
retryable: true,
|
|
342
|
+
hostRetry: continuationRetry(facts, axes),
|
|
343
|
+
history,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
else if (unknown.action === 'pass')
|
|
347
|
+
return pass('unconfirmed-limit');
|
|
348
|
+
}
|
|
349
|
+
if (alternative)
|
|
350
|
+
return reroute(alternative);
|
|
351
|
+
if (facts.phase === 'pre-send' && selectedReady)
|
|
352
|
+
return reroute(selected);
|
|
353
|
+
if (facts.host === 'opencode2' &&
|
|
354
|
+
facts.hostPhase === 'request' &&
|
|
355
|
+
axes.markedPool.requestEscape === 'ignore-timed-marks-if-all-marked' &&
|
|
356
|
+
selected &&
|
|
357
|
+
candidates.every((candidate) => !candidate.ready) &&
|
|
358
|
+
candidates.every((candidate) => candidate.blockers.length > 0 &&
|
|
359
|
+
candidate.blockers.every((blocker) => blocker.kind === 'hard')))
|
|
360
|
+
return reroute(selected);
|
|
361
|
+
const waited = poolWait(facts, axes, history, candidates);
|
|
362
|
+
if (waited)
|
|
363
|
+
return waited;
|
|
364
|
+
if (facts.phase === 'pre-send' && !candidates.length)
|
|
365
|
+
return refusal(facts, axes, history, facts.terminal?.reason ?? 'auth', facts.terminal);
|
|
366
|
+
if (facts.host === 'opencode2' &&
|
|
367
|
+
facts.hostPhase === 'retry' &&
|
|
368
|
+
!facts.terminal) {
|
|
369
|
+
if (axes.markedPool.retryExhaustion === 'host-delay')
|
|
370
|
+
return pass('host-delay');
|
|
371
|
+
if (axes.markedPool.retryExhaustion === 'same-host-delay' && current)
|
|
372
|
+
return {
|
|
373
|
+
action: 'retain',
|
|
374
|
+
account: current.account,
|
|
375
|
+
reason: 'host-delay',
|
|
376
|
+
retryable: true,
|
|
377
|
+
hostRetry: 'leave',
|
|
378
|
+
history,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
if (axes.terminalPolicy === 'status-history' &&
|
|
382
|
+
axes.terminalStatusHistory &&
|
|
383
|
+
facts.host !== 'opencode2' &&
|
|
384
|
+
!facts.terminal) {
|
|
385
|
+
const allMarked = facts.phase === 'pre-send' &&
|
|
386
|
+
candidates.length > 0 &&
|
|
387
|
+
candidates.every((candidate) => !candidate.ready);
|
|
388
|
+
const eligible = candidates.filter((candidate) => candidate.placementEligible);
|
|
389
|
+
const allBalance = facts.phase === 'pre-send' &&
|
|
390
|
+
axes.terminalStatusHistory.paymentBalance &&
|
|
391
|
+
eligible.length > 0 &&
|
|
392
|
+
eligible.every((candidate) => !candidate.ready &&
|
|
393
|
+
candidate.blockers.length > 0 &&
|
|
394
|
+
candidate.blockers.every((blocker) => blocker.cause === 'payment-balance' &&
|
|
395
|
+
finiteFuture(blocker.until, facts.now)));
|
|
396
|
+
if (allBalance ||
|
|
397
|
+
allMarked ||
|
|
398
|
+
(status === 401 && history.saw429) ||
|
|
399
|
+
(axes.terminalStatusHistory.paymentBalance &&
|
|
400
|
+
history.saw429 &&
|
|
401
|
+
history.sawPaymentBalance)) {
|
|
402
|
+
const deadlines = (allBalance ? eligible : candidates)
|
|
403
|
+
.flatMap((candidate) => candidate.blockers.map((blocker) => blocker.until))
|
|
404
|
+
.filter((until) => finiteFuture(until, facts.now));
|
|
405
|
+
return refusal(facts, axes, history, 'quota', {
|
|
406
|
+
source: 'pool',
|
|
407
|
+
reason: 'quota',
|
|
408
|
+
...(allBalance ? axes.terminalStatusHistory.paymentBalance : {}),
|
|
409
|
+
retryAfterMs: deadlines.length
|
|
410
|
+
? Math.max(axes.terminalStatusHistory.minRetryMs, Math.min(...deadlines) - facts.now)
|
|
411
|
+
: axes.terminalStatusHistory.defaultRetryMs,
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
return pass('original-response');
|
|
415
|
+
}
|
|
416
|
+
if (facts.terminal)
|
|
417
|
+
return refusal(facts, axes, history, facts.terminal.reason, facts.terminal);
|
|
418
|
+
if (signal?.kind === 'auth' || signal?.kind === 'account-access')
|
|
419
|
+
return refusal(facts, axes, history, 'auth');
|
|
420
|
+
if (signal ||
|
|
421
|
+
status === 429 ||
|
|
422
|
+
candidates.some((candidate) => candidate.blockers.length))
|
|
423
|
+
return refusal(facts, axes, history, 'quota');
|
|
424
|
+
return pass('no-override');
|
|
425
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Compact durations keep every numeric component below the hosts' retry codes. */
|
|
2
|
+
export declare function formatDuration(ms: number): string;
|
|
3
|
+
export interface ResetFormatInput {
|
|
4
|
+
readonly resetAt: number;
|
|
5
|
+
readonly now: number;
|
|
6
|
+
}
|
|
7
|
+
export type ResetFormatter = (input: ResetFormatInput) => string;
|
|
8
|
+
export declare const formatReset: ResetFormatter;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Compact durations keep every numeric component below the hosts' retry codes. */
|
|
2
|
+
export function formatDuration(ms) {
|
|
3
|
+
if (!Number.isFinite(ms))
|
|
4
|
+
return 'unknown';
|
|
5
|
+
const seconds = Math.ceil(Math.max(0, ms) / 1000);
|
|
6
|
+
if (seconds < 60)
|
|
7
|
+
return `${seconds}s`;
|
|
8
|
+
const minutes = Math.ceil(seconds / 60);
|
|
9
|
+
const days = Math.floor(minutes / (24 * 60));
|
|
10
|
+
if (days >= 100)
|
|
11
|
+
return 'over 99 days';
|
|
12
|
+
const hours = Math.floor(minutes / 60) % 24;
|
|
13
|
+
const remainingMinutes = minutes % 60;
|
|
14
|
+
if (days > 0)
|
|
15
|
+
return `${days}d ${hours}h ${remainingMinutes}m`;
|
|
16
|
+
if (hours > 0)
|
|
17
|
+
return `${hours}h ${remainingMinutes}m`;
|
|
18
|
+
return `${remainingMinutes}m`;
|
|
19
|
+
}
|
|
20
|
+
export const formatReset = ({ resetAt, now }) => formatDuration(resetAt - now);
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type LimitsNamespace } from './namespace.js';
|
|
2
|
+
/**
|
|
3
|
+
* Where a failed detached repair is reported. The limits export takes this from
|
|
4
|
+
* the caller rather than importing the shared logger, whose file sink would pull
|
|
5
|
+
* the Windows private-files module into every limits import.
|
|
6
|
+
*/
|
|
7
|
+
export interface LimitsRecoveryLogger {
|
|
8
|
+
warn(message: string, fields?: Record<string, unknown>): void;
|
|
9
|
+
}
|
|
10
|
+
/** Readers never join the write lock; notify them after a detached repair lands. */
|
|
11
|
+
export declare class EnvelopeRepair {
|
|
12
|
+
private readonly logger?;
|
|
13
|
+
private running?;
|
|
14
|
+
private readonly listeners;
|
|
15
|
+
constructor(logger?: LimitsRecoveryLogger | undefined);
|
|
16
|
+
readonly subscribe: (listener: () => void) => (() => void);
|
|
17
|
+
schedule(commit: () => Promise<unknown>): void;
|
|
18
|
+
}
|
|
19
|
+
/** Retain only durable observations; a new, empty namespace is not lost state. */
|
|
20
|
+
export declare class EnvelopeHistory {
|
|
21
|
+
private seen?;
|
|
22
|
+
private missingGeneration?;
|
|
23
|
+
private incorporated?;
|
|
24
|
+
remember(snapshot: LimitsNamespace, previous?: LimitsNamespace): void;
|
|
25
|
+
private retained;
|
|
26
|
+
lost(present: LimitsNamespace | undefined, previous?: LimitsNamespace): boolean;
|
|
27
|
+
/** Preview a restrictive merge, or apply it against the envelope under the write lock. */
|
|
28
|
+
recover(present: LimitsNamespace | undefined, previous: LimitsNamespace | undefined, namespace: string, now: number): LimitsNamespace;
|
|
29
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
+
import { mergeAccessState } from './merge-laws.js';
|
|
5
|
+
import { createLimitsNamespace, pruneLimitsNamespace, timedMarkKeyId, } from './namespace.js';
|
|
6
|
+
const detached = new AsyncResource('common-auth-limits-recovery');
|
|
7
|
+
/** Readers never join the write lock; notify them after a detached repair lands. */
|
|
8
|
+
export class EnvelopeRepair {
|
|
9
|
+
logger;
|
|
10
|
+
running;
|
|
11
|
+
listeners = new Set();
|
|
12
|
+
constructor(logger) {
|
|
13
|
+
this.logger = logger;
|
|
14
|
+
}
|
|
15
|
+
subscribe = (listener) => {
|
|
16
|
+
this.listeners.add(listener);
|
|
17
|
+
return () => this.listeners.delete(listener);
|
|
18
|
+
};
|
|
19
|
+
schedule(commit) {
|
|
20
|
+
if (this.running)
|
|
21
|
+
return;
|
|
22
|
+
this.running = detached.runInAsyncScope(async () => {
|
|
23
|
+
// Defer invocation until the single-flight promise is installed, and keep
|
|
24
|
+
// the write outside any caller context that holds the store lock.
|
|
25
|
+
await Promise.resolve();
|
|
26
|
+
let succeeded = false;
|
|
27
|
+
try {
|
|
28
|
+
await commit();
|
|
29
|
+
succeeded = true;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
try {
|
|
33
|
+
this.logger?.warn('limits envelope recovery failed; agreement remains degraded', {
|
|
34
|
+
error: error instanceof Error ? error.message : String(error),
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
// A failed diagnostic must not turn detached bookkeeping into a rejection.
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
this.running = undefined;
|
|
43
|
+
if (succeeded) {
|
|
44
|
+
for (const listener of this.listeners) {
|
|
45
|
+
try {
|
|
46
|
+
listener();
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
// Observers cannot change the result of a durable repair.
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** Retain only durable observations; a new, empty namespace is not lost state. */
|
|
58
|
+
export class EnvelopeHistory {
|
|
59
|
+
seen;
|
|
60
|
+
missingGeneration;
|
|
61
|
+
incorporated;
|
|
62
|
+
remember(snapshot, previous) {
|
|
63
|
+
if (this.seen && this.seen.revision > snapshot.revision)
|
|
64
|
+
return;
|
|
65
|
+
this.seen = structuredClone(snapshot);
|
|
66
|
+
this.missingGeneration = undefined;
|
|
67
|
+
if (previous)
|
|
68
|
+
this.incorporated = structuredClone(previous);
|
|
69
|
+
}
|
|
70
|
+
retained(previous) {
|
|
71
|
+
// A notification reload still carries the pre-repair registry snapshot.
|
|
72
|
+
// Do not mistake that already-merged generation for another loss.
|
|
73
|
+
const retainedPrevious = isDeepStrictEqual(previous, this.incorporated)
|
|
74
|
+
? undefined
|
|
75
|
+
: previous;
|
|
76
|
+
return [this.seen, retainedPrevious].filter((snapshot) => snapshot !== undefined &&
|
|
77
|
+
(snapshot === this.seen || snapshot.revision > 0));
|
|
78
|
+
}
|
|
79
|
+
lost(present, previous) {
|
|
80
|
+
return this.retained(previous).some((seen) => !present ||
|
|
81
|
+
seen.generation !== present.generation ||
|
|
82
|
+
present.revision < seen.revision ||
|
|
83
|
+
seen.tombstones.some((clear) => clear.revision > present.lowWatermark &&
|
|
84
|
+
!present.tombstones.some((current) => timedMarkKeyId(current.key) === timedMarkKeyId(clear.key) &&
|
|
85
|
+
current.revision >= clear.revision)));
|
|
86
|
+
}
|
|
87
|
+
/** Preview a restrictive merge, or apply it against the envelope under the write lock. */
|
|
88
|
+
recover(present, previous, namespace, now) {
|
|
89
|
+
if (!present)
|
|
90
|
+
this.missingGeneration ??= randomUUID();
|
|
91
|
+
const current = present ??
|
|
92
|
+
createLimitsNamespace(namespace, this.missingGeneration ?? randomUUID());
|
|
93
|
+
const sources = [...this.retained(previous), current];
|
|
94
|
+
const marks = new Map();
|
|
95
|
+
const tombstones = new Map();
|
|
96
|
+
const operations = new Map();
|
|
97
|
+
const access = new Map();
|
|
98
|
+
let highest = 0;
|
|
99
|
+
for (const source of sources) {
|
|
100
|
+
highest = Math.max(highest, source.revision, source.lowWatermark);
|
|
101
|
+
for (const mark of source.marks) {
|
|
102
|
+
const id = timedMarkKeyId(mark.key);
|
|
103
|
+
const old = marks.get(id);
|
|
104
|
+
if (old && old.mergeLaw !== mark.mergeLaw)
|
|
105
|
+
throw new Error('limits-recovery-merge-law-conflict');
|
|
106
|
+
marks.set(id, {
|
|
107
|
+
...mark,
|
|
108
|
+
entries: [...(old?.entries ?? []), ...mark.entries],
|
|
109
|
+
});
|
|
110
|
+
for (const entry of mark.entries)
|
|
111
|
+
highest = Math.max(highest, entry.baseRevision, entry.committedRevision);
|
|
112
|
+
}
|
|
113
|
+
for (const clear of source.tombstones) {
|
|
114
|
+
const id = timedMarkKeyId(clear.key);
|
|
115
|
+
const old = tombstones.get(id);
|
|
116
|
+
if (!old || old.revision < clear.revision)
|
|
117
|
+
tombstones.set(id, clear);
|
|
118
|
+
highest = Math.max(highest, clear.revision);
|
|
119
|
+
}
|
|
120
|
+
for (const op of source.operationIds) {
|
|
121
|
+
const id = JSON.stringify([op.producerId, op.operationId]);
|
|
122
|
+
const old = operations.get(id);
|
|
123
|
+
if (!old || old.revision < op.revision)
|
|
124
|
+
operations.set(id, op);
|
|
125
|
+
highest = Math.max(highest, op.revision);
|
|
126
|
+
}
|
|
127
|
+
for (const state of source.access) {
|
|
128
|
+
const { namespace, accountId, incarnation } = state.account;
|
|
129
|
+
const id = JSON.stringify([namespace, accountId, incarnation]);
|
|
130
|
+
const old = access.get(id);
|
|
131
|
+
access.set(id, old ? mergeAccessState(old, state) : state);
|
|
132
|
+
highest = Math.max(highest, state.committedRevision);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const revision = highest + 1;
|
|
136
|
+
if (!Number.isSafeInteger(revision))
|
|
137
|
+
throw new Error('limits-revision-exhausted');
|
|
138
|
+
// The barrier rejects observations predating even a clear this process never
|
|
139
|
+
// saw. Retain entry base revisions so a later restored clear can remove them.
|
|
140
|
+
return pruneLimitsNamespace({
|
|
141
|
+
...current,
|
|
142
|
+
revision,
|
|
143
|
+
lowWatermark: revision,
|
|
144
|
+
marks: [...marks.values()],
|
|
145
|
+
tombstones: [...tombstones.values()],
|
|
146
|
+
operationIds: [...operations.values()],
|
|
147
|
+
access: [...access.values()],
|
|
148
|
+
}, now);
|
|
149
|
+
}
|
|
150
|
+
}
|