@anyslate/cli 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/package.json +2 -2
- package/src/auth.mjs +379 -40
- package/src/backoff.mjs +178 -0
- package/src/commands/checkpoint.mjs +13 -3
- package/src/commands/doctor.mjs +19 -0
- package/src/commands/hook.mjs +73 -3
- package/src/commands/login.mjs +7 -0
- package/src/commands/logout.mjs +4 -0
- package/src/commands/upload-artifact.mjs +12 -4
- package/src/credentials.mjs +32 -5
- package/src/guard.mjs +381 -0
- package/src/index.mjs +12 -0
- package/src/mcp-client.mjs +16 -5
package/README.md
CHANGED
|
@@ -196,6 +196,8 @@ export ANYSLATE_HANDLE=mh_xxxxxxxx # optional, scope to one capab
|
|
|
196
196
|
export ANYSLATE_DISABLE=1 # optional, disables capture for this shell
|
|
197
197
|
export ANYSLATE_HOME=/path/to/dir # optional, overrides ~/.anyslate
|
|
198
198
|
export ANYSLATE_STDIN_TIMEOUT_MS=10000 # optional, stdin idle timeout (0 disables)
|
|
199
|
+
export ANYSLATE_HOOK_TIMEOUT_MS=5000 # optional, request budget for one hook
|
|
200
|
+
export ANYSLATE_MAX_CALLS_PER_MINUTE=60 # optional, local call ceiling (0 disables)
|
|
199
201
|
```
|
|
200
202
|
|
|
201
203
|
Env vars override `~/.anyslate/cli.json`.
|
|
@@ -214,6 +216,24 @@ Capability handles are `mh_` + a 32-character id (e.g. `mh_V1StGXR8Z5jdHi6BmyT0a
|
|
|
214
216
|
|
|
215
217
|
**If your token was minted with a memory scope, it already carries that scope server-side and `ANYSLATE_HANDLE` / `--handle` is ignored.** Scoped tokens cannot be widened per-call by design. To capture workspace-wide, mint an unscoped token. `anyslate doctor` reports which case you're in.
|
|
216
218
|
|
|
219
|
+
### When capture pauses itself
|
|
220
|
+
|
|
221
|
+
Since 0.4.0 the CLI refuses to keep calling a server that is refusing it. Three things changed:
|
|
222
|
+
|
|
223
|
+
- **No unbounded retries.** `checkpoint` and `upload-artifact` retry at most 3 times, with exponential backoff and jitter and a 15-second total sleep budget. `hook` does not retry at all: it fires on every tool call, so a retry there is a doubled request rate in exchange for one row in an activity feed.
|
|
224
|
+
- **`Retry-After` is obeyed.** A `429` or `503` that names a wait pauses this machine for exactly that long. The CLI never sleeps through a long one - the process exits and the pause is honoured by whichever process runs next.
|
|
225
|
+
- **A circuit breaker that outlives the process.** Every hook is a separate process, so an in-memory breaker would be worthless. After 3 consecutive `401`s - or one definitively dead credential, such as a refresh token the server has already replaced - capture pauses, the pause is recorded in `~/.anyslate/cli-guard.json`, and the CLI tells you once:
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
anyslate: capture paused until 2026-08-02T12:15:00.000Z — repeated authentication failures (invalid_grant). Run `anyslate login` to sign in again; no requests are sent until then.
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
After that one line, paused hooks are completely silent and make **no** network request. Repeated pauses escalate: 15 minutes, then 1 hour, 6 hours, 24 hours.
|
|
232
|
+
|
|
233
|
+
The pause is tied to the `apiUrl` + credential that earned it. `anyslate login` clears it immediately, and so does pointing `--api-url` somewhere else or swapping the token - fixing your setup is never punished by a wait. `anyslate doctor` always reports an active pause, and is itself never blocked by one.
|
|
234
|
+
|
|
235
|
+
`ANYSLATE_MAX_CALLS_PER_MINUTE` is the last line of defence: a machine-wide ceiling (default 60/minute) on capture calls, so a runaway agent loop cannot become a load test. It is far above any real session; set it to `0` to disable.
|
|
236
|
+
|
|
217
237
|
### The two URL conventions - the one thing people get wrong
|
|
218
238
|
|
|
219
239
|
AnySlate ships two clients and they want **different** URLs. This trips up nearly everyone:
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anyslate/cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "AnySlate CLI - lifecycle hooks, git/CI capture, and manual checkpoints for AI memory. Validates its connection at login, diagnoses itself with `anyslate doctor`, and fails open without failing silent.",
|
|
3
|
+
"version": "0.4.1",
|
|
4
|
+
"description": "AnySlate CLI - lifecycle hooks, git/CI capture, and manual checkpoints for AI memory. Validates its connection at login, diagnoses itself with `anyslate doctor`, backs off and circuit-breaks rather than retrying, and fails open without failing silent.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"anyslate": "./bin/anyslate.mjs"
|
package/src/auth.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// The rules that fall out of that:
|
|
10
10
|
//
|
|
11
11
|
// * Refresh PROACTIVELY, 5 minutes ahead of expiry, before the call. Waiting
|
|
12
|
-
// for the 401 spends a round trip and, on the hook path, risks the
|
|
12
|
+
// for the 401 spends a round trip and, on the hook path, risks the timeout
|
|
13
13
|
// budget.
|
|
14
14
|
// * On a 401, refresh EXACTLY ONCE and retry EXACTLY ONCE. A loop here is a
|
|
15
15
|
// self-inflicted rate limit against an endpoint that is already refusing us.
|
|
@@ -19,10 +19,37 @@
|
|
|
19
19
|
// a process that must not hang.
|
|
20
20
|
// * Never throw. Callers are fail-open paths; a rejected promise here would
|
|
21
21
|
// become an uncaught exception in a hook and a non-zero exit.
|
|
22
|
+
//
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
// 2026-08-02 — this file is where the client-side storm was possible
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// "Refresh once, retry once" was already true PER PROCESS, and it was not
|
|
27
|
+
// enough: every hook is a fresh process, so one machine with a stale refresh
|
|
28
|
+
// token re-derived the same doomed `POST /oauth/token` + `POST /mcp` pair fifty
|
|
29
|
+
// times a minute for hours, and no single invocation was misbehaving.
|
|
30
|
+
//
|
|
31
|
+
// Three additions close that, all funnelled through `callToolWithAuth` because
|
|
32
|
+
// it is the one choke point every capture command already goes through:
|
|
33
|
+
//
|
|
34
|
+
// 1. a PERSISTED circuit breaker (guard.mjs) consulted BEFORE any credential
|
|
35
|
+
// is resolved — an open breaker costs the server exactly zero requests,
|
|
36
|
+
// 2. bounded retry with jitter and a hard attempt cap (backoff.mjs), with
|
|
37
|
+
// `Retry-After` honoured rather than ignored,
|
|
38
|
+
// 3. the refresh stays paired with the ONE call that needed it, even across
|
|
39
|
+
// retries — a retried call never re-mints.
|
|
22
40
|
|
|
23
41
|
import { readConfigFile, updateConfigFile, withRefreshLock } from './credentials.mjs';
|
|
24
|
-
import { callTool } from './mcp-client.mjs';
|
|
42
|
+
import { callTool, formatCallFailure } from './mcp-client.mjs';
|
|
25
43
|
import { discover, refreshAccessToken, REFRESH_SKEW_MS } from './oauth.mjs';
|
|
44
|
+
import { DEFAULT_RETRY, classifyFailure, planRetry, retryAfterMsOf, sleep } from './backoff.mjs';
|
|
45
|
+
import {
|
|
46
|
+
breakerGate,
|
|
47
|
+
breakerNotice,
|
|
48
|
+
ceilingNotice,
|
|
49
|
+
recordCallFailure,
|
|
50
|
+
recordCallSuccess,
|
|
51
|
+
reserveCallSlot,
|
|
52
|
+
} from './guard.mjs';
|
|
26
53
|
|
|
27
54
|
export { REFRESH_SKEW_MS };
|
|
28
55
|
|
|
@@ -86,15 +113,59 @@ export function hasOauthCredentials(oauth) {
|
|
|
86
113
|
* @returns {Promise<{ok: true, tokenEndpoint: string, resource: string}
|
|
87
114
|
* | {ok: false, code: string, message: string}>}
|
|
88
115
|
*/
|
|
89
|
-
async function refreshEndpoints({ oauth, root, fetchImpl }) {
|
|
116
|
+
async function refreshEndpoints({ oauth, root, fetchImpl, timeoutMs }) {
|
|
90
117
|
if (oauth?.root === root && oauth?.token_endpoint && oauth?.resource) {
|
|
91
118
|
return { ok: true, tokenEndpoint: oauth.token_endpoint, resource: oauth.resource };
|
|
92
119
|
}
|
|
93
|
-
const found = await discover({ root, fetchImpl });
|
|
120
|
+
const found = await discover({ root, fetchImpl, timeoutMs });
|
|
94
121
|
if (!found.ok) return found;
|
|
95
122
|
return { ok: true, tokenEndpoint: found.tokenEndpoint, resource: found.resource };
|
|
96
123
|
}
|
|
97
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Prefer a peer's already-persisted OAuth session over minting a new one.
|
|
127
|
+
*
|
|
128
|
+
* Refresh tokens are single-use server-side. Concurrent hooks (each a fresh
|
|
129
|
+
* process) that both redeem the same refresh token leave the loser with
|
|
130
|
+
* `invalid_grant` even though a usable access token is already on disk —
|
|
131
|
+
* and historically that loser opened the circuit breaker and told the user
|
|
132
|
+
* to re-login. Reading disk again closes that hole.
|
|
133
|
+
*
|
|
134
|
+
* @param {{env: NodeJS.ProcessEnv, staleToken?: string|null, now?: number,
|
|
135
|
+
* startingRefresh?: string|null}} opts
|
|
136
|
+
* @returns {{ok: true, token: string, rotated: false, oauth: object}|null}
|
|
137
|
+
*/
|
|
138
|
+
export function readPeerRefreshedCredentials({ env, staleToken = null, now = Date.now(), startingRefresh = null }) {
|
|
139
|
+
const disk = readConfigFile(env);
|
|
140
|
+
const oauth = disk.oauth && typeof disk.oauth === 'object' ? disk.oauth : null;
|
|
141
|
+
if (!oauth?.access_token) return null;
|
|
142
|
+
|
|
143
|
+
// A peer rotated when the refresh token on disk changed, or when the access
|
|
144
|
+
// token differs from the one we are replacing.
|
|
145
|
+
const refreshMoved =
|
|
146
|
+
startingRefresh && oauth.refresh_token && oauth.refresh_token !== startingRefresh;
|
|
147
|
+
const accessMoved = staleToken ? oauth.access_token !== staleToken : false;
|
|
148
|
+
if (!refreshMoved && !accessMoved && staleToken) return null;
|
|
149
|
+
|
|
150
|
+
// Accept any not-hard-expired peer token. Requiring the full 5-minute skew
|
|
151
|
+
// would reject a peer refresh that landed seconds ago for no good reason.
|
|
152
|
+
if (isExpired(oauth, now)) return null;
|
|
153
|
+
if (staleToken && oauth.access_token === staleToken && !refreshMoved) return null;
|
|
154
|
+
|
|
155
|
+
return { ok: true, token: oauth.access_token, rotated: false, oauth };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const sleepMs = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
159
|
+
|
|
160
|
+
/** Default lock wait when the caller does not override (interactive commands). */
|
|
161
|
+
const LOCK_WAIT_DEFAULT = 10_000;
|
|
162
|
+
|
|
163
|
+
/** Milliseconds left before an absolute deadline, or null when unbounded. */
|
|
164
|
+
export function remainingMs(deadline, now = Date.now()) {
|
|
165
|
+
if (deadline == null || !Number.isFinite(deadline)) return null;
|
|
166
|
+
return Math.max(0, deadline - now);
|
|
167
|
+
}
|
|
168
|
+
|
|
98
169
|
/**
|
|
99
170
|
* Perform one refresh, under the lock, persisting the rotated token.
|
|
100
171
|
*
|
|
@@ -105,14 +176,42 @@ async function refreshEndpoints({ oauth, root, fetchImpl }) {
|
|
|
105
176
|
* "somebody else already refreshed" short-circuit hands the caller back the
|
|
106
177
|
* very token that just 401'd, and the retry fails identically.
|
|
107
178
|
*
|
|
179
|
+
* `lockWaitMs` exists for the hook path. The default 10s wait is fine for a
|
|
180
|
+
* user-initiated command but is too long for a lifecycle hook standing in
|
|
181
|
+
* front of the user's editor.
|
|
182
|
+
*
|
|
183
|
+
* When the lock is NOT held (wait timed out), we poll disk for a peer refresh
|
|
184
|
+
* instead of racing the single-use refresh token. Only if nothing usable
|
|
185
|
+
* appears do we attempt the network refresh — and on `invalid_grant` we
|
|
186
|
+
* re-check disk one last time before declaring the session dead.
|
|
187
|
+
*
|
|
188
|
+
* `deadline` (epoch ms) caps lock wait, unlocked peer-poll, discovery, and the
|
|
189
|
+
* refresh HTTP call so a hook's advertised budget cannot be exceeded before
|
|
190
|
+
* the MCP call even starts. When omitted, behaviour matches the unbounded
|
|
191
|
+
* interactive path (15s per HTTP call).
|
|
192
|
+
*
|
|
108
193
|
* @param {{env: NodeJS.ProcessEnv, root: string, fetchImpl?: typeof fetch, now?: number,
|
|
109
|
-
* staleToken?: string|null}} opts
|
|
194
|
+
* staleToken?: string|null, lockWaitMs?: number, deadline?: number|null}} opts
|
|
110
195
|
* @returns {Promise<{ok: true, token: string, rotated: boolean, oauth: object}
|
|
111
196
|
* | {ok: false, code: string, message: string}>}
|
|
112
197
|
*/
|
|
113
|
-
export async function refreshCredentials({
|
|
198
|
+
export async function refreshCredentials({
|
|
199
|
+
env,
|
|
200
|
+
root,
|
|
201
|
+
fetchImpl = fetch,
|
|
202
|
+
now,
|
|
203
|
+
staleToken = null,
|
|
204
|
+
lockWaitMs,
|
|
205
|
+
deadline = null,
|
|
206
|
+
}) {
|
|
207
|
+
const lockCap = remainingMs(deadline);
|
|
208
|
+
const effectiveLockWait =
|
|
209
|
+
lockCap == null
|
|
210
|
+
? lockWaitMs
|
|
211
|
+
: Math.min(typeof lockWaitMs === 'number' ? lockWaitMs : LOCK_WAIT_DEFAULT, lockCap);
|
|
212
|
+
|
|
114
213
|
return withRefreshLock(
|
|
115
|
-
async () => {
|
|
214
|
+
async ({ held }) => {
|
|
116
215
|
// Re-read INSIDE the lock. While we queued, a sibling hook may have done
|
|
117
216
|
// the whole refresh already — redeeming our (now-replaced) refresh token
|
|
118
217
|
// would fail and, worse, would overwrite theirs.
|
|
@@ -137,18 +236,107 @@ export async function refreshCredentials({ env, root, fetchImpl = fetch, now, st
|
|
|
137
236
|
};
|
|
138
237
|
}
|
|
139
238
|
|
|
140
|
-
const
|
|
239
|
+
const startingRefresh = oauth.refresh_token;
|
|
240
|
+
|
|
241
|
+
// Unlocked callers must not stampede the token endpoint. Poll briefly
|
|
242
|
+
// for a peer that held the lock; if one lands, we are done. Poll time is
|
|
243
|
+
// taken from the SHARED deadline (not an extra lockWait on top).
|
|
244
|
+
if (!held) {
|
|
245
|
+
const rem = remainingMs(deadline);
|
|
246
|
+
const pollMs =
|
|
247
|
+
rem == null
|
|
248
|
+
? Math.min(typeof lockWaitMs === 'number' ? lockWaitMs : 2_000, 1_500)
|
|
249
|
+
: Math.min(rem, 500);
|
|
250
|
+
const pollDeadline = Date.now() + pollMs;
|
|
251
|
+
while (Date.now() < pollDeadline) {
|
|
252
|
+
await sleepMs(50);
|
|
253
|
+
const peer = readPeerRefreshedCredentials({
|
|
254
|
+
env,
|
|
255
|
+
staleToken: staleToken ?? oauth.access_token,
|
|
256
|
+
now,
|
|
257
|
+
startingRefresh,
|
|
258
|
+
});
|
|
259
|
+
if (peer) return peer;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Re-read after the poll — a peer may have finished in the last tick.
|
|
264
|
+
const latest = readConfigFile(env);
|
|
265
|
+
const latestOauth = latest.oauth && typeof latest.oauth === 'object' ? latest.oauth : oauth;
|
|
266
|
+
if (
|
|
267
|
+
latestOauth?.access_token &&
|
|
268
|
+
latestOauth.access_token !== (staleToken ?? oauth.access_token) &&
|
|
269
|
+
!isExpiring(latestOauth, REFRESH_SKEW_MS, now)
|
|
270
|
+
) {
|
|
271
|
+
return { ok: true, token: latestOauth.access_token, rotated: false, oauth: latestOauth };
|
|
272
|
+
}
|
|
273
|
+
if (!latestOauth?.refresh_token) {
|
|
274
|
+
return {
|
|
275
|
+
ok: false,
|
|
276
|
+
code: 'no_refresh_token',
|
|
277
|
+
message: `anyslate: the stored OAuth session has no refresh token and its access token has expired. ${RELOGIN_HINT}`,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const httpTimeout = remainingMs(deadline) ?? 15_000;
|
|
282
|
+
if (httpTimeout <= 0) {
|
|
283
|
+
const peer = readPeerRefreshedCredentials({
|
|
284
|
+
env,
|
|
285
|
+
staleToken: staleToken ?? oauth.access_token,
|
|
286
|
+
now,
|
|
287
|
+
startingRefresh,
|
|
288
|
+
});
|
|
289
|
+
if (peer) return peer;
|
|
290
|
+
return {
|
|
291
|
+
ok: false,
|
|
292
|
+
code: 'refresh_timeout',
|
|
293
|
+
message: `anyslate: timed out before the OAuth refresh could run. ${RELOGIN_HINT}`,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const endpoints = await refreshEndpoints({
|
|
298
|
+
oauth: latestOauth,
|
|
299
|
+
root,
|
|
300
|
+
fetchImpl,
|
|
301
|
+
timeoutMs: httpTimeout,
|
|
302
|
+
});
|
|
141
303
|
if (!endpoints.ok) return endpoints;
|
|
142
304
|
|
|
305
|
+
const refreshTimeout = remainingMs(deadline) ?? httpTimeout;
|
|
306
|
+
if (refreshTimeout <= 0) {
|
|
307
|
+
const peer = readPeerRefreshedCredentials({
|
|
308
|
+
env,
|
|
309
|
+
staleToken: staleToken ?? oauth.access_token,
|
|
310
|
+
now,
|
|
311
|
+
startingRefresh,
|
|
312
|
+
});
|
|
313
|
+
if (peer) return peer;
|
|
314
|
+
return {
|
|
315
|
+
ok: false,
|
|
316
|
+
code: 'refresh_timeout',
|
|
317
|
+
message: `anyslate: timed out before the OAuth refresh could run. ${RELOGIN_HINT}`,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
143
321
|
const res = await refreshAccessToken({
|
|
144
322
|
tokenEndpoint: endpoints.tokenEndpoint,
|
|
145
|
-
refreshToken:
|
|
146
|
-
clientId:
|
|
323
|
+
refreshToken: latestOauth.refresh_token,
|
|
324
|
+
clientId: latestOauth.client_id,
|
|
147
325
|
resource: endpoints.resource,
|
|
148
326
|
fetchImpl,
|
|
149
327
|
now,
|
|
328
|
+
timeoutMs: refreshTimeout,
|
|
150
329
|
});
|
|
151
330
|
if (!res.ok) {
|
|
331
|
+
// Loser of a refresh race: peer already rotated and persisted. Prefer
|
|
332
|
+
// their tokens over "run anyslate login" + circuit-breaker pause.
|
|
333
|
+
const peer = readPeerRefreshedCredentials({
|
|
334
|
+
env,
|
|
335
|
+
staleToken: staleToken ?? oauth.access_token,
|
|
336
|
+
now,
|
|
337
|
+
startingRefresh,
|
|
338
|
+
});
|
|
339
|
+
if (peer) return peer;
|
|
152
340
|
return {
|
|
153
341
|
ok: false,
|
|
154
342
|
code: res.code || 'refresh_failed',
|
|
@@ -160,10 +348,10 @@ export async function refreshCredentials({ env, root, fetchImpl = fetch, now, st
|
|
|
160
348
|
// the instant the server answered; a crash between here and the next
|
|
161
349
|
// write would lose the only usable credential.
|
|
162
350
|
const next = {
|
|
163
|
-
...
|
|
351
|
+
...latestOauth,
|
|
164
352
|
access_token: res.tokens.access_token,
|
|
165
353
|
// A response that omits refresh_token means "keep the one you have".
|
|
166
|
-
refresh_token: res.tokens.refresh_token ||
|
|
354
|
+
refresh_token: res.tokens.refresh_token || latestOauth.refresh_token,
|
|
167
355
|
expires_at: res.tokens.expires_at,
|
|
168
356
|
token_endpoint: endpoints.tokenEndpoint,
|
|
169
357
|
resource: endpoints.resource,
|
|
@@ -180,7 +368,7 @@ export async function refreshCredentials({ env, root, fetchImpl = fetch, now, st
|
|
|
180
368
|
}
|
|
181
369
|
return { ok: true, token: next.access_token, rotated: true, oauth: next };
|
|
182
370
|
},
|
|
183
|
-
{ env },
|
|
371
|
+
{ env, waitMs: effectiveLockWait },
|
|
184
372
|
);
|
|
185
373
|
}
|
|
186
374
|
|
|
@@ -194,11 +382,12 @@ export async function refreshCredentials({ env, root, fetchImpl = fetch, now, st
|
|
|
194
382
|
* static token also exists, we fall back to it rather than failing the call —
|
|
195
383
|
* with `warning` set, so the caller can say so.
|
|
196
384
|
*
|
|
197
|
-
* @param {{cfg: object, env: NodeJS.ProcessEnv, fetchImpl?: typeof fetch, force?: boolean,
|
|
385
|
+
* @param {{cfg: object, env: NodeJS.ProcessEnv, fetchImpl?: typeof fetch, force?: boolean,
|
|
386
|
+
* now?: number, lockWaitMs?: number, deadline?: number|null}} opts
|
|
198
387
|
* @returns {Promise<{ok: true, token: string, mode: 'env'|'oauth'|'static', refreshed: boolean, warning?: string}
|
|
199
388
|
* | {ok: false, code: string, message: string, mode: string}>}
|
|
200
389
|
*/
|
|
201
|
-
export async function resolveBearer({ cfg, env, fetchImpl = fetch, force = false, now }) {
|
|
390
|
+
export async function resolveBearer({ cfg, env, fetchImpl = fetch, force = false, now, lockWaitMs, deadline = null }) {
|
|
202
391
|
if (cfg?.sources?.mcpToken === 'env' && cfg.mcpToken) {
|
|
203
392
|
return { ok: true, token: cfg.mcpToken, mode: 'env', refreshed: false };
|
|
204
393
|
}
|
|
@@ -216,6 +405,8 @@ export async function resolveBearer({ cfg, env, fetchImpl = fetch, force = false
|
|
|
216
405
|
fetchImpl,
|
|
217
406
|
now,
|
|
218
407
|
staleToken: force ? (oauth.access_token ?? null) : null,
|
|
408
|
+
lockWaitMs,
|
|
409
|
+
deadline,
|
|
219
410
|
});
|
|
220
411
|
if (refreshed.ok) {
|
|
221
412
|
return { ok: true, token: refreshed.token, mode: 'oauth', refreshed: refreshed.rotated };
|
|
@@ -246,44 +437,162 @@ export async function resolveBearer({ cfg, env, fetchImpl = fetch, force = false
|
|
|
246
437
|
}
|
|
247
438
|
|
|
248
439
|
/**
|
|
249
|
-
* `callTool` with the
|
|
250
|
-
*
|
|
440
|
+
* `callTool` with the whole availability contract wrapped around it: the
|
|
441
|
+
* persisted breaker, the machine ceiling, proactive refresh, at most one
|
|
442
|
+
* refresh-and-retry on a 401, and bounded backoff on anything transient.
|
|
443
|
+
*
|
|
444
|
+
* The 401 retry is gated on `mode === 'oauth'`: a static token that 401s is
|
|
445
|
+
* revoked or mistyped, and re-sending it cannot help.
|
|
251
446
|
*
|
|
252
|
-
*
|
|
253
|
-
*
|
|
447
|
+
* ATTEMPTS ARE PROVABLY BOUNDED. The loop condition is a counter compared to
|
|
448
|
+
* `policy.maxAttempts + 1` — the `+1` is the single post-refresh retry, which
|
|
449
|
+
* must be allowed even on the hook path (`maxAttempts: 1`) because an access
|
|
450
|
+
* token that aged out mid-session is the normal case, not an anomaly. There is
|
|
451
|
+
* no `while (true)` and no recursion anywhere on this path.
|
|
254
452
|
*
|
|
255
453
|
* @param {{cfg: object, env: NodeJS.ProcessEnv, toolName: string, args: object,
|
|
256
|
-
* fetchImpl?: typeof fetch, timeoutMs?: number
|
|
454
|
+
* fetchImpl?: typeof fetch, timeoutMs?: number, retry?: object,
|
|
455
|
+
* lockWaitMs?: number}} opts
|
|
257
456
|
* @returns {Promise<object>} the callTool result, plus `authMode` / `authWarning`,
|
|
258
|
-
* or an auth failure shaped like a callTool failure (`authError: true`)
|
|
457
|
+
* or an auth failure shaped like a callTool failure (`authError: true`), or a
|
|
458
|
+
* locally-refused call (`breakerOpen` / `rateCapped`) that never left the machine.
|
|
259
459
|
*/
|
|
260
|
-
export async function callToolWithAuth({ cfg, env, toolName, args, fetchImpl, timeoutMs }) {
|
|
261
|
-
const
|
|
460
|
+
export async function callToolWithAuth({ cfg, env, toolName, args, fetchImpl, timeoutMs, retry, lockWaitMs }) {
|
|
461
|
+
const policy = { ...DEFAULT_RETRY, ...(retry ?? {}) };
|
|
462
|
+
|
|
463
|
+
// When the caller names a timeout (hooks: 5s), treat it as a WALL budget for
|
|
464
|
+
// refresh + MCP call combined — not "15s refresh then 5s call". Interactive
|
|
465
|
+
// commands leave timeoutMs undefined and keep the unbounded refresh path.
|
|
466
|
+
const deadline = typeof timeoutMs === 'number' && timeoutMs > 0 ? Date.now() + timeoutMs : null;
|
|
467
|
+
|
|
468
|
+
// Cheapest possible check first: an open breaker resolves no credential,
|
|
469
|
+
// opens no socket, and mints no token. That is the entire point — the
|
|
470
|
+
// incident cost ~50 requests a minute for days precisely because a client
|
|
471
|
+
// with no hope of succeeding still asked.
|
|
472
|
+
const gate = breakerGate({ cfg, env });
|
|
473
|
+
if (gate.open) {
|
|
474
|
+
return {
|
|
475
|
+
ok: false,
|
|
476
|
+
status: 0,
|
|
477
|
+
data: breakerNotice(gate),
|
|
478
|
+
raw: null,
|
|
479
|
+
breakerOpen: true,
|
|
480
|
+
breaker: gate,
|
|
481
|
+
authMode: cfg?.authMode ?? 'none',
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
const slot = reserveCallSlot({ env });
|
|
486
|
+
if (!slot.allowed) {
|
|
487
|
+
return { ok: false, status: 0, data: ceilingNotice(slot), raw: null, rateCapped: true, rateSlot: slot };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const first = await resolveBearer({ cfg, env, fetchImpl, lockWaitMs, deadline });
|
|
262
491
|
if (!first.ok) {
|
|
263
|
-
|
|
492
|
+
const breaker = recordCallFailure({ cfg, env, kind: 'auth', code: first.code, detail: first.code });
|
|
493
|
+
return {
|
|
494
|
+
ok: false,
|
|
495
|
+
status: 0,
|
|
496
|
+
data: first.message,
|
|
497
|
+
raw: null,
|
|
498
|
+
authError: true,
|
|
499
|
+
authMode: first.mode,
|
|
500
|
+
breaker,
|
|
501
|
+
};
|
|
264
502
|
}
|
|
265
503
|
|
|
266
|
-
|
|
267
|
-
|
|
504
|
+
let token = first.token;
|
|
505
|
+
let mode = first.mode;
|
|
506
|
+
let warning = first.warning;
|
|
507
|
+
// A proactive refresh already spent this call's one re-mint.
|
|
508
|
+
let refreshSpent = first.refreshed;
|
|
509
|
+
let retried = false;
|
|
268
510
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
511
|
+
const hardCap = policy.maxAttempts + 1;
|
|
512
|
+
let attempt = 0;
|
|
513
|
+
let spentDelayMs = 0;
|
|
514
|
+
let res = null;
|
|
272
515
|
|
|
273
|
-
|
|
274
|
-
|
|
516
|
+
while (attempt < hardCap) {
|
|
517
|
+
attempt += 1;
|
|
518
|
+
const callTimeout = remainingMs(deadline) ?? timeoutMs;
|
|
519
|
+
if (deadline != null && (callTimeout == null || callTimeout <= 0)) {
|
|
520
|
+
res = {
|
|
521
|
+
ok: false,
|
|
522
|
+
status: 0,
|
|
523
|
+
data: `anyslate: timed out after ${timeoutMs}ms (spent on credential refresh).`,
|
|
524
|
+
raw: null,
|
|
525
|
+
networkError: true,
|
|
526
|
+
authMode: mode,
|
|
527
|
+
};
|
|
528
|
+
if (warning) res.authWarning = warning;
|
|
529
|
+
break;
|
|
530
|
+
}
|
|
531
|
+
res = await callTool({
|
|
532
|
+
apiUrl: cfg.apiUrl,
|
|
533
|
+
token,
|
|
534
|
+
toolName,
|
|
535
|
+
args,
|
|
536
|
+
fetchImpl,
|
|
537
|
+
timeoutMs: callTimeout,
|
|
538
|
+
});
|
|
539
|
+
res.authMode = mode;
|
|
540
|
+
if (warning) res.authWarning = warning;
|
|
541
|
+
if (retried) res.authRetried = true;
|
|
542
|
+
if (attempt > 1) res.attempts = attempt;
|
|
543
|
+
|
|
544
|
+
if (res.ok) break;
|
|
545
|
+
|
|
546
|
+
const kind = classifyFailure(res);
|
|
547
|
+
|
|
548
|
+
// One refresh, paired with the call that needed it — not with each attempt.
|
|
549
|
+
// Re-minting per attempt is what turned a dead credential into two requests
|
|
550
|
+
// per hook instead of one.
|
|
551
|
+
if (kind === 'auth' && mode === 'oauth' && !refreshSpent) {
|
|
552
|
+
refreshSpent = true;
|
|
553
|
+
const second = await resolveBearer({ cfg, env, fetchImpl, force: true, lockWaitMs, deadline });
|
|
554
|
+
if (!second.ok) {
|
|
555
|
+
res.authError = true;
|
|
556
|
+
res.authRefreshFailed = second.message;
|
|
557
|
+
res.authCode = second.code;
|
|
558
|
+
break;
|
|
559
|
+
}
|
|
560
|
+
token = second.token;
|
|
561
|
+
mode = second.mode;
|
|
562
|
+
warning = second.warning;
|
|
563
|
+
retried = true;
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
275
566
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
567
|
+
const retryAfterMs = retryAfterMsOf(res);
|
|
568
|
+
const plan = planRetry({ kind, attempt, spentDelayMs, retryAfterMs, policy });
|
|
569
|
+
if (!plan.retry) break;
|
|
570
|
+
await sleep(plan.delayMs);
|
|
571
|
+
spentDelayMs += plan.delayMs;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
if (res.ok) {
|
|
575
|
+
recordCallSuccess({ cfg, env });
|
|
280
576
|
return res;
|
|
281
577
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
578
|
+
|
|
579
|
+
const kind = classifyFailure(res);
|
|
580
|
+
res.breaker = recordCallFailure({
|
|
581
|
+
cfg,
|
|
582
|
+
env,
|
|
583
|
+
kind,
|
|
584
|
+
code: res.authCode,
|
|
585
|
+
detail: failureDetail(res, kind),
|
|
586
|
+
retryAfterMs: retryAfterMsOf(res),
|
|
587
|
+
});
|
|
588
|
+
return res;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/** A short, token-free reason to persist alongside the breaker state. */
|
|
592
|
+
function failureDetail(res, kind) {
|
|
593
|
+
if (kind === 'auth' && res.authRefreshFailed) return 'OAuth refresh failed';
|
|
594
|
+
if (res.networkError) return typeof res.data === 'string' ? res.data : 'request failed';
|
|
595
|
+
return `HTTP ${res.status}`;
|
|
287
596
|
}
|
|
288
597
|
|
|
289
598
|
/**
|
|
@@ -308,3 +617,33 @@ export function formatAuthFailure(prefix, res) {
|
|
|
308
617
|
}
|
|
309
618
|
return null;
|
|
310
619
|
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* The failure line for a call this machine refused to make.
|
|
623
|
+
*
|
|
624
|
+
* `formatCallFailure` would render `server 0 — …` for these, inventing an HTTP
|
|
625
|
+
* exchange that never happened. Both cases are local decisions and must read
|
|
626
|
+
* that way, or the user goes looking for a server problem that is not there.
|
|
627
|
+
*
|
|
628
|
+
* @param {string} prefix
|
|
629
|
+
* @param {object} res
|
|
630
|
+
* @returns {string|null}
|
|
631
|
+
*/
|
|
632
|
+
export function formatLocalRefusal(prefix, res) {
|
|
633
|
+
if (res?.breakerOpen || res?.rateCapped) {
|
|
634
|
+
return `${prefix}: ${typeof res.data === 'string' ? res.data : JSON.stringify(res.data)}\n`;
|
|
635
|
+
}
|
|
636
|
+
return null;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* The one failure formatter every capture command uses, so a new failure mode
|
|
641
|
+
* can never reach a command that does not know how to render it.
|
|
642
|
+
*
|
|
643
|
+
* @param {string} prefix
|
|
644
|
+
* @param {object} res
|
|
645
|
+
* @returns {string}
|
|
646
|
+
*/
|
|
647
|
+
export function formatFailure(prefix, res) {
|
|
648
|
+
return formatLocalRefusal(prefix, res) ?? formatAuthFailure(prefix, res) ?? formatCallFailure(prefix, res);
|
|
649
|
+
}
|