@juspay/neurolink 9.88.10 → 9.88.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +26 -26
- package/dist/browser/neurolink.min.js +246 -246
- package/dist/cli/commands/proxy.js +196 -60
- package/dist/lib/proxy/globalInstaller.d.ts +7 -1
- package/dist/lib/proxy/globalInstaller.js +49 -1
- package/dist/lib/proxy/updateState.d.ts +8 -0
- package/dist/lib/proxy/updateState.js +73 -1
- package/dist/lib/proxy/updaterSupervisor.d.ts +3 -0
- package/dist/lib/proxy/updaterSupervisor.js +58 -0
- package/dist/lib/proxy/usageStats.d.ts +1 -1
- package/dist/lib/proxy/usageStats.js +20 -2
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -10
- package/dist/lib/server/routes/claudeProxyRoutes.js +68 -21
- package/dist/lib/types/proxy.d.ts +69 -1
- package/dist/proxy/globalInstaller.d.ts +7 -1
- package/dist/proxy/globalInstaller.js +49 -1
- package/dist/proxy/updateState.d.ts +8 -0
- package/dist/proxy/updateState.js +73 -1
- package/dist/proxy/updaterSupervisor.d.ts +3 -0
- package/dist/proxy/updaterSupervisor.js +57 -0
- package/dist/proxy/usageStats.d.ts +1 -1
- package/dist/proxy/usageStats.js +20 -2
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -10
- package/dist/server/routes/claudeProxyRoutes.js +68 -21
- package/dist/types/proxy.d.ts +69 -1
- package/package.json +1 -1
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/** Keep the launchd updater worker alive while its owning proxy is running. */
|
|
2
|
+
export function startUpdaterWorkerSupervisor(options) {
|
|
3
|
+
let stopped = false;
|
|
4
|
+
let pid;
|
|
5
|
+
/** Return the healthy worker PID or replace an unavailable worker. */
|
|
6
|
+
const checkNow = () => {
|
|
7
|
+
if (stopped) {
|
|
8
|
+
return pid;
|
|
9
|
+
}
|
|
10
|
+
if (pid !== undefined && options.isProcessRunning(pid)) {
|
|
11
|
+
return pid;
|
|
12
|
+
}
|
|
13
|
+
const previousPid = pid;
|
|
14
|
+
try {
|
|
15
|
+
pid = options.spawnWorker();
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
pid = undefined;
|
|
19
|
+
options.log?.(`[proxy] updater worker spawn failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
20
|
+
}
|
|
21
|
+
options.onPidChange?.(pid);
|
|
22
|
+
if (pid !== undefined) {
|
|
23
|
+
options.log?.(previousPid === undefined
|
|
24
|
+
? `[proxy] updater worker started pid=${pid}`
|
|
25
|
+
: `[proxy] updater worker restarted oldPid=${previousPid} newPid=${pid}`);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
options.log?.("[proxy] updater worker unavailable; retrying later");
|
|
29
|
+
}
|
|
30
|
+
return pid;
|
|
31
|
+
};
|
|
32
|
+
checkNow();
|
|
33
|
+
const interval = setInterval(checkNow, options.intervalMs ?? 30_000);
|
|
34
|
+
interval.unref?.();
|
|
35
|
+
return {
|
|
36
|
+
currentPid: () => pid,
|
|
37
|
+
checkNow,
|
|
38
|
+
stop: () => {
|
|
39
|
+
if (stopped) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
stopped = true;
|
|
43
|
+
clearInterval(interval);
|
|
44
|
+
const activePid = pid;
|
|
45
|
+
if (activePid !== undefined && options.isProcessRunning(activePid)) {
|
|
46
|
+
try {
|
|
47
|
+
options.stopWorker(activePid);
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
options.log?.(`[proxy] updater worker stop failed pid=${activePid}: ${error instanceof Error ? error.message : String(error)}`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
pid = undefined;
|
|
54
|
+
options.onPidChange?.(undefined);
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=updaterSupervisor.js.map
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import type { AccountStats, ProxyStats } from "../types/index.js";
|
|
7
7
|
export declare function recordAttempt(accountLabel: string, accountType: string): void;
|
|
8
8
|
export declare function recordFinalSuccess(accountLabel?: string, accountType?: string): void;
|
|
9
|
-
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number): void;
|
|
9
|
+
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
10
10
|
export declare function recordFinalError(_status: number, accountLabel?: string, accountType?: string): void;
|
|
11
11
|
export declare function getStats(): ProxyStats;
|
|
12
12
|
export declare function getAccountStats(label: string): AccountStats | undefined;
|
|
@@ -6,10 +6,13 @@
|
|
|
6
6
|
const stats = {
|
|
7
7
|
startedAt: Date.now(),
|
|
8
8
|
totalAttempts: 0,
|
|
9
|
+
totalAttemptErrors: 0,
|
|
9
10
|
totalRequests: 0,
|
|
10
11
|
totalSuccess: 0,
|
|
11
12
|
totalErrors: 0,
|
|
12
13
|
totalRateLimits: 0,
|
|
14
|
+
totalTransientRateLimits: 0,
|
|
15
|
+
totalQuotaRateLimits: 0,
|
|
13
16
|
accounts: {},
|
|
14
17
|
};
|
|
15
18
|
export function recordAttempt(accountLabel, accountType) {
|
|
@@ -26,13 +29,22 @@ export function recordFinalSuccess(accountLabel, accountType) {
|
|
|
26
29
|
acct.successCount++;
|
|
27
30
|
}
|
|
28
31
|
}
|
|
29
|
-
export function recordAttemptError(accountLabel, accountType, status) {
|
|
32
|
+
export function recordAttemptError(accountLabel, accountType, status, rateLimitKind) {
|
|
30
33
|
const acct = ensureAccount(accountLabel, accountType);
|
|
31
|
-
|
|
34
|
+
stats.totalAttemptErrors++;
|
|
35
|
+
acct.attemptErrorCount++;
|
|
32
36
|
acct.lastErrorAt = Date.now();
|
|
33
37
|
if (status === 429) {
|
|
34
38
|
stats.totalRateLimits++;
|
|
35
39
|
acct.rateLimitCount++;
|
|
40
|
+
if (rateLimitKind === "transient") {
|
|
41
|
+
stats.totalTransientRateLimits++;
|
|
42
|
+
acct.transientRateLimitCount++;
|
|
43
|
+
}
|
|
44
|
+
else if (rateLimitKind === "quota") {
|
|
45
|
+
stats.totalQuotaRateLimits++;
|
|
46
|
+
acct.quotaRateLimitCount++;
|
|
47
|
+
}
|
|
36
48
|
}
|
|
37
49
|
}
|
|
38
50
|
export function recordFinalError(_status, accountLabel, accountType) {
|
|
@@ -58,10 +70,13 @@ export function getAccountStats(label) {
|
|
|
58
70
|
export function resetStats() {
|
|
59
71
|
stats.startedAt = Date.now();
|
|
60
72
|
stats.totalAttempts = 0;
|
|
73
|
+
stats.totalAttemptErrors = 0;
|
|
61
74
|
stats.totalRequests = 0;
|
|
62
75
|
stats.totalSuccess = 0;
|
|
63
76
|
stats.totalErrors = 0;
|
|
64
77
|
stats.totalRateLimits = 0;
|
|
78
|
+
stats.totalTransientRateLimits = 0;
|
|
79
|
+
stats.totalQuotaRateLimits = 0;
|
|
65
80
|
stats.accounts = {};
|
|
66
81
|
}
|
|
67
82
|
function ensureAccount(label, type) {
|
|
@@ -70,9 +85,12 @@ function ensureAccount(label, type) {
|
|
|
70
85
|
label,
|
|
71
86
|
type,
|
|
72
87
|
attemptCount: 0,
|
|
88
|
+
attemptErrorCount: 0,
|
|
73
89
|
successCount: 0,
|
|
74
90
|
errorCount: 0,
|
|
75
91
|
rateLimitCount: 0,
|
|
92
|
+
transientRateLimitCount: 0,
|
|
93
|
+
quotaRateLimitCount: 0,
|
|
76
94
|
lastAttemptAt: 0,
|
|
77
95
|
};
|
|
78
96
|
}
|
|
@@ -167,19 +167,11 @@ declare function handleAnthropicAuthRetry(args: {
|
|
|
167
167
|
};
|
|
168
168
|
enabledAccounts: ProxyPassthroughAccount[];
|
|
169
169
|
orderedAccounts: ProxyPassthroughAccount[];
|
|
170
|
-
response: Response;
|
|
171
170
|
tracer?: ProxyTracer;
|
|
172
171
|
requestStartTime: number;
|
|
173
|
-
|
|
174
|
-
attemptNumber: number;
|
|
175
|
-
finalBodyStr: string;
|
|
172
|
+
allocateAttemptNumber: () => number;
|
|
176
173
|
upstreamSpan?: import("@opentelemetry/api").Span;
|
|
177
|
-
logAttempt:
|
|
178
|
-
inputTokens?: number;
|
|
179
|
-
outputTokens?: number;
|
|
180
|
-
cacheCreationTokens?: number;
|
|
181
|
-
cacheReadTokens?: number;
|
|
182
|
-
}) => void;
|
|
174
|
+
logAttempt: AnthropicAttemptLogger;
|
|
183
175
|
logProxyBody: ProxyBodyCaptureLogger;
|
|
184
176
|
logFinalRequest: (status: number, accountLabel: string, accountType: string, errorType?: string, errorMessage?: string, extra?: {
|
|
185
177
|
inputTokens?: number;
|
|
@@ -2825,8 +2825,11 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2825
2825
|
return retryJson;
|
|
2826
2826
|
}
|
|
2827
2827
|
async function handleAnthropicAuthRetry(args) {
|
|
2828
|
-
const { ctx, body, account, accountState, headers, buildUpstreamBody, enabledAccounts, orderedAccounts,
|
|
2828
|
+
const { ctx, body, account, accountState, headers, buildUpstreamBody, enabledAccounts, orderedAccounts, tracer, requestStartTime, allocateAttemptNumber, upstreamSpan, logAttempt, logProxyBody, logFinalRequest, lastError, authFailureMessage, sawRateLimit, sawTransientFailure, sawNetworkError, } = args;
|
|
2829
2829
|
recordAttemptError(account.label, account.type, 401);
|
|
2830
|
+
logAttempt(401, "authentication_error", "received 401 from Anthropic", {
|
|
2831
|
+
retryable: true,
|
|
2832
|
+
});
|
|
2830
2833
|
let currentLastError = lastError;
|
|
2831
2834
|
let currentAuthFailureMessage = authFailureMessage;
|
|
2832
2835
|
let currentSawRateLimit = sawRateLimit;
|
|
@@ -2858,11 +2861,29 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2858
2861
|
}
|
|
2859
2862
|
await clearAuthCooldownAfterRefresh(account, accountState);
|
|
2860
2863
|
headers.authorization = `Bearer ${account.token}`;
|
|
2864
|
+
const retryAttemptNumber = allocateAttemptNumber();
|
|
2865
|
+
recordAttempt(account.label, account.type);
|
|
2866
|
+
const retryLogAttempt = (status, errorType, errorMessage, extra) => logAttempt(status, errorType, errorMessage, {
|
|
2867
|
+
...extra,
|
|
2868
|
+
attempt: retryAttemptNumber,
|
|
2869
|
+
});
|
|
2870
|
+
const retryBodyStr = buildUpstreamBody(account.token).bodyStr;
|
|
2871
|
+
const retryFetchStartMs = Date.now();
|
|
2872
|
+
logProxyBody({
|
|
2873
|
+
phase: "upstream_request",
|
|
2874
|
+
headers,
|
|
2875
|
+
body: retryBodyStr,
|
|
2876
|
+
bodySize: Buffer.byteLength(retryBodyStr, "utf8"),
|
|
2877
|
+
contentType: headers["content-type"] ?? "application/json",
|
|
2878
|
+
account: account.label,
|
|
2879
|
+
accountType: account.type,
|
|
2880
|
+
attempt: retryAttemptNumber,
|
|
2881
|
+
});
|
|
2861
2882
|
try {
|
|
2862
2883
|
const retryResp = await fetch("https://api.anthropic.com/v1/messages?beta=true", {
|
|
2863
2884
|
method: "POST",
|
|
2864
2885
|
headers,
|
|
2865
|
-
body:
|
|
2886
|
+
body: retryBodyStr,
|
|
2866
2887
|
signal: AbortSignal.timeout(UPSTREAM_FETCH_TIMEOUT_MS),
|
|
2867
2888
|
});
|
|
2868
2889
|
if (retryResp.ok) {
|
|
@@ -2877,9 +2898,9 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2877
2898
|
retryResp,
|
|
2878
2899
|
tracer,
|
|
2879
2900
|
requestStartTime,
|
|
2880
|
-
fetchStartMs,
|
|
2881
|
-
attemptNumber,
|
|
2882
|
-
finalBodyStr,
|
|
2901
|
+
fetchStartMs: retryFetchStartMs,
|
|
2902
|
+
attemptNumber: retryAttemptNumber,
|
|
2903
|
+
finalBodyStr: retryBodyStr,
|
|
2883
2904
|
upstreamSpan: currentUpstreamSpan,
|
|
2884
2905
|
logProxyBody,
|
|
2885
2906
|
logFinalRequest,
|
|
@@ -2919,9 +2940,9 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2919
2940
|
contentType: retryRespHeaders["content-type"] ?? "application/json",
|
|
2920
2941
|
account: account.label,
|
|
2921
2942
|
accountType: account.type,
|
|
2922
|
-
attempt:
|
|
2943
|
+
attempt: retryAttemptNumber,
|
|
2923
2944
|
responseStatus: retryStatus,
|
|
2924
|
-
durationMs: Date.now() -
|
|
2945
|
+
durationMs: Date.now() - retryFetchStartMs,
|
|
2925
2946
|
});
|
|
2926
2947
|
authRetryError = `retry ${authRetry + 1}/${MAX_AUTH_RETRIES} failed with status ${retryStatus}`;
|
|
2927
2948
|
currentLastError = retryBody;
|
|
@@ -2929,7 +2950,7 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2929
2950
|
if (retryStatus === 429 &&
|
|
2930
2951
|
isAntiAbuseConstruction429(retryRespHeaders, retryBody)) {
|
|
2931
2952
|
logger.always(`[proxy] ← 429 account=${account.label} anti-abuse/construction rejection after OAuth refresh — returning non-retryable request error`);
|
|
2932
|
-
|
|
2953
|
+
retryLogAttempt(429, "construction_rejection", retryBody);
|
|
2933
2954
|
tracer?.setError("construction_rejection", retryBody.slice(0, 500));
|
|
2934
2955
|
currentUpstreamSpan?.end();
|
|
2935
2956
|
return {
|
|
@@ -2938,7 +2959,7 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2938
2959
|
account,
|
|
2939
2960
|
tracer,
|
|
2940
2961
|
requestStartTime,
|
|
2941
|
-
attemptNumber,
|
|
2962
|
+
attemptNumber: retryAttemptNumber,
|
|
2942
2963
|
logProxyBody,
|
|
2943
2964
|
logFinalRequest,
|
|
2944
2965
|
}),
|
|
@@ -2951,7 +2972,6 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2951
2972
|
upstreamSpan: undefined,
|
|
2952
2973
|
};
|
|
2953
2974
|
}
|
|
2954
|
-
recordAttemptError(account.label, account.type, retryStatus);
|
|
2955
2975
|
// Construction rejections return through the terminal 400 path above.
|
|
2956
2976
|
// Every 429 reaching this branch is a genuine rate limit and must cool
|
|
2957
2977
|
// the account according to its reset window before rotating.
|
|
@@ -2965,6 +2985,13 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2965
2985
|
accountState.quota = retryQuota429;
|
|
2966
2986
|
}
|
|
2967
2987
|
const retryPlan = planCooldownFor429(retryQuota429, parseRetryAfterMs(retryRespHeaders["retry-after"] ?? null), nowRetry, getUnifiedRateLimitStatus(retryRespHeaders));
|
|
2988
|
+
const rateLimitKind = retryPlan.reason === "transient" ? "transient" : "quota";
|
|
2989
|
+
recordAttemptError(account.label, account.type, retryStatus, rateLimitKind);
|
|
2990
|
+
retryLogAttempt(429, "rate_limit_error", retryBody, {
|
|
2991
|
+
retryable: true,
|
|
2992
|
+
rateLimitKind,
|
|
2993
|
+
cooldownReason: retryPlan.reason,
|
|
2994
|
+
});
|
|
2968
2995
|
if (!accountState.coolingUntil ||
|
|
2969
2996
|
retryPlan.coolingUntil > accountState.coolingUntil) {
|
|
2970
2997
|
accountState.coolingUntil = retryPlan.coolingUntil;
|
|
@@ -2982,16 +3009,20 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2982
3009
|
break;
|
|
2983
3010
|
}
|
|
2984
3011
|
if (retryStatus === 401 || retryStatus === 402 || retryStatus === 403) {
|
|
3012
|
+
recordAttemptError(account.label, account.type, retryStatus);
|
|
3013
|
+
retryLogAttempt(retryStatus, "authentication_error", summarizeErrorMessage(retryBody), { retryable: true });
|
|
2985
3014
|
if (authRetry < MAX_AUTH_RETRIES - 1) {
|
|
2986
3015
|
await sleep(1000);
|
|
2987
3016
|
}
|
|
2988
3017
|
continue;
|
|
2989
3018
|
}
|
|
2990
3019
|
if (isTransientHttpFailure(retryStatus, retryBody)) {
|
|
3020
|
+
recordAttemptError(account.label, account.type, retryStatus);
|
|
3021
|
+
retryLogAttempt(retryStatus, "api_error", summarizeErrorMessage(retryBody), { retryable: true });
|
|
2991
3022
|
currentSawTransientFailure = true;
|
|
2992
3023
|
break;
|
|
2993
3024
|
}
|
|
2994
|
-
|
|
3025
|
+
retryLogAttempt(retryStatus, "api_error", summarizeErrorMessage(retryBody));
|
|
2995
3026
|
recordFinalError(retryStatus, account.label, account.type);
|
|
2996
3027
|
try {
|
|
2997
3028
|
logFinalRequest(retryStatus, account.label, account.type, "api_error", summarizeErrorMessage(retryBody));
|
|
@@ -3028,6 +3059,7 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
3028
3059
|
: String(retryFetchErr);
|
|
3029
3060
|
authRetryError = `network error on retry ${authRetry + 1}: ${message}`;
|
|
3030
3061
|
currentLastError = authRetryError;
|
|
3062
|
+
retryLogAttempt(502, "network_error", message, { retryable: true });
|
|
3031
3063
|
logger.debug(`[proxy] ${authRetryError}`);
|
|
3032
3064
|
break;
|
|
3033
3065
|
}
|
|
@@ -3036,7 +3068,6 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
3036
3068
|
// No persistent cooldown — just move to next account for this request.
|
|
3037
3069
|
currentLastError = authRetryError;
|
|
3038
3070
|
logger.always(`[proxy] ⚠ account=${account.label} auth retries exhausted, rotating to next account`);
|
|
3039
|
-
logAttempt(401, "authentication_error", authRetryError);
|
|
3040
3071
|
tracer?.setError("authentication_error", authRetryError);
|
|
3041
3072
|
tracer?.recordRetry(account.label, "auth_exhausted");
|
|
3042
3073
|
currentUpstreamSpan?.end();
|
|
@@ -3225,11 +3256,17 @@ async function handleAnthropicNonOkResponse(args) {
|
|
|
3225
3256
|
};
|
|
3226
3257
|
}
|
|
3227
3258
|
if (isTransientHttpFailure(response.status, errBody)) {
|
|
3228
|
-
recordAttemptError(account.label, account.type, response.status);
|
|
3259
|
+
recordAttemptError(account.label, account.type, response.status, response.status === 429 ? "transient" : undefined);
|
|
3229
3260
|
currentSawTransientFailure = true;
|
|
3230
3261
|
logger.always(`[proxy] ← ${response.status} account=${account.label} (transient)`);
|
|
3231
3262
|
currentLastError = errBody;
|
|
3232
|
-
logAttempt(response.status, "api_error", summarizeErrorMessage(errBody)
|
|
3263
|
+
logAttempt(response.status, "api_error", summarizeErrorMessage(errBody), response.status === 429
|
|
3264
|
+
? {
|
|
3265
|
+
retryable: true,
|
|
3266
|
+
rateLimitKind: "transient",
|
|
3267
|
+
cooldownReason: "transient",
|
|
3268
|
+
}
|
|
3269
|
+
: undefined);
|
|
3233
3270
|
tracer?.setError("transient_error", summarizeErrorMessage(errBody));
|
|
3234
3271
|
tracer?.recordRetry(account.label, "transient");
|
|
3235
3272
|
return {
|
|
@@ -3385,7 +3422,7 @@ function createAnthropicAttemptLogger(args) {
|
|
|
3385
3422
|
logRequestAttempt({
|
|
3386
3423
|
timestamp: new Date().toISOString(),
|
|
3387
3424
|
requestId: ctx.requestId,
|
|
3388
|
-
attempt: attemptNumber,
|
|
3425
|
+
attempt: extra?.attempt ?? attemptNumber,
|
|
3389
3426
|
method: ctx.method,
|
|
3390
3427
|
path: ctx.path,
|
|
3391
3428
|
model: body.model,
|
|
@@ -3409,6 +3446,11 @@ function createAnthropicAttemptLogger(args) {
|
|
|
3409
3446
|
...(extra?.cacheReadTokens !== undefined
|
|
3410
3447
|
? { cacheReadTokens: extra.cacheReadTokens }
|
|
3411
3448
|
: {}),
|
|
3449
|
+
...(extra?.retryable !== undefined ? { retryable: extra.retryable } : {}),
|
|
3450
|
+
...(extra?.rateLimitKind ? { rateLimitKind: extra.rateLimitKind } : {}),
|
|
3451
|
+
...(extra?.cooldownReason
|
|
3452
|
+
? { cooldownReason: extra.cooldownReason }
|
|
3453
|
+
: {}),
|
|
3412
3454
|
...(traceCtx
|
|
3413
3455
|
? { traceId: traceCtx.traceId, spanId: traceCtx.spanId }
|
|
3414
3456
|
: {}),
|
|
@@ -3677,7 +3719,6 @@ async function fetchAnthropicAccountResponse(args) {
|
|
|
3677
3719
|
};
|
|
3678
3720
|
}
|
|
3679
3721
|
sawRateLimit = true;
|
|
3680
|
-
recordAttemptError(account.label, account.type, 429);
|
|
3681
3722
|
// Parse the unified-window quota headers (present on real rate-limit 429s)
|
|
3682
3723
|
// and derive a reset-aware cooldown plan. This is the fix for "kept
|
|
3683
3724
|
// hammering the 5h/7d-exhausted account instead of switching": on an
|
|
@@ -3687,12 +3728,18 @@ async function fetchAnthropicAccountResponse(args) {
|
|
|
3687
3728
|
const quota = parseQuotaHeaders(errRespHeaders);
|
|
3688
3729
|
const unifiedStatus = getUnifiedRateLimitStatus(errRespHeaders);
|
|
3689
3730
|
const cooldownPlan = planCooldownFor429(quota, retryAfterMs, now, unifiedStatus);
|
|
3731
|
+
const rateLimitKind = cooldownPlan.reason === "transient" ? "transient" : "quota";
|
|
3732
|
+
recordAttemptError(account.label, account.type, 429, rateLimitKind);
|
|
3690
3733
|
logger.always(`[proxy] ← 429 account=${account.label} reason=${cooldownPlan.reason} ` +
|
|
3691
3734
|
`retry-after=${retryAfterMs}ms 5h-status=${errRespHeaders["anthropic-ratelimit-unified-5h-status"] ?? "unknown"} ` +
|
|
3692
3735
|
`7d-status=${errRespHeaders["anthropic-ratelimit-unified-7d-status"] ?? "unknown"} ` +
|
|
3693
3736
|
`unified-status=${unifiedStatus ?? "unknown"} ` +
|
|
3694
3737
|
`→ ${cooldownPlan.rotateImmediately ? `rotate now, cool ${minutesUntil(cooldownPlan.coolingUntil, now)}m` : "retry same account (transient)"}`);
|
|
3695
|
-
logAttempt(429, "rate_limit_error", String(lastError)
|
|
3738
|
+
logAttempt(429, "rate_limit_error", String(lastError), {
|
|
3739
|
+
retryable: true,
|
|
3740
|
+
rateLimitKind,
|
|
3741
|
+
cooldownReason: cooldownPlan.reason,
|
|
3742
|
+
});
|
|
3696
3743
|
tracer?.setError("rate_limit_error", String(lastError).slice(0, 500));
|
|
3697
3744
|
tracer?.recordRetry(account.label, "rate_limit");
|
|
3698
3745
|
currentUpstreamSpan?.end();
|
|
@@ -3940,12 +3987,12 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3940
3987
|
buildUpstreamBody: preparedAttempt.buildUpstreamBody,
|
|
3941
3988
|
enabledAccounts,
|
|
3942
3989
|
orderedAccounts,
|
|
3943
|
-
response,
|
|
3944
3990
|
tracer,
|
|
3945
3991
|
requestStartTime,
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3992
|
+
allocateAttemptNumber: () => {
|
|
3993
|
+
loopState.attemptNumber += 1;
|
|
3994
|
+
return loopState.attemptNumber;
|
|
3995
|
+
},
|
|
3949
3996
|
upstreamSpan,
|
|
3950
3997
|
logAttempt,
|
|
3951
3998
|
logProxyBody,
|
|
@@ -459,6 +459,12 @@ export type RequestAttemptLogEntry = {
|
|
|
459
459
|
responseTimeMs: number;
|
|
460
460
|
errorType?: string;
|
|
461
461
|
errorMessage?: string;
|
|
462
|
+
/** Whether this failed attempt may be retried without changing the request. */
|
|
463
|
+
retryable?: boolean;
|
|
464
|
+
/** Distinguishes short-lived admission throttles from exhausted quota windows. */
|
|
465
|
+
rateLimitKind?: "transient" | "quota";
|
|
466
|
+
/** Reset-aware cooldown reason selected for a rate-limited attempt. */
|
|
467
|
+
cooldownReason?: "transient" | "session" | "weekly" | "unified";
|
|
462
468
|
inputTokens?: number;
|
|
463
469
|
outputTokens?: number;
|
|
464
470
|
cacheCreationTokens?: number;
|
|
@@ -505,6 +511,11 @@ export type AnthropicAttemptLogger = (status: number, errorType?: string, errorM
|
|
|
505
511
|
outputTokens?: number;
|
|
506
512
|
cacheCreationTokens?: number;
|
|
507
513
|
cacheReadTokens?: number;
|
|
514
|
+
retryable?: boolean;
|
|
515
|
+
rateLimitKind?: "transient" | "quota";
|
|
516
|
+
cooldownReason?: "transient" | "session" | "weekly" | "unified";
|
|
517
|
+
/** Override used when one account selection performs an OAuth retry fetch. */
|
|
518
|
+
attempt?: number;
|
|
508
519
|
}) => void;
|
|
509
520
|
export type AnthropicLoopState = {
|
|
510
521
|
lastError: unknown;
|
|
@@ -578,8 +589,9 @@ export type PreparedAnthropicAccountAttempt = {
|
|
|
578
589
|
};
|
|
579
590
|
/** How to cool an account after a genuine (non-anti-abuse) 429, derived from
|
|
580
591
|
* the response's quota headers + retry-after. */
|
|
592
|
+
export type RateLimitCoolingReason = Exclude<AccountCoolingReason, "auth">;
|
|
581
593
|
export type AccountCooldownPlan = {
|
|
582
|
-
reason:
|
|
594
|
+
reason: RateLimitCoolingReason;
|
|
583
595
|
/** Epoch-ms until which the account should not be used. */
|
|
584
596
|
coolingUntil: number;
|
|
585
597
|
/** When true (unified/5h/7d rejected), rotate immediately — retrying the
|
|
@@ -619,19 +631,29 @@ export type AccountStats = {
|
|
|
619
631
|
label: string;
|
|
620
632
|
type: string;
|
|
621
633
|
attemptCount: number;
|
|
634
|
+
/** Failed upstream attempts, including retries that later recovered. */
|
|
635
|
+
attemptErrorCount: number;
|
|
636
|
+
/** Final requests successfully completed by this account. */
|
|
622
637
|
successCount: number;
|
|
638
|
+
/** Final requests that terminated as errors on this account. */
|
|
623
639
|
errorCount: number;
|
|
640
|
+
/** All upstream attempts that returned 429. */
|
|
624
641
|
rateLimitCount: number;
|
|
642
|
+
transientRateLimitCount: number;
|
|
643
|
+
quotaRateLimitCount: number;
|
|
625
644
|
lastAttemptAt: number;
|
|
626
645
|
lastErrorAt?: number;
|
|
627
646
|
};
|
|
628
647
|
export type ProxyStats = {
|
|
629
648
|
startedAt: number;
|
|
630
649
|
totalAttempts: number;
|
|
650
|
+
totalAttemptErrors: number;
|
|
631
651
|
totalRequests: number;
|
|
632
652
|
totalSuccess: number;
|
|
633
653
|
totalErrors: number;
|
|
634
654
|
totalRateLimits: number;
|
|
655
|
+
totalTransientRateLimits: number;
|
|
656
|
+
totalQuotaRateLimits: number;
|
|
635
657
|
accounts: Record<string, AccountStats>;
|
|
636
658
|
};
|
|
637
659
|
export type RefreshableAccount = {
|
|
@@ -1093,6 +1115,21 @@ export type UpdateState = {
|
|
|
1093
1115
|
suppressedVersions: Record<string, SuppressedVersion>;
|
|
1094
1116
|
lastUpdateAt: string | null;
|
|
1095
1117
|
lastUpdateVersion: string | null;
|
|
1118
|
+
/** Installed by the updater but not yet confirmed as the running version. */
|
|
1119
|
+
pendingRestartVersion: string | null;
|
|
1120
|
+
/** Last updater failure, retained until a successful update or replacement. */
|
|
1121
|
+
lastFailure: {
|
|
1122
|
+
at: string;
|
|
1123
|
+
version: string;
|
|
1124
|
+
stage: "check" | "install" | "validation" | "restart" | "health";
|
|
1125
|
+
message: string;
|
|
1126
|
+
} | null;
|
|
1127
|
+
};
|
|
1128
|
+
/** Result of validating a newly installed CLI through the stable trampoline. */
|
|
1129
|
+
export type InstalledVersionValidation = {
|
|
1130
|
+
version?: string;
|
|
1131
|
+
attempts: number;
|
|
1132
|
+
failure?: string;
|
|
1096
1133
|
};
|
|
1097
1134
|
/** Supported global package managers for proxy self-updates. */
|
|
1098
1135
|
export type GlobalInstallerKind = "npm" | "pnpm";
|
|
@@ -1122,6 +1159,31 @@ export type ResolveGlobalInstallerOptions = {
|
|
|
1122
1159
|
homeDir?: string;
|
|
1123
1160
|
execFileSync?: GlobalInstallerExecFile;
|
|
1124
1161
|
};
|
|
1162
|
+
/** Options for validating a newly installed CLI through its stable executable. */
|
|
1163
|
+
export type ValidateInstalledVersionOptions = {
|
|
1164
|
+
binPath: string;
|
|
1165
|
+
expectedVersion: string;
|
|
1166
|
+
maxAttempts?: number;
|
|
1167
|
+
delayMs?: number;
|
|
1168
|
+
timeoutMs?: number;
|
|
1169
|
+
execFileSync?: GlobalInstallerExecFile;
|
|
1170
|
+
sleep?: (ms: number) => Promise<void>;
|
|
1171
|
+
};
|
|
1172
|
+
/** Dependencies and callbacks used to supervise the updater worker process. */
|
|
1173
|
+
export type UpdaterWorkerSupervisorOptions = {
|
|
1174
|
+
spawnWorker: () => number | undefined;
|
|
1175
|
+
isProcessRunning: (pid: number) => boolean;
|
|
1176
|
+
stopWorker: (pid: number) => void;
|
|
1177
|
+
onPidChange?: (pid: number | undefined) => void;
|
|
1178
|
+
log?: (message: string) => void;
|
|
1179
|
+
intervalMs?: number;
|
|
1180
|
+
};
|
|
1181
|
+
/** Handle used by the proxy process to inspect and stop updater supervision. */
|
|
1182
|
+
export type UpdaterWorkerSupervisor = {
|
|
1183
|
+
currentPid: () => number | undefined;
|
|
1184
|
+
checkNow: () => number | undefined;
|
|
1185
|
+
stop: () => void;
|
|
1186
|
+
};
|
|
1125
1187
|
/** Shape of the dynamically-imported js-yaml module. `dump` is optional —
|
|
1126
1188
|
* read-only consumers (proxy config loader) only need `load`; writers
|
|
1127
1189
|
* (CLI primary-account commands) check `dump` before calling. */
|
|
@@ -1203,10 +1265,13 @@ export type ProxyStartApp = {
|
|
|
1203
1265
|
/** Stats shape consumed by the proxy status printer. */
|
|
1204
1266
|
export type StatusStats = {
|
|
1205
1267
|
totalAttempts?: number;
|
|
1268
|
+
totalAttemptErrors?: number;
|
|
1206
1269
|
totalRequests: number;
|
|
1207
1270
|
totalSuccess: number;
|
|
1208
1271
|
totalErrors: number;
|
|
1209
1272
|
totalRateLimits: number;
|
|
1273
|
+
totalTransientRateLimits?: number;
|
|
1274
|
+
totalQuotaRateLimits?: number;
|
|
1210
1275
|
accounts?: {
|
|
1211
1276
|
label: string;
|
|
1212
1277
|
type: string;
|
|
@@ -1214,7 +1279,10 @@ export type StatusStats = {
|
|
|
1214
1279
|
requests?: number;
|
|
1215
1280
|
success?: number;
|
|
1216
1281
|
errors?: number;
|
|
1282
|
+
attemptErrors?: number;
|
|
1217
1283
|
rateLimits?: number;
|
|
1284
|
+
transientRateLimits?: number;
|
|
1285
|
+
quotaRateLimits?: number;
|
|
1218
1286
|
cooling: boolean;
|
|
1219
1287
|
}[];
|
|
1220
1288
|
};
|
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import type { GlobalInstallerKind, GlobalInstallerResolution, ResolveGlobalInstallerOptions } from "../types/index.js";
|
|
1
|
+
import type { GlobalInstallerKind, GlobalInstallerResolution, InstalledVersionValidation, ResolveGlobalInstallerOptions, ValidateInstalledVersionOptions } from "../types/index.js";
|
|
2
2
|
/** Resolve a package manager that can update the installation currently running. */
|
|
3
3
|
export declare function resolveGlobalInstaller(options?: ResolveGlobalInstallerOptions): GlobalInstallerResolution;
|
|
4
4
|
export declare function getGlobalInstallArgs(kind: GlobalInstallerKind, packageSpec: string): string[];
|
|
5
|
+
/**
|
|
6
|
+
* Validate a freshly installed CLI with retries. Global package replacement
|
|
7
|
+
* can leave executable shims briefly unavailable while filesystem metadata and
|
|
8
|
+
* cold module loading settle, so a single short probe is not authoritative.
|
|
9
|
+
*/
|
|
10
|
+
export declare function validateInstalledVersion(options: ValidateInstalledVersionOptions): Promise<InstalledVersionValidation>;
|
|
5
11
|
export declare function describeInstallFailure(error: unknown): string;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { execFileSync as nodeExecFileSync } from "node:child_process";
|
|
2
2
|
import { accessSync, constants, existsSync, realpathSync } from "node:fs";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
|
-
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
|
5
5
|
function runText(execFileSync, bin, args) {
|
|
6
6
|
return String(execFileSync(bin, args, {
|
|
7
7
|
encoding: "utf8",
|
|
@@ -136,6 +136,54 @@ export function getGlobalInstallArgs(kind, packageSpec) {
|
|
|
136
136
|
? ["add", "-g", packageSpec]
|
|
137
137
|
: ["install", "--global", "--no-audit", "--no-fund", packageSpec];
|
|
138
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Validate a freshly installed CLI with retries. Global package replacement
|
|
141
|
+
* can leave executable shims briefly unavailable while filesystem metadata and
|
|
142
|
+
* cold module loading settle, so a single short probe is not authoritative.
|
|
143
|
+
*/
|
|
144
|
+
export async function validateInstalledVersion(options) {
|
|
145
|
+
if (!isAbsolute(options.binPath)) {
|
|
146
|
+
return {
|
|
147
|
+
attempts: 0,
|
|
148
|
+
failure: `binPath must be absolute: ${options.binPath}`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const execFileSync = options.execFileSync ?? nodeExecFileSync;
|
|
152
|
+
const sleepFn = options.sleep ??
|
|
153
|
+
((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
154
|
+
const maxAttempts = Math.max(1, options.maxAttempts ?? 5);
|
|
155
|
+
const delayMs = Math.max(0, options.delayMs ?? 2_000);
|
|
156
|
+
const timeoutMs = Math.max(1_000, options.timeoutMs ?? 10_000);
|
|
157
|
+
let lastVersion;
|
|
158
|
+
let lastFailure;
|
|
159
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
160
|
+
try {
|
|
161
|
+
const output = String(execFileSync(options.binPath, ["--version"], {
|
|
162
|
+
encoding: "utf8",
|
|
163
|
+
timeout: timeoutMs,
|
|
164
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
165
|
+
})).trim();
|
|
166
|
+
lastVersion = output || undefined;
|
|
167
|
+
if (lastVersion === options.expectedVersion) {
|
|
168
|
+
return { version: lastVersion, attempts: attempt };
|
|
169
|
+
}
|
|
170
|
+
lastFailure = lastVersion
|
|
171
|
+
? `resolved to v${lastVersion}; expected v${options.expectedVersion}`
|
|
172
|
+
: "version command returned empty output";
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
lastFailure = describeInstallFailure(error);
|
|
176
|
+
}
|
|
177
|
+
if (attempt < maxAttempts) {
|
|
178
|
+
await sleepFn(delayMs);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
version: lastVersion,
|
|
183
|
+
attempts: maxAttempts,
|
|
184
|
+
failure: lastFailure ?? "version validation failed",
|
|
185
|
+
};
|
|
186
|
+
}
|
|
139
187
|
function capturedOutput(error, key) {
|
|
140
188
|
if (!error || typeof error !== "object" || !(key in error)) {
|
|
141
189
|
return "";
|
|
@@ -48,6 +48,14 @@ export declare function suppressVersion(version: string, reason: string, stateFi
|
|
|
48
48
|
* @param stateFilePath - Override path for testing
|
|
49
49
|
*/
|
|
50
50
|
export declare function recordSuccessfulUpdate(version: string, stateFilePath?: string): void;
|
|
51
|
+
/** Record that package installation completed but the live restart is pending. */
|
|
52
|
+
export declare function recordUpdateInstalled(version: string, stateFilePath?: string): void;
|
|
53
|
+
/** Abandon a matching installed version so the next cycle may reinstall it. */
|
|
54
|
+
export declare function abandonPendingUpdate(version: string, stateFilePath?: string): boolean;
|
|
55
|
+
/** Persist a stage-specific updater failure so status remains actionable. */
|
|
56
|
+
export declare function recordUpdateFailure(version: string, stage: NonNullable<UpdateState["lastFailure"]>["stage"], message: string, stateFilePath?: string): void;
|
|
57
|
+
/** Complete an updater-managed install after a manual or automatic restart. */
|
|
58
|
+
export declare function reconcileRunningUpdate(runningVersion: string, stateFilePath?: string): boolean;
|
|
51
59
|
/**
|
|
52
60
|
* Record an update check: set lastCheckAt and lastCheckVersion, then persist.
|
|
53
61
|
*
|