agl 40.4.0 → 40.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +33 -0
- package/dist_serve/bundle.js +2228 -2177
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/aglapi/aglapisurface.document.js +2 -2
- package/dist_ts/classes.accountsactions.d.ts +11 -1
- package/dist_ts/classes.accountsactions.js +59 -2
- package/dist_ts/classes.controller.js +5 -5
- package/dist_ts/classes.flexaccountaccess.js +3 -2
- package/dist_ts/classes.harnessreleases.d.ts +14 -4
- package/dist_ts/classes.harnessreleases.js +28 -4
- package/dist_ts/classes.inferencegateway.d.ts +28 -3
- package/dist_ts/classes.inferencegateway.js +177 -61
- package/dist_ts/classes.inferenceplacement.d.ts +5 -3
- package/dist_ts/classes.inferenceplacement.js +2 -2
- package/dist_ts/classes.inferencerouter.d.ts +349 -91
- package/dist_ts/classes.inferencerouter.js +804 -297
- package/dist_ts/classes.inferenceservice.d.ts +14 -2
- package/dist_ts/classes.inferenceservice.js +53 -14
- package/dist_ts/classes.inferencespend.d.ts +339 -0
- package/dist_ts/classes.inferencespend.js +815 -0
- package/dist_ts/classes.inferenceusagecache.d.ts +8 -0
- package/dist_ts/classes.inferenceusagecache.js +9 -5
- package/dist_ts/classes.inferenceusagetap.d.ts +50 -9
- package/dist_ts/classes.inferenceusagetap.js +185 -31
- package/dist_ts/classes.proxycontrol.d.ts +3 -2
- package/dist_ts/classes.proxycontrol.js +1 -2
- package/dist_ts/classes.proxydaemon.js +3 -2
- package/dist_ts/classes.proxystore.d.ts +6 -0
- package/dist_ts/classes.proxystore.js +11 -2
- package/dist_ts/fleet/classes.claudeworkers.d.ts +5 -4
- package/dist_ts/fleet/classes.claudeworkers.js +10 -9
- package/dist_ts/functions.claudecoderelease.d.ts +12 -6
- package/dist_ts/functions.claudecoderelease.js +13 -7
- package/dist_ts/functions.inferencecli.js +12 -8
- package/dist_ts/interfaces.auth.d.ts +1 -1
- package/dist_ts/interfaces.auth.js +3 -1
- package/dist_ts/plugins.d.ts +3 -1
- package/dist_ts/plugins.js +7 -2
- package/dist_ts_authswitch/authority-apikeys.d.ts +32 -0
- package/dist_ts_authswitch/authority-apikeys.js +65 -0
- package/dist_ts_authswitch/authority-contract.d.ts +172 -33
- package/dist_ts_authswitch/authority-contract.js +54 -1
- package/dist_ts_authswitch/authority-overview.js +8 -1
- package/dist_ts_authswitch/authority-placement.d.ts +7 -2
- package/dist_ts_authswitch/authority-placement.js +10 -3
- package/dist_ts_authswitch/authority-runtime-contract.d.ts +2 -2
- package/dist_ts_authswitch/authority-spend.d.ts +103 -0
- package/dist_ts_authswitch/authority-spend.js +110 -0
- package/dist_ts_authswitch/authority-subscription.d.ts +12 -1
- package/dist_ts_authswitch/authority-subscription.js +24 -1
- package/dist_ts_authswitch/authority-usagelimit.d.ts +7 -4
- package/dist_ts_authswitch/authority-usagelimit.js +89 -39
- package/dist_ts_authswitch/authority-usagewindows.d.ts +4 -1
- package/dist_ts_authswitch/authority-usagewindows.js +41 -16
- package/dist_ts_authswitch/classes.authoritybroker.d.ts +30 -19
- package/dist_ts_authswitch/classes.authoritybroker.js +152 -61
- package/dist_ts_authswitch/classes.authoritycli.d.ts +9 -4
- package/dist_ts_authswitch/classes.authoritycli.js +99 -30
- package/dist_ts_authswitch/classes.authorityclient.d.ts +24 -7
- package/dist_ts_authswitch/classes.authorityclient.js +45 -15
- package/dist_ts_authswitch/classes.authoritydaemon.js +39 -12
- package/dist_ts_authswitch/classes.authoritydatabase.d.ts +28 -5
- package/dist_ts_authswitch/classes.authoritydatabase.js +171 -9
- package/dist_ts_authswitch/classes.authorityimport.js +2 -2
- package/dist_ts_authswitch/classes.authoritymodels.d.ts +71 -3
- package/dist_ts_authswitch/classes.authoritymodels.js +249 -20
- package/dist_ts_authswitch/classes.authorityregistry.js +4 -2
- package/dist_ts_authswitch/classes.authorityusage.d.ts +69 -7
- package/dist_ts_authswitch/classes.authorityusage.js +118 -4
- package/dist_ts_authswitch/classes.claudenative.d.ts +23 -8
- package/dist_ts_authswitch/classes.claudenative.js +49 -21
- package/dist_ts_authswitch/classes.claudesignin.d.ts +14 -2
- package/dist_ts_authswitch/classes.claudesignin.js +72 -23
- package/dist_ts_authswitch/classes.codexmanaged.d.ts +8 -0
- package/dist_ts_authswitch/classes.codexmanaged.js +33 -18
- package/dist_ts_authswitch/classes.harnessprocesses.d.ts +8 -0
- package/dist_ts_authswitch/classes.harnessprocesses.js +41 -9
- package/dist_ts_authswitch/testsupport.d.ts +24 -5
- package/dist_ts_authswitch/testsupport.js +56 -12
- package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.d.ts +12 -0
- package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.js +21 -0
- package/dist_ts_authswitch/ts_migration/index.js +3 -1
- package/dist_ts_interfaces/functions.authorityscope.d.ts +18 -7
- package/dist_ts_interfaces/functions.authorityscope.js +35 -8
- package/dist_ts_interfaces/interfaces.d.ts +1 -1
- package/dist_ts_interfaces/interfaces.js +1 -1
- package/dist_ts_interfaces/plugins.d.ts +2 -1
- package/dist_ts_interfaces/plugins.js +7 -4
- package/dist_ts_interfaces/requests.accounts.d.ts +56 -0
- package/dist_ts_interfaces/requests.accounts.js +11 -1
- package/dist_ts_interfaces/requests.d.ts +2 -2
- package/dist_ts_interfaces/requests.js +1 -1
- package/docs/authswitch.md +75 -12
- package/package.json +5 -5
- package/readme.md +199 -22
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +2 -2
- package/ts/classes.accountsactions.ts +75 -1
- package/ts/classes.controller.ts +4 -4
- package/ts/classes.flexaccountaccess.ts +2 -1
- package/ts/classes.harnessreleases.ts +25 -4
- package/ts/classes.inferencegateway.ts +184 -64
- package/ts/classes.inferenceplacement.ts +6 -4
- package/ts/classes.inferencerouter.ts +1026 -314
- package/ts/classes.inferenceservice.ts +56 -13
- package/ts/classes.inferencespend.ts +870 -0
- package/ts/classes.inferenceusagecache.ts +16 -4
- package/ts/classes.inferenceusagetap.ts +159 -34
- package/ts/classes.proxycontrol.ts +3 -3
- package/ts/classes.proxydaemon.ts +2 -1
- package/ts/classes.proxystore.ts +11 -1
- package/ts/fleet/classes.claudeworkers.ts +9 -8
- package/ts/functions.claudecoderelease.ts +13 -6
- package/ts/functions.inferencecli.ts +11 -7
- package/ts/interfaces.auth.ts +2 -0
- package/ts/plugins.ts +7 -0
- package/ts_authswitch/authority-apikeys.ts +85 -0
- package/ts_authswitch/authority-contract.ts +201 -34
- package/ts_authswitch/authority-overview.ts +7 -0
- package/ts_authswitch/authority-placement.ts +12 -4
- package/ts_authswitch/authority-runtime-contract.ts +2 -2
- package/ts_authswitch/authority-spend.ts +168 -0
- package/ts_authswitch/authority-subscription.ts +28 -1
- package/ts_authswitch/authority-usagelimit.ts +95 -40
- package/ts_authswitch/authority-usagewindows.ts +42 -17
- package/ts_authswitch/classes.authoritybroker.ts +161 -68
- package/ts_authswitch/classes.authoritycli.ts +112 -32
- package/ts_authswitch/classes.authorityclient.ts +66 -19
- package/ts_authswitch/classes.authoritydaemon.ts +43 -13
- package/ts_authswitch/classes.authoritydatabase.ts +178 -8
- package/ts_authswitch/classes.authorityimport.ts +1 -1
- package/ts_authswitch/classes.authoritymodels.ts +165 -19
- package/ts_authswitch/classes.authorityregistry.ts +3 -0
- package/ts_authswitch/classes.authorityusage.ts +171 -10
- package/ts_authswitch/classes.claudenative.ts +51 -22
- package/ts_authswitch/classes.claudesignin.ts +74 -24
- package/ts_authswitch/classes.codexmanaged.ts +30 -16
- package/ts_authswitch/classes.harnessprocesses.ts +39 -8
- package/ts_authswitch/native-contracts.md +82 -21
- package/ts_authswitch/readme.md +1 -1
- package/ts_authswitch/testsupport.ts +58 -11
- package/ts_authswitch/ts_migration/0011_account_api_key_fields.ts +20 -0
- package/ts_authswitch/ts_migration/index.ts +2 -0
- package/ts_interfaces/functions.authorityscope.ts +43 -8
- package/ts_interfaces/interfaces.ts +1 -1
- package/ts_interfaces/plugins.ts +6 -3
- package/ts_interfaces/requests.accounts.ts +48 -0
- package/ts_interfaces/requests.ts +4 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/accounts.plugins.ts +7 -1
- package/ts_web/accounts.templates.ts +20 -11
- package/ts_web/classes.accountsreadmodel.ts +326 -71
- package/ts_web/elements.accounts.ts +219 -20
- package/ts_web/elements.codexconnections.ts +3 -1
- package/ts_web/plugins.ts +4 -2
|
@@ -2,6 +2,7 @@ import * as plugins from './plugins.js';
|
|
|
2
2
|
import type { TInferenceCredential, TInferenceProvider } from './functions.proxycontract.js';
|
|
3
3
|
import type { IGatewayAuthorityAccess } from './classes.flexaccountaccess.js';
|
|
4
4
|
import { FlexServiceError } from './interfaces.flexipc.js';
|
|
5
|
+
import type { IInferenceSpendRecord } from './classes.inferencespend.js';
|
|
5
6
|
|
|
6
7
|
export type TInferenceModel = plugins.openAiProvider.IOpenAiChatGptModel;
|
|
7
8
|
/** A model a Claude account or an Anthropic API key may use, as the Anthropic API lists it. */
|
|
@@ -57,31 +58,52 @@ export const inferenceRetryAfterSeconds = (valueArg: string | null, nowArg: numb
|
|
|
57
58
|
return at === undefined ? undefined : Math.max(0, Math.ceil((at - nowArg) / 1000));
|
|
58
59
|
};
|
|
59
60
|
|
|
61
|
+
/**
|
|
62
|
+
* The credential tiers a pool is served from, best first: the one a policy names alone, or with none the provider's
|
|
63
|
+
* subscription sign-ins first and its API keys after them. A key takes a request only when no subscription can take it,
|
|
64
|
+
* for whatever reason, and a session placed on a key returns to a subscription at the next request one can take.
|
|
65
|
+
*/
|
|
66
|
+
export const inferenceCredentialTiers = (credentialArg: TInferenceCredential | undefined): readonly TInferenceCredential[] =>
|
|
67
|
+
credentialArg ? [credentialArg] : ['subscription', 'api_key'];
|
|
68
|
+
|
|
60
69
|
export interface IInferenceAccount {
|
|
61
70
|
id: string;
|
|
62
71
|
label: string;
|
|
63
72
|
revision: string;
|
|
64
73
|
providerId: TInferenceProvider;
|
|
65
|
-
/** What the account authenticates with: a subscription sign-in, or an
|
|
74
|
+
/** What the account authenticates with: a subscription sign-in, or an API key. */
|
|
66
75
|
credential: TInferenceCredential;
|
|
67
76
|
/** The account's usage threshold (`IAuthSwitchAccount.maxUsagePercent`); `null` when unset. */
|
|
68
77
|
maxUsagePercent: number | null;
|
|
69
78
|
reserveLiftUntil: string | null;
|
|
70
79
|
/** When the account's subscription ends (`IAuthSwitchAccount.lapsesAt`); `null` while it renews. */
|
|
71
80
|
lapsesAt: string | null;
|
|
81
|
+
/**
|
|
82
|
+
* An API key's monthly spend limit in micro-dollars (`IAuthSwitchAccount.spendLimitMicroUsd`); `null` without one,
|
|
83
|
+
* and for a sign-in. It decides only whether a key whose spend the proxy holds no reading of may take work: one with
|
|
84
|
+
* a limit may not (`spend_unread`). A spend reading carries the limit the key is judged against.
|
|
85
|
+
*/
|
|
86
|
+
spendLimitMicroUsd: number | null;
|
|
87
|
+
/**
|
|
88
|
+
* The login the account's requests are served on (the account service's gateway login of it), as the pool was read;
|
|
89
|
+
* `null` when the account service names none. An API key's spend is reported on it.
|
|
90
|
+
*/
|
|
91
|
+
loginId: string | null;
|
|
72
92
|
}
|
|
73
93
|
|
|
74
94
|
export interface IInferenceRoutingPolicy {
|
|
75
95
|
/** The pool's provider; omitted means ChatGPT accounts (`openai`). */
|
|
76
96
|
provider?: TInferenceProvider;
|
|
77
97
|
/**
|
|
78
|
-
* What the pool's accounts authenticate with
|
|
79
|
-
*
|
|
98
|
+
* What the pool's accounts authenticate with. Omitted, a Claude pool is served by its Claude accounts first and its
|
|
99
|
+
* Anthropic API keys when no Claude account can take a request (`inferenceCredentialTiers`), a ChatGPT pool by its
|
|
100
|
+
* ChatGPT accounts. `subscription` names the sign-ins alone; `api_key` is an Anthropic pool's alone: its Anthropic API
|
|
101
|
+
* keys, which Flex's Claude models run on, never a Claude account.
|
|
80
102
|
*/
|
|
81
103
|
credential?: TInferenceCredential;
|
|
82
104
|
/** Omitted means all eligible authority-owned accounts of the provider. */
|
|
83
105
|
accountIds?: readonly string[];
|
|
84
|
-
/** Exact current provider plan names; omission accepts every entitled plan. */
|
|
106
|
+
/** Exact current provider plan names; omission accepts every entitled plan. A key has no plan, so this leaves keys out. */
|
|
85
107
|
plans?: readonly string[];
|
|
86
108
|
}
|
|
87
109
|
|
|
@@ -108,19 +130,61 @@ export interface IInferenceRouterOptions {
|
|
|
108
130
|
*/
|
|
109
131
|
reportAccount?: (changeArg: IInferenceAccountChange) => void;
|
|
110
132
|
/**
|
|
111
|
-
* Tells the account service that
|
|
112
|
-
* `
|
|
113
|
-
*
|
|
133
|
+
* Tells the account service that the provider refused a request on the account for a payment problem at
|
|
134
|
+
* `observedAtArg` -- a Claude account's billing error (HTTP 402 `billing_error`), or an API key's refusal for
|
|
135
|
+
* `causeArg` (`TInferencePaymentCause`); resolves once the service holds it, so the account's usage carries it from
|
|
136
|
+
* then on (`IAuthSwitchUsageSnapshot.paymentProblem`). Unset, the router holds the account out of rotation itself.
|
|
114
137
|
*/
|
|
115
|
-
reportPaymentProblem?: (accountIdArg: string, observedAtArg: string) => Promise<void>;
|
|
138
|
+
reportPaymentProblem?: (accountIdArg: string, observedAtArg: string, causeArg: TInferencePaymentCause) => Promise<void>;
|
|
116
139
|
/**
|
|
117
140
|
* Where a payment problem the account service could not be told of is reported: once per account, never per request.
|
|
118
141
|
* One line on standard error, the proxy's journal, when unset.
|
|
119
142
|
*/
|
|
120
143
|
reportPaymentFailure?: (failureArg: IInferencePaymentReportFailure) => void;
|
|
144
|
+
/**
|
|
145
|
+
* Where the spend of the answers relayed on API keys is kept until the accounts service acknowledged it
|
|
146
|
+
* (`InferenceSpendOutbox`). A key's usage arrives judged with that spend already (`InferenceUsageCache` `view`).
|
|
147
|
+
*/
|
|
148
|
+
spend?: IInferenceSpendLedger;
|
|
121
149
|
now?: () => number;
|
|
122
150
|
}
|
|
123
151
|
|
|
152
|
+
/** What the router asks of the proxy's spend outbox (`InferenceSpendOutbox`). */
|
|
153
|
+
export interface IInferenceSpendLedger {
|
|
154
|
+
/**
|
|
155
|
+
* Counts a relayed answer's spend and keeps it until the accounts service acknowledged it, reported on the login the
|
|
156
|
+
* route was served on (`IInferenceClaudeRoute.loginId`); `null` when none could be named, which still counts.
|
|
157
|
+
*/
|
|
158
|
+
record(accountIdArg: string, loginIdArg: string | null, recordArg: IInferenceSpendRecord): void;
|
|
159
|
+
/**
|
|
160
|
+
* Whether spend relayed on the key is recorded nowhere yet: the accounts service refused a report the outbox still
|
|
161
|
+
* holds, an answer is in neither the outbox's store, whose write failed, nor the ledger, or no login of the key could
|
|
162
|
+
* be named for an answer, so it cannot be reported.
|
|
163
|
+
*/
|
|
164
|
+
held(accountIdArg: string): boolean;
|
|
165
|
+
/** Says that an answer relayed on the key ended before its provider stated its usage, so nothing of it is counted. */
|
|
166
|
+
uncounted(accountIdArg: string, modelArg: string): void;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** A payment problem the router holds an account out of rotation for: since `at`, for `cause`, until `until` (`null`: no end). */
|
|
170
|
+
interface IPaymentMark {
|
|
171
|
+
at: number;
|
|
172
|
+
cause: TInferencePaymentCause;
|
|
173
|
+
until: number | null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* What a payment problem's report adds for the account service (`IReq_AuthSwitchReportPaymentProblem`): the cause of an
|
|
178
|
+
* API key's spend cap or quota, nothing for a billing error, whose cause a missing one means -- so a Claude account's
|
|
179
|
+
* report is the one an accounts service of 40.3.0, which refuses a cause, takes.
|
|
180
|
+
*/
|
|
181
|
+
export const inferencePaymentReportOptions = (causeArg: TInferencePaymentCause):
|
|
182
|
+
{ cause?: Exclude<TInferencePaymentCause, 'billing'> } => causeArg === 'billing' ? {} : { cause: causeArg };
|
|
183
|
+
|
|
184
|
+
/** A payment mark as the refusal it makes. */
|
|
185
|
+
const paymentUnavailable = (markArg: IPaymentMark): IInferenceAccountUnavailable => ({ reason: 'payment_problem',
|
|
186
|
+
cause: markArg.cause, ...(markArg.until === null ? {} : { retryAt: new Date(markArg.until).toISOString() }) });
|
|
187
|
+
|
|
124
188
|
/** A payment problem the account service could not be told of, as the proxy reports it: the account by its id's digest. */
|
|
125
189
|
export interface IInferencePaymentReportFailure {
|
|
126
190
|
/** The first 12 hex characters of the SHA-256 digest of the account's id. */
|
|
@@ -131,27 +195,62 @@ export interface IInferencePaymentReportFailure {
|
|
|
131
195
|
|
|
132
196
|
const writePaymentReportFailure = (failureArg: IInferencePaymentReportFailure): void => {
|
|
133
197
|
process.stderr.write(`AGL proxy could not tell the account service of the payment problem of account `
|
|
134
|
-
+ `${failureArg.account}, so it keeps the account out of rotation until it restarts:
|
|
198
|
+
+ `${failureArg.account}, so it keeps the account out of rotation until it restarts, or a spend cap ends: `
|
|
199
|
+
+ `${failureArg.error}\n`);
|
|
135
200
|
};
|
|
136
201
|
|
|
137
202
|
/**
|
|
138
203
|
* Why an account takes no request although no limit says so. `usage_unread`: the account service holds no usage reading
|
|
139
204
|
* the proxy routes on. `catalog_failed`: the account's model list could not be read and none read before is held for its
|
|
140
205
|
* revision. `needs_login`: its sign-in needs renewing. `access_failed`: the account service answered no access for it.
|
|
141
|
-
* `model_missing`: its model list lacks the requested model. `payment_problem`:
|
|
142
|
-
* account
|
|
206
|
+
* `model_missing`: its model list lacks the requested model. `payment_problem`: the provider refused a request on the
|
|
207
|
+
* account for a payment problem (`TInferencePaymentCause`); it takes none until a usage read or a re-check clears it, or
|
|
208
|
+
* a spend cap ends. `spend_unread`: an API key with a monthly spend limit whose spend the proxy holds no reading of --
|
|
209
|
+
* the account service could not read its ledger (`spend_unavailable`) or did not answer -- so it may be at its limit.
|
|
210
|
+
* `spend_unrecorded`: an API key with a monthly spend limit whose spend the proxy relayed is recorded nowhere yet --
|
|
211
|
+
* the account service refused its report, which the proxy holds to send again, an answer is held in neither the
|
|
212
|
+
* proxy's store, whose write failed, nor the account service's ledger, or no login of the key could be named for an
|
|
213
|
+
* answer to report it on (`InferenceSpendOutbox.held`). `model_unpriced`: an API key with a monthly spend limit, and a
|
|
214
|
+
* request whose model, service tier, speed, region or server tool the proxy's price table cannot price (`failure` names
|
|
215
|
+
* it), so its cost could not be counted against the limit.
|
|
143
216
|
*/
|
|
144
217
|
export type TInferenceUnavailableReason = 'usage_unread' | 'catalog_failed' | 'needs_login' | 'access_failed'
|
|
145
|
-
| 'model_missing' | 'payment_problem';
|
|
218
|
+
| 'model_missing' | 'payment_problem' | 'spend_unread' | 'spend_unrecorded' | 'model_unpriced';
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Why a provider refused a request on an account for a payment problem, as the gateway reads the provider's answer.
|
|
222
|
+
* `billing`: Anthropic's billing error (HTTP 402 `billing_error`), on a Claude account's subscription or an Anthropic API
|
|
223
|
+
* key's organization. `spend_cap`: an Anthropic API key's organization reached its usage tier's monthly spend cap (HTTP
|
|
224
|
+
* 429 `rate_limit_error` with `error.details.error_code: "enforced_spend_limit_reached"`), which holds until 00:00 UTC on
|
|
225
|
+
* the first day of the next month. `quota`: OpenAI refused an API key for its quota (HTTP 429 `insufficient_quota`): no
|
|
226
|
+
* prepaid credit left, or an organization or project spend limit or the organization's usage limit reached.
|
|
227
|
+
*/
|
|
228
|
+
export type TInferencePaymentCause = plugins.authswitchAuthority.TAuthSwitchPaymentProblemCause;
|
|
146
229
|
|
|
147
230
|
export interface IInferenceAccountUnavailable {
|
|
148
231
|
reason: TInferenceUnavailableReason;
|
|
149
|
-
/**
|
|
232
|
+
/**
|
|
233
|
+
* `usage_unread`: when the account service reads the account's usage next, or `null` when it does not say.
|
|
234
|
+
* `payment_problem` of a spend cap: when the cap ends and the account takes requests again.
|
|
235
|
+
*/
|
|
150
236
|
retryAt?: string | null;
|
|
151
|
-
/**
|
|
237
|
+
/**
|
|
238
|
+
* `catalog_failed`: the HTTP status the model list answered with (`HTTP 429`), or what failed instead.
|
|
239
|
+
* `model_unpriced`: what of the request the price table cannot price.
|
|
240
|
+
*/
|
|
152
241
|
failure?: string;
|
|
242
|
+
/** `payment_problem`: what the provider refused the account for. */
|
|
243
|
+
cause?: TInferencePaymentCause;
|
|
153
244
|
}
|
|
154
245
|
|
|
246
|
+
/** A payment problem in words, as the status, the journal line and a pinned refusal state it. */
|
|
247
|
+
const paymentProblemText = (unavailableArg: IInferenceAccountUnavailable): string =>
|
|
248
|
+
unavailableArg.cause === 'spend_cap' ? 'Anthropic refused a request on it: its organization reached its usage tier\'s '
|
|
249
|
+
+ `monthly spend cap (HTTP 429)${unavailableArg.retryAt ? `; it takes requests again at ${unavailableArg.retryAt}` : ''}`
|
|
250
|
+
: unavailableArg.cause === 'quota' ? 'OpenAI refused a request on it for its quota (HTTP 429 insufficient_quota): no '
|
|
251
|
+
+ 'prepaid credit left, or a spend or usage limit reached'
|
|
252
|
+
: 'Anthropic refused a request on it with a billing error (HTTP 402), a payment problem';
|
|
253
|
+
|
|
155
254
|
/** A change of why an account is refused, as the proxy reports it: the account by its id's digest, never its id. */
|
|
156
255
|
export interface IInferenceAccountChange {
|
|
157
256
|
/** The first 12 hex characters of the SHA-256 digest of the account's id. */
|
|
@@ -171,9 +270,16 @@ const unavailableText = (unavailableArg: IInferenceAccountUnavailable): string =
|
|
|
171
270
|
: unavailableArg.reason === 'catalog_failed' ? `its model list could not be read (${unavailableArg.failure})`
|
|
172
271
|
: unavailableArg.reason === 'needs_login' ? 'it needs a fresh sign-in'
|
|
173
272
|
: unavailableArg.reason === 'access_failed' ? 'the account service answered no access for it'
|
|
174
|
-
: unavailableArg.reason === 'payment_problem'
|
|
175
|
-
|
|
176
|
-
|
|
273
|
+
: unavailableArg.reason === 'payment_problem' ? paymentProblemText(unavailableArg)
|
|
274
|
+
: unavailableArg.reason === 'spend_unread'
|
|
275
|
+
? 'it has a monthly spend limit, and its spend could not be read, so it might be at it'
|
|
276
|
+
: unavailableArg.reason === 'spend_unrecorded'
|
|
277
|
+
? 'it has a monthly spend limit, and spend the proxy relayed on it is not recorded yet -- the account service '
|
|
278
|
+
+ 'refused its report, or neither the proxy\'s store nor the account service took an answer -- so it might '
|
|
279
|
+
+ 'be at it'
|
|
280
|
+
: unavailableArg.reason === 'model_unpriced'
|
|
281
|
+
? `it has a monthly spend limit, and ${unavailableArg.failure ?? 'the request'} has no price`
|
|
282
|
+
: 'its model list lacks the requested model';
|
|
177
283
|
|
|
178
284
|
const writeAccountChange = (changeArg: IInferenceAccountChange): void => {
|
|
179
285
|
process.stderr.write(`AGL proxy account ${changeArg.account}: ${changeArg.unavailable === null
|
|
@@ -213,6 +319,38 @@ const cooldownUntil = (cooldownArg: ICooldown | undefined,
|
|
|
213
319
|
*/
|
|
214
320
|
export type TInferenceAccountLimit = 'exhausted' | 'reserved' | 'cooling';
|
|
215
321
|
|
|
322
|
+
/** An API key a request found held by its monthly spend limit: the account, and its month as the account service read it. */
|
|
323
|
+
export interface IInferenceSpendLimited {
|
|
324
|
+
accountId: string;
|
|
325
|
+
label: string;
|
|
326
|
+
spend: plugins.authswitchSpend.IAuthSwitchSpendReading;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* A key's month in words, from its spend reading: `$12.40 of $50.00 this month`, or `$12.40 this month, no spend limit`,
|
|
331
|
+
* and how many answers could not be priced, when any.
|
|
332
|
+
*/
|
|
333
|
+
const spendText = (spendArg: plugins.authswitchSpend.IAuthSwitchSpendReading): string => {
|
|
334
|
+
const usd = plugins.authswitchSpend.authSwitchUsdText;
|
|
335
|
+
const unpriced = spendArg.unpricedAnswers === 0 ? ''
|
|
336
|
+
: `, ${spendArg.unpricedAnswers} ${spendArg.unpricedAnswers === 1 ? 'answer' : 'answers'} of it could not be priced`;
|
|
337
|
+
return `${usd(spendArg.spentMicroUsd)} ${spendArg.limitMicroUsd === null ? 'this month, no spend limit'
|
|
338
|
+
: `of ${usd(spendArg.limitMicroUsd)} this month`}${unpriced}`;
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* What a refusal says of the API keys it found held by their monthly spend limit: each key with its month, and when the
|
|
343
|
+
* first of them takes requests again. Empty without any.
|
|
344
|
+
*/
|
|
345
|
+
export const inferenceSpendLimitText = (keysArg: readonly IInferenceSpendLimited[]): string => {
|
|
346
|
+
const keys = [...new Map(keysArg.map((keyArg) => [keyArg.accountId, keyArg])).values()];
|
|
347
|
+
if (!keys.length) return '';
|
|
348
|
+
const resetAt = keys.map((keyArg) => keyArg.spend.resetAt).sort()[0];
|
|
349
|
+
const named = keys.map((keyArg) => `${keyArg.label} (${spendText(keyArg.spend)})`).join(', ');
|
|
350
|
+
return `${keys.length === 1 ? 'The API key' : 'The API keys'} ${named} ${keys.length === 1 ? 'is' : 'are'} held by the `
|
|
351
|
+
+ `monthly spend limit set in AGL until ${resetAt}.`;
|
|
352
|
+
};
|
|
353
|
+
|
|
216
354
|
export class InferenceRoutingError extends Error {
|
|
217
355
|
constructor(public readonly status: number, public readonly code: string, messageArg: string,
|
|
218
356
|
public readonly retryAfterSeconds?: number,
|
|
@@ -222,11 +360,21 @@ export class InferenceRoutingError extends Error {
|
|
|
222
360
|
public readonly accountLimit?: TInferenceAccountLimit,
|
|
223
361
|
/** The account a refusal for a reason of its own (`reason`) names. */
|
|
224
362
|
public readonly accountId?: string,
|
|
225
|
-
/**
|
|
226
|
-
|
|
363
|
+
/**
|
|
364
|
+
* For `no_eligible_accounts` of a pinned account: why that account takes no request though no limit says so, in
|
|
365
|
+
* full -- a payment problem's cause, say.
|
|
366
|
+
*/
|
|
367
|
+
public readonly unavailable?: IInferenceAccountUnavailable,
|
|
368
|
+
/** For `accounts_exhausted`: the API keys found held by their monthly spend limit, which its message names. */
|
|
369
|
+
public readonly spendLimited: readonly IInferenceSpendLimited[] = []) {
|
|
227
370
|
super(messageArg);
|
|
228
371
|
this.name = 'InferenceRoutingError';
|
|
229
372
|
}
|
|
373
|
+
|
|
374
|
+
/** The reason of `unavailable`: a refusal names a reason of its own only with the refusal in full. */
|
|
375
|
+
public get reason(): TInferenceUnavailableReason | undefined {
|
|
376
|
+
return this.unavailable?.reason;
|
|
377
|
+
}
|
|
230
378
|
}
|
|
231
379
|
|
|
232
380
|
/**
|
|
@@ -237,14 +385,29 @@ const needsSignIn = (errorArg: unknown): boolean =>
|
|
|
237
385
|
(errorArg instanceof InferenceRoutingError && errorArg.code === 'account_needs_login')
|
|
238
386
|
|| (errorArg instanceof FlexServiceError && errorArg.code === 'PROVIDER_REAUTH_REQUIRED');
|
|
239
387
|
|
|
388
|
+
/**
|
|
389
|
+
* A ChatGPT account or an OpenAI API key chosen for one request: a ChatGPT sign-in through the ChatGPT backend, a key
|
|
390
|
+
* through OpenAI's API (`OPENAI_API_BASE_URL`), each through `connection`.
|
|
391
|
+
*/
|
|
240
392
|
export interface IInferenceRoute {
|
|
241
393
|
provider: 'openai';
|
|
394
|
+
/** What the chosen account authenticates with: a ChatGPT sign-in, or an OpenAI API key. */
|
|
395
|
+
credential: TInferenceCredential;
|
|
242
396
|
accountId: string;
|
|
243
397
|
plan: string;
|
|
244
398
|
/** The account's placement score when it was chosen (`IAuthSwitchPlacement.score`). */
|
|
245
399
|
score: number | null;
|
|
246
400
|
model: TInferenceModel;
|
|
247
401
|
connection: plugins.openAiProvider.IOpenAiModelConnection;
|
|
402
|
+
/** The session left its account on a later credential tier for one of a better tier (`returnToBetterTier`). */
|
|
403
|
+
returned: boolean;
|
|
404
|
+
/** The login the account was served on as the pool was read (`IInferenceAccount.loginId`): an answer's spend names it. */
|
|
405
|
+
loginId: string | null;
|
|
406
|
+
/**
|
|
407
|
+
* An OpenAI API key with a monthly spend limit: the request goes out as the price check read it, on the service tier it
|
|
408
|
+
* was checked for (`inferenceOpenAiCheckedBytes`).
|
|
409
|
+
*/
|
|
410
|
+
spendLimit: boolean;
|
|
248
411
|
release: () => void;
|
|
249
412
|
}
|
|
250
413
|
|
|
@@ -261,6 +424,18 @@ export interface IInferenceClaudeRoute {
|
|
|
261
424
|
score: number | null;
|
|
262
425
|
model: TInferenceClaudeModel;
|
|
263
426
|
connection: plugins.anthropicProvider.IAnthropicModelConnection;
|
|
427
|
+
/**
|
|
428
|
+
* The session's account is an Anthropic API key, and a Claude account of a better credential tier took the request
|
|
429
|
+
* instead (`returnToBetterTier`): the session returns to it.
|
|
430
|
+
*/
|
|
431
|
+
returned: boolean;
|
|
432
|
+
/** The login the account was served on as the pool was read (`IInferenceAccount.loginId`): an answer's spend names it. */
|
|
433
|
+
loginId: string | null;
|
|
434
|
+
/**
|
|
435
|
+
* An Anthropic API key with a monthly spend limit: the request goes out as the price check read it, on the standard
|
|
436
|
+
* service tier only (`inferenceAnthropicCheckedBytes`), the one the price table prices.
|
|
437
|
+
*/
|
|
438
|
+
spendLimit: boolean;
|
|
264
439
|
release: () => void;
|
|
265
440
|
}
|
|
266
441
|
|
|
@@ -271,55 +446,94 @@ interface ICatalog<TModel> {
|
|
|
271
446
|
}
|
|
272
447
|
|
|
273
448
|
/**
|
|
274
|
-
* How long
|
|
275
|
-
* failed sign-in holds before the next read: a sign-in does not heal
|
|
449
|
+
* How long an account's model list -- a Claude account's, an Anthropic or an OpenAI API key's -- is used before it is read
|
|
450
|
+
* again, and how long a failed model-list read or a failed sign-in holds before the next read: a sign-in does not heal
|
|
451
|
+
* itself, and a new one changes the account's revision.
|
|
276
452
|
*/
|
|
277
|
-
const
|
|
453
|
+
const modelListMs = 60_000;
|
|
278
454
|
|
|
279
455
|
/**
|
|
280
|
-
* How long
|
|
456
|
+
* How long an account whose access failed for a reason other than sign-in is refused before access is tried again.
|
|
281
457
|
* Such a failure is usually a brief outage of the local account service: retries stay bounded to one per account per 5 s
|
|
282
458
|
* without turning a short outage into a minute of refusals.
|
|
283
459
|
*/
|
|
284
|
-
const
|
|
460
|
+
const modelListAccessRetryMs = 5_000;
|
|
285
461
|
|
|
286
462
|
/**
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
463
|
+
* An account's model list for one revision of the account -- a Claude account's or an Anthropic API key's
|
|
464
|
+
* (`claudeCatalogs`), an OpenAI API key's (`keyCatalogs`): the last one read, and why the latest read of it or of the
|
|
465
|
+
* account's access failed, while it did. Neither is read again before `until`: a minute after the latest read, failed or
|
|
466
|
+
* not, except 5 s after a failed access (`access_failed`), whose retry reads the list again with it.
|
|
290
467
|
*/
|
|
291
|
-
interface
|
|
468
|
+
interface IHeldModelList<TModel> {
|
|
292
469
|
revision: string;
|
|
293
470
|
until: number;
|
|
294
471
|
/** The last list read for the revision; `null` while no read of it succeeded. */
|
|
295
|
-
models:
|
|
472
|
+
models: TModel[] | null;
|
|
296
473
|
failure: IInferenceAccountUnavailable | null;
|
|
297
474
|
}
|
|
298
475
|
|
|
476
|
+
/** Reads an account's model list with the access the account service answered for it. */
|
|
477
|
+
type TModelLister<TModel> = (accessArg: IGatewayAuthorityAccess,
|
|
478
|
+
optionsArg: { signal: AbortSignal; fetch?: typeof fetch }) => Promise<TModel[]>;
|
|
479
|
+
|
|
299
480
|
/**
|
|
300
|
-
* Why
|
|
481
|
+
* Why an account's held model list keeps it from routing: a failure of its access or sign-in always, a failed
|
|
301
482
|
* model-list read (`catalog_failed`) only while no list is held -- the account routes on the last list meanwhile.
|
|
302
483
|
*/
|
|
303
|
-
const
|
|
304
|
-
|
|
305
|
-
?
|
|
484
|
+
const heldListRefusal = <TModel>(listArg: IHeldModelList<TModel>): IInferenceAccountUnavailable | undefined =>
|
|
485
|
+
listArg.failure && (listArg.models === null || listArg.failure.reason !== 'catalog_failed')
|
|
486
|
+
? listArg.failure : undefined;
|
|
306
487
|
|
|
307
|
-
/**
|
|
308
|
-
class
|
|
488
|
+
/** An account's model list could not be read, and why (`InferenceRouter.readModelList`). */
|
|
489
|
+
class ModelListError extends Error {
|
|
309
490
|
constructor(public readonly unavailable: IInferenceAccountUnavailable) {
|
|
310
491
|
super(unavailableText(unavailable));
|
|
311
|
-
this.name = '
|
|
492
|
+
this.name = 'ModelListError';
|
|
312
493
|
}
|
|
313
494
|
}
|
|
314
495
|
|
|
315
496
|
/** A failed model list read in a few words: its HTTP status, or the kind of failure; never its message. */
|
|
316
497
|
const catalogFailure = (errorArg: unknown): string => {
|
|
317
|
-
if (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthModelListError
|
|
498
|
+
if (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthModelListError
|
|
499
|
+
|| errorArg instanceof plugins.openAiProvider.OpenAiApiModelListError) {
|
|
318
500
|
return errorArg.code === 'HTTP_ERROR' && errorArg.status !== undefined ? `HTTP ${errorArg.status}` : errorArg.code;
|
|
319
501
|
}
|
|
502
|
+
// An OpenAI key that may not read the model list (403) is refused like a list that cannot be read.
|
|
503
|
+
if (errorArg instanceof plugins.openAiProvider.OpenAiApiKeyAuthError && errorArg.status !== undefined) {
|
|
504
|
+
return `HTTP ${errorArg.status}`;
|
|
505
|
+
}
|
|
320
506
|
return errorArg instanceof Error ? errorArg.name : typeof errorArg;
|
|
321
507
|
};
|
|
322
508
|
|
|
509
|
+
/** A provider's refusal of a key or sign-in it does not accept (401): renewed once for a sign-in, ended for a key. */
|
|
510
|
+
const accessRejected = (errorArg: unknown): boolean => (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthAuthError
|
|
511
|
+
|| errorArg instanceof plugins.anthropicProvider.AnthropicApiKeyAuthError
|
|
512
|
+
|| errorArg instanceof plugins.openAiProvider.OpenAiApiKeyAuthError) && errorArg.status === 401;
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* A model an OpenAI API key lists (`GET /v1/models`), as a route and a catalog state it: what flexharness's
|
|
516
|
+
* `OPENAI_API_MODELS` describes of it -- efforts, service tiers, context window -- and nothing more. A model it does not
|
|
517
|
+
* describe stays routable, hidden from the model lists, as a ChatGPT catalog's hidden model is; its id is sent as it is
|
|
518
|
+
* listed.
|
|
519
|
+
*/
|
|
520
|
+
const openAiKeyModel = (modelIdArg: string): TInferenceModel => {
|
|
521
|
+
const described = plugins.openAiProvider.getOpenAiApiModel(modelIdArg);
|
|
522
|
+
return { providerId: 'openai', modelId: modelIdArg, wireModel: modelIdArg, displayName: described?.displayName ?? modelIdArg,
|
|
523
|
+
description: described?.description ?? '', hidden: described === undefined,
|
|
524
|
+
reasoningEfforts: described ? described.reasoningEfforts.map((effortArg) => ({ ...effortArg })) : [],
|
|
525
|
+
defaultReasoningEffort: described?.defaultReasoningEffort ?? '',
|
|
526
|
+
inputModalities: described ? [...described.inputModalities] : ['text'], supportsPersonality: false,
|
|
527
|
+
serviceTiers: described ? described.serviceTiers.map((tierArg) => ({ ...tierArg })) : [],
|
|
528
|
+
...(described ? { contextWindow: described.contextWindow } : {}), isDefault: false };
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Whether an account of a ChatGPT pool is a ChatGPT sign-in: only those reach the ChatGPT backend -- its Codex catalog,
|
|
533
|
+
* its model list and its Responses endpoint. An OpenAI API key is never sent there.
|
|
534
|
+
*/
|
|
535
|
+
const chatGptSignIn = (accountArg: IInferenceAccount): boolean => accountArg.credential === 'subscription';
|
|
536
|
+
|
|
323
537
|
/** The headroom of an account whose provider reports no usage windows: below every known headroom. */
|
|
324
538
|
const unknownHeadroom = -1;
|
|
325
539
|
|
|
@@ -370,11 +584,56 @@ const claudeLimits = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnap
|
|
|
370
584
|
};
|
|
371
585
|
|
|
372
586
|
/**
|
|
373
|
-
* An
|
|
374
|
-
*
|
|
375
|
-
*
|
|
587
|
+
* An API key's limits, OpenAI's or Anthropic's, from its month's spend as the account service answers it (reading
|
|
588
|
+
* `kind: 'spend'`), by the shared rule (`authSwitchUsageLimit`): a key with a spend limit is exhausted while its month
|
|
589
|
+
* spent it or holds an answer that could not be priced, until the month ends, and its headroom is what is left of the
|
|
590
|
+
* limit; a key without one is never stopped by its spend and has no headroom to compare (`unknownHeadroom`). A reading
|
|
591
|
+
* whose month ended is the rule's as well: available, with all of the limit left. The account service answers a key's
|
|
592
|
+
* spend as current until its month ends, and the proxy asks again at every request, so no age is judged here.
|
|
593
|
+
*
|
|
594
|
+
* Without a spend reading -- the service could not read the key's ledger (`spend_unavailable`), or did not answer -- a
|
|
595
|
+
* key with a spend limit (`IInferenceAccount.spendLimitMicroUsd`) is refused (`undefined`, `spend_unread`), as it may be
|
|
596
|
+
* at its limit; one without a limit routes with unknown headroom. Rate limits are the provider's: a 429 rests a key.
|
|
597
|
+
*/
|
|
598
|
+
const keyLimits = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined,
|
|
599
|
+
accountArg: IInferenceAccount, nowArg: number): IAccountLimits | undefined => {
|
|
600
|
+
// The proxy hands a key's usage over judged (`InferenceSpendOutbox.judged`): its month's spend with the answers the
|
|
601
|
+
// account service has not acknowledged yet, in the month now falls in.
|
|
602
|
+
const reading = usageArg?.state === 'current' || usageArg?.state === 'stale' ? usageArg.reading : null;
|
|
603
|
+
if (reading?.kind === 'spend') return accountLimits(reading, '', accountArg, unknownHeadroom, nowArg);
|
|
604
|
+
return accountArg.spendLimitMicroUsd === null
|
|
605
|
+
? { plan: '', state: 'available', headroom: unknownHeadroom, limit: null } : undefined;
|
|
606
|
+
};
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* Whether an API key has a monthly spend limit: by its spend reading's limit, which the account service answered with
|
|
610
|
+
* the reading, or without a reading by the limit the account carries.
|
|
376
611
|
*/
|
|
377
|
-
const
|
|
612
|
+
const keyHasSpendLimit = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined,
|
|
613
|
+
accountArg: IInferenceAccount): boolean => usageArg?.reading?.kind === 'spend'
|
|
614
|
+
? usageArg.reading.spend.limitMicroUsd !== null : accountArg.spendLimitMicroUsd !== null;
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* An API key's state in words, as `agl proxy status` shows it on its limits (`keyLimits`), with its month: available, at
|
|
618
|
+
* its spend limit, or held below it until the month ends for an answer that could not be priced; available after its
|
|
619
|
+
* month ended with nothing relayed on it since, or without a limit and a spend reading.
|
|
620
|
+
*/
|
|
621
|
+
const keyStateText = (limitsArg: IAccountLimits,
|
|
622
|
+
usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined, nowArg: number): string => {
|
|
623
|
+
const readingArg = usageArg?.reading;
|
|
624
|
+
if (readingArg?.kind !== 'spend' || !limitsArg.limit) return 'available (no spend limit, and no spend reading)';
|
|
625
|
+
const spend = readingArg.spend;
|
|
626
|
+
// A reading the account service observed before the judged month began is last month's, judged as the new month's.
|
|
627
|
+
const newMonth = usageArg?.observedAt !== null && usageArg?.observedAt !== undefined
|
|
628
|
+
&& Date.parse(usageArg.observedAt) < Date.parse(spend.periodStart);
|
|
629
|
+
if (plugins.authswitchSpend.authSwitchSpendMonthEnded(spend, nowArg) || (newMonth && spend.answers === 0)) {
|
|
630
|
+
return 'available (a new month began since its last spend reading)';
|
|
631
|
+
}
|
|
632
|
+
const state = limitsArg.state !== 'exhausted' ? 'available'
|
|
633
|
+
: spend.limitMicroUsd !== null && spend.spentMicroUsd < spend.limitMicroUsd ? 'held until its month ends'
|
|
634
|
+
: 'at its spend limit';
|
|
635
|
+
return `${state} (${spendText(spend)})`;
|
|
636
|
+
};
|
|
378
637
|
|
|
379
638
|
/**
|
|
380
639
|
* A ChatGPT account's limits from its usage reading (see `authSwitchUsageLimit`): unset, the main limit decides --
|
|
@@ -516,14 +775,110 @@ interface ICandidate<TModel> {
|
|
|
516
775
|
reading: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot['reading'];
|
|
517
776
|
ignoreLimits: boolean;
|
|
518
777
|
account: IInferenceAccount;
|
|
778
|
+
/** The account's credential tier in its pool, 0 the best (`inferenceCredentialTiers`): placement takes the best first. */
|
|
779
|
+
tier: number;
|
|
519
780
|
model: TModel;
|
|
520
781
|
plan: string;
|
|
521
782
|
headroom: number;
|
|
522
783
|
/** The account's placement score (`IAuthSwitchPlacement.score`). */
|
|
523
784
|
score: number | null;
|
|
524
785
|
lastResort: boolean;
|
|
786
|
+
/** An API key with a monthly spend limit. */
|
|
787
|
+
spendLimit: boolean;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/** How a request is placed on a Claude pool (`InferenceRouter.acquireClaude`). */
|
|
791
|
+
export interface IInferenceClaudeAcquireOptions {
|
|
792
|
+
/** The session's account: the request is judged on it alone, unless `returnToBetterTier` finds a better one. */
|
|
793
|
+
accountId?: string;
|
|
794
|
+
/** The accounts this request left already; it never goes back to them. */
|
|
795
|
+
exclude?: ReadonlySet<string>;
|
|
796
|
+
/**
|
|
797
|
+
* The accounts of `exclude` the request left for a reason of their own, and that reason: each counts as its own
|
|
798
|
+
* judgement counts the reason, never as at its limit. An account of `exclude` it does not name counts as at its limit.
|
|
799
|
+
*/
|
|
800
|
+
excludedBecause?: ReadonlyMap<string, IInferenceAccountUnavailable>;
|
|
801
|
+
/** A request that uses no quota (`count_tokens`): an account at its limit takes it as well. */
|
|
802
|
+
ignoreLimits?: boolean;
|
|
803
|
+
/**
|
|
804
|
+
* What of the request the proxy's price table cannot price (`inferenceAnthropicUnpricedRequest`): an API key with a
|
|
805
|
+
* monthly spend limit takes no such request (`model_unpriced`). A request that uses no quota ignores it.
|
|
806
|
+
*/
|
|
807
|
+
unpriced?: string;
|
|
808
|
+
/**
|
|
809
|
+
* The session may move: when its account is of a later credential tier -- an Anthropic API key of a pool that names
|
|
810
|
+
* no credential -- a Claude account that can take the request takes it instead, and the route is `returned`.
|
|
811
|
+
*/
|
|
812
|
+
returnToBetterTier?: boolean;
|
|
525
813
|
}
|
|
526
814
|
|
|
815
|
+
/** How a request is placed on a ChatGPT pool (`InferenceRouter.acquire`). */
|
|
816
|
+
export interface IInferenceAcquireOptions {
|
|
817
|
+
/** The session's account: the request is judged on it alone, unless `returnToBetterTier` finds a better one. */
|
|
818
|
+
accountId?: string;
|
|
819
|
+
/** The accounts this request left already; it never goes back to them. */
|
|
820
|
+
exclude?: ReadonlySet<string>;
|
|
821
|
+
/** The accounts of `exclude` the request left for a reason of their own, as for a Claude pool (`IInferenceClaudeAcquireOptions`). */
|
|
822
|
+
excludedBecause?: ReadonlyMap<string, IInferenceAccountUnavailable>;
|
|
823
|
+
/** The request's `reasoning.effort`, which a ChatGPT account's model must accept; a key's model is OpenAI's to judge. */
|
|
824
|
+
reasoningEffort?: string;
|
|
825
|
+
/** The request's `service_tier`, which a ChatGPT account's model must offer (`auto` and `default` always). */
|
|
826
|
+
serviceTier?: string;
|
|
827
|
+
/** The Codex release the client read its catalog as: the model it names is checked against the catalog it was offered. */
|
|
828
|
+
clientVersion?: string;
|
|
829
|
+
/** A Responses request, which an OpenAI API key may take; a translated Messages request never goes to a key. */
|
|
830
|
+
apiKeys?: boolean;
|
|
831
|
+
/**
|
|
832
|
+
* What of the request the proxy's price table cannot price (`inferenceOpenAiUnpricedRequest`): an OpenAI API key with a
|
|
833
|
+
* monthly spend limit takes no such request (`model_unpriced`).
|
|
834
|
+
*/
|
|
835
|
+
unpriced?: string;
|
|
836
|
+
/**
|
|
837
|
+
* The session may move: when its account is an OpenAI API key of a pool that names no credential, a ChatGPT account
|
|
838
|
+
* that can take the request takes it instead, and the route is `returned`. The gateway asks it only of a request that
|
|
839
|
+
* carries no account state, which never leaves its account.
|
|
840
|
+
*/
|
|
841
|
+
returnToBetterTier?: boolean;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/** An account a request finds held with a payment problem, for the 402 that names it when nothing else is left. */
|
|
845
|
+
interface IPaymentRefusal {
|
|
846
|
+
label: string;
|
|
847
|
+
credential: TInferenceCredential;
|
|
848
|
+
unavailable: IInferenceAccountUnavailable;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* The 402 a request gets when the accounts it could go to are all held with payment problems: what each was refused for,
|
|
853
|
+
* and what ends it.
|
|
854
|
+
*/
|
|
855
|
+
const paymentRefusalText = (refusalsArg: readonly IPaymentRefusal[]): string => {
|
|
856
|
+
const named = (matchArg: (refusalArg: IPaymentRefusal) => boolean, nounArg: string): string | undefined => {
|
|
857
|
+
const labels = refusalsArg.filter(matchArg).map((refusalArg) => refusalArg.label);
|
|
858
|
+
return labels.length ? `${nounArg}${labels.length === 1 ? '' : 's'} ${labels.join(', ')}` : undefined;
|
|
859
|
+
};
|
|
860
|
+
const subscriptions = named((refusalArg) => refusalArg.unavailable.cause === 'billing'
|
|
861
|
+
&& refusalArg.credential === 'subscription', 'Claude account');
|
|
862
|
+
const billedKeys = named((refusalArg) => refusalArg.unavailable.cause === 'billing' && refusalArg.credential === 'api_key',
|
|
863
|
+
'Anthropic API key');
|
|
864
|
+
const cappedKeys = named((refusalArg) => refusalArg.unavailable.cause === 'spend_cap', 'Anthropic API key');
|
|
865
|
+
const capEnds = refusalsArg.flatMap((refusalArg) => refusalArg.unavailable.cause === 'spend_cap'
|
|
866
|
+
&& refusalArg.unavailable.retryAt ? [refusalArg.unavailable.retryAt] : []).sort()[0];
|
|
867
|
+
const quotaKeys = named((refusalArg) => refusalArg.unavailable.cause === 'quota', 'OpenAI API key');
|
|
868
|
+
return [
|
|
869
|
+
subscriptions && `Anthropic answered a billing error (HTTP 402) for the ${subscriptions}: a payment problem with the`
|
|
870
|
+
+ ' subscription. Settle the billing, then re-check the account in AGL Accounts or with'
|
|
871
|
+
+ ' `authswitch account recheck <account-id>`.',
|
|
872
|
+
billedKeys && `Anthropic answered a billing error (HTTP 402) for the ${billedKeys}: settle the billing of its`
|
|
873
|
+
+ ' organization in the Claude Console, then re-check the key in AGL Accounts.',
|
|
874
|
+
cappedKeys && `The organization of the ${cappedKeys} reached its usage tier's monthly spend cap (HTTP 429)`
|
|
875
|
+
+ `${capEnds ? `; it takes requests again at ${capEnds}` : ''}, or once Anthropic grants a higher tier and the key is`
|
|
876
|
+
+ ' re-checked in AGL Accounts.',
|
|
877
|
+
quotaKeys && `OpenAI refused the ${quotaKeys} for its quota (HTTP 429 insufficient_quota): add credit or raise the`
|
|
878
|
+
+ ' organization\'s or project\'s limit, then re-check the key in AGL Accounts.',
|
|
879
|
+
].filter((sentenceArg): sentenceArg is string => !!sentenceArg).join(' ');
|
|
880
|
+
};
|
|
881
|
+
|
|
527
882
|
/** What judging a request's accounts found, for the refusal `InferenceRouter.select` answers when none is chosen. */
|
|
528
883
|
interface ISelectionState {
|
|
529
884
|
exhausted: boolean;
|
|
@@ -537,10 +892,28 @@ interface ISelectionState {
|
|
|
537
892
|
accountLimit: TInferenceAccountLimit | undefined;
|
|
538
893
|
/** A pinned account that takes no request for a reason of its own, and why. */
|
|
539
894
|
refusal?: { accountId: string; unavailable: IInferenceAccountUnavailable };
|
|
540
|
-
/** The
|
|
541
|
-
payment:
|
|
895
|
+
/** The accounts left out for a payment problem, and why: never counted as at their limit. */
|
|
896
|
+
payment: IPaymentRefusal[];
|
|
897
|
+
/** The API keys at their monthly spend limit, which an `accounts_exhausted` refusal names. */
|
|
898
|
+
spendLimited: IInferenceSpendLimited[];
|
|
899
|
+
/** The labels of the API keys with a spend limit whose spend could not be read, which a `no_eligible_accounts` names. */
|
|
900
|
+
spendUnread: string[];
|
|
901
|
+
/**
|
|
902
|
+
* The labels of the API keys with a spend limit whose spend is recorded nowhere yet, which a `no_eligible_accounts`
|
|
903
|
+
* names: a report the account service refused, or an answer in neither the outbox's store, whose write failed, nor the
|
|
904
|
+
* ledger, or one no login of the key could be named for (`IInferenceSpendLedger.held`).
|
|
905
|
+
*/
|
|
906
|
+
spendUnrecorded: string[];
|
|
907
|
+
/** The API keys with a spend limit the request was refused on for want of a price, and what lacked one. */
|
|
908
|
+
unpriced: { labels: string[]; what: string | undefined };
|
|
542
909
|
}
|
|
543
910
|
|
|
911
|
+
/** What judging a request's accounts finds before any account is judged: nothing yet. */
|
|
912
|
+
const emptySelection = (accountsArg: number, pinnedArg: boolean): ISelectionState => ({ exhausted: false, reserved: false,
|
|
913
|
+
unavailable: false, modelKnown: false, earliestReset: Infinity, exhaustedPlans: new Set(), accounts: accountsArg,
|
|
914
|
+
pinned: pinnedArg, accountLimit: undefined, payment: [], spendLimited: [], spendUnread: [], spendUnrecorded: [],
|
|
915
|
+
unpriced: { labels: [], what: undefined } });
|
|
916
|
+
|
|
544
917
|
/** An account's placement by the shared rule, on the reading the router holds for it. */
|
|
545
918
|
const accountPlacement = (readingArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot['reading'],
|
|
546
919
|
accountArg: IInferenceAccount, nowArg: number): plugins.authswitchPlacement.IAuthSwitchPlacement =>
|
|
@@ -551,21 +924,25 @@ export class InferenceRouter {
|
|
|
551
924
|
/** ChatGPT catalogs by account and the Codex release they were read as. */
|
|
552
925
|
private readonly catalogs = new Map<string, ICatalog<TInferenceModel>>();
|
|
553
926
|
private readonly nativeCatalogs = new Map<string, INativeCatalog>();
|
|
554
|
-
|
|
927
|
+
/** The model lists of the Claude accounts and the Anthropic API keys (`claudeCatalog`). */
|
|
928
|
+
private readonly claudeCatalogs = new Map<string, IHeldModelList<TInferenceClaudeModel>>();
|
|
929
|
+
/** The model lists of the OpenAI API keys (`keyCatalog`). */
|
|
930
|
+
private readonly keyCatalogs = new Map<string, IHeldModelList<TInferenceModel>>();
|
|
555
931
|
/** Why each Claude account was last reported refused, or routed on its last model list (`noteAccount`). */
|
|
556
932
|
private readonly noted = new Map<string, { unavailable: IInferenceAccountUnavailable; lastCatalog: boolean }>();
|
|
557
933
|
private readonly active = new Map<string, number>();
|
|
558
934
|
private readonly cooldowns = new Map<string, ICooldown>();
|
|
559
935
|
/**
|
|
560
|
-
* The
|
|
561
|
-
* holds the problem
|
|
936
|
+
* The accounts a payment problem took out of rotation (`reportPaymentProblem`): a Claude account until the account
|
|
937
|
+
* service holds the problem -- one it could not be told of stays here for the proxy's lifetime -- and an API key until
|
|
938
|
+
* its spend cap ends, or for the proxy's lifetime.
|
|
562
939
|
*/
|
|
563
|
-
private readonly paymentMarks = new Map<string,
|
|
940
|
+
private readonly paymentMarks = new Map<string, IPaymentMark>();
|
|
564
941
|
/** The accounts whose payment problem the account service could not be told of, each reported once. */
|
|
565
942
|
private readonly paymentReportsFailed = new Set<string>();
|
|
566
943
|
/**
|
|
567
|
-
* The
|
|
568
|
-
* so
|
|
944
|
+
* The API keys, Anthropic's and OpenAI's, among the accounts the last pool read listed: the account service reads no
|
|
945
|
+
* subscription usage for them, so no usage a request on one shows -- headers, tokens, a 429 -- is handed to it.
|
|
569
946
|
*/
|
|
570
947
|
private apiKeyAccounts = new Set<string>();
|
|
571
948
|
private readonly now: () => number;
|
|
@@ -579,21 +956,30 @@ export class InferenceRouter {
|
|
|
579
956
|
}
|
|
580
957
|
|
|
581
958
|
/**
|
|
582
|
-
* The
|
|
583
|
-
*
|
|
584
|
-
|
|
585
|
-
|
|
959
|
+
* The credential tiers of the policy's pool, best first (`inferenceCredentialTiers`). A plan restriction leaves the
|
|
960
|
+
* API keys out, as a key has no plan, so a lease with `plans` never reaches a key.
|
|
961
|
+
*/
|
|
962
|
+
private tiers(policyArg: IInferenceRoutingPolicy): readonly TInferenceCredential[] {
|
|
963
|
+
return inferenceCredentialTiers(policyArg.credential).filter((tierArg) => !policyArg.plans || tierArg !== 'api_key');
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* The policy's accounts of its provider and credential tiers (`tiers`), and every eligible account's id of that pool:
|
|
968
|
+
* an account absent from both left the pool. An account whose lapse date passed is in neither -- it is never routed to
|
|
969
|
+
* again -- and is named in `lapsed`. A pool that names its credential serves that kind alone: Flex's API-key pool
|
|
970
|
+
* never takes a Claude account. One that names none spans both, and a session placed on a key returns to a Claude
|
|
971
|
+
* account once one can take its request.
|
|
586
972
|
*/
|
|
587
973
|
private async pool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<{ accounts: IInferenceAccount[];
|
|
588
|
-
eligible: Set<string>; lapsed: Set<string
|
|
974
|
+
eligible: Set<string>; lapsed: Set<string>; tiers: readonly TInferenceCredential[] }> {
|
|
589
975
|
const everyAccount = await this.options.accounts(signalArg);
|
|
590
976
|
this.apiKeyAccounts = new Set(everyAccount.filter((account) => account.credential === 'api_key')
|
|
591
977
|
.map((account) => account.id));
|
|
592
978
|
const provider = policyArg.provider ?? 'openai';
|
|
593
|
-
const
|
|
979
|
+
const tiers = this.tiers(policyArg);
|
|
594
980
|
const now = this.now();
|
|
595
981
|
const inPool = (accountArg: IInferenceAccount) => accountArg.providerId === provider
|
|
596
|
-
&& accountArg.credential
|
|
982
|
+
&& tiers.includes(accountArg.credential);
|
|
597
983
|
const lapsed = new Set(everyAccount.filter((account) => inPool(account)
|
|
598
984
|
&& plugins.authswitchPlacement.authSwitchAccountLifecycle(account.lapsesAt, now) === 'lapsed').map((account) => account.id));
|
|
599
985
|
const allAccounts = everyAccount.filter((account) => inPool(account) && !lapsed.has(account.id));
|
|
@@ -601,12 +987,12 @@ export class InferenceRouter {
|
|
|
601
987
|
if (accounts.length > 64) throw new InferenceRoutingError(503, 'pool_too_large',
|
|
602
988
|
'Select at most 64 accounts for this gateway.');
|
|
603
989
|
const known = new Set(everyAccount.map((account) => account.id));
|
|
604
|
-
for (const cache of [this.catalogs, this.nativeCatalogs, this.claudeCatalogs]) {
|
|
990
|
+
for (const cache of [this.catalogs, this.nativeCatalogs, this.claudeCatalogs, this.keyCatalogs]) {
|
|
605
991
|
for (const cacheKey of cache.keys()) if (!known.has(cacheKey.slice(0, 64))) cache.delete(cacheKey);
|
|
606
992
|
}
|
|
607
993
|
for (const id of this.cooldowns.keys()) if (!known.has(id)) this.cooldowns.delete(id);
|
|
608
994
|
for (const id of this.noted.keys()) if (!known.has(id)) this.noted.delete(id);
|
|
609
|
-
return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed };
|
|
995
|
+
return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed, tiers };
|
|
610
996
|
}
|
|
611
997
|
|
|
612
998
|
/**
|
|
@@ -617,7 +1003,8 @@ export class InferenceRouter {
|
|
|
617
1003
|
* the session waiting.
|
|
618
1004
|
*/
|
|
619
1005
|
private async sessionPool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
|
|
620
|
-
accountIdArg: string | undefined): Promise<{ accounts: IInferenceAccount[]; eligible: Set<string
|
|
1006
|
+
accountIdArg: string | undefined): Promise<{ accounts: IInferenceAccount[]; eligible: Set<string>;
|
|
1007
|
+
tiers: readonly TInferenceCredential[] }> {
|
|
621
1008
|
const pool = await this.pool(policyArg, signalArg);
|
|
622
1009
|
if (!accountIdArg) return pool;
|
|
623
1010
|
if (pool.lapsed.has(accountIdArg)) {
|
|
@@ -662,84 +1049,94 @@ export class InferenceRouter {
|
|
|
662
1049
|
return models;
|
|
663
1050
|
}
|
|
664
1051
|
|
|
1052
|
+
/** The models a Claude account or an Anthropic API key may use (`heldModels`). */
|
|
1053
|
+
private claudeCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
|
|
1054
|
+
return this.heldModels(this.claudeCatalogs, accountArg, signalArg, (accessArg, optionsArg) => accountArg.credential === 'api_key'
|
|
1055
|
+
? plugins.anthropicProvider.listAnthropicApiKeyModels({ apiKey: accessArg.accessToken }, optionsArg)
|
|
1056
|
+
: plugins.anthropicProvider.listAnthropicOAuthModels(accessArg, optionsArg));
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/** The models an OpenAI API key may use, as its `GET /v1/models` lists them (`heldModels`, `openAiKeyModel`). */
|
|
1060
|
+
private keyCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceModel[]> {
|
|
1061
|
+
return this.heldModels(this.keyCatalogs, accountArg, signalArg, async (accessArg, optionsArg) =>
|
|
1062
|
+
(await plugins.openAiProvider.listOpenAiApiKeyModels({ apiKey: accessArg.accessToken }, optionsArg))
|
|
1063
|
+
.map((modelArg) => openAiKeyModel(modelArg.modelId)));
|
|
1064
|
+
}
|
|
1065
|
+
|
|
665
1066
|
/**
|
|
666
|
-
* The models
|
|
1067
|
+
* The models an account may use, from its list in `listsArg`: the list read within the last minute, else read now.
|
|
667
1068
|
* A model-list read that fails keeps the list read before for the account's revision, which the account routes on
|
|
668
1069
|
* meanwhile; with none, the account is unavailable for the failure's reason. A failure of the account's access or
|
|
669
1070
|
* sign-in makes it unavailable whether a list is held or not. Either way the failure is noted and its account is not
|
|
670
|
-
* read again, however many requests ask, for a minute -- or 5 s after a failed access (`
|
|
1071
|
+
* read again, however many requests ask, for a minute -- or 5 s after a failed access (`holdListFailure`).
|
|
671
1072
|
*/
|
|
672
|
-
private async
|
|
673
|
-
|
|
1073
|
+
private async heldModels<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount,
|
|
1074
|
+
signalArg: AbortSignal, listArg: TModelLister<TModel>): Promise<TModel[]> {
|
|
1075
|
+
const cached = listsArg.get(accountArg.id);
|
|
674
1076
|
if (cached?.revision === accountArg.revision && cached.until > this.now()) {
|
|
675
|
-
const refusal =
|
|
676
|
-
if (refusal) throw new
|
|
1077
|
+
const refusal = heldListRefusal(cached);
|
|
1078
|
+
if (refusal) throw new ModelListError(refusal);
|
|
677
1079
|
return cached.models!;
|
|
678
1080
|
}
|
|
679
1081
|
try {
|
|
680
|
-
const models = await this.
|
|
681
|
-
|
|
682
|
-
failure: null });
|
|
1082
|
+
const models = await this.readModelList(accountArg, signalArg, listArg);
|
|
1083
|
+
listsArg.set(accountArg.id, { revision: accountArg.revision, until: this.now() + modelListMs, models, failure: null });
|
|
683
1084
|
this.noteAccount(accountArg.id, null);
|
|
684
1085
|
return models;
|
|
685
1086
|
} catch (errorArg) {
|
|
686
1087
|
signalArg.throwIfAborted();
|
|
687
|
-
const failure = errorArg instanceof
|
|
1088
|
+
const failure = errorArg instanceof ModelListError ? errorArg.unavailable
|
|
688
1089
|
: { reason: 'catalog_failed' as const, failure: catalogFailure(errorArg) };
|
|
689
|
-
const models = this.
|
|
1090
|
+
const models = this.holdListFailure(listsArg, accountArg, failure);
|
|
690
1091
|
if (models) return models;
|
|
691
|
-
throw new
|
|
1092
|
+
throw new ModelListError(failure);
|
|
692
1093
|
}
|
|
693
1094
|
}
|
|
694
1095
|
|
|
695
1096
|
/**
|
|
696
|
-
* Records why
|
|
1097
|
+
* Records why an account's model list or access failed, keeping the list held for its revision, and notes it:
|
|
697
1098
|
* neither is read again for a minute after a failed model-list read or sign-in, or for 5 s after a failed access
|
|
698
1099
|
* (`access_failed`). The access is then tried again by reading the list again, even one read within the minute: a
|
|
699
1100
|
* single list read on recovery. Answers the held list when the account routes on it meanwhile -- only after a failed
|
|
700
|
-
* model-list read (`
|
|
1101
|
+
* model-list read (`heldListRefusal`).
|
|
701
1102
|
*/
|
|
702
|
-
private
|
|
703
|
-
failureArg: IInferenceAccountUnavailable):
|
|
704
|
-
const
|
|
705
|
-
until: this.now() + (failureArg.reason === 'access_failed' ?
|
|
706
|
-
models: this.
|
|
707
|
-
|
|
708
|
-
const routed =
|
|
1103
|
+
private holdListFailure<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount,
|
|
1104
|
+
failureArg: IInferenceAccountUnavailable): TModel[] | undefined {
|
|
1105
|
+
const list: IHeldModelList<TModel> = { revision: accountArg.revision,
|
|
1106
|
+
until: this.now() + (failureArg.reason === 'access_failed' ? modelListAccessRetryMs : modelListMs),
|
|
1107
|
+
models: this.heldList(listsArg, accountArg) ?? null, failure: failureArg };
|
|
1108
|
+
listsArg.set(accountArg.id, list);
|
|
1109
|
+
const routed = heldListRefusal(list) ? undefined : list.models!;
|
|
709
1110
|
this.noteAccount(accountArg.id, failureArg, routed !== undefined);
|
|
710
1111
|
return routed;
|
|
711
1112
|
}
|
|
712
1113
|
|
|
713
1114
|
/**
|
|
714
|
-
* Reads the models
|
|
715
|
-
* renewed, while a key is ended, so the account service refuses the second read. Every failure but an abort
|
|
716
|
-
* `
|
|
1115
|
+
* Reads the models an account may use through `listArg`. A rejected access (`accessRejected`) is reported once: a
|
|
1116
|
+
* sign-in is renewed, while a key is ended, so the account service refuses the second read. Every failure but an abort
|
|
1117
|
+
* is a `ModelListError` naming its reason.
|
|
717
1118
|
*/
|
|
718
|
-
private async
|
|
719
|
-
|
|
1119
|
+
private async readModelList<TModel>(accountArg: IInferenceAccount, signalArg: AbortSignal,
|
|
1120
|
+
listArg: TModelLister<TModel>): Promise<TModel[]> {
|
|
1121
|
+
const read = async (rejectedGenerationArg?: number): Promise<{ models?: TModel[]; rejected?: number }> => {
|
|
720
1122
|
let access: IGatewayAuthorityAccess;
|
|
721
1123
|
try {
|
|
722
1124
|
access = await this.options.resolveAccess(accountArg.id, signalArg, rejectedGenerationArg);
|
|
723
1125
|
} catch (errorArg) {
|
|
724
1126
|
signalArg.throwIfAborted();
|
|
725
|
-
throw new
|
|
1127
|
+
throw new ModelListError({ reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' });
|
|
726
1128
|
}
|
|
727
|
-
const listOptions = { signal: signalArg, ...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}) };
|
|
728
1129
|
try {
|
|
729
|
-
return { models:
|
|
730
|
-
?
|
|
731
|
-
: await plugins.anthropicProvider.listAnthropicOAuthModels(access, listOptions) };
|
|
1130
|
+
return { models: await listArg(access, { signal: signalArg,
|
|
1131
|
+
...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}) }) };
|
|
732
1132
|
} catch (errorArg) {
|
|
733
|
-
if ((errorArg
|
|
734
|
-
|| errorArg instanceof plugins.anthropicProvider.AnthropicApiKeyAuthError) && errorArg.status === 401) {
|
|
735
|
-
return { rejected: access.generation };
|
|
736
|
-
}
|
|
1133
|
+
if (accessRejected(errorArg)) return { rejected: access.generation };
|
|
737
1134
|
throw errorArg;
|
|
738
1135
|
}
|
|
739
1136
|
};
|
|
740
1137
|
let result = await read();
|
|
741
1138
|
if (result.rejected !== undefined) result = await read(result.rejected);
|
|
742
|
-
if (!result.models) throw new
|
|
1139
|
+
if (!result.models) throw new ModelListError({ reason: 'needs_login' });
|
|
743
1140
|
return result.models;
|
|
744
1141
|
}
|
|
745
1142
|
|
|
@@ -750,7 +1147,8 @@ export class InferenceRouter {
|
|
|
750
1147
|
private noteAccount(accountIdArg: string, unavailableArg: IInferenceAccountUnavailable | null, lastCatalogArg = false): void {
|
|
751
1148
|
const previous = this.noted.get(accountIdArg);
|
|
752
1149
|
if (unavailableArg === null ? !previous : previous?.lastCatalog === lastCatalogArg
|
|
753
|
-
&& previous.unavailable.reason === unavailableArg.reason && previous.unavailable.failure === unavailableArg.failure
|
|
1150
|
+
&& previous.unavailable.reason === unavailableArg.reason && previous.unavailable.failure === unavailableArg.failure
|
|
1151
|
+
&& previous.unavailable.cause === unavailableArg.cause) return;
|
|
754
1152
|
if (unavailableArg === null) this.noted.delete(accountIdArg);
|
|
755
1153
|
else this.noted.set(accountIdArg, { unavailable: unavailableArg, lastCatalog: lastCatalogArg });
|
|
756
1154
|
try {
|
|
@@ -760,37 +1158,57 @@ export class InferenceRouter {
|
|
|
760
1158
|
} catch {}
|
|
761
1159
|
}
|
|
762
1160
|
|
|
763
|
-
/** The model list
|
|
764
|
-
private
|
|
765
|
-
const cached =
|
|
1161
|
+
/** The model list an account's revision was last read with, without reading it: `undefined` when none is held. */
|
|
1162
|
+
private heldList<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount): TModel[] | undefined {
|
|
1163
|
+
const cached = listsArg.get(accountArg.id);
|
|
766
1164
|
return cached?.revision === accountArg.revision ? cached.models ?? undefined : undefined;
|
|
767
1165
|
}
|
|
768
1166
|
|
|
769
1167
|
/**
|
|
770
|
-
* Why
|
|
771
|
-
*
|
|
1168
|
+
* Why an account's model list or access keeps it from routing, as a request would find it now, before its next read is
|
|
1169
|
+
* due: a failed access or sign-in, or a failed model-list read with no list held for its revision. Only the accounts
|
|
1170
|
+
* whose model list the router reads have one: Claude accounts and API keys.
|
|
772
1171
|
*/
|
|
773
|
-
private
|
|
774
|
-
const
|
|
775
|
-
|
|
1172
|
+
private listRefusal(accountArg: IInferenceAccount): IInferenceAccountUnavailable | undefined {
|
|
1173
|
+
const refusal = <TModel>(cachedArg: IHeldModelList<TModel> | undefined) => cachedArg?.revision === accountArg.revision
|
|
1174
|
+
&& cachedArg.until > this.now() ? heldListRefusal(cachedArg) : undefined;
|
|
1175
|
+
return accountArg.providerId === 'anthropic' ? refusal(this.claudeCatalogs.get(accountArg.id))
|
|
1176
|
+
: accountArg.credential === 'api_key' ? refusal(this.keyCatalogs.get(accountArg.id)) : undefined;
|
|
776
1177
|
}
|
|
777
1178
|
|
|
778
1179
|
/**
|
|
779
|
-
* The
|
|
780
|
-
*
|
|
781
|
-
*
|
|
782
|
-
*
|
|
1180
|
+
* The Claude pool's models, from its best credential tier with a readable account (`inferenceCredentialTiers`): the
|
|
1181
|
+
* Claude accounts' while one of them can read its model list, else the Anthropic API keys'. Within the tier, the
|
|
1182
|
+
* models every readable account offers: a session may move to any of them, so a model one account lacks is not
|
|
1183
|
+
* offered. Limits are the smallest any of them states; efforts are those all of them accept, and adaptive thinking is
|
|
1184
|
+
* offered when all of them accept it. What one account does not state, the pool does not state.
|
|
783
1185
|
*/
|
|
784
1186
|
public async claudeModels(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
|
|
785
1187
|
if (policyArg.provider !== 'anthropic') throw new Error('The Claude catalog belongs to a Claude pool.');
|
|
786
|
-
const accounts = await this.
|
|
1188
|
+
const { accounts, tiers } = await this.pool(policyArg, signalArg);
|
|
1189
|
+
for (const tier of tiers) {
|
|
1190
|
+
const models = await this.tierClaudeModels(accounts.filter((account) => account.credential === tier), policyArg,
|
|
1191
|
+
signalArg);
|
|
1192
|
+
if (models?.size) return [...models.values()];
|
|
1193
|
+
}
|
|
1194
|
+
throw new InferenceRoutingError(503, 'no_eligible_accounts', policyArg.credential === 'api_key'
|
|
1195
|
+
? 'No Anthropic API key could supply a model list. Add one with `authswitch account add anthropic-api-key`.'
|
|
1196
|
+
: policyArg.credential === 'subscription' || policyArg.plans
|
|
1197
|
+
? 'No eligible Claude account could supply a model list. Check the AGL account service and sign-ins.'
|
|
1198
|
+
: 'No eligible Claude account or Anthropic API key could supply a model list. Check the AGL account service and '
|
|
1199
|
+
+ 'sign-ins, or add a key with `authswitch account add anthropic-api-key`.');
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
/** The models every readable account of one credential tier of a Claude pool offers; `undefined` when none is readable. */
|
|
1203
|
+
private async tierClaudeModels(accountsArg: readonly IInferenceAccount[], policyArg: IInferenceRoutingPolicy,
|
|
1204
|
+
signalArg: AbortSignal): Promise<Map<string, TInferenceClaudeModel> | undefined> {
|
|
787
1205
|
let models: Map<string, TInferenceClaudeModel> | undefined;
|
|
788
|
-
for (const account of
|
|
1206
|
+
for (const account of accountsArg) {
|
|
789
1207
|
signalArg.throwIfAborted();
|
|
790
1208
|
try {
|
|
1209
|
+
// A plan restriction leaves the keys out of the pool: only Claude accounts are judged on it.
|
|
791
1210
|
if (policyArg.plans) {
|
|
792
|
-
const limits = account.
|
|
793
|
-
: claudeLimits(await this.options.usage(account.id, signalArg), account, this.now());
|
|
1211
|
+
const limits = claudeLimits(await this.options.usage(account.id, signalArg), account, this.now());
|
|
794
1212
|
if (!limits || !policyArg.plans.includes(limits.plan)) continue;
|
|
795
1213
|
}
|
|
796
1214
|
const offered = new Map((await this.claudeCatalog(account, signalArg)).map((model) => [model.modelId, model]));
|
|
@@ -816,24 +1234,46 @@ export class InferenceRouter {
|
|
|
816
1234
|
// An unreadable account is unavailable; it never supplies guessed models.
|
|
817
1235
|
}
|
|
818
1236
|
}
|
|
819
|
-
|
|
820
|
-
? 'No Anthropic API key could supply a model list. Add one with `authswitch account add anthropic-api-key`.'
|
|
821
|
-
: 'No eligible Claude account could supply a model list. Check the AGL account service and sign-ins.');
|
|
822
|
-
return [...models.values()];
|
|
1237
|
+
return models;
|
|
823
1238
|
}
|
|
824
1239
|
|
|
825
1240
|
/**
|
|
826
|
-
*
|
|
827
|
-
*
|
|
1241
|
+
* The ChatGPT pool's models, from its best credential tier with a readable account (`inferenceCredentialTiers`): the
|
|
1242
|
+
* ChatGPT sign-ins' current entitlements while one of them can read its catalog, else the OpenAI API keys' models
|
|
1243
|
+
* (`keyTierModels`). A pool whose every ChatGPT account read needs a fresh sign-in -- ChatGPT rejected its access even
|
|
1244
|
+
* when renewed, or the account service could not renew it -- and no key to fall back to answers
|
|
828
1245
|
* `account_needs_login`, so its caller asks for a sign-in rather than for the proxy.
|
|
829
1246
|
*/
|
|
830
1247
|
public async models(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<TInferenceModel[]> {
|
|
831
1248
|
if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('The ChatGPT catalog belongs to a ChatGPT pool.');
|
|
832
|
-
const accounts = await this.
|
|
1249
|
+
const { accounts, tiers } = await this.pool(policyArg, signalArg);
|
|
1250
|
+
let signInNeeded = false;
|
|
1251
|
+
for (const tier of tiers) {
|
|
1252
|
+
if (tier === 'subscription') {
|
|
1253
|
+
const read = await this.signInModels(accounts.filter(chatGptSignIn), policyArg, signalArg);
|
|
1254
|
+
if (read.models.size) return [...read.models.values()];
|
|
1255
|
+
signInNeeded = read.signInNeeded;
|
|
1256
|
+
} else {
|
|
1257
|
+
const models = await this.keyTierModels(accounts.filter((account) => !chatGptSignIn(account)), signalArg);
|
|
1258
|
+
if (models?.size) return [...models.values()];
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
if (signInNeeded) throw new InferenceRoutingError(503, 'account_needs_login', 'An account needs a fresh sign-in.');
|
|
1262
|
+
throw new InferenceRoutingError(503, 'no_eligible_accounts', policyArg.credential === 'api_key'
|
|
1263
|
+
? 'No OpenAI API key could supply a model list. Add one in AGL Accounts or with `authswitch account add openai-api-key`.'
|
|
1264
|
+
: 'No eligible account could supply a model catalog. Check the AGL account service and sign-ins.');
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
/**
|
|
1268
|
+
* Current model entitlements, merged conservatively across the pool's ChatGPT sign-ins, and whether every account read
|
|
1269
|
+
* needed a fresh sign-in.
|
|
1270
|
+
*/
|
|
1271
|
+
private async signInModels(accountsArg: readonly IInferenceAccount[], policyArg: IInferenceRoutingPolicy,
|
|
1272
|
+
signalArg: AbortSignal): Promise<{ models: Map<string, TInferenceModel>; signInNeeded: boolean }> {
|
|
833
1273
|
const models = new Map<string, TInferenceModel>();
|
|
834
1274
|
let read = 0;
|
|
835
1275
|
let signInNeeded = 0;
|
|
836
|
-
for (const account of
|
|
1276
|
+
for (const account of accountsArg) {
|
|
837
1277
|
signalArg.throwIfAborted();
|
|
838
1278
|
try {
|
|
839
1279
|
if (policyArg.plans) {
|
|
@@ -859,22 +1299,44 @@ export class InferenceRouter {
|
|
|
859
1299
|
if (needsSignIn(errorArg)) signInNeeded += 1;
|
|
860
1300
|
}
|
|
861
1301
|
}
|
|
862
|
-
|
|
863
|
-
|
|
1302
|
+
return { models, signInNeeded: !models.size && read > 0 && signInNeeded === read };
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
/**
|
|
1306
|
+
* The models every readable OpenAI API key of the pool lists and flexharness describes (`openAiKeyModel`), newest first
|
|
1307
|
+
* as `OPENAI_API_MODELS` orders them, the first the default: a session may move to any of the keys, so a model one key
|
|
1308
|
+
* lacks is not offered. `undefined` when no key is readable.
|
|
1309
|
+
*/
|
|
1310
|
+
private async keyTierModels(accountsArg: readonly IInferenceAccount[],
|
|
1311
|
+
signalArg: AbortSignal): Promise<Map<string, TInferenceModel> | undefined> {
|
|
1312
|
+
let offered: Set<string> | undefined;
|
|
1313
|
+
for (const account of accountsArg) {
|
|
1314
|
+
signalArg.throwIfAborted();
|
|
1315
|
+
try {
|
|
1316
|
+
const listed = new Set((await this.keyCatalog(account, signalArg)).filter((model) => !model.hidden).map((model) => model.modelId));
|
|
1317
|
+
offered = offered ? new Set([...offered].filter((modelId) => listed.has(modelId))) : listed;
|
|
1318
|
+
} catch {
|
|
1319
|
+
signalArg.throwIfAborted();
|
|
1320
|
+
// An unreadable key is unavailable; it never supplies guessed models.
|
|
1321
|
+
}
|
|
864
1322
|
}
|
|
865
|
-
if (!
|
|
866
|
-
|
|
867
|
-
|
|
1323
|
+
if (!offered) return undefined;
|
|
1324
|
+
const models = plugins.openAiProvider.OPENAI_API_MODELS.filter((model) => offered!.has(model.modelId))
|
|
1325
|
+
.map((model, index) => ({ ...openAiKeyModel(model.modelId), isDefault: index === 0 }));
|
|
1326
|
+
return new Map(models.map((model) => [model.modelId, model]));
|
|
868
1327
|
}
|
|
869
1328
|
|
|
870
1329
|
/**
|
|
871
|
-
* The pool's Codex model catalog in the Codex backend's own entry shape, read for the client's version
|
|
872
|
-
*
|
|
1330
|
+
* The pool's Codex model catalog in the Codex backend's own entry shape, read for the client's version through its
|
|
1331
|
+
* ChatGPT sign-ins alone: an OpenAI API key never reaches the ChatGPT catalog connection, so a pool of keys alone has
|
|
1332
|
+
* no Codex catalog here (`no_eligible_accounts`; Codex then uses its bundled one). An entry shared by several
|
|
1333
|
+
* accounts keeps only what every one of them offers.
|
|
873
1334
|
*/
|
|
874
1335
|
public async nativeModels(policyArg: IInferenceRoutingPolicy, clientVersionArg: string,
|
|
875
1336
|
signalArg: AbortSignal): Promise<TInferenceNativeModel[]> {
|
|
876
1337
|
if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('The Codex catalog belongs to a ChatGPT pool.');
|
|
877
|
-
|
|
1338
|
+
// The Codex catalog is the ChatGPT backend's: only a ChatGPT sign-in reads it, whatever credential the policy names.
|
|
1339
|
+
const accounts = (await this.accounts(policyArg, signalArg)).filter(chatGptSignIn);
|
|
878
1340
|
const models = new Map<string, TInferenceNativeModel>();
|
|
879
1341
|
for (const account of accounts) {
|
|
880
1342
|
signalArg.throwIfAborted();
|
|
@@ -927,92 +1389,162 @@ export class InferenceRouter {
|
|
|
927
1389
|
}
|
|
928
1390
|
|
|
929
1391
|
/**
|
|
930
|
-
* A ChatGPT account
|
|
931
|
-
*
|
|
1392
|
+
* A ChatGPT account or an OpenAI API key for one request, chosen as `acquireClaude` chooses: the pinned session account
|
|
1393
|
+
* if it is still eligible, else the best credential tier's least busy account with the most headroom
|
|
1394
|
+
* (`inferenceCredentialTiers`). A ChatGPT account is judged on its usage reading and the Codex catalog of the release
|
|
1395
|
+
* the client read it as (`clientVersion`). An OpenAI API key takes only a Responses request (`apiKeys`), never a
|
|
1396
|
+
* translated Messages request, and is judged as an Anthropic API key is (`judgeListed`): a payment problem -- OpenAI's
|
|
1397
|
+
* quota refusal -- or its month's spend at its limit keeps it out, a key with a limit takes no request it cannot price
|
|
1398
|
+
* (`unpriced`, `model_unpriced`), and its model list (`GET /v1/models`, held a minute) must list the model. The accounts
|
|
1399
|
+
* of `exclude` count as `excludedBecause` says, as for a Claude pool. `returnToBetterTier`: a session pinned to a key
|
|
1400
|
+
* goes back to a ChatGPT account as soon as one can take its request, and the route says so (`returned`).
|
|
932
1401
|
*/
|
|
933
1402
|
public async acquire(modelIdArg: string, policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
|
|
934
|
-
optionsArg:
|
|
935
|
-
clientVersion?: string } = {}): Promise<IInferenceRoute> {
|
|
1403
|
+
optionsArg: IInferenceAcquireOptions = {}): Promise<IInferenceRoute> {
|
|
936
1404
|
if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('A ChatGPT route belongs to a ChatGPT pool.');
|
|
1405
|
+
const { accounts, tiers } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
|
|
1406
|
+
const pinned = accounts.find((account) => account.id === optionsArg.accountId);
|
|
1407
|
+
if (pinned && optionsArg.returnToBetterTier) {
|
|
1408
|
+
// The session's account is of a later tier: an account of a better one that can take the request takes it, as a
|
|
1409
|
+
// new session would be placed. When none can, the session stays where it is, judged as before.
|
|
1410
|
+
const better = accounts.filter((account) => tiers.indexOf(account.credential) < tiers.indexOf(pinned.credential));
|
|
1411
|
+
if (better.length) {
|
|
1412
|
+
try {
|
|
1413
|
+
const { accountId: _pinned, ...unpinned } = optionsArg;
|
|
1414
|
+
return { ...await this.routeOpenAi(modelIdArg, policyArg, better, tiers, signalArg, unpinned), returned: true };
|
|
1415
|
+
} catch (errorArg) {
|
|
1416
|
+
signalArg.throwIfAborted();
|
|
1417
|
+
if (!(errorArg instanceof InferenceRoutingError)) throw errorArg;
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
return this.routeOpenAi(modelIdArg, policyArg, accounts, tiers, signalArg, optionsArg);
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
/** `acquire` among `accountsArg`, the pool's accounts or those of its better tiers. */
|
|
1425
|
+
private async routeOpenAi(modelIdArg: string, policyArg: IInferenceRoutingPolicy, accountsArg: readonly IInferenceAccount[],
|
|
1426
|
+
tiersArg: readonly TInferenceCredential[], signalArg: AbortSignal, optionsArg: IInferenceAcquireOptions): Promise<IInferenceRoute> {
|
|
937
1427
|
const candidates: ICandidate<TInferenceModel>[] = [];
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
let
|
|
941
|
-
|
|
942
|
-
let earliestReset = Infinity;
|
|
943
|
-
let accountLimit: TInferenceAccountLimit | undefined;
|
|
944
|
-
const exhaustedPlans = new Set<string>();
|
|
945
|
-
const { accounts } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
|
|
946
|
-
for (const account of accounts) {
|
|
1428
|
+
const state = emptySelection(accountsArg.length, optionsArg.accountId !== undefined);
|
|
1429
|
+
/** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
|
|
1430
|
+
let refusal: IInferenceAccountUnavailable | undefined;
|
|
1431
|
+
for (const account of accountsArg) {
|
|
947
1432
|
signalArg.throwIfAborted();
|
|
948
1433
|
if (optionsArg.accountId && optionsArg.accountId !== account.id) continue;
|
|
949
|
-
if (optionsArg.exclude?.has(account.id)) {
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
|
|
953
|
-
continue;
|
|
954
|
-
}
|
|
1434
|
+
if (optionsArg.exclude?.has(account.id)) { this.countLeft(state, account, optionsArg.excludedBecause?.get(account.id)); continue; }
|
|
1435
|
+
// A translated Messages request never goes to an OpenAI API key: only a Responses request does.
|
|
1436
|
+
if (!chatGptSignIn(account) && !optionsArg.apiKeys) { state.unavailable = true; continue; }
|
|
955
1437
|
try {
|
|
1438
|
+
if (!chatGptSignIn(account)) {
|
|
1439
|
+
// A key's usage is its month's spend, with the payment problem the account service holds on it.
|
|
1440
|
+
const usage = await this.options.usage(account.id, signalArg)
|
|
1441
|
+
.catch(() => { signalArg.throwIfAborted(); return undefined; });
|
|
1442
|
+
const judged = await this.judgeListed(state, account, usage, keyLimits(usage, account, this.now()), {
|
|
1443
|
+
modelId: modelIdArg, lists: this.keyCatalogs, read: (accountArg, innerSignalArg) => this.keyCatalog(accountArg, innerSignalArg),
|
|
1444
|
+
...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }) }, signalArg);
|
|
1445
|
+
if (judged?.kind !== 'candidate') { if (judged) refusal = judged.unavailable; continue; }
|
|
1446
|
+
// The request goes to OpenAI as the client sent it: its effort and service tier are OpenAI's to judge, a limited
|
|
1447
|
+
// key's tier the price check's (`unpriced`).
|
|
1448
|
+
const placement = accountPlacement(judged.limits.limit ? usage!.reading : null, account, this.now());
|
|
1449
|
+
candidates.push({ account, tier: tiersArg.indexOf(account.credential), model: judged.model,
|
|
1450
|
+
reading: judged.limits.limit ? usage!.reading : null, ignoreLimits: false, plan: judged.limits.plan,
|
|
1451
|
+
headroom: judged.limits.headroom, score: placement.score, lastResort: placement.eligibility === 'last_resort',
|
|
1452
|
+
spendLimit: judged.spendLimit });
|
|
1453
|
+
continue;
|
|
1454
|
+
}
|
|
956
1455
|
const usage = await this.options.usage(account.id, signalArg);
|
|
957
1456
|
if (!usableReading(usage, this.now()) || usage.reading?.kind !== 'openai') {
|
|
958
|
-
unavailable = true;
|
|
1457
|
+
state.unavailable = true;
|
|
959
1458
|
continue;
|
|
960
1459
|
}
|
|
961
1460
|
const plan = usage.reading.rateLimits.plan;
|
|
962
1461
|
if (policyArg.plans && !policyArg.plans.includes(plan)) continue;
|
|
963
1462
|
const model = (await this.catalog(account, signalArg, optionsArg.clientVersion)).find((entry) => entry.modelId === modelIdArg);
|
|
964
1463
|
if (!model) continue;
|
|
965
|
-
modelKnown = true;
|
|
1464
|
+
state.modelKnown = true;
|
|
966
1465
|
if (optionsArg.reasoningEffort && !model.reasoningEfforts.some((effort) => effort.effort === optionsArg.reasoningEffort)) continue;
|
|
967
1466
|
if (optionsArg.serviceTier && !['auto', 'default'].includes(optionsArg.serviceTier)
|
|
968
1467
|
&& !model.serviceTiers.some((tier) => tier.id === optionsArg.serviceTier)) continue;
|
|
969
1468
|
const limits = chatGptLimits(usage, account, this.now());
|
|
970
|
-
if (!limits) { unavailable = true; continue; }
|
|
1469
|
+
if (!limits) { state.unavailable = true; continue; }
|
|
971
1470
|
const cooldown = cooldownUntil(this.cooldowns.get(account.id), usage);
|
|
972
1471
|
if (limits.state !== 'available' || cooldown > this.now()) {
|
|
973
|
-
exhausted = true;
|
|
974
|
-
reserved ||= limits.state === 'reserved';
|
|
975
|
-
accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
|
|
976
|
-
exhaustedPlans.add(limits.plan);
|
|
1472
|
+
state.exhausted = true;
|
|
1473
|
+
state.reserved ||= limits.state === 'reserved';
|
|
1474
|
+
state.accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
|
|
1475
|
+
state.exhaustedPlans.add(limits.plan);
|
|
977
1476
|
const reset = Math.max(cooldown, limits.resetAt ?? 0);
|
|
978
|
-
if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
|
|
1477
|
+
if (reset > this.now()) state.earliestReset = Math.min(state.earliestReset, reset);
|
|
979
1478
|
continue;
|
|
980
1479
|
}
|
|
981
1480
|
const placement = accountPlacement(usage.reading, account, this.now());
|
|
982
|
-
candidates.push({ account, model, reading: usage.reading, ignoreLimits: false,
|
|
983
|
-
lastResort: placement.eligibility === 'last_resort'
|
|
984
|
-
|
|
1481
|
+
candidates.push({ account, tier: tiersArg.indexOf(account.credential), model, reading: usage.reading, ignoreLimits: false,
|
|
1482
|
+
plan: limits.plan, headroom: limits.headroom, score: placement.score, lastResort: placement.eligibility === 'last_resort',
|
|
1483
|
+
spendLimit: false });
|
|
1484
|
+
} catch (errorArg) {
|
|
1485
|
+
signalArg.throwIfAborted();
|
|
1486
|
+
state.unavailable = true;
|
|
1487
|
+
if (errorArg instanceof ModelListError) refusal = errorArg.unavailable;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
if (optionsArg.accountId !== undefined && refusal) state.refusal = { accountId: optionsArg.accountId, unavailable: refusal };
|
|
1491
|
+
// The chosen account's access is resolved before anything is sent. An OpenAI API key whose access fails takes no
|
|
1492
|
+
// request, as an Anthropic key's does (`routeClaude`): the failure is held for the key (`keyCatalogs`) and noted, and
|
|
1493
|
+
// the request goes to the next candidate -- a pinned request has none, so it is refused naming the key and the reason.
|
|
1494
|
+
// A ChatGPT account's failure fails the request.
|
|
1495
|
+
let selected: ICandidate<TInferenceModel>;
|
|
1496
|
+
let initial: IGatewayAuthorityAccess;
|
|
1497
|
+
let release: () => void;
|
|
1498
|
+
for (;;) {
|
|
1499
|
+
selected = this.select(candidates, state);
|
|
1500
|
+
release = this.reserve(selected.account.id);
|
|
1501
|
+
try {
|
|
1502
|
+
initial = await this.options.resolveAccess(selected.account.id, signalArg);
|
|
1503
|
+
break;
|
|
1504
|
+
} catch (errorArg) {
|
|
1505
|
+
release();
|
|
1506
|
+
if (chatGptSignIn(selected.account)) throw errorArg;
|
|
985
1507
|
signalArg.throwIfAborted();
|
|
986
|
-
|
|
1508
|
+
const failure: IInferenceAccountUnavailable = { reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' };
|
|
1509
|
+
this.holdListFailure(this.keyCatalogs, selected.account, failure);
|
|
1510
|
+
candidates.splice(candidates.indexOf(selected), 1);
|
|
1511
|
+
state.unavailable = true;
|
|
1512
|
+
if (state.pinned) state.refusal = { accountId: selected.account.id, unavailable: failure };
|
|
987
1513
|
}
|
|
988
1514
|
}
|
|
989
|
-
const selectionState: ISelectionState = { exhausted, reserved, unavailable, modelKnown, earliestReset, exhaustedPlans,
|
|
990
|
-
accounts: accounts.length, pinned: optionsArg.accountId !== undefined, accountLimit, payment: [] };
|
|
991
|
-
const selected = this.select(candidates, selectionState);
|
|
992
1515
|
const id = selected.account.id;
|
|
993
|
-
const release = this.reserve(id);
|
|
994
1516
|
try {
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1517
|
+
this.select([selected], state);
|
|
1518
|
+
const connection = chatGptSignIn(selected.account)
|
|
1519
|
+
? plugins.openAiProvider.createOpenAiChatGptResolvingModelConnection({
|
|
1520
|
+
expectedIdentity: initial,
|
|
1521
|
+
resolveAccess: async ({ signal, rejectedGeneration }) => {
|
|
1522
|
+
const access = await this.options.resolveAccess(id, signal, rejectedGeneration);
|
|
1523
|
+
this.select([selected], state);
|
|
1524
|
+
return access;
|
|
1525
|
+
},
|
|
1526
|
+
fetch: async (inputArg, initArg) => {
|
|
1527
|
+
const response = await (this.options.providerFetch ?? fetch)(inputArg, initArg);
|
|
1528
|
+
this.observe(id, 'openai', response.headers, response.status);
|
|
1529
|
+
if (response.status === 429) {
|
|
1530
|
+
this.reportQuota(id, this.retryAfterSeconds(response.headers));
|
|
1531
|
+
}
|
|
1532
|
+
return response;
|
|
1533
|
+
},
|
|
1534
|
+
originator: 'agl',
|
|
1535
|
+
})
|
|
1536
|
+
: plugins.openAiProvider.createOpenAiApiKeyConnection({
|
|
1537
|
+
apiKey: initial.accessToken,
|
|
1538
|
+
// A key is never renewed: OpenAI's 401 is reported before the connection fails the request with it.
|
|
1539
|
+
fetch: async (inputArg, initArg) => {
|
|
1540
|
+
const response = await (this.options.providerFetch ?? fetch)(inputArg, initArg);
|
|
1541
|
+
if (response.status === 401) await this.rejectApiKey(id, initial.generation, signalArg);
|
|
1542
|
+
return response;
|
|
1543
|
+
},
|
|
1544
|
+
});
|
|
1545
|
+
return { provider: 'openai', credential: selected.account.credential, accountId: id, plan: selected.plan,
|
|
1546
|
+
score: selected.score, model: selected.model, connection, returned: false, loginId: selected.account.loginId,
|
|
1547
|
+
spendLimit: selected.spendLimit, release };
|
|
1016
1548
|
} catch (errorArg) {
|
|
1017
1549
|
release();
|
|
1018
1550
|
throw errorArg;
|
|
@@ -1020,106 +1552,229 @@ export class InferenceRouter {
|
|
|
1020
1552
|
}
|
|
1021
1553
|
|
|
1022
1554
|
/**
|
|
1023
|
-
*
|
|
1024
|
-
*
|
|
1025
|
-
*
|
|
1026
|
-
*
|
|
1027
|
-
*
|
|
1555
|
+
* Counts an account the request left already (`exclude`) as its own judgement counts the reason the request left it
|
|
1556
|
+
* for (`excludedBecause`): a payment problem joins the 402 list, a key whose spend could not be read or recorded is
|
|
1557
|
+
* named in the 503, one that could not price the request joins the 400 list, one without the model leads to the 404,
|
|
1558
|
+
* and every other reason to the 503 for want of an account. An account left for no reason of its own counts as at its
|
|
1559
|
+
* limit (429). The switch is exhaustive: a new reason must be placed here.
|
|
1560
|
+
*/
|
|
1561
|
+
private countLeft(stateArg: ISelectionState, accountArg: IInferenceAccount,
|
|
1562
|
+
leftArg: IInferenceAccountUnavailable | undefined): void {
|
|
1563
|
+
if (!leftArg) {
|
|
1564
|
+
stateArg.exhausted = true;
|
|
1565
|
+
const reset = this.cooldowns.get(accountArg.id)?.until ?? 0;
|
|
1566
|
+
if (reset > this.now()) stateArg.earliestReset = Math.min(stateArg.earliestReset, reset);
|
|
1567
|
+
return;
|
|
1568
|
+
}
|
|
1569
|
+
switch (leftArg.reason) {
|
|
1570
|
+
case 'payment_problem':
|
|
1571
|
+
stateArg.payment.push({ label: accountArg.label, credential: accountArg.credential, unavailable: leftArg });
|
|
1572
|
+
return;
|
|
1573
|
+
case 'model_missing':
|
|
1574
|
+
return;
|
|
1575
|
+
case 'model_unpriced':
|
|
1576
|
+
stateArg.unpriced.labels.push(accountArg.label);
|
|
1577
|
+
stateArg.unpriced.what ??= leftArg.failure;
|
|
1578
|
+
return;
|
|
1579
|
+
case 'spend_unread':
|
|
1580
|
+
stateArg.unavailable = true;
|
|
1581
|
+
stateArg.spendUnread.push(accountArg.label);
|
|
1582
|
+
return;
|
|
1583
|
+
case 'spend_unrecorded':
|
|
1584
|
+
stateArg.unavailable = true;
|
|
1585
|
+
stateArg.spendUnrecorded.push(accountArg.label);
|
|
1586
|
+
return;
|
|
1587
|
+
case 'usage_unread':
|
|
1588
|
+
case 'catalog_failed':
|
|
1589
|
+
case 'needs_login':
|
|
1590
|
+
case 'access_failed':
|
|
1591
|
+
stateArg.unavailable = true;
|
|
1592
|
+
return;
|
|
1593
|
+
default: {
|
|
1594
|
+
const unhandledReason: never = leftArg.reason;
|
|
1595
|
+
return unhandledReason;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Judges an account whose model list the router reads -- a Claude account, an Anthropic or an OpenAI API key -- for one
|
|
1602
|
+
* request, as placement takes it, recording on `stateArg` what keeps it out. A payment problem refuses it before anything
|
|
1603
|
+
* else; without limits to route on (`limitsArg`) it is unavailable (`usage_unread`, a key's `spend_unread`); a key with
|
|
1604
|
+
* a spend limit whose spend is recorded nowhere yet is refused (`spend_unrecorded`). Limits and cool-down are judged
|
|
1605
|
+
* before any model list is read: an account that cannot take the request now is never read for it, its held list says
|
|
1606
|
+
* whether it would serve the model, and without one it counts as at its limit, so a session on it moves rather than
|
|
1607
|
+
* waiting for a list. A key's spend reading says nothing of the provider's rate limits, so only its own end ends a key's
|
|
1608
|
+
* cool-down. A key with a spend limit then takes no request whose cost the price table cannot count (`unpriced`,
|
|
1609
|
+
* Decision Q4), and the model list (`read`) must list the model. Answers the candidate's model and limits, or the refusal
|
|
1610
|
+
* of an account that takes no request for a reason of its own -- a pinned refusal names it -- or `undefined`.
|
|
1611
|
+
*/
|
|
1612
|
+
private async judgeListed<TModel extends { modelId: string }>(stateArg: ISelectionState, accountArg: IInferenceAccount,
|
|
1613
|
+
usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined, limitsArg: IAccountLimits | undefined,
|
|
1614
|
+
optionsArg: { modelId: string; lists: Map<string, IHeldModelList<TModel>>;
|
|
1615
|
+
read: (accountArg: IInferenceAccount, signalArg: AbortSignal) => Promise<TModel[]>; plans?: readonly string[];
|
|
1616
|
+
unpriced?: string; ignoreLimits?: boolean },
|
|
1617
|
+
signalArg: AbortSignal): Promise<{ kind: 'candidate'; model: TModel; limits: IAccountLimits; spendLimit: boolean }
|
|
1618
|
+
| { kind: 'refused'; unavailable: IInferenceAccountUnavailable } | undefined> {
|
|
1619
|
+
const apiKey = accountArg.credential === 'api_key';
|
|
1620
|
+
const refused = (unavailableArg: IInferenceAccountUnavailable) => ({ kind: 'refused' as const, unavailable: unavailableArg });
|
|
1621
|
+
// A payment problem refuses the account before anything else is judged, whatever limits a request ignores.
|
|
1622
|
+
const problem = this.paymentProblem(accountArg.id, usageArg);
|
|
1623
|
+
if (problem) {
|
|
1624
|
+
stateArg.payment.push({ label: accountArg.label, credential: accountArg.credential, unavailable: problem });
|
|
1625
|
+
this.noteAccount(accountArg.id, problem);
|
|
1626
|
+
return refused(problem);
|
|
1627
|
+
}
|
|
1628
|
+
if (this.noted.get(accountArg.id)?.unavailable.reason === 'payment_problem') this.noteAccount(accountArg.id, null);
|
|
1629
|
+
if (!limitsArg) {
|
|
1630
|
+
stateArg.unavailable = true;
|
|
1631
|
+
const unavailable: IInferenceAccountUnavailable = apiKey ? { reason: 'spend_unread' }
|
|
1632
|
+
: { reason: 'usage_unread', retryAt: usageArg?.nextAttemptAt ?? null };
|
|
1633
|
+
if (apiKey) stateArg.spendUnread.push(accountArg.label);
|
|
1634
|
+
this.noteAccount(accountArg.id, unavailable);
|
|
1635
|
+
return refused(unavailable);
|
|
1636
|
+
}
|
|
1637
|
+
// A key with a spend limit whose spend the proxy relayed is recorded nowhere yet takes nothing while it is not,
|
|
1638
|
+
// whatever its spend: the account service's ledger lacks those answers.
|
|
1639
|
+
const spendLimit = apiKey && keyHasSpendLimit(usageArg, accountArg);
|
|
1640
|
+
if (spendLimit && this.options.spend?.held(accountArg.id)) {
|
|
1641
|
+
stateArg.unavailable = true;
|
|
1642
|
+
stateArg.spendUnrecorded.push(accountArg.label);
|
|
1643
|
+
const unavailable: IInferenceAccountUnavailable = { reason: 'spend_unrecorded' };
|
|
1644
|
+
this.noteAccount(accountArg.id, unavailable);
|
|
1645
|
+
return refused(unavailable);
|
|
1646
|
+
}
|
|
1647
|
+
const notedReason = this.noted.get(accountArg.id)?.unavailable.reason;
|
|
1648
|
+
if (notedReason === 'usage_unread' || notedReason === 'spend_unread' || notedReason === 'spend_unrecorded') {
|
|
1649
|
+
this.noteAccount(accountArg.id, null);
|
|
1650
|
+
}
|
|
1651
|
+
if (optionsArg.plans && !optionsArg.plans.includes(limitsArg.plan)) return undefined;
|
|
1652
|
+
const cooldown = cooldownUntil(this.cooldowns.get(accountArg.id), apiKey ? undefined : usageArg);
|
|
1653
|
+
if (!optionsArg.ignoreLimits && (limitsArg.state !== 'available' || cooldown > this.now())) {
|
|
1654
|
+
const held = this.heldList(optionsArg.lists, accountArg);
|
|
1655
|
+
if (held && !held.some((entry) => entry.modelId === optionsArg.modelId)) return refused({ reason: 'model_missing' });
|
|
1656
|
+
if (held) stateArg.modelKnown = true;
|
|
1657
|
+
stateArg.exhausted = true;
|
|
1658
|
+
stateArg.reserved ||= limitsArg.state === 'reserved';
|
|
1659
|
+
stateArg.accountLimit = limitsArg.state === 'available' ? 'cooling' : limitsArg.state;
|
|
1660
|
+
// A key has no plan to name.
|
|
1661
|
+
if (limitsArg.plan) stateArg.exhaustedPlans.add(limitsArg.plan);
|
|
1662
|
+
if (limitsArg.state === 'exhausted' && usageArg?.reading?.kind === 'spend') {
|
|
1663
|
+
stateArg.spendLimited.push({ accountId: accountArg.id, label: accountArg.label, spend: usageArg.reading.spend });
|
|
1664
|
+
}
|
|
1665
|
+
const reset = Math.max(cooldown, limitsArg.resetAt ?? 0);
|
|
1666
|
+
if (reset > this.now()) stateArg.earliestReset = Math.min(stateArg.earliestReset, reset);
|
|
1667
|
+
return undefined;
|
|
1668
|
+
}
|
|
1669
|
+
if (spendLimit && optionsArg.unpriced !== undefined && !optionsArg.ignoreLimits) {
|
|
1670
|
+
stateArg.unpriced.labels.push(accountArg.label);
|
|
1671
|
+
stateArg.unpriced.what ??= optionsArg.unpriced;
|
|
1672
|
+
return refused({ reason: 'model_unpriced', failure: optionsArg.unpriced });
|
|
1673
|
+
}
|
|
1674
|
+
const model = (await optionsArg.read(accountArg, signalArg)).find((entry) => entry.modelId === optionsArg.modelId);
|
|
1675
|
+
if (!model) return refused({ reason: 'model_missing' });
|
|
1676
|
+
stateArg.modelKnown = true;
|
|
1677
|
+
return { kind: 'candidate', model, limits: limitsArg, spendLimit };
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
/**
|
|
1681
|
+
* A Claude account or an Anthropic API key for one of Claude Code's requests, or an Anthropic API key for one of
|
|
1682
|
+
* Flex's, chosen as `acquire` chooses: the pinned session account if it is still eligible, else the least busy account
|
|
1683
|
+
* with the most headroom whose current reading shows it available -- every counted window under its threshold, no
|
|
1684
|
+
* provider retry ahead -- with no cooldown after a provider 429, and which lists the model. A key is judged on its
|
|
1685
|
+
* month's spend (`keyLimits`): at its spend limit it takes no request until the month ends, as an account at its limit,
|
|
1686
|
+
* and a session on it moves; with a limit and no readable spend it is refused (`spend_unread`). A pool that names no
|
|
1687
|
+
* credential places on its Claude accounts first and on its keys only when no Claude account can take the request, for
|
|
1688
|
+
* whatever reason (`inferenceCredentialTiers`).
|
|
1028
1689
|
* `ignoreLimits` serves a request that uses no quota (`count_tokens`) on an account at its limit as well. A pinned
|
|
1029
1690
|
* account that takes no request for a reason of its own -- no usage reading, no model list, no access, or no such
|
|
1030
1691
|
* model -- is refused naming the account and the reason (`InferenceRoutingError.reason`). An unpinned request whose
|
|
1031
|
-
* chosen account's access fails goes to the next candidate.
|
|
1692
|
+
* chosen account's access fails goes to the next candidate. `returnToBetterTier`: a session pinned to a key goes back
|
|
1693
|
+
* to a Claude account as soon as one can take its request, and the route says so (`returned`).
|
|
1032
1694
|
*
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
1035
|
-
*
|
|
1036
|
-
*
|
|
1037
|
-
*
|
|
1695
|
+
* An account held with a payment problem -- Anthropic answered a request on it with a billing error (HTTP 402), or on
|
|
1696
|
+
* a key with its organization's spend cap (HTTP 429 `enforced_spend_limit_reached`), as its usage reading says or this
|
|
1697
|
+
* proxy saw (`reportPaymentProblem`) -- takes no request, `count_tokens` included, and is refused for the reason
|
|
1698
|
+
* `payment_problem`. So is an account of `exclude` that `excludedBecause` names with a payment problem: the request
|
|
1699
|
+
* left it for one. Neither counts as at its limit: when such accounts are all that is left the refusal is a 402
|
|
1700
|
+
* `payment_problem` naming them, and while another account is merely at its limit the 429 stands. Any other account
|
|
1701
|
+
* of `exclude` that `excludedBecause` names counts as its reason counts on the account's own judgement: a key whose
|
|
1702
|
+
* spend could not be read is named in the 503, an account without the model leads to the 404, and every other reason
|
|
1703
|
+
* to the 503 for want of an account -- never to the 429 of accounts at their limits.
|
|
1038
1704
|
*/
|
|
1039
1705
|
public async acquireClaude(modelIdArg: string, policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
|
|
1040
|
-
optionsArg:
|
|
1041
|
-
ignoreLimits?: boolean } = {}): Promise<IInferenceClaudeRoute> {
|
|
1706
|
+
optionsArg: IInferenceClaudeAcquireOptions = {}): Promise<IInferenceClaudeRoute> {
|
|
1042
1707
|
if (policyArg.provider !== 'anthropic') throw new Error('A Claude route belongs to a Claude pool.');
|
|
1043
|
-
const
|
|
1044
|
-
let exhausted = false;
|
|
1045
|
-
let reserved = false;
|
|
1046
|
-
let modelKnown = false;
|
|
1047
|
-
let unavailable = false;
|
|
1048
|
-
let earliestReset = Infinity;
|
|
1049
|
-
let accountLimit: TInferenceAccountLimit | undefined;
|
|
1050
|
-
/** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
|
|
1051
|
-
let refusal: IInferenceAccountUnavailable | undefined;
|
|
1052
|
-
const exhaustedPlans = new Set<string>();
|
|
1053
|
-
const payment: string[] = [];
|
|
1054
|
-
const { accounts } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
|
|
1708
|
+
const { accounts, tiers } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
|
|
1055
1709
|
if (policyArg.credential === 'api_key' && !accounts.length) throw new InferenceRoutingError(503, 'no_eligible_accounts',
|
|
1056
1710
|
'No Anthropic API key is added, and this pool runs on API keys alone, never on a Claude sign-in. Add one with '
|
|
1057
1711
|
+ '`authswitch account add anthropic-api-key`.');
|
|
1058
|
-
|
|
1712
|
+
const pinned = accounts.find((account) => account.id === optionsArg.accountId);
|
|
1713
|
+
if (pinned && optionsArg.returnToBetterTier && !optionsArg.ignoreLimits) {
|
|
1714
|
+
// The session's account is of a later tier: an account of a better one that can take the request takes it, as a
|
|
1715
|
+
// new session would be placed. When none can, the session stays where it is, judged as before.
|
|
1716
|
+
const better = accounts.filter((account) => tiers.indexOf(account.credential) < tiers.indexOf(pinned.credential));
|
|
1717
|
+
if (better.length) {
|
|
1718
|
+
try {
|
|
1719
|
+
const route = await this.routeClaude(modelIdArg, policyArg, better, tiers, signalArg, {
|
|
1720
|
+
...(optionsArg.exclude ? { exclude: optionsArg.exclude } : {}),
|
|
1721
|
+
...(optionsArg.excludedBecause ? { excludedBecause: optionsArg.excludedBecause } : {}),
|
|
1722
|
+
...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }) });
|
|
1723
|
+
return { ...route, returned: true };
|
|
1724
|
+
} catch (errorArg) {
|
|
1725
|
+
signalArg.throwIfAborted();
|
|
1726
|
+
if (!(errorArg instanceof InferenceRoutingError)) throw errorArg;
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
return this.routeClaude(modelIdArg, policyArg, accounts, tiers, signalArg, optionsArg);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
/** `acquireClaude` among `accountsArg`, the pool's accounts or those of its better tiers. */
|
|
1734
|
+
private async routeClaude(modelIdArg: string, policyArg: IInferenceRoutingPolicy, accountsArg: readonly IInferenceAccount[],
|
|
1735
|
+
tiersArg: readonly TInferenceCredential[], signalArg: AbortSignal,
|
|
1736
|
+
optionsArg: IInferenceClaudeAcquireOptions): Promise<IInferenceClaudeRoute> {
|
|
1737
|
+
const candidates: ICandidate<TInferenceClaudeModel>[] = [];
|
|
1738
|
+
const selectionState = emptySelection(accountsArg.length, optionsArg.accountId !== undefined);
|
|
1739
|
+
/** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
|
|
1740
|
+
let refusal: IInferenceAccountUnavailable | undefined;
|
|
1741
|
+
for (const account of accountsArg) {
|
|
1059
1742
|
signalArg.throwIfAborted();
|
|
1060
1743
|
if (optionsArg.accountId && optionsArg.accountId !== account.id) continue;
|
|
1061
1744
|
if (optionsArg.exclude?.has(account.id)) {
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
const reset = this.cooldowns.get(account.id)?.until ?? 0;
|
|
1065
|
-
if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
|
|
1745
|
+
// An account left for a reason of its own counts as that reason counts on its own judgement.
|
|
1746
|
+
this.countLeft(selectionState, account, optionsArg.excludedBecause?.get(account.id));
|
|
1066
1747
|
continue;
|
|
1067
1748
|
}
|
|
1068
1749
|
try {
|
|
1069
1750
|
// A usage read the account service did not answer leaves the account as unread as a reading it cannot route on.
|
|
1070
|
-
|
|
1751
|
+
// A key's usage is its month's spend, with the payment problem the account service holds on it.
|
|
1752
|
+
const usage = await this.options.usage(account.id, signalArg)
|
|
1071
1753
|
.catch(() => { signalArg.throwIfAborted(); return undefined; });
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
if (
|
|
1080
|
-
const
|
|
1081
|
-
|
|
1082
|
-
if (!limits) {
|
|
1083
|
-
unavailable = true;
|
|
1084
|
-
refusal = { reason: 'usage_unread', retryAt: usage?.nextAttemptAt ?? null };
|
|
1085
|
-
this.noteAccount(account.id, refusal);
|
|
1086
|
-
continue;
|
|
1087
|
-
}
|
|
1088
|
-
if (this.noted.get(account.id)?.unavailable.reason === 'usage_unread') this.noteAccount(account.id, null);
|
|
1089
|
-
if (policyArg.plans && !policyArg.plans.includes(limits.plan)) continue;
|
|
1090
|
-
// Limits and cool-down are judged before any model list is read: an account that cannot take the request now
|
|
1091
|
-
// is never read for it. Its last model list, if one is held, says whether it would serve the model; without
|
|
1092
|
-
// one it counts as limited, so a session on it moves rather than waiting for a list.
|
|
1093
|
-
const cooldown = cooldownUntil(this.cooldowns.get(account.id), usage);
|
|
1094
|
-
if (!optionsArg.ignoreLimits && (limits.state !== 'available' || cooldown > this.now())) {
|
|
1095
|
-
const held = this.heldClaudeCatalog(account);
|
|
1096
|
-
if (held && !held.some((entry) => entry.modelId === modelIdArg)) { refusal = { reason: 'model_missing' }; continue; }
|
|
1097
|
-
if (held) modelKnown = true;
|
|
1098
|
-
exhausted = true;
|
|
1099
|
-
reserved ||= limits.state === 'reserved';
|
|
1100
|
-
accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
|
|
1101
|
-
exhaustedPlans.add(limits.plan);
|
|
1102
|
-
const reset = Math.max(cooldown, limits.resetAt ?? 0);
|
|
1103
|
-
if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
|
|
1104
|
-
continue;
|
|
1105
|
-
}
|
|
1106
|
-
const model = (await this.claudeCatalog(account, signalArg)).find((entry) => entry.modelId === modelIdArg);
|
|
1107
|
-
if (!model) { refusal = { reason: 'model_missing' }; continue; }
|
|
1108
|
-
modelKnown = true;
|
|
1109
|
-
// A reading the router does not route on (`usage_unavailable`), and a key's absent one, score nothing.
|
|
1754
|
+
const apiKey = account.credential === 'api_key';
|
|
1755
|
+
const judged = await this.judgeListed(selectionState, account, usage, apiKey ? keyLimits(usage, account, this.now())
|
|
1756
|
+
: usage ? claudeLimits(usage, account, this.now()) : undefined, {
|
|
1757
|
+
modelId: modelIdArg, lists: this.claudeCatalogs, read: (accountArg, innerSignalArg) => this.claudeCatalog(accountArg, innerSignalArg),
|
|
1758
|
+
...(policyArg.plans ? { plans: policyArg.plans } : {}),
|
|
1759
|
+
...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }),
|
|
1760
|
+
...(optionsArg.ignoreLimits ? { ignoreLimits: true } : {}) }, signalArg);
|
|
1761
|
+
if (judged?.kind !== 'candidate') { if (judged) refusal = judged.unavailable; continue; }
|
|
1762
|
+
const { model, limits, spendLimit } = judged;
|
|
1763
|
+
// A reading the router does not route on (`usage_unavailable`), and a key's without a spend reading, score nothing.
|
|
1110
1764
|
const placement = accountPlacement(limits.limit ? usage!.reading : null, account, this.now());
|
|
1111
|
-
candidates.push({ account,
|
|
1112
|
-
|
|
1113
|
-
|
|
1765
|
+
candidates.push({ account, tier: tiersArg.indexOf(account.credential), model,
|
|
1766
|
+
reading: apiKey ? (limits.limit ? usage!.reading : null) : usage?.reading ?? null,
|
|
1767
|
+
ignoreLimits: optionsArg.ignoreLimits === true, plan: limits.plan, headroom: limits.headroom,
|
|
1768
|
+
score: placement.score, lastResort: placement.eligibility === 'last_resort', spendLimit });
|
|
1114
1769
|
} catch (errorArg) {
|
|
1115
1770
|
signalArg.throwIfAborted();
|
|
1116
|
-
unavailable = true;
|
|
1117
|
-
if (errorArg instanceof
|
|
1771
|
+
selectionState.unavailable = true;
|
|
1772
|
+
if (errorArg instanceof ModelListError) refusal = errorArg.unavailable;
|
|
1118
1773
|
}
|
|
1119
1774
|
}
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1775
|
+
if (optionsArg.accountId !== undefined && refusal) {
|
|
1776
|
+
selectionState.refusal = { accountId: optionsArg.accountId, unavailable: refusal };
|
|
1777
|
+
}
|
|
1123
1778
|
// The chosen account's access is resolved before anything is sent. One that fails takes no request, as a failed
|
|
1124
1779
|
// access found reading its model list does: it is recorded and noted for the account, and the request goes to the
|
|
1125
1780
|
// next candidate -- a pinned request has none, so it is refused naming the account and the reason.
|
|
@@ -1136,7 +1791,7 @@ export class InferenceRouter {
|
|
|
1136
1791
|
release();
|
|
1137
1792
|
signalArg.throwIfAborted();
|
|
1138
1793
|
const failure: IInferenceAccountUnavailable = { reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' };
|
|
1139
|
-
this.
|
|
1794
|
+
this.holdListFailure(this.claudeCatalogs, selected.account, failure);
|
|
1140
1795
|
candidates.splice(candidates.indexOf(selected), 1);
|
|
1141
1796
|
selectionState.unavailable = true;
|
|
1142
1797
|
if (selectionState.pinned) selectionState.refusal = { accountId: selected.account.id, unavailable: failure };
|
|
@@ -1167,7 +1822,8 @@ export class InferenceRouter {
|
|
|
1167
1822
|
...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}),
|
|
1168
1823
|
});
|
|
1169
1824
|
return { provider: 'anthropic', credential, accountId: id, plan: selected.plan, score: selected.score,
|
|
1170
|
-
model: selected.model, connection,
|
|
1825
|
+
model: selected.model, connection, returned: false, loginId: selected.account.loginId, spendLimit: selected.spendLimit,
|
|
1826
|
+
release };
|
|
1171
1827
|
} catch (errorArg) {
|
|
1172
1828
|
release();
|
|
1173
1829
|
throw errorArg;
|
|
@@ -1189,9 +1845,9 @@ export class InferenceRouter {
|
|
|
1189
1845
|
}
|
|
1190
1846
|
|
|
1191
1847
|
/**
|
|
1192
|
-
* The candidate a request goes to
|
|
1193
|
-
* none. A pinned request has its session's
|
|
1194
|
-
* An account at its reserve is refused like one at its limit, and the refusal names the reserve. A pinned account
|
|
1848
|
+
* The candidate a request goes to -- of the best credential tier, then by the placement order
|
|
1849
|
+
* (`authSwitchPlacementOrder`) -- or the refusal that says why there is none. A pinned request has its session's
|
|
1850
|
+
* account as its one candidate, so the order only ever places new sessions. An account at its reserve is refused like one at its limit, and the refusal names the reserve. A pinned account
|
|
1195
1851
|
* that takes no request for a reason of its own (`refusal`) is refused with that account and reason.
|
|
1196
1852
|
*/
|
|
1197
1853
|
private select<TModel>(candidatesArg: ICandidate<TModel>[], stateArg: ISelectionState): ICandidate<TModel> {
|
|
@@ -1213,51 +1869,70 @@ export class InferenceRouter {
|
|
|
1213
1869
|
candidate.score = placement.score;
|
|
1214
1870
|
return true;
|
|
1215
1871
|
});
|
|
1872
|
+
// The best credential tier first, then the placement order within it.
|
|
1216
1873
|
const order = plugins.authswitchPlacement.authSwitchPlacementOrder((accountIdArg) => this.active.get(accountIdArg) ?? 0);
|
|
1217
|
-
candidatesArg.sort((left, right) => order({ id: left.account.id, score: left.score,
|
|
1218
|
-
headroom: left.headroom }, { id: right.account.id, score: right.score,
|
|
1219
|
-
headroom: right.headroom }));
|
|
1874
|
+
candidatesArg.sort((left, right) => left.tier - right.tier || order({ id: left.account.id, score: left.score,
|
|
1875
|
+
lastResort: left.lastResort, headroom: left.headroom }, { id: right.account.id, score: right.score,
|
|
1876
|
+
lastResort: right.lastResort, headroom: right.headroom }));
|
|
1220
1877
|
const selected = candidatesArg[0];
|
|
1221
1878
|
if (selected) return selected;
|
|
1222
|
-
|
|
1879
|
+
// A refusal names the API keys held by their spend limit, and those with a limit whose spend could not be read.
|
|
1880
|
+
const named = (messageArg: string, sentenceArg: string) => sentenceArg ? `${messageArg} ${sentenceArg}` : messageArg;
|
|
1881
|
+
if (stateArg.exhausted) throw new InferenceRoutingError(429, 'accounts_exhausted', named(
|
|
1223
1882
|
stateArg.pinned
|
|
1224
1883
|
? `The account that owns this conversation state is at its ${stateArg.reserved ? 'reserve' : 'limit'}.`
|
|
1225
1884
|
: `All eligible accounts are at their limits${stateArg.reserved ? ' or their reserve' : ''}.`,
|
|
1885
|
+
inferenceSpendLimitText(stateArg.spendLimited)),
|
|
1226
1886
|
Number.isFinite(stateArg.earliestReset) ? Math.max(1, Math.ceil((stateArg.earliestReset - this.now()) / 1000)) : undefined,
|
|
1227
|
-
[...stateArg.exhaustedPlans], stateArg.pinned ? stateArg.accountLimit : undefined
|
|
1887
|
+
[...stateArg.exhaustedPlans], stateArg.pinned ? stateArg.accountLimit : undefined, undefined, undefined,
|
|
1888
|
+
stateArg.spendLimited);
|
|
1228
1889
|
if (stateArg.pinned && stateArg.refusal) {
|
|
1229
1890
|
const { accountId, unavailable } = stateArg.refusal;
|
|
1230
1891
|
throw new InferenceRoutingError(503, 'no_eligible_accounts',
|
|
1231
1892
|
`The account that owns this conversation state (${accountId}) takes no request: ${unavailableText(unavailable)}.`,
|
|
1232
|
-
undefined, [], undefined, accountId, unavailable
|
|
1893
|
+
undefined, [], undefined, accountId, unavailable);
|
|
1233
1894
|
}
|
|
1234
1895
|
if (stateArg.payment.length && !stateArg.unavailable) {
|
|
1235
|
-
throw new InferenceRoutingError(402, 'payment_problem',
|
|
1236
|
-
+ ` Claude ${stateArg.payment.length === 1 ? 'account' : 'accounts'} ${stateArg.payment.join(', ')}: a payment`
|
|
1237
|
-
+ ' problem with the subscription. Settle the billing, then re-check the account in AGL Accounts or with'
|
|
1238
|
-
+ ' `authswitch account recheck <account-id>`.');
|
|
1896
|
+
throw new InferenceRoutingError(402, 'payment_problem', paymentRefusalText(stateArg.payment));
|
|
1239
1897
|
}
|
|
1898
|
+
if (stateArg.unpriced.labels.length && !stateArg.unavailable) {
|
|
1899
|
+
const keys = [...new Set(stateArg.unpriced.labels)];
|
|
1900
|
+
throw new InferenceRoutingError(400, 'model_unpriced', `The API ${keys.length === 1 ? 'key' : 'keys'} ${keys.join(', ')} `
|
|
1901
|
+
+ `${keys.length === 1 ? 'has' : 'have'} a monthly spend limit set in AGL, and AGL's price table has no price for `
|
|
1902
|
+
+ `${stateArg.unpriced.what ?? 'this request'}, so the cost of the answer could not be counted against the limit. `
|
|
1903
|
+
+ 'Send it with a model, service tier, speed and tools the price table prices, or remove the spend limit in AGL '
|
|
1904
|
+
+ 'Accounts.');
|
|
1905
|
+
}
|
|
1906
|
+
const keysNamed = (labelsArg: readonly string[], whatArg: string): string => labelsArg.length
|
|
1907
|
+
? `The spend of the API ${labelsArg.length === 1 ? 'key' : 'keys'} ${labelsArg.join(', ')} ${whatArg}, and AGL `
|
|
1908
|
+
+ 'places nothing on a key with a monthly spend limit until it can.' : '';
|
|
1240
1909
|
if (!stateArg.accounts || stateArg.unavailable || stateArg.pinned) throw new InferenceRoutingError(503, 'no_eligible_accounts',
|
|
1241
|
-
'No eligible account has current usage, an available sign-in, and access to the requested model.'
|
|
1910
|
+
named('No eligible account has current usage, an available sign-in, and access to the requested model.',
|
|
1911
|
+
[keysNamed(stateArg.spendUnread, 'could not be read'),
|
|
1912
|
+
keysNamed(stateArg.spendUnrecorded, 'could not be recorded')]
|
|
1913
|
+
.filter(Boolean).join(' ')));
|
|
1242
1914
|
throw new InferenceRoutingError(404, 'model_not_available', stateArg.modelKnown
|
|
1243
1915
|
? 'No eligible account supports the requested model options.' : 'The requested model is not available to this account pool.');
|
|
1244
1916
|
}
|
|
1245
1917
|
|
|
1246
1918
|
/**
|
|
1247
1919
|
* The accounts as `agl proxy status` shows them: each judged as placement would judge it on the reading `heldArg`
|
|
1248
|
-
* answers for it, which the caller read beforehand -- an
|
|
1920
|
+
* answers for it, which the caller read beforehand -- an API key on its month's spend (`keyLimits`), in dollars -- with
|
|
1249
1921
|
* its cooldown, the requests it serves now, its lifecycle and where a new session of an unrestricted client of its
|
|
1250
|
-
*
|
|
1922
|
+
* provider would go (`placementRank`): a client that names no credential, so a provider's subscriptions rank before its
|
|
1923
|
+
* keys (`inferenceCredentialTiers`).
|
|
1251
1924
|
*/
|
|
1252
1925
|
public accountStatus(accountsArg: readonly IInferenceAccount[],
|
|
1253
1926
|
heldArg: (accountIdArg: string) => plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): IInferenceAccountStatus[] {
|
|
1254
1927
|
const now = this.now();
|
|
1255
1928
|
const statuses = accountsArg.map((account): IInferenceAccountStatus => {
|
|
1256
1929
|
const apiKey = account.credential === 'api_key';
|
|
1257
|
-
|
|
1258
|
-
const
|
|
1930
|
+
// A key's usage is its month's spend, with the payment problem the account service holds on it.
|
|
1931
|
+
const held = heldArg(account.id);
|
|
1932
|
+
const limits = apiKey ? keyLimits(held, account, now)
|
|
1259
1933
|
: held ? (account.providerId === 'anthropic' ? claudeLimits : chatGptLimits)(held, account, now) : undefined;
|
|
1260
|
-
|
|
1934
|
+
// A key's spend reading says nothing of the provider's rate limits, so only its own end ends a key's cool-down.
|
|
1935
|
+
const cooldown = cooldownUntil(this.cooldowns.get(account.id), apiKey ? undefined : held);
|
|
1261
1936
|
const placement = accountPlacement(limits?.limit ? held!.reading : null, account, now);
|
|
1262
1937
|
const base = { id: account.id, label: account.label, provider: account.providerId, credential: account.credential,
|
|
1263
1938
|
maxUsagePercent: account.maxUsagePercent, reserveLiftUntil: account.reserveLiftUntil, inFlight: this.active.get(account.id) ?? 0,
|
|
@@ -1270,39 +1945,43 @@ export class InferenceRouter {
|
|
|
1270
1945
|
.some((windowArg) => windowArg.estimatedUsedPercent !== null),
|
|
1271
1946
|
lastReadRateLimited: held?.lastReadError?.rateLimited ?? false };
|
|
1272
1947
|
// A payment problem refuses the account whatever its reading says, as a request finds it.
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(
|
|
1948
|
+
const unpaid = this.paymentProblem(account.id, held);
|
|
1949
|
+
if (unpaid) {
|
|
1950
|
+
return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(unpaid)})`, unavailable: unpaid,
|
|
1276
1951
|
plan: limits?.plan || null, headroom: null, crossing: [], resetAt: null };
|
|
1277
1952
|
}
|
|
1278
|
-
|
|
1279
|
-
|
|
1953
|
+
const unrecorded = apiKey && limits && keyHasSpendLimit(held, account) && this.options.spend?.held(account.id) === true;
|
|
1954
|
+
if (!limits || unrecorded) {
|
|
1955
|
+
const unavailable: IInferenceAccountUnavailable = unrecorded ? { reason: 'spend_unrecorded' }
|
|
1956
|
+
: apiKey ? { reason: 'spend_unread' } : { reason: 'usage_unread', retryAt: held?.nextAttemptAt ?? null };
|
|
1280
1957
|
return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(unavailable)})`, unavailable,
|
|
1281
1958
|
plan: null, headroom: null, crossing: [], resetAt: null };
|
|
1282
1959
|
}
|
|
1283
1960
|
// An account whose model list a request could not read takes no request until it is read again; one at its limit
|
|
1284
1961
|
// or cooling down is judged on that alone, as a request judges it.
|
|
1285
|
-
const refusal =
|
|
1962
|
+
const refusal = this.listRefusal(account);
|
|
1286
1963
|
const state = limits.state === 'available' && cooldown > now ? 'cooling'
|
|
1287
1964
|
: limits.state === 'available' && refusal ? 'unavailable' : limits.state;
|
|
1288
1965
|
return { ...base, state, ...(state === 'unavailable' && refusal ? { unavailable: refusal } : {}),
|
|
1289
1966
|
stateText: placement.lifecycle === 'lapsed' ? 'lapsed (its subscription ended; never routed to again)'
|
|
1290
1967
|
: state === 'cooling' ? 'cooling down after a provider 429'
|
|
1291
1968
|
: state === 'unavailable' && refusal ? `unavailable (${unavailableText(refusal)})`
|
|
1292
|
-
:
|
|
1293
|
-
:
|
|
1969
|
+
: apiKey ? keyStateText(limits, held, now)
|
|
1970
|
+
: limits.limit ? plugins.authswitchUsageLimit.authSwitchUsageLimitText(limits.limit)
|
|
1971
|
+
: 'available (no usage reported)',
|
|
1294
1972
|
plan: limits.plan || null, headroom: limits.limit?.headroom ?? null, crossing: limits.limit?.crossing ?? [],
|
|
1295
1973
|
resetAt: state === 'cooling' ? new Date(cooldown).toISOString() : limits.limit?.resetAt ?? null };
|
|
1296
1974
|
});
|
|
1297
|
-
// Each
|
|
1975
|
+
// Each provider's accounts that take a new session now, in the order placement takes them: the best credential tier
|
|
1976
|
+
// first.
|
|
1298
1977
|
const order = plugins.authswitchPlacement.authSwitchPlacementOrder((accountIdArg) => this.active.get(accountIdArg) ?? 0);
|
|
1299
|
-
const
|
|
1300
|
-
for (const
|
|
1301
|
-
statuses.filter((statusArg) =>
|
|
1978
|
+
const tiers = inferenceCredentialTiers(undefined);
|
|
1979
|
+
for (const provider of new Set(statuses.map((statusArg) => statusArg.provider))) {
|
|
1980
|
+
statuses.filter((statusArg) => statusArg.provider === provider && statusArg.state === 'available'
|
|
1302
1981
|
&& statusArg.lifecycle !== 'lapsed')
|
|
1303
|
-
.map((statusArg) => ({ status: statusArg, rank: { id: statusArg.id,
|
|
1304
|
-
lastResort: statusArg.lastResort, headroom: statusArg.headroom ?? unknownHeadroom } }))
|
|
1305
|
-
.sort((left, right) => order(left.rank, right.rank))
|
|
1982
|
+
.map((statusArg) => ({ status: statusArg, tier: tiers.indexOf(statusArg.credential), rank: { id: statusArg.id,
|
|
1983
|
+
score: statusArg.placementScore, lastResort: statusArg.lastResort, headroom: statusArg.headroom ?? unknownHeadroom } }))
|
|
1984
|
+
.sort((left, right) => left.tier - right.tier || order(left.rank, right.rank))
|
|
1306
1985
|
.forEach((entryArg, indexArg) => { entryArg.status.placementRank = indexArg + 1; });
|
|
1307
1986
|
}
|
|
1308
1987
|
return statuses;
|
|
@@ -1333,21 +2012,25 @@ export class InferenceRouter {
|
|
|
1333
2012
|
}
|
|
1334
2013
|
|
|
1335
2014
|
/**
|
|
1336
|
-
*
|
|
1337
|
-
*
|
|
1338
|
-
* account's usage carries it, and that decides from then on
|
|
1339
|
-
*
|
|
1340
|
-
*
|
|
2015
|
+
* A provider refused a request on the account for a payment problem (`TInferencePaymentCause`): the account is out of
|
|
2016
|
+
* rotation at once, and the refusal it now carries is answered. The problem is reported to the account service
|
|
2017
|
+
* (`reportPaymentProblem`); once it holds the problem the account's usage carries it, and that decides from then on --
|
|
2018
|
+
* until a re-check clears it, or a spend cap's month ends -- so the router's own mark is dropped. When the service
|
|
2019
|
+
* cannot be told -- it is not running, or an older release that does not know the report or the account's kind --
|
|
2020
|
+
* the mark holds, a spend cap until 00:00 UTC on the first day of the next month (`authSwitchSpendPeriod`), anything
|
|
2021
|
+
* else for as long as the proxy runs, and the failure is reported once.
|
|
1341
2022
|
*/
|
|
1342
|
-
public reportPaymentProblem(accountIdArg: string):
|
|
1343
|
-
if (this.apiKeyAccounts.has(accountIdArg)) return;
|
|
2023
|
+
public reportPaymentProblem(accountIdArg: string, causeArg: TInferencePaymentCause): IInferenceAccountUnavailable {
|
|
1344
2024
|
const at = this.now();
|
|
1345
|
-
|
|
1346
|
-
|
|
2025
|
+
const mark: IPaymentMark = { at, cause: causeArg,
|
|
2026
|
+
until: causeArg === 'spend_cap' ? Date.parse(plugins.authswitchSpend.authSwitchSpendPeriod(at).end) : null };
|
|
2027
|
+
this.paymentMarks.set(accountIdArg, mark);
|
|
2028
|
+
const unavailable = paymentUnavailable(mark);
|
|
2029
|
+
this.noteAccount(accountIdArg, unavailable);
|
|
1347
2030
|
const report = this.options.reportPaymentProblem;
|
|
1348
|
-
if (!report) return;
|
|
1349
|
-
void (async () => report(accountIdArg, new Date(at).toISOString()))().then(() => {
|
|
1350
|
-
if (this.paymentMarks.get(accountIdArg) ===
|
|
2031
|
+
if (!report) return unavailable;
|
|
2032
|
+
void (async () => report(accountIdArg, new Date(at).toISOString(), causeArg))().then(() => {
|
|
2033
|
+
if (this.paymentMarks.get(accountIdArg) === mark) this.paymentMarks.delete(accountIdArg);
|
|
1351
2034
|
}, (errorArg: unknown) => {
|
|
1352
2035
|
if (this.paymentReportsFailed.has(accountIdArg)) return;
|
|
1353
2036
|
this.paymentReportsFailed.add(accountIdArg);
|
|
@@ -1357,12 +2040,24 @@ export class InferenceRouter {
|
|
|
1357
2040
|
error: (errorArg instanceof Error ? `${errorArg.name}: ${errorArg.message}` : String(errorArg)).replace(/\s+/gu, ' ') });
|
|
1358
2041
|
} catch {}
|
|
1359
2042
|
});
|
|
2043
|
+
return unavailable;
|
|
1360
2044
|
}
|
|
1361
2045
|
|
|
1362
|
-
/**
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
2046
|
+
/**
|
|
2047
|
+
* The payment problem an account is held with, as a refusal: by this proxy's own mark while it holds -- a spend cap's
|
|
2048
|
+
* ends at its time -- or by the account's usage reading, which carries it once the account service holds it, with its
|
|
2049
|
+
* cause (absent: `billing`) and, for a spend cap, when it ends; one whose spend cap ended holds nothing
|
|
2050
|
+
* (`authSwitchPaymentProblemHeld`, the rule the account service holds it by).
|
|
2051
|
+
*/
|
|
2052
|
+
private paymentProblem(accountIdArg: string,
|
|
2053
|
+
usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): IInferenceAccountUnavailable | undefined {
|
|
2054
|
+
const now = this.now();
|
|
2055
|
+
const mark = this.paymentMarks.get(accountIdArg);
|
|
2056
|
+
if (mark && (mark.until === null || mark.until > now)) return paymentUnavailable(mark);
|
|
2057
|
+
if (mark) this.paymentMarks.delete(accountIdArg);
|
|
2058
|
+
const held = usageArg?.paymentProblem;
|
|
2059
|
+
return held === undefined || !plugins.authswitchSubscription.authSwitchPaymentProblemHeld(held, now) ? undefined
|
|
2060
|
+
: { reason: 'payment_problem', cause: held.cause ?? 'billing', ...(held.until === undefined ? {} : { retryAt: held.until }) };
|
|
1366
2061
|
}
|
|
1367
2062
|
|
|
1368
2063
|
/** A provider answer's `Retry-After`, in seconds from now, in either of its forms. */
|
|
@@ -1379,11 +2074,28 @@ export class InferenceRouter {
|
|
|
1379
2074
|
this.options.tokens?.(accountIdArg, usageArg);
|
|
1380
2075
|
}
|
|
1381
2076
|
|
|
2077
|
+
/**
|
|
2078
|
+
* Takes the spend of an answer an API key served on the login `loginIdArg` (`inferenceAnthropicSpend`): it counts
|
|
2079
|
+
* against the key's limit from now on and is reported to the account service (`spend`).
|
|
2080
|
+
*/
|
|
2081
|
+
public recordSpend(accountIdArg: string, loginIdArg: string | null, recordArg: IInferenceSpendRecord): void {
|
|
2082
|
+
this.options.spend?.record(accountIdArg, loginIdArg, recordArg);
|
|
2083
|
+
}
|
|
2084
|
+
|
|
2085
|
+
/**
|
|
2086
|
+
* Says that an answer an API key served ended before its provider stated its usage, so nothing of it counts against
|
|
2087
|
+
* the key's limit (`IInferenceSpendLedger.uncounted`).
|
|
2088
|
+
*/
|
|
2089
|
+
public uncountedSpend(accountIdArg: string, modelArg: string): void {
|
|
2090
|
+
this.options.spend?.uncounted(accountIdArg, modelArg);
|
|
2091
|
+
}
|
|
2092
|
+
|
|
1382
2093
|
/**
|
|
1383
2094
|
* A provider 429 on the account. Only a definitive pre-stream refusal permits another account to be tried. The
|
|
1384
2095
|
* account cools down for the provider's `Retry-After`, at most a day; without one, or with one already due, for 60
|
|
1385
2096
|
* seconds or until the account service holds a usage read made after the 429 (see `cooldownUntil`), which the 429
|
|
1386
|
-
* asks it to make at once. An API key, whose
|
|
2097
|
+
* asks it to make at once. An API key, whose spend reading says nothing of the provider's rate limits, cools down
|
|
2098
|
+
* the 60 seconds.
|
|
1387
2099
|
*/
|
|
1388
2100
|
public reportQuota(accountIdArg: string, retryAfterSecondsArg?: number): void {
|
|
1389
2101
|
const explicit = Number.isFinite(retryAfterSecondsArg) && retryAfterSecondsArg! > 0;
|