agl 40.4.0 → 40.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/changelog.md +33 -0
  2. package/dist_serve/bundle.js +2228 -2177
  3. package/dist_serve/bundle.js.map +2 -2
  4. package/dist_ts/00_commitinfo_data.js +1 -1
  5. package/dist_ts/aglapi/aglapisurface.document.js +2 -2
  6. package/dist_ts/classes.accountsactions.d.ts +11 -1
  7. package/dist_ts/classes.accountsactions.js +59 -2
  8. package/dist_ts/classes.controller.js +5 -5
  9. package/dist_ts/classes.flexaccountaccess.js +3 -2
  10. package/dist_ts/classes.harnessreleases.d.ts +14 -4
  11. package/dist_ts/classes.harnessreleases.js +28 -4
  12. package/dist_ts/classes.inferencegateway.d.ts +28 -3
  13. package/dist_ts/classes.inferencegateway.js +177 -61
  14. package/dist_ts/classes.inferenceplacement.d.ts +5 -3
  15. package/dist_ts/classes.inferenceplacement.js +2 -2
  16. package/dist_ts/classes.inferencerouter.d.ts +349 -91
  17. package/dist_ts/classes.inferencerouter.js +804 -297
  18. package/dist_ts/classes.inferenceservice.d.ts +14 -2
  19. package/dist_ts/classes.inferenceservice.js +53 -14
  20. package/dist_ts/classes.inferencespend.d.ts +339 -0
  21. package/dist_ts/classes.inferencespend.js +815 -0
  22. package/dist_ts/classes.inferenceusagecache.d.ts +8 -0
  23. package/dist_ts/classes.inferenceusagecache.js +9 -5
  24. package/dist_ts/classes.inferenceusagetap.d.ts +50 -9
  25. package/dist_ts/classes.inferenceusagetap.js +185 -31
  26. package/dist_ts/classes.proxycontrol.d.ts +3 -2
  27. package/dist_ts/classes.proxycontrol.js +1 -2
  28. package/dist_ts/classes.proxydaemon.js +3 -2
  29. package/dist_ts/classes.proxystore.d.ts +6 -0
  30. package/dist_ts/classes.proxystore.js +11 -2
  31. package/dist_ts/fleet/classes.claudeworkers.d.ts +5 -4
  32. package/dist_ts/fleet/classes.claudeworkers.js +10 -9
  33. package/dist_ts/functions.claudecoderelease.d.ts +12 -6
  34. package/dist_ts/functions.claudecoderelease.js +13 -7
  35. package/dist_ts/functions.inferencecli.js +12 -8
  36. package/dist_ts/interfaces.auth.d.ts +1 -1
  37. package/dist_ts/interfaces.auth.js +3 -1
  38. package/dist_ts/plugins.d.ts +3 -1
  39. package/dist_ts/plugins.js +7 -2
  40. package/dist_ts_authswitch/authority-apikeys.d.ts +32 -0
  41. package/dist_ts_authswitch/authority-apikeys.js +65 -0
  42. package/dist_ts_authswitch/authority-contract.d.ts +172 -33
  43. package/dist_ts_authswitch/authority-contract.js +54 -1
  44. package/dist_ts_authswitch/authority-overview.js +8 -1
  45. package/dist_ts_authswitch/authority-placement.d.ts +7 -2
  46. package/dist_ts_authswitch/authority-placement.js +10 -3
  47. package/dist_ts_authswitch/authority-runtime-contract.d.ts +2 -2
  48. package/dist_ts_authswitch/authority-spend.d.ts +103 -0
  49. package/dist_ts_authswitch/authority-spend.js +110 -0
  50. package/dist_ts_authswitch/authority-subscription.d.ts +12 -1
  51. package/dist_ts_authswitch/authority-subscription.js +24 -1
  52. package/dist_ts_authswitch/authority-usagelimit.d.ts +7 -4
  53. package/dist_ts_authswitch/authority-usagelimit.js +89 -39
  54. package/dist_ts_authswitch/authority-usagewindows.d.ts +4 -1
  55. package/dist_ts_authswitch/authority-usagewindows.js +41 -16
  56. package/dist_ts_authswitch/classes.authoritybroker.d.ts +30 -19
  57. package/dist_ts_authswitch/classes.authoritybroker.js +152 -61
  58. package/dist_ts_authswitch/classes.authoritycli.d.ts +9 -4
  59. package/dist_ts_authswitch/classes.authoritycli.js +99 -30
  60. package/dist_ts_authswitch/classes.authorityclient.d.ts +24 -7
  61. package/dist_ts_authswitch/classes.authorityclient.js +45 -15
  62. package/dist_ts_authswitch/classes.authoritydaemon.js +39 -12
  63. package/dist_ts_authswitch/classes.authoritydatabase.d.ts +28 -5
  64. package/dist_ts_authswitch/classes.authoritydatabase.js +171 -9
  65. package/dist_ts_authswitch/classes.authorityimport.js +2 -2
  66. package/dist_ts_authswitch/classes.authoritymodels.d.ts +71 -3
  67. package/dist_ts_authswitch/classes.authoritymodels.js +249 -20
  68. package/dist_ts_authswitch/classes.authorityregistry.js +4 -2
  69. package/dist_ts_authswitch/classes.authorityusage.d.ts +69 -7
  70. package/dist_ts_authswitch/classes.authorityusage.js +118 -4
  71. package/dist_ts_authswitch/classes.claudenative.d.ts +23 -8
  72. package/dist_ts_authswitch/classes.claudenative.js +49 -21
  73. package/dist_ts_authswitch/classes.claudesignin.d.ts +14 -2
  74. package/dist_ts_authswitch/classes.claudesignin.js +72 -23
  75. package/dist_ts_authswitch/classes.codexmanaged.d.ts +8 -0
  76. package/dist_ts_authswitch/classes.codexmanaged.js +33 -18
  77. package/dist_ts_authswitch/classes.harnessprocesses.d.ts +8 -0
  78. package/dist_ts_authswitch/classes.harnessprocesses.js +41 -9
  79. package/dist_ts_authswitch/testsupport.d.ts +24 -5
  80. package/dist_ts_authswitch/testsupport.js +56 -12
  81. package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.d.ts +12 -0
  82. package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.js +21 -0
  83. package/dist_ts_authswitch/ts_migration/index.js +3 -1
  84. package/dist_ts_interfaces/functions.authorityscope.d.ts +18 -7
  85. package/dist_ts_interfaces/functions.authorityscope.js +35 -8
  86. package/dist_ts_interfaces/interfaces.d.ts +1 -1
  87. package/dist_ts_interfaces/interfaces.js +1 -1
  88. package/dist_ts_interfaces/plugins.d.ts +2 -1
  89. package/dist_ts_interfaces/plugins.js +7 -4
  90. package/dist_ts_interfaces/requests.accounts.d.ts +56 -0
  91. package/dist_ts_interfaces/requests.accounts.js +11 -1
  92. package/dist_ts_interfaces/requests.d.ts +2 -2
  93. package/dist_ts_interfaces/requests.js +1 -1
  94. package/docs/authswitch.md +75 -12
  95. package/package.json +5 -5
  96. package/readme.md +199 -22
  97. package/ts/00_commitinfo_data.ts +1 -1
  98. package/ts/aglapi/aglapisurface.document.ts +2 -2
  99. package/ts/classes.accountsactions.ts +75 -1
  100. package/ts/classes.controller.ts +4 -4
  101. package/ts/classes.flexaccountaccess.ts +2 -1
  102. package/ts/classes.harnessreleases.ts +25 -4
  103. package/ts/classes.inferencegateway.ts +184 -64
  104. package/ts/classes.inferenceplacement.ts +6 -4
  105. package/ts/classes.inferencerouter.ts +1026 -314
  106. package/ts/classes.inferenceservice.ts +56 -13
  107. package/ts/classes.inferencespend.ts +870 -0
  108. package/ts/classes.inferenceusagecache.ts +16 -4
  109. package/ts/classes.inferenceusagetap.ts +159 -34
  110. package/ts/classes.proxycontrol.ts +3 -3
  111. package/ts/classes.proxydaemon.ts +2 -1
  112. package/ts/classes.proxystore.ts +11 -1
  113. package/ts/fleet/classes.claudeworkers.ts +9 -8
  114. package/ts/functions.claudecoderelease.ts +13 -6
  115. package/ts/functions.inferencecli.ts +11 -7
  116. package/ts/interfaces.auth.ts +2 -0
  117. package/ts/plugins.ts +7 -0
  118. package/ts_authswitch/authority-apikeys.ts +85 -0
  119. package/ts_authswitch/authority-contract.ts +201 -34
  120. package/ts_authswitch/authority-overview.ts +7 -0
  121. package/ts_authswitch/authority-placement.ts +12 -4
  122. package/ts_authswitch/authority-runtime-contract.ts +2 -2
  123. package/ts_authswitch/authority-spend.ts +168 -0
  124. package/ts_authswitch/authority-subscription.ts +28 -1
  125. package/ts_authswitch/authority-usagelimit.ts +95 -40
  126. package/ts_authswitch/authority-usagewindows.ts +42 -17
  127. package/ts_authswitch/classes.authoritybroker.ts +161 -68
  128. package/ts_authswitch/classes.authoritycli.ts +112 -32
  129. package/ts_authswitch/classes.authorityclient.ts +66 -19
  130. package/ts_authswitch/classes.authoritydaemon.ts +43 -13
  131. package/ts_authswitch/classes.authoritydatabase.ts +178 -8
  132. package/ts_authswitch/classes.authorityimport.ts +1 -1
  133. package/ts_authswitch/classes.authoritymodels.ts +165 -19
  134. package/ts_authswitch/classes.authorityregistry.ts +3 -0
  135. package/ts_authswitch/classes.authorityusage.ts +171 -10
  136. package/ts_authswitch/classes.claudenative.ts +51 -22
  137. package/ts_authswitch/classes.claudesignin.ts +74 -24
  138. package/ts_authswitch/classes.codexmanaged.ts +30 -16
  139. package/ts_authswitch/classes.harnessprocesses.ts +39 -8
  140. package/ts_authswitch/native-contracts.md +82 -21
  141. package/ts_authswitch/readme.md +1 -1
  142. package/ts_authswitch/testsupport.ts +58 -11
  143. package/ts_authswitch/ts_migration/0011_account_api_key_fields.ts +20 -0
  144. package/ts_authswitch/ts_migration/index.ts +2 -0
  145. package/ts_interfaces/functions.authorityscope.ts +43 -8
  146. package/ts_interfaces/interfaces.ts +1 -1
  147. package/ts_interfaces/plugins.ts +6 -3
  148. package/ts_interfaces/requests.accounts.ts +48 -0
  149. package/ts_interfaces/requests.ts +4 -0
  150. package/ts_web/00_commitinfo_data.ts +1 -1
  151. package/ts_web/accounts.plugins.ts +7 -1
  152. package/ts_web/accounts.templates.ts +20 -11
  153. package/ts_web/classes.accountsreadmodel.ts +326 -71
  154. package/ts_web/elements.accounts.ts +219 -20
  155. package/ts_web/elements.codexconnections.ts +3 -1
  156. package/ts_web/plugins.ts +4 -2
@@ -2,6 +2,7 @@ import * as plugins from './plugins.js';
2
2
  import type { TInferenceCredential, TInferenceProvider } from './functions.proxycontract.js';
3
3
  import type { IGatewayAuthorityAccess } from './classes.flexaccountaccess.js';
4
4
  import { FlexServiceError } from './interfaces.flexipc.js';
5
+ import type { IInferenceSpendRecord } from './classes.inferencespend.js';
5
6
 
6
7
  export type TInferenceModel = plugins.openAiProvider.IOpenAiChatGptModel;
7
8
  /** A model a Claude account or an Anthropic API key may use, as the Anthropic API lists it. */
@@ -57,31 +58,52 @@ export const inferenceRetryAfterSeconds = (valueArg: string | null, nowArg: numb
57
58
  return at === undefined ? undefined : Math.max(0, Math.ceil((at - nowArg) / 1000));
58
59
  };
59
60
 
61
+ /**
62
+ * The credential tiers a pool is served from, best first: the one a policy names alone, or with none the provider's
63
+ * subscription sign-ins first and its API keys after them. A key takes a request only when no subscription can take it,
64
+ * for whatever reason, and a session placed on a key returns to a subscription at the next request one can take.
65
+ */
66
+ export const inferenceCredentialTiers = (credentialArg: TInferenceCredential | undefined): readonly TInferenceCredential[] =>
67
+ credentialArg ? [credentialArg] : ['subscription', 'api_key'];
68
+
60
69
  export interface IInferenceAccount {
61
70
  id: string;
62
71
  label: string;
63
72
  revision: string;
64
73
  providerId: TInferenceProvider;
65
- /** What the account authenticates with: a subscription sign-in, or an Anthropic API key. */
74
+ /** What the account authenticates with: a subscription sign-in, or an API key. */
66
75
  credential: TInferenceCredential;
67
76
  /** The account's usage threshold (`IAuthSwitchAccount.maxUsagePercent`); `null` when unset. */
68
77
  maxUsagePercent: number | null;
69
78
  reserveLiftUntil: string | null;
70
79
  /** When the account's subscription ends (`IAuthSwitchAccount.lapsesAt`); `null` while it renews. */
71
80
  lapsesAt: string | null;
81
+ /**
82
+ * An API key's monthly spend limit in micro-dollars (`IAuthSwitchAccount.spendLimitMicroUsd`); `null` without one,
83
+ * and for a sign-in. It decides only whether a key whose spend the proxy holds no reading of may take work: one with
84
+ * a limit may not (`spend_unread`). A spend reading carries the limit the key is judged against.
85
+ */
86
+ spendLimitMicroUsd: number | null;
87
+ /**
88
+ * The login the account's requests are served on (the account service's gateway login of it), as the pool was read;
89
+ * `null` when the account service names none. An API key's spend is reported on it.
90
+ */
91
+ loginId: string | null;
72
92
  }
73
93
 
74
94
  export interface IInferenceRoutingPolicy {
75
95
  /** The pool's provider; omitted means ChatGPT accounts (`openai`). */
76
96
  provider?: TInferenceProvider;
77
97
  /**
78
- * What the pool's accounts authenticate with; omitted means subscription sign-ins. `api_key` is an Anthropic pool's
79
- * alone: its Anthropic API keys, which Flex's Claude models run on, never a Claude account.
98
+ * What the pool's accounts authenticate with. Omitted, a Claude pool is served by its Claude accounts first and its
99
+ * Anthropic API keys when no Claude account can take a request (`inferenceCredentialTiers`), a ChatGPT pool by its
100
+ * ChatGPT accounts. `subscription` names the sign-ins alone; `api_key` is an Anthropic pool's alone: its Anthropic API
101
+ * keys, which Flex's Claude models run on, never a Claude account.
80
102
  */
81
103
  credential?: TInferenceCredential;
82
104
  /** Omitted means all eligible authority-owned accounts of the provider. */
83
105
  accountIds?: readonly string[];
84
- /** Exact current provider plan names; omission accepts every entitled plan. */
106
+ /** Exact current provider plan names; omission accepts every entitled plan. A key has no plan, so this leaves keys out. */
85
107
  plans?: readonly string[];
86
108
  }
87
109
 
@@ -108,19 +130,61 @@ export interface IInferenceRouterOptions {
108
130
  */
109
131
  reportAccount?: (changeArg: IInferenceAccountChange) => void;
110
132
  /**
111
- * Tells the account service that Anthropic refused a request on the Claude account with a billing error (HTTP 402
112
- * `billing_error`) at `observedAtArg`; resolves once the service holds it, so the account's usage carries it from then
113
- * on (`IAuthSwitchUsageSnapshot.paymentProblem`). Unset, the router holds the account out of rotation itself.
133
+ * Tells the account service that the provider refused a request on the account for a payment problem at
134
+ * `observedAtArg` -- a Claude account's billing error (HTTP 402 `billing_error`), or an API key's refusal for
135
+ * `causeArg` (`TInferencePaymentCause`); resolves once the service holds it, so the account's usage carries it from
136
+ * then on (`IAuthSwitchUsageSnapshot.paymentProblem`). Unset, the router holds the account out of rotation itself.
114
137
  */
115
- reportPaymentProblem?: (accountIdArg: string, observedAtArg: string) => Promise<void>;
138
+ reportPaymentProblem?: (accountIdArg: string, observedAtArg: string, causeArg: TInferencePaymentCause) => Promise<void>;
116
139
  /**
117
140
  * Where a payment problem the account service could not be told of is reported: once per account, never per request.
118
141
  * One line on standard error, the proxy's journal, when unset.
119
142
  */
120
143
  reportPaymentFailure?: (failureArg: IInferencePaymentReportFailure) => void;
144
+ /**
145
+ * Where the spend of the answers relayed on API keys is kept until the accounts service acknowledged it
146
+ * (`InferenceSpendOutbox`). A key's usage arrives judged with that spend already (`InferenceUsageCache` `view`).
147
+ */
148
+ spend?: IInferenceSpendLedger;
121
149
  now?: () => number;
122
150
  }
123
151
 
152
+ /** What the router asks of the proxy's spend outbox (`InferenceSpendOutbox`). */
153
+ export interface IInferenceSpendLedger {
154
+ /**
155
+ * Counts a relayed answer's spend and keeps it until the accounts service acknowledged it, reported on the login the
156
+ * route was served on (`IInferenceClaudeRoute.loginId`); `null` when none could be named, which still counts.
157
+ */
158
+ record(accountIdArg: string, loginIdArg: string | null, recordArg: IInferenceSpendRecord): void;
159
+ /**
160
+ * Whether spend relayed on the key is recorded nowhere yet: the accounts service refused a report the outbox still
161
+ * holds, an answer is in neither the outbox's store, whose write failed, nor the ledger, or no login of the key could
162
+ * be named for an answer, so it cannot be reported.
163
+ */
164
+ held(accountIdArg: string): boolean;
165
+ /** Says that an answer relayed on the key ended before its provider stated its usage, so nothing of it is counted. */
166
+ uncounted(accountIdArg: string, modelArg: string): void;
167
+ }
168
+
169
+ /** A payment problem the router holds an account out of rotation for: since `at`, for `cause`, until `until` (`null`: no end). */
170
+ interface IPaymentMark {
171
+ at: number;
172
+ cause: TInferencePaymentCause;
173
+ until: number | null;
174
+ }
175
+
176
+ /**
177
+ * What a payment problem's report adds for the account service (`IReq_AuthSwitchReportPaymentProblem`): the cause of an
178
+ * API key's spend cap or quota, nothing for a billing error, whose cause a missing one means -- so a Claude account's
179
+ * report is the one an accounts service of 40.3.0, which refuses a cause, takes.
180
+ */
181
+ export const inferencePaymentReportOptions = (causeArg: TInferencePaymentCause):
182
+ { cause?: Exclude<TInferencePaymentCause, 'billing'> } => causeArg === 'billing' ? {} : { cause: causeArg };
183
+
184
+ /** A payment mark as the refusal it makes. */
185
+ const paymentUnavailable = (markArg: IPaymentMark): IInferenceAccountUnavailable => ({ reason: 'payment_problem',
186
+ cause: markArg.cause, ...(markArg.until === null ? {} : { retryAt: new Date(markArg.until).toISOString() }) });
187
+
124
188
  /** A payment problem the account service could not be told of, as the proxy reports it: the account by its id's digest. */
125
189
  export interface IInferencePaymentReportFailure {
126
190
  /** The first 12 hex characters of the SHA-256 digest of the account's id. */
@@ -131,27 +195,62 @@ export interface IInferencePaymentReportFailure {
131
195
 
132
196
  const writePaymentReportFailure = (failureArg: IInferencePaymentReportFailure): void => {
133
197
  process.stderr.write(`AGL proxy could not tell the account service of the payment problem of account `
134
- + `${failureArg.account}, so it keeps the account out of rotation until it restarts: ${failureArg.error}\n`);
198
+ + `${failureArg.account}, so it keeps the account out of rotation until it restarts, or a spend cap ends: `
199
+ + `${failureArg.error}\n`);
135
200
  };
136
201
 
137
202
  /**
138
203
  * Why an account takes no request although no limit says so. `usage_unread`: the account service holds no usage reading
139
204
  * the proxy routes on. `catalog_failed`: the account's model list could not be read and none read before is held for its
140
205
  * revision. `needs_login`: its sign-in needs renewing. `access_failed`: the account service answered no access for it.
141
- * `model_missing`: its model list lacks the requested model. `payment_problem`: Anthropic refused a request on the Claude
142
- * account with a billing error (HTTP 402 `billing_error`); it takes none until a usage read or a re-check clears it.
206
+ * `model_missing`: its model list lacks the requested model. `payment_problem`: the provider refused a request on the
207
+ * account for a payment problem (`TInferencePaymentCause`); it takes none until a usage read or a re-check clears it, or
208
+ * a spend cap ends. `spend_unread`: an API key with a monthly spend limit whose spend the proxy holds no reading of --
209
+ * the account service could not read its ledger (`spend_unavailable`) or did not answer -- so it may be at its limit.
210
+ * `spend_unrecorded`: an API key with a monthly spend limit whose spend the proxy relayed is recorded nowhere yet --
211
+ * the account service refused its report, which the proxy holds to send again, an answer is held in neither the
212
+ * proxy's store, whose write failed, nor the account service's ledger, or no login of the key could be named for an
213
+ * answer to report it on (`InferenceSpendOutbox.held`). `model_unpriced`: an API key with a monthly spend limit, and a
214
+ * request whose model, service tier, speed, region or server tool the proxy's price table cannot price (`failure` names
215
+ * it), so its cost could not be counted against the limit.
143
216
  */
144
217
  export type TInferenceUnavailableReason = 'usage_unread' | 'catalog_failed' | 'needs_login' | 'access_failed'
145
- | 'model_missing' | 'payment_problem';
218
+ | 'model_missing' | 'payment_problem' | 'spend_unread' | 'spend_unrecorded' | 'model_unpriced';
219
+
220
+ /**
221
+ * Why a provider refused a request on an account for a payment problem, as the gateway reads the provider's answer.
222
+ * `billing`: Anthropic's billing error (HTTP 402 `billing_error`), on a Claude account's subscription or an Anthropic API
223
+ * key's organization. `spend_cap`: an Anthropic API key's organization reached its usage tier's monthly spend cap (HTTP
224
+ * 429 `rate_limit_error` with `error.details.error_code: "enforced_spend_limit_reached"`), which holds until 00:00 UTC on
225
+ * the first day of the next month. `quota`: OpenAI refused an API key for its quota (HTTP 429 `insufficient_quota`): no
226
+ * prepaid credit left, or an organization or project spend limit or the organization's usage limit reached.
227
+ */
228
+ export type TInferencePaymentCause = plugins.authswitchAuthority.TAuthSwitchPaymentProblemCause;
146
229
 
147
230
  export interface IInferenceAccountUnavailable {
148
231
  reason: TInferenceUnavailableReason;
149
- /** `usage_unread`: when the account service reads the account's usage next, or `null` when it does not say. */
232
+ /**
233
+ * `usage_unread`: when the account service reads the account's usage next, or `null` when it does not say.
234
+ * `payment_problem` of a spend cap: when the cap ends and the account takes requests again.
235
+ */
150
236
  retryAt?: string | null;
151
- /** `catalog_failed`: the HTTP status the model list answered with (`HTTP 429`), or what failed instead. */
237
+ /**
238
+ * `catalog_failed`: the HTTP status the model list answered with (`HTTP 429`), or what failed instead.
239
+ * `model_unpriced`: what of the request the price table cannot price.
240
+ */
152
241
  failure?: string;
242
+ /** `payment_problem`: what the provider refused the account for. */
243
+ cause?: TInferencePaymentCause;
153
244
  }
154
245
 
246
+ /** A payment problem in words, as the status, the journal line and a pinned refusal state it. */
247
+ const paymentProblemText = (unavailableArg: IInferenceAccountUnavailable): string =>
248
+ unavailableArg.cause === 'spend_cap' ? 'Anthropic refused a request on it: its organization reached its usage tier\'s '
249
+ + `monthly spend cap (HTTP 429)${unavailableArg.retryAt ? `; it takes requests again at ${unavailableArg.retryAt}` : ''}`
250
+ : unavailableArg.cause === 'quota' ? 'OpenAI refused a request on it for its quota (HTTP 429 insufficient_quota): no '
251
+ + 'prepaid credit left, or a spend or usage limit reached'
252
+ : 'Anthropic refused a request on it with a billing error (HTTP 402), a payment problem';
253
+
155
254
  /** A change of why an account is refused, as the proxy reports it: the account by its id's digest, never its id. */
156
255
  export interface IInferenceAccountChange {
157
256
  /** The first 12 hex characters of the SHA-256 digest of the account's id. */
@@ -171,9 +270,16 @@ const unavailableText = (unavailableArg: IInferenceAccountUnavailable): string =
171
270
  : unavailableArg.reason === 'catalog_failed' ? `its model list could not be read (${unavailableArg.failure})`
172
271
  : unavailableArg.reason === 'needs_login' ? 'it needs a fresh sign-in'
173
272
  : unavailableArg.reason === 'access_failed' ? 'the account service answered no access for it'
174
- : unavailableArg.reason === 'payment_problem'
175
- ? 'Anthropic refused a request on it with a billing error (HTTP 402), a payment problem'
176
- : 'its model list lacks the requested model';
273
+ : unavailableArg.reason === 'payment_problem' ? paymentProblemText(unavailableArg)
274
+ : unavailableArg.reason === 'spend_unread'
275
+ ? 'it has a monthly spend limit, and its spend could not be read, so it might be at it'
276
+ : unavailableArg.reason === 'spend_unrecorded'
277
+ ? 'it has a monthly spend limit, and spend the proxy relayed on it is not recorded yet -- the account service '
278
+ + 'refused its report, or neither the proxy\'s store nor the account service took an answer -- so it might '
279
+ + 'be at it'
280
+ : unavailableArg.reason === 'model_unpriced'
281
+ ? `it has a monthly spend limit, and ${unavailableArg.failure ?? 'the request'} has no price`
282
+ : 'its model list lacks the requested model';
177
283
 
178
284
  const writeAccountChange = (changeArg: IInferenceAccountChange): void => {
179
285
  process.stderr.write(`AGL proxy account ${changeArg.account}: ${changeArg.unavailable === null
@@ -213,6 +319,38 @@ const cooldownUntil = (cooldownArg: ICooldown | undefined,
213
319
  */
214
320
  export type TInferenceAccountLimit = 'exhausted' | 'reserved' | 'cooling';
215
321
 
322
+ /** An API key a request found held by its monthly spend limit: the account, and its month as the account service read it. */
323
+ export interface IInferenceSpendLimited {
324
+ accountId: string;
325
+ label: string;
326
+ spend: plugins.authswitchSpend.IAuthSwitchSpendReading;
327
+ }
328
+
329
+ /**
330
+ * A key's month in words, from its spend reading: `$12.40 of $50.00 this month`, or `$12.40 this month, no spend limit`,
331
+ * and how many answers could not be priced, when any.
332
+ */
333
+ const spendText = (spendArg: plugins.authswitchSpend.IAuthSwitchSpendReading): string => {
334
+ const usd = plugins.authswitchSpend.authSwitchUsdText;
335
+ const unpriced = spendArg.unpricedAnswers === 0 ? ''
336
+ : `, ${spendArg.unpricedAnswers} ${spendArg.unpricedAnswers === 1 ? 'answer' : 'answers'} of it could not be priced`;
337
+ return `${usd(spendArg.spentMicroUsd)} ${spendArg.limitMicroUsd === null ? 'this month, no spend limit'
338
+ : `of ${usd(spendArg.limitMicroUsd)} this month`}${unpriced}`;
339
+ };
340
+
341
+ /**
342
+ * What a refusal says of the API keys it found held by their monthly spend limit: each key with its month, and when the
343
+ * first of them takes requests again. Empty without any.
344
+ */
345
+ export const inferenceSpendLimitText = (keysArg: readonly IInferenceSpendLimited[]): string => {
346
+ const keys = [...new Map(keysArg.map((keyArg) => [keyArg.accountId, keyArg])).values()];
347
+ if (!keys.length) return '';
348
+ const resetAt = keys.map((keyArg) => keyArg.spend.resetAt).sort()[0];
349
+ const named = keys.map((keyArg) => `${keyArg.label} (${spendText(keyArg.spend)})`).join(', ');
350
+ return `${keys.length === 1 ? 'The API key' : 'The API keys'} ${named} ${keys.length === 1 ? 'is' : 'are'} held by the `
351
+ + `monthly spend limit set in AGL until ${resetAt}.`;
352
+ };
353
+
216
354
  export class InferenceRoutingError extends Error {
217
355
  constructor(public readonly status: number, public readonly code: string, messageArg: string,
218
356
  public readonly retryAfterSeconds?: number,
@@ -222,11 +360,21 @@ export class InferenceRoutingError extends Error {
222
360
  public readonly accountLimit?: TInferenceAccountLimit,
223
361
  /** The account a refusal for a reason of its own (`reason`) names. */
224
362
  public readonly accountId?: string,
225
- /** For `no_eligible_accounts` of a pinned account: why that account takes no request, though no limit says so. */
226
- public readonly reason?: TInferenceUnavailableReason) {
363
+ /**
364
+ * For `no_eligible_accounts` of a pinned account: why that account takes no request though no limit says so, in
365
+ * full -- a payment problem's cause, say.
366
+ */
367
+ public readonly unavailable?: IInferenceAccountUnavailable,
368
+ /** For `accounts_exhausted`: the API keys found held by their monthly spend limit, which its message names. */
369
+ public readonly spendLimited: readonly IInferenceSpendLimited[] = []) {
227
370
  super(messageArg);
228
371
  this.name = 'InferenceRoutingError';
229
372
  }
373
+
374
+ /** The reason of `unavailable`: a refusal names a reason of its own only with the refusal in full. */
375
+ public get reason(): TInferenceUnavailableReason | undefined {
376
+ return this.unavailable?.reason;
377
+ }
230
378
  }
231
379
 
232
380
  /**
@@ -237,14 +385,29 @@ const needsSignIn = (errorArg: unknown): boolean =>
237
385
  (errorArg instanceof InferenceRoutingError && errorArg.code === 'account_needs_login')
238
386
  || (errorArg instanceof FlexServiceError && errorArg.code === 'PROVIDER_REAUTH_REQUIRED');
239
387
 
388
+ /**
389
+ * A ChatGPT account or an OpenAI API key chosen for one request: a ChatGPT sign-in through the ChatGPT backend, a key
390
+ * through OpenAI's API (`OPENAI_API_BASE_URL`), each through `connection`.
391
+ */
240
392
  export interface IInferenceRoute {
241
393
  provider: 'openai';
394
+ /** What the chosen account authenticates with: a ChatGPT sign-in, or an OpenAI API key. */
395
+ credential: TInferenceCredential;
242
396
  accountId: string;
243
397
  plan: string;
244
398
  /** The account's placement score when it was chosen (`IAuthSwitchPlacement.score`). */
245
399
  score: number | null;
246
400
  model: TInferenceModel;
247
401
  connection: plugins.openAiProvider.IOpenAiModelConnection;
402
+ /** The session left its account on a later credential tier for one of a better tier (`returnToBetterTier`). */
403
+ returned: boolean;
404
+ /** The login the account was served on as the pool was read (`IInferenceAccount.loginId`): an answer's spend names it. */
405
+ loginId: string | null;
406
+ /**
407
+ * An OpenAI API key with a monthly spend limit: the request goes out as the price check read it, on the service tier it
408
+ * was checked for (`inferenceOpenAiCheckedBytes`).
409
+ */
410
+ spendLimit: boolean;
248
411
  release: () => void;
249
412
  }
250
413
 
@@ -261,6 +424,18 @@ export interface IInferenceClaudeRoute {
261
424
  score: number | null;
262
425
  model: TInferenceClaudeModel;
263
426
  connection: plugins.anthropicProvider.IAnthropicModelConnection;
427
+ /**
428
+ * The session's account is an Anthropic API key, and a Claude account of a better credential tier took the request
429
+ * instead (`returnToBetterTier`): the session returns to it.
430
+ */
431
+ returned: boolean;
432
+ /** The login the account was served on as the pool was read (`IInferenceAccount.loginId`): an answer's spend names it. */
433
+ loginId: string | null;
434
+ /**
435
+ * An Anthropic API key with a monthly spend limit: the request goes out as the price check read it, on the standard
436
+ * service tier only (`inferenceAnthropicCheckedBytes`), the one the price table prices.
437
+ */
438
+ spendLimit: boolean;
264
439
  release: () => void;
265
440
  }
266
441
 
@@ -271,55 +446,94 @@ interface ICatalog<TModel> {
271
446
  }
272
447
 
273
448
  /**
274
- * How long a Claude account's model list is used before it is read again, and how long a failed model-list read or a
275
- * failed sign-in holds before the next read: a sign-in does not heal itself, and a new one changes the account's revision.
449
+ * How long an account's model list -- a Claude account's, an Anthropic or an OpenAI API key's -- is used before it is read
450
+ * again, and how long a failed model-list read or a failed sign-in holds before the next read: a sign-in does not heal
451
+ * itself, and a new one changes the account's revision.
276
452
  */
277
- const claudeCatalogMs = 60_000;
453
+ const modelListMs = 60_000;
278
454
 
279
455
  /**
280
- * How long a Claude account whose access failed for a reason other than sign-in is refused before access is tried again.
456
+ * How long an account whose access failed for a reason other than sign-in is refused before access is tried again.
281
457
  * Such a failure is usually a brief outage of the local account service: retries stay bounded to one per account per 5 s
282
458
  * without turning a short outage into a minute of refusals.
283
459
  */
284
- const claudeAccessRetryMs = 5_000;
460
+ const modelListAccessRetryMs = 5_000;
285
461
 
286
462
  /**
287
- * A Claude account's model list for one revision of the account: the last one read, and why the latest read of it or of
288
- * the account's access failed, while it did. Neither is read again before `until`: a minute after the latest read,
289
- * failed or not, except 5 s after a failed access (`access_failed`), whose retry reads the list again with it.
463
+ * An account's model list for one revision of the account -- a Claude account's or an Anthropic API key's
464
+ * (`claudeCatalogs`), an OpenAI API key's (`keyCatalogs`): the last one read, and why the latest read of it or of the
465
+ * account's access failed, while it did. Neither is read again before `until`: a minute after the latest read, failed or
466
+ * not, except 5 s after a failed access (`access_failed`), whose retry reads the list again with it.
290
467
  */
291
- interface IClaudeCatalog {
468
+ interface IHeldModelList<TModel> {
292
469
  revision: string;
293
470
  until: number;
294
471
  /** The last list read for the revision; `null` while no read of it succeeded. */
295
- models: TInferenceClaudeModel[] | null;
472
+ models: TModel[] | null;
296
473
  failure: IInferenceAccountUnavailable | null;
297
474
  }
298
475
 
476
+ /** Reads an account's model list with the access the account service answered for it. */
477
+ type TModelLister<TModel> = (accessArg: IGatewayAuthorityAccess,
478
+ optionsArg: { signal: AbortSignal; fetch?: typeof fetch }) => Promise<TModel[]>;
479
+
299
480
  /**
300
- * Why a Claude account's held model list keeps it from routing: a failure of its access or sign-in always, a failed
481
+ * Why an account's held model list keeps it from routing: a failure of its access or sign-in always, a failed
301
482
  * model-list read (`catalog_failed`) only while no list is held -- the account routes on the last list meanwhile.
302
483
  */
303
- const claudeCatalogRefusal = (catalogArg: IClaudeCatalog): IInferenceAccountUnavailable | undefined =>
304
- catalogArg.failure && (catalogArg.models === null || catalogArg.failure.reason !== 'catalog_failed')
305
- ? catalogArg.failure : undefined;
484
+ const heldListRefusal = <TModel>(listArg: IHeldModelList<TModel>): IInferenceAccountUnavailable | undefined =>
485
+ listArg.failure && (listArg.models === null || listArg.failure.reason !== 'catalog_failed')
486
+ ? listArg.failure : undefined;
306
487
 
307
- /** A Claude account's model list could not be read, and why (`InferenceRouter.readClaudeCatalog`). */
308
- class ClaudeCatalogError extends Error {
488
+ /** An account's model list could not be read, and why (`InferenceRouter.readModelList`). */
489
+ class ModelListError extends Error {
309
490
  constructor(public readonly unavailable: IInferenceAccountUnavailable) {
310
491
  super(unavailableText(unavailable));
311
- this.name = 'ClaudeCatalogError';
492
+ this.name = 'ModelListError';
312
493
  }
313
494
  }
314
495
 
315
496
  /** A failed model list read in a few words: its HTTP status, or the kind of failure; never its message. */
316
497
  const catalogFailure = (errorArg: unknown): string => {
317
- if (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthModelListError) {
498
+ if (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthModelListError
499
+ || errorArg instanceof plugins.openAiProvider.OpenAiApiModelListError) {
318
500
  return errorArg.code === 'HTTP_ERROR' && errorArg.status !== undefined ? `HTTP ${errorArg.status}` : errorArg.code;
319
501
  }
502
+ // An OpenAI key that may not read the model list (403) is refused like a list that cannot be read.
503
+ if (errorArg instanceof plugins.openAiProvider.OpenAiApiKeyAuthError && errorArg.status !== undefined) {
504
+ return `HTTP ${errorArg.status}`;
505
+ }
320
506
  return errorArg instanceof Error ? errorArg.name : typeof errorArg;
321
507
  };
322
508
 
509
+ /** A provider's refusal of a key or sign-in it does not accept (401): renewed once for a sign-in, ended for a key. */
510
+ const accessRejected = (errorArg: unknown): boolean => (errorArg instanceof plugins.anthropicProvider.AnthropicOAuthAuthError
511
+ || errorArg instanceof plugins.anthropicProvider.AnthropicApiKeyAuthError
512
+ || errorArg instanceof plugins.openAiProvider.OpenAiApiKeyAuthError) && errorArg.status === 401;
513
+
514
+ /**
515
+ * A model an OpenAI API key lists (`GET /v1/models`), as a route and a catalog state it: what flexharness's
516
+ * `OPENAI_API_MODELS` describes of it -- efforts, service tiers, context window -- and nothing more. A model it does not
517
+ * describe stays routable, hidden from the model lists, as a ChatGPT catalog's hidden model is; its id is sent as it is
518
+ * listed.
519
+ */
520
+ const openAiKeyModel = (modelIdArg: string): TInferenceModel => {
521
+ const described = plugins.openAiProvider.getOpenAiApiModel(modelIdArg);
522
+ return { providerId: 'openai', modelId: modelIdArg, wireModel: modelIdArg, displayName: described?.displayName ?? modelIdArg,
523
+ description: described?.description ?? '', hidden: described === undefined,
524
+ reasoningEfforts: described ? described.reasoningEfforts.map((effortArg) => ({ ...effortArg })) : [],
525
+ defaultReasoningEffort: described?.defaultReasoningEffort ?? '',
526
+ inputModalities: described ? [...described.inputModalities] : ['text'], supportsPersonality: false,
527
+ serviceTiers: described ? described.serviceTiers.map((tierArg) => ({ ...tierArg })) : [],
528
+ ...(described ? { contextWindow: described.contextWindow } : {}), isDefault: false };
529
+ };
530
+
531
+ /**
532
+ * Whether an account of a ChatGPT pool is a ChatGPT sign-in: only those reach the ChatGPT backend -- its Codex catalog,
533
+ * its model list and its Responses endpoint. An OpenAI API key is never sent there.
534
+ */
535
+ const chatGptSignIn = (accountArg: IInferenceAccount): boolean => accountArg.credential === 'subscription';
536
+
323
537
  /** The headroom of an account whose provider reports no usage windows: below every known headroom. */
324
538
  const unknownHeadroom = -1;
325
539
 
@@ -370,11 +584,56 @@ const claudeLimits = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnap
370
584
  };
371
585
 
372
586
  /**
373
- * An Anthropic API key's limits. The account service reads no usage for a key -- its spend and rate limits are its
374
- * Claude Console organization's -- so a key routes with unknown headroom, as an account that reports no windows does,
375
- * and only a provider 429 rests it. A usage threshold has nothing to count against.
587
+ * An API key's limits, OpenAI's or Anthropic's, from its month's spend as the account service answers it (reading
588
+ * `kind: 'spend'`), by the shared rule (`authSwitchUsageLimit`): a key with a spend limit is exhausted while its month
589
+ * spent it or holds an answer that could not be priced, until the month ends, and its headroom is what is left of the
590
+ * limit; a key without one is never stopped by its spend and has no headroom to compare (`unknownHeadroom`). A reading
591
+ * whose month ended is the rule's as well: available, with all of the limit left. The account service answers a key's
592
+ * spend as current until its month ends, and the proxy asks again at every request, so no age is judged here.
593
+ *
594
+ * Without a spend reading -- the service could not read the key's ledger (`spend_unavailable`), or did not answer -- a
595
+ * key with a spend limit (`IInferenceAccount.spendLimitMicroUsd`) is refused (`undefined`, `spend_unread`), as it may be
596
+ * at its limit; one without a limit routes with unknown headroom. Rate limits are the provider's: a 429 rests a key.
597
+ */
598
+ const keyLimits = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined,
599
+ accountArg: IInferenceAccount, nowArg: number): IAccountLimits | undefined => {
600
+ // The proxy hands a key's usage over judged (`InferenceSpendOutbox.judged`): its month's spend with the answers the
601
+ // account service has not acknowledged yet, in the month now falls in.
602
+ const reading = usageArg?.state === 'current' || usageArg?.state === 'stale' ? usageArg.reading : null;
603
+ if (reading?.kind === 'spend') return accountLimits(reading, '', accountArg, unknownHeadroom, nowArg);
604
+ return accountArg.spendLimitMicroUsd === null
605
+ ? { plan: '', state: 'available', headroom: unknownHeadroom, limit: null } : undefined;
606
+ };
607
+
608
+ /**
609
+ * Whether an API key has a monthly spend limit: by its spend reading's limit, which the account service answered with
610
+ * the reading, or without a reading by the limit the account carries.
376
611
  */
377
- const apiKeyLimits: IAccountLimits = { plan: '', state: 'available', headroom: unknownHeadroom, limit: null };
612
+ const keyHasSpendLimit = (usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined,
613
+ accountArg: IInferenceAccount): boolean => usageArg?.reading?.kind === 'spend'
614
+ ? usageArg.reading.spend.limitMicroUsd !== null : accountArg.spendLimitMicroUsd !== null;
615
+
616
+ /**
617
+ * An API key's state in words, as `agl proxy status` shows it on its limits (`keyLimits`), with its month: available, at
618
+ * its spend limit, or held below it until the month ends for an answer that could not be priced; available after its
619
+ * month ended with nothing relayed on it since, or without a limit and a spend reading.
620
+ */
621
+ const keyStateText = (limitsArg: IAccountLimits,
622
+ usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined, nowArg: number): string => {
623
+ const readingArg = usageArg?.reading;
624
+ if (readingArg?.kind !== 'spend' || !limitsArg.limit) return 'available (no spend limit, and no spend reading)';
625
+ const spend = readingArg.spend;
626
+ // A reading the account service observed before the judged month began is last month's, judged as the new month's.
627
+ const newMonth = usageArg?.observedAt !== null && usageArg?.observedAt !== undefined
628
+ && Date.parse(usageArg.observedAt) < Date.parse(spend.periodStart);
629
+ if (plugins.authswitchSpend.authSwitchSpendMonthEnded(spend, nowArg) || (newMonth && spend.answers === 0)) {
630
+ return 'available (a new month began since its last spend reading)';
631
+ }
632
+ const state = limitsArg.state !== 'exhausted' ? 'available'
633
+ : spend.limitMicroUsd !== null && spend.spentMicroUsd < spend.limitMicroUsd ? 'held until its month ends'
634
+ : 'at its spend limit';
635
+ return `${state} (${spendText(spend)})`;
636
+ };
378
637
 
379
638
  /**
380
639
  * A ChatGPT account's limits from its usage reading (see `authSwitchUsageLimit`): unset, the main limit decides --
@@ -516,14 +775,110 @@ interface ICandidate<TModel> {
516
775
  reading: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot['reading'];
517
776
  ignoreLimits: boolean;
518
777
  account: IInferenceAccount;
778
+ /** The account's credential tier in its pool, 0 the best (`inferenceCredentialTiers`): placement takes the best first. */
779
+ tier: number;
519
780
  model: TModel;
520
781
  plan: string;
521
782
  headroom: number;
522
783
  /** The account's placement score (`IAuthSwitchPlacement.score`). */
523
784
  score: number | null;
524
785
  lastResort: boolean;
786
+ /** An API key with a monthly spend limit. */
787
+ spendLimit: boolean;
788
+ }
789
+
790
+ /** How a request is placed on a Claude pool (`InferenceRouter.acquireClaude`). */
791
+ export interface IInferenceClaudeAcquireOptions {
792
+ /** The session's account: the request is judged on it alone, unless `returnToBetterTier` finds a better one. */
793
+ accountId?: string;
794
+ /** The accounts this request left already; it never goes back to them. */
795
+ exclude?: ReadonlySet<string>;
796
+ /**
797
+ * The accounts of `exclude` the request left for a reason of their own, and that reason: each counts as its own
798
+ * judgement counts the reason, never as at its limit. An account of `exclude` it does not name counts as at its limit.
799
+ */
800
+ excludedBecause?: ReadonlyMap<string, IInferenceAccountUnavailable>;
801
+ /** A request that uses no quota (`count_tokens`): an account at its limit takes it as well. */
802
+ ignoreLimits?: boolean;
803
+ /**
804
+ * What of the request the proxy's price table cannot price (`inferenceAnthropicUnpricedRequest`): an API key with a
805
+ * monthly spend limit takes no such request (`model_unpriced`). A request that uses no quota ignores it.
806
+ */
807
+ unpriced?: string;
808
+ /**
809
+ * The session may move: when its account is of a later credential tier -- an Anthropic API key of a pool that names
810
+ * no credential -- a Claude account that can take the request takes it instead, and the route is `returned`.
811
+ */
812
+ returnToBetterTier?: boolean;
525
813
  }
526
814
 
815
+ /** How a request is placed on a ChatGPT pool (`InferenceRouter.acquire`). */
816
+ export interface IInferenceAcquireOptions {
817
+ /** The session's account: the request is judged on it alone, unless `returnToBetterTier` finds a better one. */
818
+ accountId?: string;
819
+ /** The accounts this request left already; it never goes back to them. */
820
+ exclude?: ReadonlySet<string>;
821
+ /** The accounts of `exclude` the request left for a reason of their own, as for a Claude pool (`IInferenceClaudeAcquireOptions`). */
822
+ excludedBecause?: ReadonlyMap<string, IInferenceAccountUnavailable>;
823
+ /** The request's `reasoning.effort`, which a ChatGPT account's model must accept; a key's model is OpenAI's to judge. */
824
+ reasoningEffort?: string;
825
+ /** The request's `service_tier`, which a ChatGPT account's model must offer (`auto` and `default` always). */
826
+ serviceTier?: string;
827
+ /** The Codex release the client read its catalog as: the model it names is checked against the catalog it was offered. */
828
+ clientVersion?: string;
829
+ /** A Responses request, which an OpenAI API key may take; a translated Messages request never goes to a key. */
830
+ apiKeys?: boolean;
831
+ /**
832
+ * What of the request the proxy's price table cannot price (`inferenceOpenAiUnpricedRequest`): an OpenAI API key with a
833
+ * monthly spend limit takes no such request (`model_unpriced`).
834
+ */
835
+ unpriced?: string;
836
+ /**
837
+ * The session may move: when its account is an OpenAI API key of a pool that names no credential, a ChatGPT account
838
+ * that can take the request takes it instead, and the route is `returned`. The gateway asks it only of a request that
839
+ * carries no account state, which never leaves its account.
840
+ */
841
+ returnToBetterTier?: boolean;
842
+ }
843
+
844
+ /** An account a request finds held with a payment problem, for the 402 that names it when nothing else is left. */
845
+ interface IPaymentRefusal {
846
+ label: string;
847
+ credential: TInferenceCredential;
848
+ unavailable: IInferenceAccountUnavailable;
849
+ }
850
+
851
+ /**
852
+ * The 402 a request gets when the accounts it could go to are all held with payment problems: what each was refused for,
853
+ * and what ends it.
854
+ */
855
+ const paymentRefusalText = (refusalsArg: readonly IPaymentRefusal[]): string => {
856
+ const named = (matchArg: (refusalArg: IPaymentRefusal) => boolean, nounArg: string): string | undefined => {
857
+ const labels = refusalsArg.filter(matchArg).map((refusalArg) => refusalArg.label);
858
+ return labels.length ? `${nounArg}${labels.length === 1 ? '' : 's'} ${labels.join(', ')}` : undefined;
859
+ };
860
+ const subscriptions = named((refusalArg) => refusalArg.unavailable.cause === 'billing'
861
+ && refusalArg.credential === 'subscription', 'Claude account');
862
+ const billedKeys = named((refusalArg) => refusalArg.unavailable.cause === 'billing' && refusalArg.credential === 'api_key',
863
+ 'Anthropic API key');
864
+ const cappedKeys = named((refusalArg) => refusalArg.unavailable.cause === 'spend_cap', 'Anthropic API key');
865
+ const capEnds = refusalsArg.flatMap((refusalArg) => refusalArg.unavailable.cause === 'spend_cap'
866
+ && refusalArg.unavailable.retryAt ? [refusalArg.unavailable.retryAt] : []).sort()[0];
867
+ const quotaKeys = named((refusalArg) => refusalArg.unavailable.cause === 'quota', 'OpenAI API key');
868
+ return [
869
+ subscriptions && `Anthropic answered a billing error (HTTP 402) for the ${subscriptions}: a payment problem with the`
870
+ + ' subscription. Settle the billing, then re-check the account in AGL Accounts or with'
871
+ + ' `authswitch account recheck <account-id>`.',
872
+ billedKeys && `Anthropic answered a billing error (HTTP 402) for the ${billedKeys}: settle the billing of its`
873
+ + ' organization in the Claude Console, then re-check the key in AGL Accounts.',
874
+ cappedKeys && `The organization of the ${cappedKeys} reached its usage tier's monthly spend cap (HTTP 429)`
875
+ + `${capEnds ? `; it takes requests again at ${capEnds}` : ''}, or once Anthropic grants a higher tier and the key is`
876
+ + ' re-checked in AGL Accounts.',
877
+ quotaKeys && `OpenAI refused the ${quotaKeys} for its quota (HTTP 429 insufficient_quota): add credit or raise the`
878
+ + ' organization\'s or project\'s limit, then re-check the key in AGL Accounts.',
879
+ ].filter((sentenceArg): sentenceArg is string => !!sentenceArg).join(' ');
880
+ };
881
+
527
882
  /** What judging a request's accounts found, for the refusal `InferenceRouter.select` answers when none is chosen. */
528
883
  interface ISelectionState {
529
884
  exhausted: boolean;
@@ -537,10 +892,28 @@ interface ISelectionState {
537
892
  accountLimit: TInferenceAccountLimit | undefined;
538
893
  /** A pinned account that takes no request for a reason of its own, and why. */
539
894
  refusal?: { accountId: string; unavailable: IInferenceAccountUnavailable };
540
- /** The labels of the Claude accounts left out for a payment problem: never counted as at their limit. */
541
- payment: string[];
895
+ /** The accounts left out for a payment problem, and why: never counted as at their limit. */
896
+ payment: IPaymentRefusal[];
897
+ /** The API keys at their monthly spend limit, which an `accounts_exhausted` refusal names. */
898
+ spendLimited: IInferenceSpendLimited[];
899
+ /** The labels of the API keys with a spend limit whose spend could not be read, which a `no_eligible_accounts` names. */
900
+ spendUnread: string[];
901
+ /**
902
+ * The labels of the API keys with a spend limit whose spend is recorded nowhere yet, which a `no_eligible_accounts`
903
+ * names: a report the account service refused, or an answer in neither the outbox's store, whose write failed, nor the
904
+ * ledger, or one no login of the key could be named for (`IInferenceSpendLedger.held`).
905
+ */
906
+ spendUnrecorded: string[];
907
+ /** The API keys with a spend limit the request was refused on for want of a price, and what lacked one. */
908
+ unpriced: { labels: string[]; what: string | undefined };
542
909
  }
543
910
 
911
+ /** What judging a request's accounts finds before any account is judged: nothing yet. */
912
+ const emptySelection = (accountsArg: number, pinnedArg: boolean): ISelectionState => ({ exhausted: false, reserved: false,
913
+ unavailable: false, modelKnown: false, earliestReset: Infinity, exhaustedPlans: new Set(), accounts: accountsArg,
914
+ pinned: pinnedArg, accountLimit: undefined, payment: [], spendLimited: [], spendUnread: [], spendUnrecorded: [],
915
+ unpriced: { labels: [], what: undefined } });
916
+
544
917
  /** An account's placement by the shared rule, on the reading the router holds for it. */
545
918
  const accountPlacement = (readingArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot['reading'],
546
919
  accountArg: IInferenceAccount, nowArg: number): plugins.authswitchPlacement.IAuthSwitchPlacement =>
@@ -551,21 +924,25 @@ export class InferenceRouter {
551
924
  /** ChatGPT catalogs by account and the Codex release they were read as. */
552
925
  private readonly catalogs = new Map<string, ICatalog<TInferenceModel>>();
553
926
  private readonly nativeCatalogs = new Map<string, INativeCatalog>();
554
- private readonly claudeCatalogs = new Map<string, IClaudeCatalog>();
927
+ /** The model lists of the Claude accounts and the Anthropic API keys (`claudeCatalog`). */
928
+ private readonly claudeCatalogs = new Map<string, IHeldModelList<TInferenceClaudeModel>>();
929
+ /** The model lists of the OpenAI API keys (`keyCatalog`). */
930
+ private readonly keyCatalogs = new Map<string, IHeldModelList<TInferenceModel>>();
555
931
  /** Why each Claude account was last reported refused, or routed on its last model list (`noteAccount`). */
556
932
  private readonly noted = new Map<string, { unavailable: IInferenceAccountUnavailable; lastCatalog: boolean }>();
557
933
  private readonly active = new Map<string, number>();
558
934
  private readonly cooldowns = new Map<string, ICooldown>();
559
935
  /**
560
- * The Claude accounts a billing error (HTTP 402) took out of rotation, by when it was seen, until the account service
561
- * holds the problem; one it could not be told of stays here for the proxy's lifetime (`reportPaymentProblem`).
936
+ * The accounts a payment problem took out of rotation (`reportPaymentProblem`): a Claude account until the account
937
+ * service holds the problem -- one it could not be told of stays here for the proxy's lifetime -- and an API key until
938
+ * its spend cap ends, or for the proxy's lifetime.
562
939
  */
563
- private readonly paymentMarks = new Map<string, number>();
940
+ private readonly paymentMarks = new Map<string, IPaymentMark>();
564
941
  /** The accounts whose payment problem the account service could not be told of, each reported once. */
565
942
  private readonly paymentReportsFailed = new Set<string>();
566
943
  /**
567
- * The Anthropic API keys among the accounts the last pool read listed: the account service reads no usage for them,
568
- * so nothing a request on one shows is handed to it.
944
+ * The API keys, Anthropic's and OpenAI's, among the accounts the last pool read listed: the account service reads no
945
+ * subscription usage for them, so no usage a request on one shows -- headers, tokens, a 429 -- is handed to it.
569
946
  */
570
947
  private apiKeyAccounts = new Set<string>();
571
948
  private readonly now: () => number;
@@ -579,21 +956,30 @@ export class InferenceRouter {
579
956
  }
580
957
 
581
958
  /**
582
- * The policy's accounts of its provider and credential, and every eligible account's id of that pool: an account
583
- * absent from both left the pool. An account whose lapse date passed is in neither -- it is never routed to again --
584
- * and is named in `lapsed`. A Claude account and an Anthropic API key never share a pool, so a session placed on one
585
- * kind never moves to the other.
959
+ * The credential tiers of the policy's pool, best first (`inferenceCredentialTiers`). A plan restriction leaves the
960
+ * API keys out, as a key has no plan, so a lease with `plans` never reaches a key.
961
+ */
962
+ private tiers(policyArg: IInferenceRoutingPolicy): readonly TInferenceCredential[] {
963
+ return inferenceCredentialTiers(policyArg.credential).filter((tierArg) => !policyArg.plans || tierArg !== 'api_key');
964
+ }
965
+
966
+ /**
967
+ * The policy's accounts of its provider and credential tiers (`tiers`), and every eligible account's id of that pool:
968
+ * an account absent from both left the pool. An account whose lapse date passed is in neither -- it is never routed to
969
+ * again -- and is named in `lapsed`. A pool that names its credential serves that kind alone: Flex's API-key pool
970
+ * never takes a Claude account. One that names none spans both, and a session placed on a key returns to a Claude
971
+ * account once one can take its request.
586
972
  */
587
973
  private async pool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<{ accounts: IInferenceAccount[];
588
- eligible: Set<string>; lapsed: Set<string> }> {
974
+ eligible: Set<string>; lapsed: Set<string>; tiers: readonly TInferenceCredential[] }> {
589
975
  const everyAccount = await this.options.accounts(signalArg);
590
976
  this.apiKeyAccounts = new Set(everyAccount.filter((account) => account.credential === 'api_key')
591
977
  .map((account) => account.id));
592
978
  const provider = policyArg.provider ?? 'openai';
593
- const credential = policyArg.credential ?? 'subscription';
979
+ const tiers = this.tiers(policyArg);
594
980
  const now = this.now();
595
981
  const inPool = (accountArg: IInferenceAccount) => accountArg.providerId === provider
596
- && accountArg.credential === credential;
982
+ && tiers.includes(accountArg.credential);
597
983
  const lapsed = new Set(everyAccount.filter((account) => inPool(account)
598
984
  && plugins.authswitchPlacement.authSwitchAccountLifecycle(account.lapsesAt, now) === 'lapsed').map((account) => account.id));
599
985
  const allAccounts = everyAccount.filter((account) => inPool(account) && !lapsed.has(account.id));
@@ -601,12 +987,12 @@ export class InferenceRouter {
601
987
  if (accounts.length > 64) throw new InferenceRoutingError(503, 'pool_too_large',
602
988
  'Select at most 64 accounts for this gateway.');
603
989
  const known = new Set(everyAccount.map((account) => account.id));
604
- for (const cache of [this.catalogs, this.nativeCatalogs, this.claudeCatalogs]) {
990
+ for (const cache of [this.catalogs, this.nativeCatalogs, this.claudeCatalogs, this.keyCatalogs]) {
605
991
  for (const cacheKey of cache.keys()) if (!known.has(cacheKey.slice(0, 64))) cache.delete(cacheKey);
606
992
  }
607
993
  for (const id of this.cooldowns.keys()) if (!known.has(id)) this.cooldowns.delete(id);
608
994
  for (const id of this.noted.keys()) if (!known.has(id)) this.noted.delete(id);
609
- return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed };
995
+ return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed, tiers };
610
996
  }
611
997
 
612
998
  /**
@@ -617,7 +1003,8 @@ export class InferenceRouter {
617
1003
  * the session waiting.
618
1004
  */
619
1005
  private async sessionPool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
620
- accountIdArg: string | undefined): Promise<{ accounts: IInferenceAccount[]; eligible: Set<string> }> {
1006
+ accountIdArg: string | undefined): Promise<{ accounts: IInferenceAccount[]; eligible: Set<string>;
1007
+ tiers: readonly TInferenceCredential[] }> {
621
1008
  const pool = await this.pool(policyArg, signalArg);
622
1009
  if (!accountIdArg) return pool;
623
1010
  if (pool.lapsed.has(accountIdArg)) {
@@ -662,84 +1049,94 @@ export class InferenceRouter {
662
1049
  return models;
663
1050
  }
664
1051
 
1052
+ /** The models a Claude account or an Anthropic API key may use (`heldModels`). */
1053
+ private claudeCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
1054
+ return this.heldModels(this.claudeCatalogs, accountArg, signalArg, (accessArg, optionsArg) => accountArg.credential === 'api_key'
1055
+ ? plugins.anthropicProvider.listAnthropicApiKeyModels({ apiKey: accessArg.accessToken }, optionsArg)
1056
+ : plugins.anthropicProvider.listAnthropicOAuthModels(accessArg, optionsArg));
1057
+ }
1058
+
1059
+ /** The models an OpenAI API key may use, as its `GET /v1/models` lists them (`heldModels`, `openAiKeyModel`). */
1060
+ private keyCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceModel[]> {
1061
+ return this.heldModels(this.keyCatalogs, accountArg, signalArg, async (accessArg, optionsArg) =>
1062
+ (await plugins.openAiProvider.listOpenAiApiKeyModels({ apiKey: accessArg.accessToken }, optionsArg))
1063
+ .map((modelArg) => openAiKeyModel(modelArg.modelId)));
1064
+ }
1065
+
665
1066
  /**
666
- * The models a Claude account or an Anthropic API key may use: the list read within the last minute, else read now.
1067
+ * The models an account may use, from its list in `listsArg`: the list read within the last minute, else read now.
667
1068
  * A model-list read that fails keeps the list read before for the account's revision, which the account routes on
668
1069
  * meanwhile; with none, the account is unavailable for the failure's reason. A failure of the account's access or
669
1070
  * sign-in makes it unavailable whether a list is held or not. Either way the failure is noted and its account is not
670
- * read again, however many requests ask, for a minute -- or 5 s after a failed access (`holdClaudeFailure`).
1071
+ * read again, however many requests ask, for a minute -- or 5 s after a failed access (`holdListFailure`).
671
1072
  */
672
- private async claudeCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
673
- const cached = this.claudeCatalogs.get(accountArg.id);
1073
+ private async heldModels<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount,
1074
+ signalArg: AbortSignal, listArg: TModelLister<TModel>): Promise<TModel[]> {
1075
+ const cached = listsArg.get(accountArg.id);
674
1076
  if (cached?.revision === accountArg.revision && cached.until > this.now()) {
675
- const refusal = claudeCatalogRefusal(cached);
676
- if (refusal) throw new ClaudeCatalogError(refusal);
1077
+ const refusal = heldListRefusal(cached);
1078
+ if (refusal) throw new ModelListError(refusal);
677
1079
  return cached.models!;
678
1080
  }
679
1081
  try {
680
- const models = await this.readClaudeCatalog(accountArg, signalArg);
681
- this.claudeCatalogs.set(accountArg.id, { revision: accountArg.revision, until: this.now() + claudeCatalogMs, models,
682
- failure: null });
1082
+ const models = await this.readModelList(accountArg, signalArg, listArg);
1083
+ listsArg.set(accountArg.id, { revision: accountArg.revision, until: this.now() + modelListMs, models, failure: null });
683
1084
  this.noteAccount(accountArg.id, null);
684
1085
  return models;
685
1086
  } catch (errorArg) {
686
1087
  signalArg.throwIfAborted();
687
- const failure = errorArg instanceof ClaudeCatalogError ? errorArg.unavailable
1088
+ const failure = errorArg instanceof ModelListError ? errorArg.unavailable
688
1089
  : { reason: 'catalog_failed' as const, failure: catalogFailure(errorArg) };
689
- const models = this.holdClaudeFailure(accountArg, failure);
1090
+ const models = this.holdListFailure(listsArg, accountArg, failure);
690
1091
  if (models) return models;
691
- throw new ClaudeCatalogError(failure);
1092
+ throw new ModelListError(failure);
692
1093
  }
693
1094
  }
694
1095
 
695
1096
  /**
696
- * Records why a Claude account's model list or access failed, keeping the list held for its revision, and notes it:
1097
+ * Records why an account's model list or access failed, keeping the list held for its revision, and notes it:
697
1098
  * neither is read again for a minute after a failed model-list read or sign-in, or for 5 s after a failed access
698
1099
  * (`access_failed`). The access is then tried again by reading the list again, even one read within the minute: a
699
1100
  * single list read on recovery. Answers the held list when the account routes on it meanwhile -- only after a failed
700
- * model-list read (`claudeCatalogRefusal`).
1101
+ * model-list read (`heldListRefusal`).
701
1102
  */
702
- private holdClaudeFailure(accountArg: IInferenceAccount,
703
- failureArg: IInferenceAccountUnavailable): TInferenceClaudeModel[] | undefined {
704
- const catalog: IClaudeCatalog = { revision: accountArg.revision,
705
- until: this.now() + (failureArg.reason === 'access_failed' ? claudeAccessRetryMs : claudeCatalogMs),
706
- models: this.heldClaudeCatalog(accountArg) ?? null, failure: failureArg };
707
- this.claudeCatalogs.set(accountArg.id, catalog);
708
- const routed = claudeCatalogRefusal(catalog) ? undefined : catalog.models!;
1103
+ private holdListFailure<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount,
1104
+ failureArg: IInferenceAccountUnavailable): TModel[] | undefined {
1105
+ const list: IHeldModelList<TModel> = { revision: accountArg.revision,
1106
+ until: this.now() + (failureArg.reason === 'access_failed' ? modelListAccessRetryMs : modelListMs),
1107
+ models: this.heldList(listsArg, accountArg) ?? null, failure: failureArg };
1108
+ listsArg.set(accountArg.id, list);
1109
+ const routed = heldListRefusal(list) ? undefined : list.models!;
709
1110
  this.noteAccount(accountArg.id, failureArg, routed !== undefined);
710
1111
  return routed;
711
1112
  }
712
1113
 
713
1114
  /**
714
- * Reads the models a Claude account or an Anthropic API key may use. A rejected access is reported once: a sign-in is
715
- * renewed, while a key is ended, so the account service refuses the second read. Every failure but an abort is a
716
- * `ClaudeCatalogError` naming its reason.
1115
+ * Reads the models an account may use through `listArg`. A rejected access (`accessRejected`) is reported once: a
1116
+ * sign-in is renewed, while a key is ended, so the account service refuses the second read. Every failure but an abort
1117
+ * is a `ModelListError` naming its reason.
717
1118
  */
718
- private async readClaudeCatalog(accountArg: IInferenceAccount, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
719
- const read = async (rejectedGenerationArg?: number) => {
1119
+ private async readModelList<TModel>(accountArg: IInferenceAccount, signalArg: AbortSignal,
1120
+ listArg: TModelLister<TModel>): Promise<TModel[]> {
1121
+ const read = async (rejectedGenerationArg?: number): Promise<{ models?: TModel[]; rejected?: number }> => {
720
1122
  let access: IGatewayAuthorityAccess;
721
1123
  try {
722
1124
  access = await this.options.resolveAccess(accountArg.id, signalArg, rejectedGenerationArg);
723
1125
  } catch (errorArg) {
724
1126
  signalArg.throwIfAborted();
725
- throw new ClaudeCatalogError({ reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' });
1127
+ throw new ModelListError({ reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' });
726
1128
  }
727
- const listOptions = { signal: signalArg, ...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}) };
728
1129
  try {
729
- return { models: accountArg.credential === 'api_key'
730
- ? await plugins.anthropicProvider.listAnthropicApiKeyModels({ apiKey: access.accessToken }, listOptions)
731
- : await plugins.anthropicProvider.listAnthropicOAuthModels(access, listOptions) };
1130
+ return { models: await listArg(access, { signal: signalArg,
1131
+ ...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}) }) };
732
1132
  } catch (errorArg) {
733
- if ((errorArg instanceof plugins.anthropicProvider.AnthropicOAuthAuthError
734
- || errorArg instanceof plugins.anthropicProvider.AnthropicApiKeyAuthError) && errorArg.status === 401) {
735
- return { rejected: access.generation };
736
- }
1133
+ if (accessRejected(errorArg)) return { rejected: access.generation };
737
1134
  throw errorArg;
738
1135
  }
739
1136
  };
740
1137
  let result = await read();
741
1138
  if (result.rejected !== undefined) result = await read(result.rejected);
742
- if (!result.models) throw new ClaudeCatalogError({ reason: 'needs_login' });
1139
+ if (!result.models) throw new ModelListError({ reason: 'needs_login' });
743
1140
  return result.models;
744
1141
  }
745
1142
 
@@ -750,7 +1147,8 @@ export class InferenceRouter {
750
1147
  private noteAccount(accountIdArg: string, unavailableArg: IInferenceAccountUnavailable | null, lastCatalogArg = false): void {
751
1148
  const previous = this.noted.get(accountIdArg);
752
1149
  if (unavailableArg === null ? !previous : previous?.lastCatalog === lastCatalogArg
753
- && previous.unavailable.reason === unavailableArg.reason && previous.unavailable.failure === unavailableArg.failure) return;
1150
+ && previous.unavailable.reason === unavailableArg.reason && previous.unavailable.failure === unavailableArg.failure
1151
+ && previous.unavailable.cause === unavailableArg.cause) return;
754
1152
  if (unavailableArg === null) this.noted.delete(accountIdArg);
755
1153
  else this.noted.set(accountIdArg, { unavailable: unavailableArg, lastCatalog: lastCatalogArg });
756
1154
  try {
@@ -760,37 +1158,57 @@ export class InferenceRouter {
760
1158
  } catch {}
761
1159
  }
762
1160
 
763
- /** The model list a Claude account's revision was last read with, without reading it: `undefined` when none is held. */
764
- private heldClaudeCatalog(accountArg: IInferenceAccount): TInferenceClaudeModel[] | undefined {
765
- const cached = this.claudeCatalogs.get(accountArg.id);
1161
+ /** The model list an account's revision was last read with, without reading it: `undefined` when none is held. */
1162
+ private heldList<TModel>(listsArg: Map<string, IHeldModelList<TModel>>, accountArg: IInferenceAccount): TModel[] | undefined {
1163
+ const cached = listsArg.get(accountArg.id);
766
1164
  return cached?.revision === accountArg.revision ? cached.models ?? undefined : undefined;
767
1165
  }
768
1166
 
769
1167
  /**
770
- * Why a Claude account's model list or access keeps it from routing, as a request would find it now, before its next
771
- * read is due: a failed access or sign-in, or a failed model-list read with no list held for its revision.
1168
+ * Why an account's model list or access keeps it from routing, as a request would find it now, before its next read is
1169
+ * due: a failed access or sign-in, or a failed model-list read with no list held for its revision. Only the accounts
1170
+ * whose model list the router reads have one: Claude accounts and API keys.
772
1171
  */
773
- private catalogRefusal(accountArg: IInferenceAccount): IInferenceAccountUnavailable | undefined {
774
- const cached = this.claudeCatalogs.get(accountArg.id);
775
- return cached?.revision === accountArg.revision && cached.until > this.now() ? claudeCatalogRefusal(cached) : undefined;
1172
+ private listRefusal(accountArg: IInferenceAccount): IInferenceAccountUnavailable | undefined {
1173
+ const refusal = <TModel>(cachedArg: IHeldModelList<TModel> | undefined) => cachedArg?.revision === accountArg.revision
1174
+ && cachedArg.until > this.now() ? heldListRefusal(cachedArg) : undefined;
1175
+ return accountArg.providerId === 'anthropic' ? refusal(this.claudeCatalogs.get(accountArg.id))
1176
+ : accountArg.credential === 'api_key' ? refusal(this.keyCatalogs.get(accountArg.id)) : undefined;
776
1177
  }
777
1178
 
778
1179
  /**
779
- * The models every readable account of the Claude pool -- its Claude accounts, or its Anthropic API keys -- offers: a
780
- * session may move to any of them, so a model one account lacks is not offered. Limits are the smallest any of them
781
- * states; efforts are those all of them accept, and adaptive thinking is offered when all of them accept it. What one
782
- * account does not state, the pool does not state.
1180
+ * The Claude pool's models, from its best credential tier with a readable account (`inferenceCredentialTiers`): the
1181
+ * Claude accounts' while one of them can read its model list, else the Anthropic API keys'. Within the tier, the
1182
+ * models every readable account offers: a session may move to any of them, so a model one account lacks is not
1183
+ * offered. Limits are the smallest any of them states; efforts are those all of them accept, and adaptive thinking is
1184
+ * offered when all of them accept it. What one account does not state, the pool does not state.
783
1185
  */
784
1186
  public async claudeModels(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<TInferenceClaudeModel[]> {
785
1187
  if (policyArg.provider !== 'anthropic') throw new Error('The Claude catalog belongs to a Claude pool.');
786
- const accounts = await this.accounts(policyArg, signalArg);
1188
+ const { accounts, tiers } = await this.pool(policyArg, signalArg);
1189
+ for (const tier of tiers) {
1190
+ const models = await this.tierClaudeModels(accounts.filter((account) => account.credential === tier), policyArg,
1191
+ signalArg);
1192
+ if (models?.size) return [...models.values()];
1193
+ }
1194
+ throw new InferenceRoutingError(503, 'no_eligible_accounts', policyArg.credential === 'api_key'
1195
+ ? 'No Anthropic API key could supply a model list. Add one with `authswitch account add anthropic-api-key`.'
1196
+ : policyArg.credential === 'subscription' || policyArg.plans
1197
+ ? 'No eligible Claude account could supply a model list. Check the AGL account service and sign-ins.'
1198
+ : 'No eligible Claude account or Anthropic API key could supply a model list. Check the AGL account service and '
1199
+ + 'sign-ins, or add a key with `authswitch account add anthropic-api-key`.');
1200
+ }
1201
+
1202
+ /** The models every readable account of one credential tier of a Claude pool offers; `undefined` when none is readable. */
1203
+ private async tierClaudeModels(accountsArg: readonly IInferenceAccount[], policyArg: IInferenceRoutingPolicy,
1204
+ signalArg: AbortSignal): Promise<Map<string, TInferenceClaudeModel> | undefined> {
787
1205
  let models: Map<string, TInferenceClaudeModel> | undefined;
788
- for (const account of accounts) {
1206
+ for (const account of accountsArg) {
789
1207
  signalArg.throwIfAborted();
790
1208
  try {
1209
+ // A plan restriction leaves the keys out of the pool: only Claude accounts are judged on it.
791
1210
  if (policyArg.plans) {
792
- const limits = account.credential === 'api_key' ? apiKeyLimits
793
- : claudeLimits(await this.options.usage(account.id, signalArg), account, this.now());
1211
+ const limits = claudeLimits(await this.options.usage(account.id, signalArg), account, this.now());
794
1212
  if (!limits || !policyArg.plans.includes(limits.plan)) continue;
795
1213
  }
796
1214
  const offered = new Map((await this.claudeCatalog(account, signalArg)).map((model) => [model.modelId, model]));
@@ -816,24 +1234,46 @@ export class InferenceRouter {
816
1234
  // An unreadable account is unavailable; it never supplies guessed models.
817
1235
  }
818
1236
  }
819
- if (!models?.size) throw new InferenceRoutingError(503, 'no_eligible_accounts', policyArg.credential === 'api_key'
820
- ? 'No Anthropic API key could supply a model list. Add one with `authswitch account add anthropic-api-key`.'
821
- : 'No eligible Claude account could supply a model list. Check the AGL account service and sign-ins.');
822
- return [...models.values()];
1237
+ return models;
823
1238
  }
824
1239
 
825
1240
  /**
826
- * Current model entitlements, merged conservatively across the account pool. A pool whose every account read needs a
827
- * fresh sign-in -- ChatGPT rejected its access even when renewed, or the account service could not renew it -- answers
1241
+ * The ChatGPT pool's models, from its best credential tier with a readable account (`inferenceCredentialTiers`): the
1242
+ * ChatGPT sign-ins' current entitlements while one of them can read its catalog, else the OpenAI API keys' models
1243
+ * (`keyTierModels`). A pool whose every ChatGPT account read needs a fresh sign-in -- ChatGPT rejected its access even
1244
+ * when renewed, or the account service could not renew it -- and no key to fall back to answers
828
1245
  * `account_needs_login`, so its caller asks for a sign-in rather than for the proxy.
829
1246
  */
830
1247
  public async models(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<TInferenceModel[]> {
831
1248
  if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('The ChatGPT catalog belongs to a ChatGPT pool.');
832
- const accounts = await this.accounts(policyArg, signalArg);
1249
+ const { accounts, tiers } = await this.pool(policyArg, signalArg);
1250
+ let signInNeeded = false;
1251
+ for (const tier of tiers) {
1252
+ if (tier === 'subscription') {
1253
+ const read = await this.signInModels(accounts.filter(chatGptSignIn), policyArg, signalArg);
1254
+ if (read.models.size) return [...read.models.values()];
1255
+ signInNeeded = read.signInNeeded;
1256
+ } else {
1257
+ const models = await this.keyTierModels(accounts.filter((account) => !chatGptSignIn(account)), signalArg);
1258
+ if (models?.size) return [...models.values()];
1259
+ }
1260
+ }
1261
+ if (signInNeeded) throw new InferenceRoutingError(503, 'account_needs_login', 'An account needs a fresh sign-in.');
1262
+ throw new InferenceRoutingError(503, 'no_eligible_accounts', policyArg.credential === 'api_key'
1263
+ ? 'No OpenAI API key could supply a model list. Add one in AGL Accounts or with `authswitch account add openai-api-key`.'
1264
+ : 'No eligible account could supply a model catalog. Check the AGL account service and sign-ins.');
1265
+ }
1266
+
1267
+ /**
1268
+ * Current model entitlements, merged conservatively across the pool's ChatGPT sign-ins, and whether every account read
1269
+ * needed a fresh sign-in.
1270
+ */
1271
+ private async signInModels(accountsArg: readonly IInferenceAccount[], policyArg: IInferenceRoutingPolicy,
1272
+ signalArg: AbortSignal): Promise<{ models: Map<string, TInferenceModel>; signInNeeded: boolean }> {
833
1273
  const models = new Map<string, TInferenceModel>();
834
1274
  let read = 0;
835
1275
  let signInNeeded = 0;
836
- for (const account of accounts) {
1276
+ for (const account of accountsArg) {
837
1277
  signalArg.throwIfAborted();
838
1278
  try {
839
1279
  if (policyArg.plans) {
@@ -859,22 +1299,44 @@ export class InferenceRouter {
859
1299
  if (needsSignIn(errorArg)) signInNeeded += 1;
860
1300
  }
861
1301
  }
862
- if (!models.size && read > 0 && signInNeeded === read) {
863
- throw new InferenceRoutingError(503, 'account_needs_login', 'An account needs a fresh sign-in.');
1302
+ return { models, signInNeeded: !models.size && read > 0 && signInNeeded === read };
1303
+ }
1304
+
1305
+ /**
1306
+ * The models every readable OpenAI API key of the pool lists and flexharness describes (`openAiKeyModel`), newest first
1307
+ * as `OPENAI_API_MODELS` orders them, the first the default: a session may move to any of the keys, so a model one key
1308
+ * lacks is not offered. `undefined` when no key is readable.
1309
+ */
1310
+ private async keyTierModels(accountsArg: readonly IInferenceAccount[],
1311
+ signalArg: AbortSignal): Promise<Map<string, TInferenceModel> | undefined> {
1312
+ let offered: Set<string> | undefined;
1313
+ for (const account of accountsArg) {
1314
+ signalArg.throwIfAborted();
1315
+ try {
1316
+ const listed = new Set((await this.keyCatalog(account, signalArg)).filter((model) => !model.hidden).map((model) => model.modelId));
1317
+ offered = offered ? new Set([...offered].filter((modelId) => listed.has(modelId))) : listed;
1318
+ } catch {
1319
+ signalArg.throwIfAborted();
1320
+ // An unreadable key is unavailable; it never supplies guessed models.
1321
+ }
864
1322
  }
865
- if (!models.size) throw new InferenceRoutingError(503, 'no_eligible_accounts',
866
- 'No eligible account could supply a model catalog. Check the AGL account service and sign-ins.');
867
- return [...models.values()];
1323
+ if (!offered) return undefined;
1324
+ const models = plugins.openAiProvider.OPENAI_API_MODELS.filter((model) => offered!.has(model.modelId))
1325
+ .map((model, index) => ({ ...openAiKeyModel(model.modelId), isDefault: index === 0 }));
1326
+ return new Map(models.map((model) => [model.modelId, model]));
868
1327
  }
869
1328
 
870
1329
  /**
871
- * The pool's Codex model catalog in the Codex backend's own entry shape, read for the client's version.
872
- * An entry shared by several accounts keeps only what every one of them offers.
1330
+ * The pool's Codex model catalog in the Codex backend's own entry shape, read for the client's version through its
1331
+ * ChatGPT sign-ins alone: an OpenAI API key never reaches the ChatGPT catalog connection, so a pool of keys alone has
1332
+ * no Codex catalog here (`no_eligible_accounts`; Codex then uses its bundled one). An entry shared by several
1333
+ * accounts keeps only what every one of them offers.
873
1334
  */
874
1335
  public async nativeModels(policyArg: IInferenceRoutingPolicy, clientVersionArg: string,
875
1336
  signalArg: AbortSignal): Promise<TInferenceNativeModel[]> {
876
1337
  if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('The Codex catalog belongs to a ChatGPT pool.');
877
- const accounts = await this.accounts(policyArg, signalArg);
1338
+ // The Codex catalog is the ChatGPT backend's: only a ChatGPT sign-in reads it, whatever credential the policy names.
1339
+ const accounts = (await this.accounts(policyArg, signalArg)).filter(chatGptSignIn);
878
1340
  const models = new Map<string, TInferenceNativeModel>();
879
1341
  for (const account of accounts) {
880
1342
  signalArg.throwIfAborted();
@@ -927,92 +1389,162 @@ export class InferenceRouter {
927
1389
  }
928
1390
 
929
1391
  /**
930
- * A ChatGPT account for one request. `clientVersion` is the Codex release the client read its catalog as, so the
931
- * model it names is checked against the catalog it was offered.
1392
+ * A ChatGPT account or an OpenAI API key for one request, chosen as `acquireClaude` chooses: the pinned session account
1393
+ * if it is still eligible, else the best credential tier's least busy account with the most headroom
1394
+ * (`inferenceCredentialTiers`). A ChatGPT account is judged on its usage reading and the Codex catalog of the release
1395
+ * the client read it as (`clientVersion`). An OpenAI API key takes only a Responses request (`apiKeys`), never a
1396
+ * translated Messages request, and is judged as an Anthropic API key is (`judgeListed`): a payment problem -- OpenAI's
1397
+ * quota refusal -- or its month's spend at its limit keeps it out, a key with a limit takes no request it cannot price
1398
+ * (`unpriced`, `model_unpriced`), and its model list (`GET /v1/models`, held a minute) must list the model. The accounts
1399
+ * of `exclude` count as `excludedBecause` says, as for a Claude pool. `returnToBetterTier`: a session pinned to a key
1400
+ * goes back to a ChatGPT account as soon as one can take its request, and the route says so (`returned`).
932
1401
  */
933
1402
  public async acquire(modelIdArg: string, policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
934
- optionsArg: { accountId?: string; exclude?: ReadonlySet<string>; reasoningEffort?: string; serviceTier?: string;
935
- clientVersion?: string } = {}): Promise<IInferenceRoute> {
1403
+ optionsArg: IInferenceAcquireOptions = {}): Promise<IInferenceRoute> {
936
1404
  if ((policyArg.provider ?? 'openai') !== 'openai') throw new Error('A ChatGPT route belongs to a ChatGPT pool.');
1405
+ const { accounts, tiers } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
1406
+ const pinned = accounts.find((account) => account.id === optionsArg.accountId);
1407
+ if (pinned && optionsArg.returnToBetterTier) {
1408
+ // The session's account is of a later tier: an account of a better one that can take the request takes it, as a
1409
+ // new session would be placed. When none can, the session stays where it is, judged as before.
1410
+ const better = accounts.filter((account) => tiers.indexOf(account.credential) < tiers.indexOf(pinned.credential));
1411
+ if (better.length) {
1412
+ try {
1413
+ const { accountId: _pinned, ...unpinned } = optionsArg;
1414
+ return { ...await this.routeOpenAi(modelIdArg, policyArg, better, tiers, signalArg, unpinned), returned: true };
1415
+ } catch (errorArg) {
1416
+ signalArg.throwIfAborted();
1417
+ if (!(errorArg instanceof InferenceRoutingError)) throw errorArg;
1418
+ }
1419
+ }
1420
+ }
1421
+ return this.routeOpenAi(modelIdArg, policyArg, accounts, tiers, signalArg, optionsArg);
1422
+ }
1423
+
1424
+ /** `acquire` among `accountsArg`, the pool's accounts or those of its better tiers. */
1425
+ private async routeOpenAi(modelIdArg: string, policyArg: IInferenceRoutingPolicy, accountsArg: readonly IInferenceAccount[],
1426
+ tiersArg: readonly TInferenceCredential[], signalArg: AbortSignal, optionsArg: IInferenceAcquireOptions): Promise<IInferenceRoute> {
937
1427
  const candidates: ICandidate<TInferenceModel>[] = [];
938
- let exhausted = false;
939
- let reserved = false;
940
- let modelKnown = false;
941
- let unavailable = false;
942
- let earliestReset = Infinity;
943
- let accountLimit: TInferenceAccountLimit | undefined;
944
- const exhaustedPlans = new Set<string>();
945
- const { accounts } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
946
- for (const account of accounts) {
1428
+ const state = emptySelection(accountsArg.length, optionsArg.accountId !== undefined);
1429
+ /** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
1430
+ let refusal: IInferenceAccountUnavailable | undefined;
1431
+ for (const account of accountsArg) {
947
1432
  signalArg.throwIfAborted();
948
1433
  if (optionsArg.accountId && optionsArg.accountId !== account.id) continue;
949
- if (optionsArg.exclude?.has(account.id)) {
950
- exhausted = true;
951
- const reset = this.cooldowns.get(account.id)?.until ?? 0;
952
- if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
953
- continue;
954
- }
1434
+ if (optionsArg.exclude?.has(account.id)) { this.countLeft(state, account, optionsArg.excludedBecause?.get(account.id)); continue; }
1435
+ // A translated Messages request never goes to an OpenAI API key: only a Responses request does.
1436
+ if (!chatGptSignIn(account) && !optionsArg.apiKeys) { state.unavailable = true; continue; }
955
1437
  try {
1438
+ if (!chatGptSignIn(account)) {
1439
+ // A key's usage is its month's spend, with the payment problem the account service holds on it.
1440
+ const usage = await this.options.usage(account.id, signalArg)
1441
+ .catch(() => { signalArg.throwIfAborted(); return undefined; });
1442
+ const judged = await this.judgeListed(state, account, usage, keyLimits(usage, account, this.now()), {
1443
+ modelId: modelIdArg, lists: this.keyCatalogs, read: (accountArg, innerSignalArg) => this.keyCatalog(accountArg, innerSignalArg),
1444
+ ...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }) }, signalArg);
1445
+ if (judged?.kind !== 'candidate') { if (judged) refusal = judged.unavailable; continue; }
1446
+ // The request goes to OpenAI as the client sent it: its effort and service tier are OpenAI's to judge, a limited
1447
+ // key's tier the price check's (`unpriced`).
1448
+ const placement = accountPlacement(judged.limits.limit ? usage!.reading : null, account, this.now());
1449
+ candidates.push({ account, tier: tiersArg.indexOf(account.credential), model: judged.model,
1450
+ reading: judged.limits.limit ? usage!.reading : null, ignoreLimits: false, plan: judged.limits.plan,
1451
+ headroom: judged.limits.headroom, score: placement.score, lastResort: placement.eligibility === 'last_resort',
1452
+ spendLimit: judged.spendLimit });
1453
+ continue;
1454
+ }
956
1455
  const usage = await this.options.usage(account.id, signalArg);
957
1456
  if (!usableReading(usage, this.now()) || usage.reading?.kind !== 'openai') {
958
- unavailable = true;
1457
+ state.unavailable = true;
959
1458
  continue;
960
1459
  }
961
1460
  const plan = usage.reading.rateLimits.plan;
962
1461
  if (policyArg.plans && !policyArg.plans.includes(plan)) continue;
963
1462
  const model = (await this.catalog(account, signalArg, optionsArg.clientVersion)).find((entry) => entry.modelId === modelIdArg);
964
1463
  if (!model) continue;
965
- modelKnown = true;
1464
+ state.modelKnown = true;
966
1465
  if (optionsArg.reasoningEffort && !model.reasoningEfforts.some((effort) => effort.effort === optionsArg.reasoningEffort)) continue;
967
1466
  if (optionsArg.serviceTier && !['auto', 'default'].includes(optionsArg.serviceTier)
968
1467
  && !model.serviceTiers.some((tier) => tier.id === optionsArg.serviceTier)) continue;
969
1468
  const limits = chatGptLimits(usage, account, this.now());
970
- if (!limits) { unavailable = true; continue; }
1469
+ if (!limits) { state.unavailable = true; continue; }
971
1470
  const cooldown = cooldownUntil(this.cooldowns.get(account.id), usage);
972
1471
  if (limits.state !== 'available' || cooldown > this.now()) {
973
- exhausted = true;
974
- reserved ||= limits.state === 'reserved';
975
- accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
976
- exhaustedPlans.add(limits.plan);
1472
+ state.exhausted = true;
1473
+ state.reserved ||= limits.state === 'reserved';
1474
+ state.accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
1475
+ state.exhaustedPlans.add(limits.plan);
977
1476
  const reset = Math.max(cooldown, limits.resetAt ?? 0);
978
- if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
1477
+ if (reset > this.now()) state.earliestReset = Math.min(state.earliestReset, reset);
979
1478
  continue;
980
1479
  }
981
1480
  const placement = accountPlacement(usage.reading, account, this.now());
982
- candidates.push({ account, model, reading: usage.reading, ignoreLimits: false, plan: limits.plan, headroom: limits.headroom, score: placement.score,
983
- lastResort: placement.eligibility === 'last_resort' });
984
- } catch {
1481
+ candidates.push({ account, tier: tiersArg.indexOf(account.credential), model, reading: usage.reading, ignoreLimits: false,
1482
+ plan: limits.plan, headroom: limits.headroom, score: placement.score, lastResort: placement.eligibility === 'last_resort',
1483
+ spendLimit: false });
1484
+ } catch (errorArg) {
1485
+ signalArg.throwIfAborted();
1486
+ state.unavailable = true;
1487
+ if (errorArg instanceof ModelListError) refusal = errorArg.unavailable;
1488
+ }
1489
+ }
1490
+ if (optionsArg.accountId !== undefined && refusal) state.refusal = { accountId: optionsArg.accountId, unavailable: refusal };
1491
+ // The chosen account's access is resolved before anything is sent. An OpenAI API key whose access fails takes no
1492
+ // request, as an Anthropic key's does (`routeClaude`): the failure is held for the key (`keyCatalogs`) and noted, and
1493
+ // the request goes to the next candidate -- a pinned request has none, so it is refused naming the key and the reason.
1494
+ // A ChatGPT account's failure fails the request.
1495
+ let selected: ICandidate<TInferenceModel>;
1496
+ let initial: IGatewayAuthorityAccess;
1497
+ let release: () => void;
1498
+ for (;;) {
1499
+ selected = this.select(candidates, state);
1500
+ release = this.reserve(selected.account.id);
1501
+ try {
1502
+ initial = await this.options.resolveAccess(selected.account.id, signalArg);
1503
+ break;
1504
+ } catch (errorArg) {
1505
+ release();
1506
+ if (chatGptSignIn(selected.account)) throw errorArg;
985
1507
  signalArg.throwIfAborted();
986
- unavailable = true;
1508
+ const failure: IInferenceAccountUnavailable = { reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' };
1509
+ this.holdListFailure(this.keyCatalogs, selected.account, failure);
1510
+ candidates.splice(candidates.indexOf(selected), 1);
1511
+ state.unavailable = true;
1512
+ if (state.pinned) state.refusal = { accountId: selected.account.id, unavailable: failure };
987
1513
  }
988
1514
  }
989
- const selectionState: ISelectionState = { exhausted, reserved, unavailable, modelKnown, earliestReset, exhaustedPlans,
990
- accounts: accounts.length, pinned: optionsArg.accountId !== undefined, accountLimit, payment: [] };
991
- const selected = this.select(candidates, selectionState);
992
1515
  const id = selected.account.id;
993
- const release = this.reserve(id);
994
1516
  try {
995
- const initial = await this.options.resolveAccess(id, signalArg);
996
- this.select([selected], selectionState);
997
- const connection = plugins.openAiProvider.createOpenAiChatGptResolvingModelConnection({
998
- expectedIdentity: initial,
999
- resolveAccess: async ({ signal, rejectedGeneration }) => {
1000
- const access = await this.options.resolveAccess(id, signal, rejectedGeneration);
1001
- this.select([selected], selectionState);
1002
- return access;
1003
- },
1004
- fetch: async (inputArg, initArg) => {
1005
- const response = await (this.options.providerFetch ?? fetch)(inputArg, initArg);
1006
- this.observe(id, 'openai', response.headers, response.status);
1007
- if (response.status === 429) {
1008
- this.reportQuota(id, this.retryAfterSeconds(response.headers));
1009
- }
1010
- return response;
1011
- },
1012
- originator: 'agl',
1013
- });
1014
- return { provider: 'openai', accountId: id, plan: selected.plan, score: selected.score, model: selected.model,
1015
- connection, release };
1517
+ this.select([selected], state);
1518
+ const connection = chatGptSignIn(selected.account)
1519
+ ? plugins.openAiProvider.createOpenAiChatGptResolvingModelConnection({
1520
+ expectedIdentity: initial,
1521
+ resolveAccess: async ({ signal, rejectedGeneration }) => {
1522
+ const access = await this.options.resolveAccess(id, signal, rejectedGeneration);
1523
+ this.select([selected], state);
1524
+ return access;
1525
+ },
1526
+ fetch: async (inputArg, initArg) => {
1527
+ const response = await (this.options.providerFetch ?? fetch)(inputArg, initArg);
1528
+ this.observe(id, 'openai', response.headers, response.status);
1529
+ if (response.status === 429) {
1530
+ this.reportQuota(id, this.retryAfterSeconds(response.headers));
1531
+ }
1532
+ return response;
1533
+ },
1534
+ originator: 'agl',
1535
+ })
1536
+ : plugins.openAiProvider.createOpenAiApiKeyConnection({
1537
+ apiKey: initial.accessToken,
1538
+ // A key is never renewed: OpenAI's 401 is reported before the connection fails the request with it.
1539
+ fetch: async (inputArg, initArg) => {
1540
+ const response = await (this.options.providerFetch ?? fetch)(inputArg, initArg);
1541
+ if (response.status === 401) await this.rejectApiKey(id, initial.generation, signalArg);
1542
+ return response;
1543
+ },
1544
+ });
1545
+ return { provider: 'openai', credential: selected.account.credential, accountId: id, plan: selected.plan,
1546
+ score: selected.score, model: selected.model, connection, returned: false, loginId: selected.account.loginId,
1547
+ spendLimit: selected.spendLimit, release };
1016
1548
  } catch (errorArg) {
1017
1549
  release();
1018
1550
  throw errorArg;
@@ -1020,106 +1552,229 @@ export class InferenceRouter {
1020
1552
  }
1021
1553
 
1022
1554
  /**
1023
- * A Claude account for one of Claude Code's requests, or an Anthropic API key for one of Flex's, chosen as `acquire`
1024
- * chooses: the pinned session account if it is still eligible, else the least busy account with the most headroom
1025
- * whose current reading shows it available -- every counted window under its threshold, no provider retry ahead --
1026
- * with no cooldown after a provider 429, and which lists the model. A key has no reading (`apiKeyLimits`): it is
1027
- * available unless it cools down after a 429.
1555
+ * Counts an account the request left already (`exclude`) as its own judgement counts the reason the request left it
1556
+ * for (`excludedBecause`): a payment problem joins the 402 list, a key whose spend could not be read or recorded is
1557
+ * named in the 503, one that could not price the request joins the 400 list, one without the model leads to the 404,
1558
+ * and every other reason to the 503 for want of an account. An account left for no reason of its own counts as at its
1559
+ * limit (429). The switch is exhaustive: a new reason must be placed here.
1560
+ */
1561
+ private countLeft(stateArg: ISelectionState, accountArg: IInferenceAccount,
1562
+ leftArg: IInferenceAccountUnavailable | undefined): void {
1563
+ if (!leftArg) {
1564
+ stateArg.exhausted = true;
1565
+ const reset = this.cooldowns.get(accountArg.id)?.until ?? 0;
1566
+ if (reset > this.now()) stateArg.earliestReset = Math.min(stateArg.earliestReset, reset);
1567
+ return;
1568
+ }
1569
+ switch (leftArg.reason) {
1570
+ case 'payment_problem':
1571
+ stateArg.payment.push({ label: accountArg.label, credential: accountArg.credential, unavailable: leftArg });
1572
+ return;
1573
+ case 'model_missing':
1574
+ return;
1575
+ case 'model_unpriced':
1576
+ stateArg.unpriced.labels.push(accountArg.label);
1577
+ stateArg.unpriced.what ??= leftArg.failure;
1578
+ return;
1579
+ case 'spend_unread':
1580
+ stateArg.unavailable = true;
1581
+ stateArg.spendUnread.push(accountArg.label);
1582
+ return;
1583
+ case 'spend_unrecorded':
1584
+ stateArg.unavailable = true;
1585
+ stateArg.spendUnrecorded.push(accountArg.label);
1586
+ return;
1587
+ case 'usage_unread':
1588
+ case 'catalog_failed':
1589
+ case 'needs_login':
1590
+ case 'access_failed':
1591
+ stateArg.unavailable = true;
1592
+ return;
1593
+ default: {
1594
+ const unhandledReason: never = leftArg.reason;
1595
+ return unhandledReason;
1596
+ }
1597
+ }
1598
+ }
1599
+
1600
+ /**
1601
+ * Judges an account whose model list the router reads -- a Claude account, an Anthropic or an OpenAI API key -- for one
1602
+ * request, as placement takes it, recording on `stateArg` what keeps it out. A payment problem refuses it before anything
1603
+ * else; without limits to route on (`limitsArg`) it is unavailable (`usage_unread`, a key's `spend_unread`); a key with
1604
+ * a spend limit whose spend is recorded nowhere yet is refused (`spend_unrecorded`). Limits and cool-down are judged
1605
+ * before any model list is read: an account that cannot take the request now is never read for it, its held list says
1606
+ * whether it would serve the model, and without one it counts as at its limit, so a session on it moves rather than
1607
+ * waiting for a list. A key's spend reading says nothing of the provider's rate limits, so only its own end ends a key's
1608
+ * cool-down. A key with a spend limit then takes no request whose cost the price table cannot count (`unpriced`,
1609
+ * Decision Q4), and the model list (`read`) must list the model. Answers the candidate's model and limits, or the refusal
1610
+ * of an account that takes no request for a reason of its own -- a pinned refusal names it -- or `undefined`.
1611
+ */
1612
+ private async judgeListed<TModel extends { modelId: string }>(stateArg: ISelectionState, accountArg: IInferenceAccount,
1613
+ usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined, limitsArg: IAccountLimits | undefined,
1614
+ optionsArg: { modelId: string; lists: Map<string, IHeldModelList<TModel>>;
1615
+ read: (accountArg: IInferenceAccount, signalArg: AbortSignal) => Promise<TModel[]>; plans?: readonly string[];
1616
+ unpriced?: string; ignoreLimits?: boolean },
1617
+ signalArg: AbortSignal): Promise<{ kind: 'candidate'; model: TModel; limits: IAccountLimits; spendLimit: boolean }
1618
+ | { kind: 'refused'; unavailable: IInferenceAccountUnavailable } | undefined> {
1619
+ const apiKey = accountArg.credential === 'api_key';
1620
+ const refused = (unavailableArg: IInferenceAccountUnavailable) => ({ kind: 'refused' as const, unavailable: unavailableArg });
1621
+ // A payment problem refuses the account before anything else is judged, whatever limits a request ignores.
1622
+ const problem = this.paymentProblem(accountArg.id, usageArg);
1623
+ if (problem) {
1624
+ stateArg.payment.push({ label: accountArg.label, credential: accountArg.credential, unavailable: problem });
1625
+ this.noteAccount(accountArg.id, problem);
1626
+ return refused(problem);
1627
+ }
1628
+ if (this.noted.get(accountArg.id)?.unavailable.reason === 'payment_problem') this.noteAccount(accountArg.id, null);
1629
+ if (!limitsArg) {
1630
+ stateArg.unavailable = true;
1631
+ const unavailable: IInferenceAccountUnavailable = apiKey ? { reason: 'spend_unread' }
1632
+ : { reason: 'usage_unread', retryAt: usageArg?.nextAttemptAt ?? null };
1633
+ if (apiKey) stateArg.spendUnread.push(accountArg.label);
1634
+ this.noteAccount(accountArg.id, unavailable);
1635
+ return refused(unavailable);
1636
+ }
1637
+ // A key with a spend limit whose spend the proxy relayed is recorded nowhere yet takes nothing while it is not,
1638
+ // whatever its spend: the account service's ledger lacks those answers.
1639
+ const spendLimit = apiKey && keyHasSpendLimit(usageArg, accountArg);
1640
+ if (spendLimit && this.options.spend?.held(accountArg.id)) {
1641
+ stateArg.unavailable = true;
1642
+ stateArg.spendUnrecorded.push(accountArg.label);
1643
+ const unavailable: IInferenceAccountUnavailable = { reason: 'spend_unrecorded' };
1644
+ this.noteAccount(accountArg.id, unavailable);
1645
+ return refused(unavailable);
1646
+ }
1647
+ const notedReason = this.noted.get(accountArg.id)?.unavailable.reason;
1648
+ if (notedReason === 'usage_unread' || notedReason === 'spend_unread' || notedReason === 'spend_unrecorded') {
1649
+ this.noteAccount(accountArg.id, null);
1650
+ }
1651
+ if (optionsArg.plans && !optionsArg.plans.includes(limitsArg.plan)) return undefined;
1652
+ const cooldown = cooldownUntil(this.cooldowns.get(accountArg.id), apiKey ? undefined : usageArg);
1653
+ if (!optionsArg.ignoreLimits && (limitsArg.state !== 'available' || cooldown > this.now())) {
1654
+ const held = this.heldList(optionsArg.lists, accountArg);
1655
+ if (held && !held.some((entry) => entry.modelId === optionsArg.modelId)) return refused({ reason: 'model_missing' });
1656
+ if (held) stateArg.modelKnown = true;
1657
+ stateArg.exhausted = true;
1658
+ stateArg.reserved ||= limitsArg.state === 'reserved';
1659
+ stateArg.accountLimit = limitsArg.state === 'available' ? 'cooling' : limitsArg.state;
1660
+ // A key has no plan to name.
1661
+ if (limitsArg.plan) stateArg.exhaustedPlans.add(limitsArg.plan);
1662
+ if (limitsArg.state === 'exhausted' && usageArg?.reading?.kind === 'spend') {
1663
+ stateArg.spendLimited.push({ accountId: accountArg.id, label: accountArg.label, spend: usageArg.reading.spend });
1664
+ }
1665
+ const reset = Math.max(cooldown, limitsArg.resetAt ?? 0);
1666
+ if (reset > this.now()) stateArg.earliestReset = Math.min(stateArg.earliestReset, reset);
1667
+ return undefined;
1668
+ }
1669
+ if (spendLimit && optionsArg.unpriced !== undefined && !optionsArg.ignoreLimits) {
1670
+ stateArg.unpriced.labels.push(accountArg.label);
1671
+ stateArg.unpriced.what ??= optionsArg.unpriced;
1672
+ return refused({ reason: 'model_unpriced', failure: optionsArg.unpriced });
1673
+ }
1674
+ const model = (await optionsArg.read(accountArg, signalArg)).find((entry) => entry.modelId === optionsArg.modelId);
1675
+ if (!model) return refused({ reason: 'model_missing' });
1676
+ stateArg.modelKnown = true;
1677
+ return { kind: 'candidate', model, limits: limitsArg, spendLimit };
1678
+ }
1679
+
1680
+ /**
1681
+ * A Claude account or an Anthropic API key for one of Claude Code's requests, or an Anthropic API key for one of
1682
+ * Flex's, chosen as `acquire` chooses: the pinned session account if it is still eligible, else the least busy account
1683
+ * with the most headroom whose current reading shows it available -- every counted window under its threshold, no
1684
+ * provider retry ahead -- with no cooldown after a provider 429, and which lists the model. A key is judged on its
1685
+ * month's spend (`keyLimits`): at its spend limit it takes no request until the month ends, as an account at its limit,
1686
+ * and a session on it moves; with a limit and no readable spend it is refused (`spend_unread`). A pool that names no
1687
+ * credential places on its Claude accounts first and on its keys only when no Claude account can take the request, for
1688
+ * whatever reason (`inferenceCredentialTiers`).
1028
1689
  * `ignoreLimits` serves a request that uses no quota (`count_tokens`) on an account at its limit as well. A pinned
1029
1690
  * account that takes no request for a reason of its own -- no usage reading, no model list, no access, or no such
1030
1691
  * model -- is refused naming the account and the reason (`InferenceRoutingError.reason`). An unpinned request whose
1031
- * chosen account's access fails goes to the next candidate.
1692
+ * chosen account's access fails goes to the next candidate. `returnToBetterTier`: a session pinned to a key goes back
1693
+ * to a Claude account as soon as one can take its request, and the route says so (`returned`).
1032
1694
  *
1033
- * A Claude account held with a payment problem -- Anthropic answered a request on it with a billing error (HTTP 402),
1034
- * as its usage reading says or this proxy saw (`reportPaymentProblem`) -- takes no request, `count_tokens` included,
1035
- * and is refused for the reason `payment_problem`. So is an account of `exclude` that `paymentExcluded` names: the
1036
- * request left it for a payment problem. Neither counts as at its limit: when such accounts are all that is left the
1037
- * refusal is a 402 `payment_problem` naming them, and while another account is merely at its limit the 429 stands.
1695
+ * An account held with a payment problem -- Anthropic answered a request on it with a billing error (HTTP 402), or on
1696
+ * a key with its organization's spend cap (HTTP 429 `enforced_spend_limit_reached`), as its usage reading says or this
1697
+ * proxy saw (`reportPaymentProblem`) -- takes no request, `count_tokens` included, and is refused for the reason
1698
+ * `payment_problem`. So is an account of `exclude` that `excludedBecause` names with a payment problem: the request
1699
+ * left it for one. Neither counts as at its limit: when such accounts are all that is left the refusal is a 402
1700
+ * `payment_problem` naming them, and while another account is merely at its limit the 429 stands. Any other account
1701
+ * of `exclude` that `excludedBecause` names counts as its reason counts on the account's own judgement: a key whose
1702
+ * spend could not be read is named in the 503, an account without the model leads to the 404, and every other reason
1703
+ * to the 503 for want of an account -- never to the 429 of accounts at their limits.
1038
1704
  */
1039
1705
  public async acquireClaude(modelIdArg: string, policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
1040
- optionsArg: { accountId?: string; exclude?: ReadonlySet<string>; paymentExcluded?: ReadonlySet<string>;
1041
- ignoreLimits?: boolean } = {}): Promise<IInferenceClaudeRoute> {
1706
+ optionsArg: IInferenceClaudeAcquireOptions = {}): Promise<IInferenceClaudeRoute> {
1042
1707
  if (policyArg.provider !== 'anthropic') throw new Error('A Claude route belongs to a Claude pool.');
1043
- const candidates: ICandidate<TInferenceClaudeModel>[] = [];
1044
- let exhausted = false;
1045
- let reserved = false;
1046
- let modelKnown = false;
1047
- let unavailable = false;
1048
- let earliestReset = Infinity;
1049
- let accountLimit: TInferenceAccountLimit | undefined;
1050
- /** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
1051
- let refusal: IInferenceAccountUnavailable | undefined;
1052
- const exhaustedPlans = new Set<string>();
1053
- const payment: string[] = [];
1054
- const { accounts } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
1708
+ const { accounts, tiers } = await this.sessionPool(policyArg, signalArg, optionsArg.accountId);
1055
1709
  if (policyArg.credential === 'api_key' && !accounts.length) throw new InferenceRoutingError(503, 'no_eligible_accounts',
1056
1710
  'No Anthropic API key is added, and this pool runs on API keys alone, never on a Claude sign-in. Add one with '
1057
1711
  + '`authswitch account add anthropic-api-key`.');
1058
- for (const account of accounts) {
1712
+ const pinned = accounts.find((account) => account.id === optionsArg.accountId);
1713
+ if (pinned && optionsArg.returnToBetterTier && !optionsArg.ignoreLimits) {
1714
+ // The session's account is of a later tier: an account of a better one that can take the request takes it, as a
1715
+ // new session would be placed. When none can, the session stays where it is, judged as before.
1716
+ const better = accounts.filter((account) => tiers.indexOf(account.credential) < tiers.indexOf(pinned.credential));
1717
+ if (better.length) {
1718
+ try {
1719
+ const route = await this.routeClaude(modelIdArg, policyArg, better, tiers, signalArg, {
1720
+ ...(optionsArg.exclude ? { exclude: optionsArg.exclude } : {}),
1721
+ ...(optionsArg.excludedBecause ? { excludedBecause: optionsArg.excludedBecause } : {}),
1722
+ ...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }) });
1723
+ return { ...route, returned: true };
1724
+ } catch (errorArg) {
1725
+ signalArg.throwIfAborted();
1726
+ if (!(errorArg instanceof InferenceRoutingError)) throw errorArg;
1727
+ }
1728
+ }
1729
+ }
1730
+ return this.routeClaude(modelIdArg, policyArg, accounts, tiers, signalArg, optionsArg);
1731
+ }
1732
+
1733
+ /** `acquireClaude` among `accountsArg`, the pool's accounts or those of its better tiers. */
1734
+ private async routeClaude(modelIdArg: string, policyArg: IInferenceRoutingPolicy, accountsArg: readonly IInferenceAccount[],
1735
+ tiersArg: readonly TInferenceCredential[], signalArg: AbortSignal,
1736
+ optionsArg: IInferenceClaudeAcquireOptions): Promise<IInferenceClaudeRoute> {
1737
+ const candidates: ICandidate<TInferenceClaudeModel>[] = [];
1738
+ const selectionState = emptySelection(accountsArg.length, optionsArg.accountId !== undefined);
1739
+ /** Why the last account judged takes no request though no limit says so: a pinned refusal names it. */
1740
+ let refusal: IInferenceAccountUnavailable | undefined;
1741
+ for (const account of accountsArg) {
1059
1742
  signalArg.throwIfAborted();
1060
1743
  if (optionsArg.accountId && optionsArg.accountId !== account.id) continue;
1061
1744
  if (optionsArg.exclude?.has(account.id)) {
1062
- if (optionsArg.paymentExcluded?.has(account.id)) { payment.push(account.label); continue; }
1063
- exhausted = true;
1064
- const reset = this.cooldowns.get(account.id)?.until ?? 0;
1065
- if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
1745
+ // An account left for a reason of its own counts as that reason counts on its own judgement.
1746
+ this.countLeft(selectionState, account, optionsArg.excludedBecause?.get(account.id));
1066
1747
  continue;
1067
1748
  }
1068
1749
  try {
1069
1750
  // A usage read the account service did not answer leaves the account as unread as a reading it cannot route on.
1070
- const usage = account.credential === 'api_key' ? undefined : await this.options.usage(account.id, signalArg)
1751
+ // A key's usage is its month's spend, with the payment problem the account service holds on it.
1752
+ const usage = await this.options.usage(account.id, signalArg)
1071
1753
  .catch(() => { signalArg.throwIfAborted(); return undefined; });
1072
- // A payment problem refuses the account before anything else is judged, whatever limits a request ignores.
1073
- if (account.credential !== 'api_key' && this.paymentRefused(account.id, usage)) {
1074
- refusal = { reason: 'payment_problem' };
1075
- payment.push(account.label);
1076
- this.noteAccount(account.id, refusal);
1077
- continue;
1078
- }
1079
- if (this.noted.get(account.id)?.unavailable.reason === 'payment_problem') this.noteAccount(account.id, null);
1080
- const limits = account.credential === 'api_key' ? apiKeyLimits
1081
- : usage ? claudeLimits(usage, account, this.now()) : undefined;
1082
- if (!limits) {
1083
- unavailable = true;
1084
- refusal = { reason: 'usage_unread', retryAt: usage?.nextAttemptAt ?? null };
1085
- this.noteAccount(account.id, refusal);
1086
- continue;
1087
- }
1088
- if (this.noted.get(account.id)?.unavailable.reason === 'usage_unread') this.noteAccount(account.id, null);
1089
- if (policyArg.plans && !policyArg.plans.includes(limits.plan)) continue;
1090
- // Limits and cool-down are judged before any model list is read: an account that cannot take the request now
1091
- // is never read for it. Its last model list, if one is held, says whether it would serve the model; without
1092
- // one it counts as limited, so a session on it moves rather than waiting for a list.
1093
- const cooldown = cooldownUntil(this.cooldowns.get(account.id), usage);
1094
- if (!optionsArg.ignoreLimits && (limits.state !== 'available' || cooldown > this.now())) {
1095
- const held = this.heldClaudeCatalog(account);
1096
- if (held && !held.some((entry) => entry.modelId === modelIdArg)) { refusal = { reason: 'model_missing' }; continue; }
1097
- if (held) modelKnown = true;
1098
- exhausted = true;
1099
- reserved ||= limits.state === 'reserved';
1100
- accountLimit = limits.state === 'available' ? 'cooling' : limits.state;
1101
- exhaustedPlans.add(limits.plan);
1102
- const reset = Math.max(cooldown, limits.resetAt ?? 0);
1103
- if (reset > this.now()) earliestReset = Math.min(earliestReset, reset);
1104
- continue;
1105
- }
1106
- const model = (await this.claudeCatalog(account, signalArg)).find((entry) => entry.modelId === modelIdArg);
1107
- if (!model) { refusal = { reason: 'model_missing' }; continue; }
1108
- modelKnown = true;
1109
- // A reading the router does not route on (`usage_unavailable`), and a key's absent one, score nothing.
1754
+ const apiKey = account.credential === 'api_key';
1755
+ const judged = await this.judgeListed(selectionState, account, usage, apiKey ? keyLimits(usage, account, this.now())
1756
+ : usage ? claudeLimits(usage, account, this.now()) : undefined, {
1757
+ modelId: modelIdArg, lists: this.claudeCatalogs, read: (accountArg, innerSignalArg) => this.claudeCatalog(accountArg, innerSignalArg),
1758
+ ...(policyArg.plans ? { plans: policyArg.plans } : {}),
1759
+ ...(optionsArg.unpriced === undefined ? {} : { unpriced: optionsArg.unpriced }),
1760
+ ...(optionsArg.ignoreLimits ? { ignoreLimits: true } : {}) }, signalArg);
1761
+ if (judged?.kind !== 'candidate') { if (judged) refusal = judged.unavailable; continue; }
1762
+ const { model, limits, spendLimit } = judged;
1763
+ // A reading the router does not route on (`usage_unavailable`), and a key's without a spend reading, score nothing.
1110
1764
  const placement = accountPlacement(limits.limit ? usage!.reading : null, account, this.now());
1111
- candidates.push({ account, model, reading: usage?.reading ?? null, ignoreLimits: optionsArg.ignoreLimits === true,
1112
- plan: limits.plan, headroom: limits.headroom, score: placement.score,
1113
- lastResort: placement.eligibility === 'last_resort' });
1765
+ candidates.push({ account, tier: tiersArg.indexOf(account.credential), model,
1766
+ reading: apiKey ? (limits.limit ? usage!.reading : null) : usage?.reading ?? null,
1767
+ ignoreLimits: optionsArg.ignoreLimits === true, plan: limits.plan, headroom: limits.headroom,
1768
+ score: placement.score, lastResort: placement.eligibility === 'last_resort', spendLimit });
1114
1769
  } catch (errorArg) {
1115
1770
  signalArg.throwIfAborted();
1116
- unavailable = true;
1117
- if (errorArg instanceof ClaudeCatalogError) refusal = errorArg.unavailable;
1771
+ selectionState.unavailable = true;
1772
+ if (errorArg instanceof ModelListError) refusal = errorArg.unavailable;
1118
1773
  }
1119
1774
  }
1120
- const selectionState: ISelectionState = { exhausted, reserved, unavailable, modelKnown, payment,
1121
- earliestReset, exhaustedPlans, accounts: accounts.length, pinned: optionsArg.accountId !== undefined, accountLimit,
1122
- ...(optionsArg.accountId !== undefined && refusal ? { refusal: { accountId: optionsArg.accountId, unavailable: refusal } } : {}) };
1775
+ if (optionsArg.accountId !== undefined && refusal) {
1776
+ selectionState.refusal = { accountId: optionsArg.accountId, unavailable: refusal };
1777
+ }
1123
1778
  // The chosen account's access is resolved before anything is sent. One that fails takes no request, as a failed
1124
1779
  // access found reading its model list does: it is recorded and noted for the account, and the request goes to the
1125
1780
  // next candidate -- a pinned request has none, so it is refused naming the account and the reason.
@@ -1136,7 +1791,7 @@ export class InferenceRouter {
1136
1791
  release();
1137
1792
  signalArg.throwIfAborted();
1138
1793
  const failure: IInferenceAccountUnavailable = { reason: needsSignIn(errorArg) ? 'needs_login' : 'access_failed' };
1139
- this.holdClaudeFailure(selected.account, failure);
1794
+ this.holdListFailure(this.claudeCatalogs, selected.account, failure);
1140
1795
  candidates.splice(candidates.indexOf(selected), 1);
1141
1796
  selectionState.unavailable = true;
1142
1797
  if (selectionState.pinned) selectionState.refusal = { accountId: selected.account.id, unavailable: failure };
@@ -1167,7 +1822,8 @@ export class InferenceRouter {
1167
1822
  ...(this.options.providerFetch ? { fetch: this.options.providerFetch } : {}),
1168
1823
  });
1169
1824
  return { provider: 'anthropic', credential, accountId: id, plan: selected.plan, score: selected.score,
1170
- model: selected.model, connection, release };
1825
+ model: selected.model, connection, returned: false, loginId: selected.account.loginId, spendLimit: selected.spendLimit,
1826
+ release };
1171
1827
  } catch (errorArg) {
1172
1828
  release();
1173
1829
  throw errorArg;
@@ -1189,9 +1845,9 @@ export class InferenceRouter {
1189
1845
  }
1190
1846
 
1191
1847
  /**
1192
- * The candidate a request goes to by the placement order (`authSwitchPlacementOrder`), or the refusal that says why there is
1193
- * none. A pinned request has its session's account as its one candidate, so the order only ever places new sessions.
1194
- * An account at its reserve is refused like one at its limit, and the refusal names the reserve. A pinned account
1848
+ * The candidate a request goes to -- of the best credential tier, then by the placement order
1849
+ * (`authSwitchPlacementOrder`) -- or the refusal that says why there is none. A pinned request has its session's
1850
+ * account as its one candidate, so the order only ever places new sessions. An account at its reserve is refused like one at its limit, and the refusal names the reserve. A pinned account
1195
1851
  * that takes no request for a reason of its own (`refusal`) is refused with that account and reason.
1196
1852
  */
1197
1853
  private select<TModel>(candidatesArg: ICandidate<TModel>[], stateArg: ISelectionState): ICandidate<TModel> {
@@ -1213,51 +1869,70 @@ export class InferenceRouter {
1213
1869
  candidate.score = placement.score;
1214
1870
  return true;
1215
1871
  });
1872
+ // The best credential tier first, then the placement order within it.
1216
1873
  const order = plugins.authswitchPlacement.authSwitchPlacementOrder((accountIdArg) => this.active.get(accountIdArg) ?? 0);
1217
- candidatesArg.sort((left, right) => order({ id: left.account.id, score: left.score, lastResort: left.lastResort,
1218
- headroom: left.headroom }, { id: right.account.id, score: right.score, lastResort: right.lastResort,
1219
- headroom: right.headroom }));
1874
+ candidatesArg.sort((left, right) => left.tier - right.tier || order({ id: left.account.id, score: left.score,
1875
+ lastResort: left.lastResort, headroom: left.headroom }, { id: right.account.id, score: right.score,
1876
+ lastResort: right.lastResort, headroom: right.headroom }));
1220
1877
  const selected = candidatesArg[0];
1221
1878
  if (selected) return selected;
1222
- if (stateArg.exhausted) throw new InferenceRoutingError(429, 'accounts_exhausted',
1879
+ // A refusal names the API keys held by their spend limit, and those with a limit whose spend could not be read.
1880
+ const named = (messageArg: string, sentenceArg: string) => sentenceArg ? `${messageArg} ${sentenceArg}` : messageArg;
1881
+ if (stateArg.exhausted) throw new InferenceRoutingError(429, 'accounts_exhausted', named(
1223
1882
  stateArg.pinned
1224
1883
  ? `The account that owns this conversation state is at its ${stateArg.reserved ? 'reserve' : 'limit'}.`
1225
1884
  : `All eligible accounts are at their limits${stateArg.reserved ? ' or their reserve' : ''}.`,
1885
+ inferenceSpendLimitText(stateArg.spendLimited)),
1226
1886
  Number.isFinite(stateArg.earliestReset) ? Math.max(1, Math.ceil((stateArg.earliestReset - this.now()) / 1000)) : undefined,
1227
- [...stateArg.exhaustedPlans], stateArg.pinned ? stateArg.accountLimit : undefined);
1887
+ [...stateArg.exhaustedPlans], stateArg.pinned ? stateArg.accountLimit : undefined, undefined, undefined,
1888
+ stateArg.spendLimited);
1228
1889
  if (stateArg.pinned && stateArg.refusal) {
1229
1890
  const { accountId, unavailable } = stateArg.refusal;
1230
1891
  throw new InferenceRoutingError(503, 'no_eligible_accounts',
1231
1892
  `The account that owns this conversation state (${accountId}) takes no request: ${unavailableText(unavailable)}.`,
1232
- undefined, [], undefined, accountId, unavailable.reason);
1893
+ undefined, [], undefined, accountId, unavailable);
1233
1894
  }
1234
1895
  if (stateArg.payment.length && !stateArg.unavailable) {
1235
- throw new InferenceRoutingError(402, 'payment_problem', `Anthropic answered a billing error (HTTP 402) for the`
1236
- + ` Claude ${stateArg.payment.length === 1 ? 'account' : 'accounts'} ${stateArg.payment.join(', ')}: a payment`
1237
- + ' problem with the subscription. Settle the billing, then re-check the account in AGL Accounts or with'
1238
- + ' `authswitch account recheck <account-id>`.');
1896
+ throw new InferenceRoutingError(402, 'payment_problem', paymentRefusalText(stateArg.payment));
1239
1897
  }
1898
+ if (stateArg.unpriced.labels.length && !stateArg.unavailable) {
1899
+ const keys = [...new Set(stateArg.unpriced.labels)];
1900
+ throw new InferenceRoutingError(400, 'model_unpriced', `The API ${keys.length === 1 ? 'key' : 'keys'} ${keys.join(', ')} `
1901
+ + `${keys.length === 1 ? 'has' : 'have'} a monthly spend limit set in AGL, and AGL's price table has no price for `
1902
+ + `${stateArg.unpriced.what ?? 'this request'}, so the cost of the answer could not be counted against the limit. `
1903
+ + 'Send it with a model, service tier, speed and tools the price table prices, or remove the spend limit in AGL '
1904
+ + 'Accounts.');
1905
+ }
1906
+ const keysNamed = (labelsArg: readonly string[], whatArg: string): string => labelsArg.length
1907
+ ? `The spend of the API ${labelsArg.length === 1 ? 'key' : 'keys'} ${labelsArg.join(', ')} ${whatArg}, and AGL `
1908
+ + 'places nothing on a key with a monthly spend limit until it can.' : '';
1240
1909
  if (!stateArg.accounts || stateArg.unavailable || stateArg.pinned) throw new InferenceRoutingError(503, 'no_eligible_accounts',
1241
- 'No eligible account has current usage, an available sign-in, and access to the requested model.');
1910
+ named('No eligible account has current usage, an available sign-in, and access to the requested model.',
1911
+ [keysNamed(stateArg.spendUnread, 'could not be read'),
1912
+ keysNamed(stateArg.spendUnrecorded, 'could not be recorded')]
1913
+ .filter(Boolean).join(' ')));
1242
1914
  throw new InferenceRoutingError(404, 'model_not_available', stateArg.modelKnown
1243
1915
  ? 'No eligible account supports the requested model options.' : 'The requested model is not available to this account pool.');
1244
1916
  }
1245
1917
 
1246
1918
  /**
1247
1919
  * The accounts as `agl proxy status` shows them: each judged as placement would judge it on the reading `heldArg`
1248
- * answers for it, which the caller read beforehand -- an Anthropic API key, which has none, as `apiKeyLimits` judges it -- with
1920
+ * answers for it, which the caller read beforehand -- an API key on its month's spend (`keyLimits`), in dollars -- with
1249
1921
  * its cooldown, the requests it serves now, its lifecycle and where a new session of an unrestricted client of its
1250
- * pool would go (`placementRank`).
1922
+ * provider would go (`placementRank`): a client that names no credential, so a provider's subscriptions rank before its
1923
+ * keys (`inferenceCredentialTiers`).
1251
1924
  */
1252
1925
  public accountStatus(accountsArg: readonly IInferenceAccount[],
1253
1926
  heldArg: (accountIdArg: string) => plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): IInferenceAccountStatus[] {
1254
1927
  const now = this.now();
1255
1928
  const statuses = accountsArg.map((account): IInferenceAccountStatus => {
1256
1929
  const apiKey = account.credential === 'api_key';
1257
- const held = apiKey ? undefined : heldArg(account.id);
1258
- const limits = apiKey ? apiKeyLimits
1930
+ // A key's usage is its month's spend, with the payment problem the account service holds on it.
1931
+ const held = heldArg(account.id);
1932
+ const limits = apiKey ? keyLimits(held, account, now)
1259
1933
  : held ? (account.providerId === 'anthropic' ? claudeLimits : chatGptLimits)(held, account, now) : undefined;
1260
- const cooldown = cooldownUntil(this.cooldowns.get(account.id), held);
1934
+ // A key's spend reading says nothing of the provider's rate limits, so only its own end ends a key's cool-down.
1935
+ const cooldown = cooldownUntil(this.cooldowns.get(account.id), apiKey ? undefined : held);
1261
1936
  const placement = accountPlacement(limits?.limit ? held!.reading : null, account, now);
1262
1937
  const base = { id: account.id, label: account.label, provider: account.providerId, credential: account.credential,
1263
1938
  maxUsagePercent: account.maxUsagePercent, reserveLiftUntil: account.reserveLiftUntil, inFlight: this.active.get(account.id) ?? 0,
@@ -1270,39 +1945,43 @@ export class InferenceRouter {
1270
1945
  .some((windowArg) => windowArg.estimatedUsedPercent !== null),
1271
1946
  lastReadRateLimited: held?.lastReadError?.rateLimited ?? false };
1272
1947
  // A payment problem refuses the account whatever its reading says, as a request finds it.
1273
- if (!apiKey && account.providerId === 'anthropic' && this.paymentRefused(account.id, held)) {
1274
- const unavailable: IInferenceAccountUnavailable = { reason: 'payment_problem' };
1275
- return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(unavailable)})`, unavailable,
1948
+ const unpaid = this.paymentProblem(account.id, held);
1949
+ if (unpaid) {
1950
+ return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(unpaid)})`, unavailable: unpaid,
1276
1951
  plan: limits?.plan || null, headroom: null, crossing: [], resetAt: null };
1277
1952
  }
1278
- if (!limits) {
1279
- const unavailable: IInferenceAccountUnavailable = { reason: 'usage_unread', retryAt: held?.nextAttemptAt ?? null };
1953
+ const unrecorded = apiKey && limits && keyHasSpendLimit(held, account) && this.options.spend?.held(account.id) === true;
1954
+ if (!limits || unrecorded) {
1955
+ const unavailable: IInferenceAccountUnavailable = unrecorded ? { reason: 'spend_unrecorded' }
1956
+ : apiKey ? { reason: 'spend_unread' } : { reason: 'usage_unread', retryAt: held?.nextAttemptAt ?? null };
1280
1957
  return { ...base, state: 'unavailable', stateText: `unavailable (${unavailableText(unavailable)})`, unavailable,
1281
1958
  plan: null, headroom: null, crossing: [], resetAt: null };
1282
1959
  }
1283
1960
  // An account whose model list a request could not read takes no request until it is read again; one at its limit
1284
1961
  // or cooling down is judged on that alone, as a request judges it.
1285
- const refusal = account.providerId === 'anthropic' ? this.catalogRefusal(account) : undefined;
1962
+ const refusal = this.listRefusal(account);
1286
1963
  const state = limits.state === 'available' && cooldown > now ? 'cooling'
1287
1964
  : limits.state === 'available' && refusal ? 'unavailable' : limits.state;
1288
1965
  return { ...base, state, ...(state === 'unavailable' && refusal ? { unavailable: refusal } : {}),
1289
1966
  stateText: placement.lifecycle === 'lapsed' ? 'lapsed (its subscription ended; never routed to again)'
1290
1967
  : state === 'cooling' ? 'cooling down after a provider 429'
1291
1968
  : state === 'unavailable' && refusal ? `unavailable (${unavailableText(refusal)})`
1292
- : limits.limit ? plugins.authswitchUsageLimit.authSwitchUsageLimitText(limits.limit)
1293
- : apiKey ? 'available (an API key; no usage is read for it)' : 'available (no usage reported)',
1969
+ : apiKey ? keyStateText(limits, held, now)
1970
+ : limits.limit ? plugins.authswitchUsageLimit.authSwitchUsageLimitText(limits.limit)
1971
+ : 'available (no usage reported)',
1294
1972
  plan: limits.plan || null, headroom: limits.limit?.headroom ?? null, crossing: limits.limit?.crossing ?? [],
1295
1973
  resetAt: state === 'cooling' ? new Date(cooldown).toISOString() : limits.limit?.resetAt ?? null };
1296
1974
  });
1297
- // Each pool's accounts that take a new session now, in the order placement takes them.
1975
+ // Each provider's accounts that take a new session now, in the order placement takes them: the best credential tier
1976
+ // first.
1298
1977
  const order = plugins.authswitchPlacement.authSwitchPlacementOrder((accountIdArg) => this.active.get(accountIdArg) ?? 0);
1299
- const poolOf = (statusArg: IInferenceAccountStatus) => `${statusArg.provider}:${statusArg.credential}`;
1300
- for (const pool of new Set(statuses.map(poolOf))) {
1301
- statuses.filter((statusArg) => poolOf(statusArg) === pool && statusArg.state === 'available'
1978
+ const tiers = inferenceCredentialTiers(undefined);
1979
+ for (const provider of new Set(statuses.map((statusArg) => statusArg.provider))) {
1980
+ statuses.filter((statusArg) => statusArg.provider === provider && statusArg.state === 'available'
1302
1981
  && statusArg.lifecycle !== 'lapsed')
1303
- .map((statusArg) => ({ status: statusArg, rank: { id: statusArg.id, score: statusArg.placementScore,
1304
- lastResort: statusArg.lastResort, headroom: statusArg.headroom ?? unknownHeadroom } }))
1305
- .sort((left, right) => order(left.rank, right.rank))
1982
+ .map((statusArg) => ({ status: statusArg, tier: tiers.indexOf(statusArg.credential), rank: { id: statusArg.id,
1983
+ score: statusArg.placementScore, lastResort: statusArg.lastResort, headroom: statusArg.headroom ?? unknownHeadroom } }))
1984
+ .sort((left, right) => left.tier - right.tier || order(left.rank, right.rank))
1306
1985
  .forEach((entryArg, indexArg) => { entryArg.status.placementRank = indexArg + 1; });
1307
1986
  }
1308
1987
  return statuses;
@@ -1333,21 +2012,25 @@ export class InferenceRouter {
1333
2012
  }
1334
2013
 
1335
2014
  /**
1336
- * Anthropic refused a request on the Claude account with a billing error (HTTP 402 `billing_error`): the account is out
1337
- * of rotation at once, and the account service is told (`reportPaymentProblem`). Once it holds the problem the
1338
- * account's usage carries it, and that decides from then on, so the router's own mark is dropped; when the service
1339
- * cannot be told -- it is not running, or an older release that does not know the report -- the mark holds for as long
1340
- * as the proxy runs, and the failure is reported once. An API key's 402 is no payment problem the service holds.
2015
+ * A provider refused a request on the account for a payment problem (`TInferencePaymentCause`): the account is out of
2016
+ * rotation at once, and the refusal it now carries is answered. The problem is reported to the account service
2017
+ * (`reportPaymentProblem`); once it holds the problem the account's usage carries it, and that decides from then on --
2018
+ * until a re-check clears it, or a spend cap's month ends -- so the router's own mark is dropped. When the service
2019
+ * cannot be told -- it is not running, or an older release that does not know the report or the account's kind --
2020
+ * the mark holds, a spend cap until 00:00 UTC on the first day of the next month (`authSwitchSpendPeriod`), anything
2021
+ * else for as long as the proxy runs, and the failure is reported once.
1341
2022
  */
1342
- public reportPaymentProblem(accountIdArg: string): void {
1343
- if (this.apiKeyAccounts.has(accountIdArg)) return;
2023
+ public reportPaymentProblem(accountIdArg: string, causeArg: TInferencePaymentCause): IInferenceAccountUnavailable {
1344
2024
  const at = this.now();
1345
- this.paymentMarks.set(accountIdArg, at);
1346
- this.noteAccount(accountIdArg, { reason: 'payment_problem' });
2025
+ const mark: IPaymentMark = { at, cause: causeArg,
2026
+ until: causeArg === 'spend_cap' ? Date.parse(plugins.authswitchSpend.authSwitchSpendPeriod(at).end) : null };
2027
+ this.paymentMarks.set(accountIdArg, mark);
2028
+ const unavailable = paymentUnavailable(mark);
2029
+ this.noteAccount(accountIdArg, unavailable);
1347
2030
  const report = this.options.reportPaymentProblem;
1348
- if (!report) return;
1349
- void (async () => report(accountIdArg, new Date(at).toISOString()))().then(() => {
1350
- if (this.paymentMarks.get(accountIdArg) === at) this.paymentMarks.delete(accountIdArg);
2031
+ if (!report) return unavailable;
2032
+ void (async () => report(accountIdArg, new Date(at).toISOString(), causeArg))().then(() => {
2033
+ if (this.paymentMarks.get(accountIdArg) === mark) this.paymentMarks.delete(accountIdArg);
1351
2034
  }, (errorArg: unknown) => {
1352
2035
  if (this.paymentReportsFailed.has(accountIdArg)) return;
1353
2036
  this.paymentReportsFailed.add(accountIdArg);
@@ -1357,12 +2040,24 @@ export class InferenceRouter {
1357
2040
  error: (errorArg instanceof Error ? `${errorArg.name}: ${errorArg.message}` : String(errorArg)).replace(/\s+/gu, ' ') });
1358
2041
  } catch {}
1359
2042
  });
2043
+ return unavailable;
1360
2044
  }
1361
2045
 
1362
- /** Whether a Claude account is held with a payment problem: by its usage reading, or by this proxy's own mark. */
1363
- private paymentRefused(accountIdArg: string,
1364
- usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): boolean {
1365
- return this.paymentMarks.has(accountIdArg) || usageArg?.paymentProblem !== undefined;
2046
+ /**
2047
+ * The payment problem an account is held with, as a refusal: by this proxy's own mark while it holds -- a spend cap's
2048
+ * ends at its time -- or by the account's usage reading, which carries it once the account service holds it, with its
2049
+ * cause (absent: `billing`) and, for a spend cap, when it ends; one whose spend cap ended holds nothing
2050
+ * (`authSwitchPaymentProblemHeld`, the rule the account service holds it by).
2051
+ */
2052
+ private paymentProblem(accountIdArg: string,
2053
+ usageArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): IInferenceAccountUnavailable | undefined {
2054
+ const now = this.now();
2055
+ const mark = this.paymentMarks.get(accountIdArg);
2056
+ if (mark && (mark.until === null || mark.until > now)) return paymentUnavailable(mark);
2057
+ if (mark) this.paymentMarks.delete(accountIdArg);
2058
+ const held = usageArg?.paymentProblem;
2059
+ return held === undefined || !plugins.authswitchSubscription.authSwitchPaymentProblemHeld(held, now) ? undefined
2060
+ : { reason: 'payment_problem', cause: held.cause ?? 'billing', ...(held.until === undefined ? {} : { retryAt: held.until }) };
1366
2061
  }
1367
2062
 
1368
2063
  /** A provider answer's `Retry-After`, in seconds from now, in either of its forms. */
@@ -1379,11 +2074,28 @@ export class InferenceRouter {
1379
2074
  this.options.tokens?.(accountIdArg, usageArg);
1380
2075
  }
1381
2076
 
2077
+ /**
2078
+ * Takes the spend of an answer an API key served on the login `loginIdArg` (`inferenceAnthropicSpend`): it counts
2079
+ * against the key's limit from now on and is reported to the account service (`spend`).
2080
+ */
2081
+ public recordSpend(accountIdArg: string, loginIdArg: string | null, recordArg: IInferenceSpendRecord): void {
2082
+ this.options.spend?.record(accountIdArg, loginIdArg, recordArg);
2083
+ }
2084
+
2085
+ /**
2086
+ * Says that an answer an API key served ended before its provider stated its usage, so nothing of it counts against
2087
+ * the key's limit (`IInferenceSpendLedger.uncounted`).
2088
+ */
2089
+ public uncountedSpend(accountIdArg: string, modelArg: string): void {
2090
+ this.options.spend?.uncounted(accountIdArg, modelArg);
2091
+ }
2092
+
1382
2093
  /**
1383
2094
  * A provider 429 on the account. Only a definitive pre-stream refusal permits another account to be tried. The
1384
2095
  * account cools down for the provider's `Retry-After`, at most a day; without one, or with one already due, for 60
1385
2096
  * seconds or until the account service holds a usage read made after the 429 (see `cooldownUntil`), which the 429
1386
- * asks it to make at once. An API key, whose usage the account service does not read, cools down the 60 seconds.
2097
+ * asks it to make at once. An API key, whose spend reading says nothing of the provider's rate limits, cools down
2098
+ * the 60 seconds.
1387
2099
  */
1388
2100
  public reportQuota(accountIdArg: string, retryAfterSecondsArg?: number): void {
1389
2101
  const explicit = Number.isFinite(retryAfterSecondsArg) && retryAfterSecondsArg! > 0;