agl 40.4.0 → 40.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/changelog.md +33 -0
  2. package/dist_serve/bundle.js +2228 -2177
  3. package/dist_serve/bundle.js.map +2 -2
  4. package/dist_ts/00_commitinfo_data.js +1 -1
  5. package/dist_ts/aglapi/aglapisurface.document.js +2 -2
  6. package/dist_ts/classes.accountsactions.d.ts +11 -1
  7. package/dist_ts/classes.accountsactions.js +59 -2
  8. package/dist_ts/classes.controller.js +5 -5
  9. package/dist_ts/classes.flexaccountaccess.js +3 -2
  10. package/dist_ts/classes.harnessreleases.d.ts +14 -4
  11. package/dist_ts/classes.harnessreleases.js +28 -4
  12. package/dist_ts/classes.inferencegateway.d.ts +28 -3
  13. package/dist_ts/classes.inferencegateway.js +177 -61
  14. package/dist_ts/classes.inferenceplacement.d.ts +5 -3
  15. package/dist_ts/classes.inferenceplacement.js +2 -2
  16. package/dist_ts/classes.inferencerouter.d.ts +349 -91
  17. package/dist_ts/classes.inferencerouter.js +804 -297
  18. package/dist_ts/classes.inferenceservice.d.ts +14 -2
  19. package/dist_ts/classes.inferenceservice.js +53 -14
  20. package/dist_ts/classes.inferencespend.d.ts +339 -0
  21. package/dist_ts/classes.inferencespend.js +815 -0
  22. package/dist_ts/classes.inferenceusagecache.d.ts +8 -0
  23. package/dist_ts/classes.inferenceusagecache.js +9 -5
  24. package/dist_ts/classes.inferenceusagetap.d.ts +50 -9
  25. package/dist_ts/classes.inferenceusagetap.js +185 -31
  26. package/dist_ts/classes.proxycontrol.d.ts +3 -2
  27. package/dist_ts/classes.proxycontrol.js +1 -2
  28. package/dist_ts/classes.proxydaemon.js +3 -2
  29. package/dist_ts/classes.proxystore.d.ts +6 -0
  30. package/dist_ts/classes.proxystore.js +11 -2
  31. package/dist_ts/fleet/classes.claudeworkers.d.ts +5 -4
  32. package/dist_ts/fleet/classes.claudeworkers.js +10 -9
  33. package/dist_ts/functions.claudecoderelease.d.ts +12 -6
  34. package/dist_ts/functions.claudecoderelease.js +13 -7
  35. package/dist_ts/functions.inferencecli.js +12 -8
  36. package/dist_ts/interfaces.auth.d.ts +1 -1
  37. package/dist_ts/interfaces.auth.js +3 -1
  38. package/dist_ts/plugins.d.ts +3 -1
  39. package/dist_ts/plugins.js +7 -2
  40. package/dist_ts_authswitch/authority-apikeys.d.ts +32 -0
  41. package/dist_ts_authswitch/authority-apikeys.js +65 -0
  42. package/dist_ts_authswitch/authority-contract.d.ts +172 -33
  43. package/dist_ts_authswitch/authority-contract.js +54 -1
  44. package/dist_ts_authswitch/authority-overview.js +8 -1
  45. package/dist_ts_authswitch/authority-placement.d.ts +7 -2
  46. package/dist_ts_authswitch/authority-placement.js +10 -3
  47. package/dist_ts_authswitch/authority-runtime-contract.d.ts +2 -2
  48. package/dist_ts_authswitch/authority-spend.d.ts +103 -0
  49. package/dist_ts_authswitch/authority-spend.js +110 -0
  50. package/dist_ts_authswitch/authority-subscription.d.ts +12 -1
  51. package/dist_ts_authswitch/authority-subscription.js +24 -1
  52. package/dist_ts_authswitch/authority-usagelimit.d.ts +7 -4
  53. package/dist_ts_authswitch/authority-usagelimit.js +89 -39
  54. package/dist_ts_authswitch/authority-usagewindows.d.ts +4 -1
  55. package/dist_ts_authswitch/authority-usagewindows.js +41 -16
  56. package/dist_ts_authswitch/classes.authoritybroker.d.ts +30 -19
  57. package/dist_ts_authswitch/classes.authoritybroker.js +152 -61
  58. package/dist_ts_authswitch/classes.authoritycli.d.ts +9 -4
  59. package/dist_ts_authswitch/classes.authoritycli.js +99 -30
  60. package/dist_ts_authswitch/classes.authorityclient.d.ts +24 -7
  61. package/dist_ts_authswitch/classes.authorityclient.js +45 -15
  62. package/dist_ts_authswitch/classes.authoritydaemon.js +39 -12
  63. package/dist_ts_authswitch/classes.authoritydatabase.d.ts +28 -5
  64. package/dist_ts_authswitch/classes.authoritydatabase.js +171 -9
  65. package/dist_ts_authswitch/classes.authorityimport.js +2 -2
  66. package/dist_ts_authswitch/classes.authoritymodels.d.ts +71 -3
  67. package/dist_ts_authswitch/classes.authoritymodels.js +249 -20
  68. package/dist_ts_authswitch/classes.authorityregistry.js +4 -2
  69. package/dist_ts_authswitch/classes.authorityusage.d.ts +69 -7
  70. package/dist_ts_authswitch/classes.authorityusage.js +118 -4
  71. package/dist_ts_authswitch/classes.claudenative.d.ts +23 -8
  72. package/dist_ts_authswitch/classes.claudenative.js +49 -21
  73. package/dist_ts_authswitch/classes.claudesignin.d.ts +14 -2
  74. package/dist_ts_authswitch/classes.claudesignin.js +72 -23
  75. package/dist_ts_authswitch/classes.codexmanaged.d.ts +8 -0
  76. package/dist_ts_authswitch/classes.codexmanaged.js +33 -18
  77. package/dist_ts_authswitch/classes.harnessprocesses.d.ts +8 -0
  78. package/dist_ts_authswitch/classes.harnessprocesses.js +41 -9
  79. package/dist_ts_authswitch/testsupport.d.ts +24 -5
  80. package/dist_ts_authswitch/testsupport.js +56 -12
  81. package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.d.ts +12 -0
  82. package/dist_ts_authswitch/ts_migration/0011_account_api_key_fields.js +21 -0
  83. package/dist_ts_authswitch/ts_migration/index.js +3 -1
  84. package/dist_ts_interfaces/functions.authorityscope.d.ts +18 -7
  85. package/dist_ts_interfaces/functions.authorityscope.js +35 -8
  86. package/dist_ts_interfaces/interfaces.d.ts +1 -1
  87. package/dist_ts_interfaces/interfaces.js +1 -1
  88. package/dist_ts_interfaces/plugins.d.ts +2 -1
  89. package/dist_ts_interfaces/plugins.js +7 -4
  90. package/dist_ts_interfaces/requests.accounts.d.ts +56 -0
  91. package/dist_ts_interfaces/requests.accounts.js +11 -1
  92. package/dist_ts_interfaces/requests.d.ts +2 -2
  93. package/dist_ts_interfaces/requests.js +1 -1
  94. package/docs/authswitch.md +75 -12
  95. package/package.json +5 -5
  96. package/readme.md +199 -22
  97. package/ts/00_commitinfo_data.ts +1 -1
  98. package/ts/aglapi/aglapisurface.document.ts +2 -2
  99. package/ts/classes.accountsactions.ts +75 -1
  100. package/ts/classes.controller.ts +4 -4
  101. package/ts/classes.flexaccountaccess.ts +2 -1
  102. package/ts/classes.harnessreleases.ts +25 -4
  103. package/ts/classes.inferencegateway.ts +184 -64
  104. package/ts/classes.inferenceplacement.ts +6 -4
  105. package/ts/classes.inferencerouter.ts +1026 -314
  106. package/ts/classes.inferenceservice.ts +56 -13
  107. package/ts/classes.inferencespend.ts +870 -0
  108. package/ts/classes.inferenceusagecache.ts +16 -4
  109. package/ts/classes.inferenceusagetap.ts +159 -34
  110. package/ts/classes.proxycontrol.ts +3 -3
  111. package/ts/classes.proxydaemon.ts +2 -1
  112. package/ts/classes.proxystore.ts +11 -1
  113. package/ts/fleet/classes.claudeworkers.ts +9 -8
  114. package/ts/functions.claudecoderelease.ts +13 -6
  115. package/ts/functions.inferencecli.ts +11 -7
  116. package/ts/interfaces.auth.ts +2 -0
  117. package/ts/plugins.ts +7 -0
  118. package/ts_authswitch/authority-apikeys.ts +85 -0
  119. package/ts_authswitch/authority-contract.ts +201 -34
  120. package/ts_authswitch/authority-overview.ts +7 -0
  121. package/ts_authswitch/authority-placement.ts +12 -4
  122. package/ts_authswitch/authority-runtime-contract.ts +2 -2
  123. package/ts_authswitch/authority-spend.ts +168 -0
  124. package/ts_authswitch/authority-subscription.ts +28 -1
  125. package/ts_authswitch/authority-usagelimit.ts +95 -40
  126. package/ts_authswitch/authority-usagewindows.ts +42 -17
  127. package/ts_authswitch/classes.authoritybroker.ts +161 -68
  128. package/ts_authswitch/classes.authoritycli.ts +112 -32
  129. package/ts_authswitch/classes.authorityclient.ts +66 -19
  130. package/ts_authswitch/classes.authoritydaemon.ts +43 -13
  131. package/ts_authswitch/classes.authoritydatabase.ts +178 -8
  132. package/ts_authswitch/classes.authorityimport.ts +1 -1
  133. package/ts_authswitch/classes.authoritymodels.ts +165 -19
  134. package/ts_authswitch/classes.authorityregistry.ts +3 -0
  135. package/ts_authswitch/classes.authorityusage.ts +171 -10
  136. package/ts_authswitch/classes.claudenative.ts +51 -22
  137. package/ts_authswitch/classes.claudesignin.ts +74 -24
  138. package/ts_authswitch/classes.codexmanaged.ts +30 -16
  139. package/ts_authswitch/classes.harnessprocesses.ts +39 -8
  140. package/ts_authswitch/native-contracts.md +82 -21
  141. package/ts_authswitch/readme.md +1 -1
  142. package/ts_authswitch/testsupport.ts +58 -11
  143. package/ts_authswitch/ts_migration/0011_account_api_key_fields.ts +20 -0
  144. package/ts_authswitch/ts_migration/index.ts +2 -0
  145. package/ts_interfaces/functions.authorityscope.ts +43 -8
  146. package/ts_interfaces/interfaces.ts +1 -1
  147. package/ts_interfaces/plugins.ts +6 -3
  148. package/ts_interfaces/requests.accounts.ts +48 -0
  149. package/ts_interfaces/requests.ts +4 -0
  150. package/ts_web/00_commitinfo_data.ts +1 -1
  151. package/ts_web/accounts.plugins.ts +7 -1
  152. package/ts_web/accounts.templates.ts +20 -11
  153. package/ts_web/classes.accountsreadmodel.ts +326 -71
  154. package/ts_web/elements.accounts.ts +219 -20
  155. package/ts_web/elements.codexconnections.ts +3 -1
  156. package/ts_web/plugins.ts +4 -2
@@ -0,0 +1,870 @@
1
+ import * as plugins from './plugins.js';
2
+
3
+ type TSpendAnswer = plugins.authswitchAuthority.IAuthSwitchSpendAnswer;
4
+ type TUsageSnapshot = plugins.authswitchAuthority.IAuthSwitchUsageSnapshot;
5
+
6
+ const hex64 = /^[a-f0-9]{64}$/;
7
+ /** A service tier as a spend answer names it (`IAuthSwitchSpendAnswer.serviceTier`). */
8
+ const serviceTierPattern = /^[a-z][a-z0-9_-]{0,63}$/;
9
+
10
+ const isRecord = (valueArg: unknown): valueArg is Record<string, unknown> =>
11
+ typeof valueArg === 'object' && valueArg !== null && !Array.isArray(valueArg);
12
+
13
+ /** The record's entry for a name the record itself holds, never one inherited from `Object.prototype`. */
14
+ const own = <T>(recordArg: Readonly<Record<string, T>> | undefined, keyArg: string): T | undefined =>
15
+ recordArg !== undefined && Object.prototype.hasOwnProperty.call(recordArg, keyArg) ? recordArg[keyArg] : undefined;
16
+
17
+ /** An error in one line, as the proxy's journal states it: its class and its own words, which carry no credential. */
18
+ const oneLine = (errorArg: unknown): string =>
19
+ (errorArg instanceof Error ? `${errorArg.name}: ${errorArg.message}` : String(errorArg)).replace(/\s+/gu, ' ');
20
+
21
+ /** An account as the journal names it: the first 12 hex characters of the SHA-256 digest of its id. */
22
+ const accountDigest = (accountIdArg: string): string =>
23
+ plugins.crypto.createHash('sha256').update(accountIdArg).digest('hex').slice(0, 12);
24
+
25
+ // Pricing ---------------------------------------------------------------------------------------------------------
26
+
27
+ /**
28
+ * The tool types of the Messages API whose calls the caller runs itself, so a request carrying them costs tokens only:
29
+ * a tool the caller defines (no `type`, or `custom`) and Anthropic's client tools and client toolsets, as Anthropic's
30
+ * tool reference lists them (https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-reference, read
31
+ * 2026-10-22). Every other type is a tool Anthropic runs, a server tool, priced by its exact type
32
+ * (`getAnthropicServerToolPrice`) or not at all. Claude Code 2.1.306 declares its own tools, its tool search included,
33
+ * as `custom` ones; its WebSearch runs in a request of its own with `web_search_20250305`, and `advisor_20260301` comes
34
+ * only with an `advisorModel` setting.
35
+ */
36
+ const anthropicClientToolTypes: ReadonlySet<string> = new Set(['custom', 'memory_20250818', 'bash_20250124',
37
+ 'text_editor_20250728', 'text_editor_20250124', 'computer_toolset_20260801', 'computer_20251124', 'computer_20250124',
38
+ 'browser_toolset_20260801']);
39
+
40
+ /**
41
+ * The `type` of every server tool a Messages request declares: each tool whose type is neither absent nor one of the
42
+ * client tools (`anthropicClientToolTypes`); a type that is no string is named as it reads.
43
+ */
44
+ export const inferenceAnthropicServerToolTypes = (bodyArg: Record<string, unknown>): string[] =>
45
+ (Array.isArray(bodyArg.tools) ? bodyArg.tools : []).flatMap((toolArg): string[] => {
46
+ const type = isRecord(toolArg) ? toolArg.type : undefined;
47
+ if (type === undefined || type === null || (typeof type === 'string' && anthropicClientToolTypes.has(type))) return [];
48
+ return [typeof type === 'string' ? type : JSON.stringify(type)];
49
+ });
50
+
51
+ /**
52
+ * The price of a request's answers: the model's, with the call fees of the server tools the request declares
53
+ * (`getAnthropicRequestPrice`); `undefined` when the model or one of the tools has no price.
54
+ */
55
+ const anthropicRequestPrice = (modelArg: string, serverToolTypesArg: readonly string[]): plugins.flexModels.IModelPrice | undefined => {
56
+ const price = plugins.anthropicProvider.getAnthropicModelPrice(modelArg);
57
+ return price && plugins.anthropicProvider.getAnthropicRequestPrice(price, serverToolTypesArg);
58
+ };
59
+
60
+ /**
61
+ * What of an Anthropic Messages request the proxy's price table (flexharness `getAnthropicModelPrice`,
62
+ * `getAnthropicRequestPrice`, `getAnthropicServerToolPrice`) cannot price, in words, or `undefined` when all of it can
63
+ * be: the model; a service tier other than `auto` and `standard_only`, the two a request may name
64
+ * (https://platform.claude.com/docs/en/api/service-tiers, read 2026-10-22), as the proxy sends a key with a spend limit
65
+ * on the standard tier only; a speed the model has no price for (`speed: "fast"` is Fast mode); an `inference_geo` the
66
+ * model has no surcharge for; a server tool whose exact type has no price, Anthropic's MCP connector included. A key
67
+ * with a monthly spend limit takes no such request, since its answer could not be priced (Decision Q4). The answer is
68
+ * still priced by the model that answered and what its usage states, never by this.
69
+ */
70
+ export const inferenceAnthropicUnpricedRequest = (bodyArg: Record<string, unknown>): string | undefined => {
71
+ const model = typeof bodyArg.model === 'string' ? bodyArg.model : '';
72
+ const price = plugins.anthropicProvider.getAnthropicModelPrice(model);
73
+ if (!price) return `the model ${model}, which AGL's price table does not list`;
74
+ const tier = bodyArg.service_tier;
75
+ if (tier !== undefined && tier !== null && tier !== 'auto' && tier !== 'standard_only') {
76
+ return `the service tier ${JSON.stringify(tier)}, of which only "auto" and "standard_only" go to a key with a spend limit`;
77
+ }
78
+ const speed = bodyArg.speed;
79
+ const service = speed === undefined || speed === null || speed === 'standard' ? 'standard' : speed === 'fast' ? 'fast' : undefined;
80
+ if (service === undefined || own(price.services, service) === undefined) {
81
+ return `the speed ${JSON.stringify(speed)} of ${model}, which AGL's price table does not price`;
82
+ }
83
+ const geo = bodyArg.inference_geo;
84
+ if (geo !== undefined && geo !== null && geo !== 'global'
85
+ && (typeof geo !== 'string' || own(price.surcharges, `inference_geo:${geo}`) === undefined)) {
86
+ return `inference_geo ${JSON.stringify(geo)} on ${model}, which AGL's price table does not price`;
87
+ }
88
+ if (Array.isArray(bodyArg.mcp_servers) && bodyArg.mcp_servers.length > 0) {
89
+ return 'the MCP connector (mcp_servers), whose calls AGL\'s price table does not price';
90
+ }
91
+ const serverTools = inferenceAnthropicServerToolTypes(bodyArg);
92
+ let requestPrice: plugins.flexModels.IModelPrice | undefined;
93
+ try {
94
+ requestPrice = plugins.anthropicProvider.getAnthropicRequestPrice(price, serverTools);
95
+ } catch {
96
+ return `the server tools ${serverTools.map((typeArg) => JSON.stringify(typeArg)).join(', ')}, whose fees AGL's price `
97
+ + 'table states twice';
98
+ }
99
+ if (requestPrice) return undefined;
100
+ const unpricedTool = serverTools.find((typeArg) => plugins.anthropicProvider.getAnthropicServerToolPrice(typeArg) === undefined);
101
+ return `the server tool ${JSON.stringify(unpricedTool)}, whose calls AGL's price table does not price`;
102
+ };
103
+
104
+ /**
105
+ * The bytes a request goes to an Anthropic API key with a spend limit as: the body the price check read
106
+ * (`inferenceAnthropicUnpricedRequest`), encoded again with `service_tier: "standard_only"`, so it is served on the
107
+ * standard tier the price table prices and never on Priority Tier, which has no published price (lane A ruling 6), and
108
+ * Anthropic receives exactly what was checked -- also when the client named `standard_only` itself.
109
+ */
110
+ export const inferenceAnthropicCheckedBytes = (bodyArg: Record<string, unknown>): Uint8Array<ArrayBuffer> =>
111
+ new TextEncoder().encode(JSON.stringify({ ...bodyArg, service_tier: 'standard_only' }));
112
+
113
+ /** What a relayed answer stated about its usage, as the gateway's usage tap read it while the answer passed. */
114
+ export interface IInferenceAnsweredUsage {
115
+ /** The model that answered, as the answer named it; absent when it named none the tap could read. */
116
+ model?: string;
117
+ /**
118
+ * The answer's usage object as the provider sent it -- a stream's merged from `message_start` and the `message_delta`s
119
+ * after it -- absent when the answer stated none.
120
+ */
121
+ usage?: Record<string, unknown>;
122
+ /** The answer arrived whole: a JSON body read to its end, or a stream through `message_stop`. */
123
+ complete: boolean;
124
+ }
125
+
126
+ /** One answer's spend as the outbox keeps it, and why it could not be priced when it could not. */
127
+ export interface IInferenceSpendRecord {
128
+ answer: TSpendAnswer;
129
+ /** Why the answer could not be priced (`answer.microUsd` is then null). */
130
+ unpriced?: string;
131
+ }
132
+
133
+ /**
134
+ * The spend of an answer an Anthropic API key served, priced once by the model that answered and the usage it stated
135
+ * (`readAnthropicMessagesUsage`, `priceModelCall`), never by the requested model, with the call fees of the server tools
136
+ * the request declared (`serverToolTypesArg`, `getAnthropicRequestPrice`). An answer whose model has no price, whose usage
137
+ * cannot be read completely or priced -- a tier, a cache lifetime, a server tool or a region the price lacks -- counts
138
+ * as one that could not be priced (`microUsd: null`), which holds a key with a spend limit until its month ends. A stream
139
+ * cut before it stated any usage records nothing: the client received no token of it; one cut later is priced on the
140
+ * usage it stated by then. `requestedModelArg` names the answer when it named no model.
141
+ */
142
+ export const inferenceAnthropicSpend = (answeredArg: IInferenceAnsweredUsage, requestedModelArg: string,
143
+ nowArg: number, serverToolTypesArg: readonly string[] = []): IInferenceSpendRecord | undefined => {
144
+ if (!answeredArg.usage && !answeredArg.complete) return undefined;
145
+ const model = [answeredArg.model, requestedModelArg].find((modelArg) => plugins.authswitchUsageTokens.isAuthSwitchModelId(modelArg));
146
+ if (model === undefined) throw new Error('A relayed answer names no model a spend report takes.');
147
+ const reportedTier = answeredArg.usage?.service_tier;
148
+ const answer = { answerId: plugins.crypto.randomUUID(), endedAt: new Date(nowArg).toISOString(), model,
149
+ serviceTier: typeof reportedTier === 'string' && serviceTierPattern.test(reportedTier) ? reportedTier : null };
150
+ const unpriced = (reasonArg: string): IInferenceSpendRecord =>
151
+ ({ answer: { ...answer, microUsd: null, priceSource: null }, unpriced: reasonArg });
152
+ if (!answeredArg.usage) return unpriced('the answer stated no usage');
153
+ let usage: plugins.flexModels.IModelPricedUsage;
154
+ try {
155
+ usage = plugins.anthropicProvider.readAnthropicMessagesUsage(answeredArg.usage);
156
+ } catch (errorArg) {
157
+ if (errorArg instanceof plugins.flexModels.ModelUsageReadError) return unpriced(errorArg.message);
158
+ throw errorArg;
159
+ }
160
+ if (!plugins.anthropicProvider.getAnthropicModelPrice(model)) return unpriced(`AGL's price table does not list the model ${model}`);
161
+ let price: plugins.flexModels.IModelPrice | undefined;
162
+ try {
163
+ price = anthropicRequestPrice(model, serverToolTypesArg);
164
+ } catch (errorArg) {
165
+ return unpriced(`AGL's price table disagrees with itself: ${oneLine(errorArg)}`);
166
+ }
167
+ if (!price) return unpriced(`AGL's price table has no price for a server tool the request declared (${serverToolTypesArg.join(', ')})`);
168
+ const cost = plugins.flexModels.priceModelCall(price, usage);
169
+ if (!cost.priced) return unpriced(`AGL's price table could not price it (${cost.reason}, service ${usage.service})`);
170
+ if (cost.microUsd > plugins.authswitchAuthority.authSwitchSpendAnswerMaxMicroUsd) {
171
+ return unpriced(`its price of ${cost.microUsd} micro-dollars is beyond what one answer can cost`);
172
+ }
173
+ return { answer: { ...answer, microUsd: cost.microUsd, priceSource: price.source.checkedAt } };
174
+ };
175
+
176
+ /**
177
+ * The tool types of the Responses API whose calls the caller runs itself, so a request declaring them costs tokens only,
178
+ * as OpenAI's Responses reference lists its tools (https://developers.openai.com/api/reference/resources/responses/methods/create
179
+ * and https://developers.openai.com/api/docs/guides/tools, read 2026-10-22): a function (`function`), a custom tool
180
+ * (`custom`), the local shell (`local_shell`), apply patch (`apply_patch`) and computer use (`computer`,
181
+ * `computer_use_preview`), whose calls the caller executes and answers. Two types are client tools in one mode only:
182
+ * `shell` with `environment.type: "local"` -- in a container OpenAI runs it
183
+ * (https://developers.openai.com/api/docs/guides/tools-shell) -- and `tool_search` with `execution: "client"`, whose
184
+ * search the caller runs and answers with a `tool_search_output` item; hosted, OpenAI runs it
185
+ * (https://developers.openai.com/api/docs/guides/tools-tool-search, read 2026-10-23). A `namespace` groups function and
186
+ * custom tools, each judged on its own. Every other type is a tool OpenAI runs, a server tool, priced only by the fee of
187
+ * its exact type (`IModelPrice.toolCallFees`: `web_search`, `file_search`), so `web_search_preview` and a dated
188
+ * `web_search_2025_08_26` have no price. Codex 0.163 sends `function`, `custom`, `tool_search` (`execution: "client"`)
189
+ * and `web_search` (`test/e2e/e2e.proxyspendcodex.node.ts`).
190
+ */
191
+ const openAiClientToolTypes: ReadonlySet<string> = new Set(['function', 'custom', 'local_shell', 'apply_patch', 'computer',
192
+ 'computer_use_preview']);
193
+
194
+ /**
195
+ * The `type` of every server tool a Responses request declares, in `tools` and in the tools its input adds
196
+ * (`additional_tools` and `tool_search_output` items), a namespace's own tools included: each tool that is no client
197
+ * tool (`openAiClientToolTypes`, a local `shell`); a type that is no string is named as it reads.
198
+ */
199
+ export const inferenceOpenAiServerToolTypes = (bodyArg: Record<string, unknown>): string[] => {
200
+ const declared = (toolsArg: unknown): string[] => (Array.isArray(toolsArg) ? toolsArg : []).flatMap((toolArg): string[] => {
201
+ const tool: Record<string, unknown> = isRecord(toolArg) ? toolArg : {};
202
+ const type = tool.type;
203
+ if (type === 'namespace') return declared(tool.tools);
204
+ if ((typeof type === 'string' && openAiClientToolTypes.has(type))
205
+ || (type === 'shell' && isRecord(tool.environment) && tool.environment.type === 'local')
206
+ || (type === 'tool_search' && tool.execution === 'client')) return [];
207
+ return [typeof type === 'string' ? type : String(JSON.stringify(type))];
208
+ });
209
+ const added = (Array.isArray(bodyArg.input) ? bodyArg.input : []).flatMap((itemArg) => isRecord(itemArg)
210
+ && (itemArg.type === 'additional_tools' || itemArg.type === 'tool_search_output') ? declared(itemArg.tools) : []);
211
+ return [...declared(bodyArg.tools), ...added];
212
+ };
213
+
214
+ /**
215
+ * The service tier a Responses request goes to an OpenAI API key with a spend limit on: the one the client named when it
216
+ * named a tier the price table prices for the model, else Standard (`default`), never `auto`, which a project set to Fast
217
+ * serves at Fast prices (`readme` of flexharness-providers' OpenAI provider).
218
+ */
219
+ const openAiCheckedTier = (bodyArg: Record<string, unknown>): string =>
220
+ typeof bodyArg.service_tier === 'string' && bodyArg.service_tier !== 'auto' ? bodyArg.service_tier : 'default';
221
+
222
+ /**
223
+ * What of a Responses request the proxy's price table (flexharness `getOpenAiModelPrice`) cannot price, in words, or
224
+ * `undefined` when all of it can be: the model, by its exact id or a listed snapshot (a dated id the price lists not is
225
+ * unpriced); an explicit service tier the model has no price for (`auto` and none go out as `default`; `fast` names
226
+ * Fast mode; `scale` has no published price); a background response (`background: true`), whose answer never passes the
227
+ * proxy; a server tool whose exact type has no call fee (`inferenceOpenAiServerToolTypes`). An OpenAI API key with a
228
+ * monthly spend limit takes no such request (Decision Q4).
229
+ */
230
+ export const inferenceOpenAiUnpricedRequest = (bodyArg: Record<string, unknown>): string | undefined => {
231
+ const model = typeof bodyArg.model === 'string' ? bodyArg.model : '';
232
+ const price = plugins.openAiProvider.getOpenAiModelPrice(model);
233
+ if (!price) return `the model ${model}, which AGL's price table does not list`;
234
+ const tier = bodyArg.service_tier;
235
+ if (tier !== undefined && tier !== null && tier !== 'auto'
236
+ && (typeof tier !== 'string' || own(price.services, tier) === undefined)) {
237
+ return `the service tier ${JSON.stringify(tier)} of ${model}, which AGL's price table does not price`;
238
+ }
239
+ if (bodyArg.background === true) {
240
+ return 'a background response (background: true), whose answer does not pass the proxy to be priced';
241
+ }
242
+ const unpricedTool = inferenceOpenAiServerToolTypes(bodyArg).find((typeArg) => own(price.toolCallFees, typeArg) === undefined);
243
+ return unpricedTool === undefined ? undefined
244
+ : `the server tool ${JSON.stringify(unpricedTool)}, whose calls AGL's price table does not price`;
245
+ };
246
+
247
+ /**
248
+ * The bytes a Responses request goes to an OpenAI API key with a spend limit as: the body the price check read
249
+ * (`inferenceOpenAiUnpricedRequest`), the model rewritten to `wireModelArg`, encoded again with the service tier it was
250
+ * checked for (`default` when the client named `auto` or none), so OpenAI serves it on a tier the price table prices.
251
+ */
252
+ export const inferenceOpenAiCheckedBytes = (bodyArg: Record<string, unknown>, wireModelArg: string): Uint8Array<ArrayBuffer> =>
253
+ new TextEncoder().encode(JSON.stringify({ ...bodyArg, model: wireModelArg, service_tier: openAiCheckedTier(bodyArg) }));
254
+
255
+ /** What a relayed Responses answer stated for its price, as the gateway's usage tap read it while the answer passed. */
256
+ export interface IInferenceResponsesAnswered {
257
+ /**
258
+ * How the answer ended: `answered`, its terminal event (`response.completed`, `response.incomplete`,
259
+ * `response.failed`) or its JSON body was read; `unreadable`, it arrived but was too large to read (over 8 MiB);
260
+ * `cut`, it ended before either arrived -- the client left, or the provider's stream ended or failed.
261
+ */
262
+ outcome: 'answered' | 'unreadable' | 'cut';
263
+ /** The response's `status` as stated (`completed`, `incomplete`, `failed`). */
264
+ status?: string;
265
+ /** The model that answered (`response.model`). */
266
+ model?: string;
267
+ /** The service tier that served it (`response.service_tier`), as stated. */
268
+ serviceTier?: unknown;
269
+ /** The response's `usage` object as sent; absent when it stated none. */
270
+ usage?: Record<string, unknown>;
271
+ /** The ids of the answer's `web_search_call` and `file_search_call` items, each once, whatever its action. */
272
+ webSearchCalls: ReadonlySet<string>;
273
+ fileSearchCalls: ReadonlySet<string>;
274
+ /** A web or file search call without an id, which cannot be counted once. */
275
+ unidentifiedCalls: boolean;
276
+ }
277
+
278
+ /**
279
+ * The spend of a Responses answer an OpenAI API key served, priced once by the model that answered, the tier that served
280
+ * it and the usage it stated (`readOpenAiResponsesUsage`, `priceModelCall`), with every `web_search_call` item one web
281
+ * search and every `file_search_call` item one file search, counted once per item id whatever its action -- so AGL's
282
+ * figure can exceed OpenAI's bill. A completed, incomplete or failed answer that states its usage is priced; one whose
283
+ * model, tier or usage the price table cannot price counts as one that could not be priced (`microUsd: null`), which
284
+ * holds a key with a spend limit until its month ends, and so does one that arrived too large to read, or a completed
285
+ * or incomplete one that states no usage. A failed answer that states none, and one cut before its terminal event,
286
+ * record nothing (`undefined`): no usage is estimated, and the caller says so in the journal.
287
+ */
288
+ export const inferenceOpenAiSpend = (answeredArg: IInferenceResponsesAnswered, requestedModelArg: string,
289
+ nowArg: number): IInferenceSpendRecord | undefined => {
290
+ if (answeredArg.outcome === 'cut' || (answeredArg.outcome === 'answered' && !answeredArg.usage && answeredArg.status === 'failed')) {
291
+ return undefined;
292
+ }
293
+ const model = [answeredArg.model, requestedModelArg].find((modelArg) => plugins.authswitchUsageTokens.isAuthSwitchModelId(modelArg));
294
+ if (model === undefined) throw new Error('A relayed answer names no model a spend report takes.');
295
+ const tier = answeredArg.serviceTier;
296
+ const answer = { answerId: plugins.crypto.randomUUID(), endedAt: new Date(nowArg).toISOString(), model,
297
+ serviceTier: typeof tier === 'string' && serviceTierPattern.test(tier) ? tier : null };
298
+ const unpriced = (reasonArg: string): IInferenceSpendRecord =>
299
+ ({ answer: { ...answer, microUsd: null, priceSource: null }, unpriced: reasonArg });
300
+ if (answeredArg.outcome === 'unreadable') return unpriced('its final event was too large to read');
301
+ if (!answeredArg.usage) return unpriced('the answer stated no usage');
302
+ if (answeredArg.unidentifiedCalls) return unpriced('it holds a web or file search call without an id, which cannot be counted once');
303
+ let usage: plugins.flexModels.IModelPricedUsage;
304
+ try {
305
+ usage = plugins.openAiProvider.readOpenAiResponsesUsage(answeredArg.usage, tier);
306
+ } catch (errorArg) {
307
+ if (errorArg instanceof plugins.flexModels.ModelUsageReadError) return unpriced(errorArg.message);
308
+ throw errorArg;
309
+ }
310
+ const price = plugins.openAiProvider.getOpenAiModelPrice(model);
311
+ if (!price) return unpriced(`AGL's price table does not list the model ${model}`);
312
+ const toolCalls: Record<string, number> = {
313
+ ...(answeredArg.webSearchCalls.size ? { web_search: answeredArg.webSearchCalls.size } : {}),
314
+ ...(answeredArg.fileSearchCalls.size ? { file_search: answeredArg.fileSearchCalls.size } : {}),
315
+ };
316
+ const cost = plugins.flexModels.priceModelCall(price, { ...usage, toolCalls });
317
+ if (!cost.priced) return unpriced(`AGL's price table could not price it (${cost.reason}, service ${usage.service})`);
318
+ if (cost.microUsd > plugins.authswitchAuthority.authSwitchSpendAnswerMaxMicroUsd) {
319
+ return unpriced(`its price of ${cost.microUsd} micro-dollars is beyond what one answer can cost`);
320
+ }
321
+ return { answer: { ...answer, microUsd: cost.microUsd, priceSource: price.source.checkedAt } };
322
+ };
323
+
324
+ // The outbox -------------------------------------------------------------------------------------------------------
325
+
326
+ /**
327
+ * One answer the proxy relayed on an API key, as its outbox keeps it until the accounts service acknowledged its cost:
328
+ * the spend report answer (`IAuthSwitchSpendAnswer`, its `answerId` as `id`) and the key's account. No prompt, answer
329
+ * content, token count or credential is part of it.
330
+ */
331
+ export interface IInferenceSpendDocument {
332
+ /** The answer's id (`IAuthSwitchSpendAnswer.answerId`), a UUID. */
333
+ id: string;
334
+ /** The API key's account. */
335
+ accountId: string;
336
+ /**
337
+ * The key's login the answer was served on, which its report names, also once the key is removed; `null` when none
338
+ * could be named, so the answer waits, counted, until one can (`IInferenceSpendOutboxOptions.attribute`).
339
+ */
340
+ loginId: string | null;
341
+ endedAt: string;
342
+ model: string;
343
+ serviceTier: string | null;
344
+ microUsd: number | null;
345
+ priceSource: string | null;
346
+ }
347
+
348
+ const answerOf = (documentArg: IInferenceSpendDocument): TSpendAnswer => ({ answerId: documentArg.id,
349
+ endedAt: documentArg.endedAt, model: documentArg.model, serviceTier: documentArg.serviceTier,
350
+ microUsd: documentArg.microUsd, priceSource: documentArg.priceSource });
351
+
352
+ const assertDocument: (valueArg: unknown) => asserts valueArg is IInferenceSpendDocument = (valueArg) => {
353
+ const value = valueArg as IInferenceSpendDocument | null;
354
+ if (!isRecord(value) || Object.keys(value).length !== 8 || typeof value.accountId !== 'string' || !hex64.test(value.accountId)
355
+ || (value.loginId !== null && (typeof value.loginId !== 'string' || !hex64.test(value.loginId)))
356
+ || !plugins.authswitchAuthority.isAuthSwitchSpendAnswer(answerOf(value))) {
357
+ throw new Error('Invalid inference spend answer.');
358
+ }
359
+ };
360
+
361
+ @plugins.nosqldb.managed({ collectionName: 'agl_inference_spend' })
362
+ @plugins.nosqldb.exactPersistence({ assertDocument })
363
+ export class InferenceSpendModel extends plugins.nosqldb.SmartDataDbDoc<InferenceSpendModel, IInferenceSpendDocument> {
364
+ declare static exact: plugins.nosqldb.TExact<InferenceSpendModel>;
365
+ @plugins.nosqldb.unI() public id!: string;
366
+ @plugins.nosqldb.svDb() public accountId!: string;
367
+ @plugins.nosqldb.svDb() public loginId!: string | null;
368
+ @plugins.nosqldb.svDb() public endedAt!: string;
369
+ @plugins.nosqldb.svDb() public model!: string;
370
+ @plugins.nosqldb.svDb() public serviceTier!: string | null;
371
+ @plugins.nosqldb.svDb() public microUsd!: number | null;
372
+ @plugins.nosqldb.svDb() public priceSource!: string | null;
373
+ }
374
+
375
+ /** An answer the outbox keeps, with its key's account and the login it was served on (`null`: none named yet). */
376
+ export interface IInferenceSpendEntry {
377
+ accountId: string;
378
+ loginId: string | null;
379
+ answer: TSpendAnswer;
380
+ }
381
+
382
+ /** The outbox's durable side: the answers kept in the proxy's own database until acknowledged. */
383
+ export interface IInferenceSpendStore {
384
+ /** Every answer kept. */
385
+ load(signalArg: AbortSignal): Promise<IInferenceSpendEntry[]>;
386
+ /** Keeps an answer; one kept already with the same content is kept. */
387
+ insert(entryArg: IInferenceSpendEntry, signalArg: AbortSignal): Promise<void>;
388
+ /** Forgets answers by id; one forgotten already is forgotten. */
389
+ delete(answerIdsArg: readonly string[], signalArg: AbortSignal): Promise<void>;
390
+ }
391
+
392
+ /** How long one store write may take before it counts as failed. */
393
+ const storeWriteTimeoutMs = 30_000;
394
+
395
+ /** How many kept answers one read of the store loads at a time. */
396
+ const loadPage = 256;
397
+
398
+ /** The outbox's store in the proxy's database (`agl_inference_spend`, a model of `proxyModels`). */
399
+ export class InferenceSpendStore implements IInferenceSpendStore {
400
+ public async load(signalArg: AbortSignal): Promise<IInferenceSpendEntry[]> {
401
+ const entries: IInferenceSpendEntry[] = [];
402
+ let after: string | undefined;
403
+ for (;;) {
404
+ const stored = await InferenceSpendModel.exact.findStored({ ...(after === undefined ? {} : { filter: { id: { $gt: after } } }),
405
+ sort: { id: 1 }, limit: loadPage, signal: signalArg });
406
+ for (const entry of stored) {
407
+ const document = InferenceSpendModel.exact.toPersisted(entry);
408
+ entries.push({ accountId: document.accountId, loginId: document.loginId, answer: answerOf(document) });
409
+ }
410
+ if (stored.length < loadPage) return entries;
411
+ after = entries.at(-1)!.answer.answerId;
412
+ }
413
+ }
414
+
415
+ public async insert(entryArg: IInferenceSpendEntry, signalArg: AbortSignal): Promise<void> {
416
+ const { answer } = entryArg;
417
+ const document: IInferenceSpendDocument = { id: answer.answerId, accountId: entryArg.accountId, loginId: entryArg.loginId,
418
+ endedAt: answer.endedAt,
419
+ model: answer.model, serviceTier: answer.serviceTier, microUsd: answer.microUsd, priceSource: answer.priceSource };
420
+ const inserted = await InferenceSpendModel.exact.insert(document, { signal: signalArg });
421
+ if (inserted.status !== 'conflict') return;
422
+ const held = InferenceSpendModel.exact.toPersisted(inserted.document);
423
+ if (JSON.stringify(held) !== JSON.stringify(document)) throw new Error('The proxy keeps another answer under this answer id.');
424
+ }
425
+
426
+ public async delete(answerIdsArg: readonly string[], signalArg: AbortSignal): Promise<void> {
427
+ for (const id of answerIdsArg) {
428
+ const current = await InferenceSpendModel.exact.findStoredOne({ id }, { signal: signalArg });
429
+ if (!current) continue;
430
+ const deleted = await InferenceSpendModel.exact.delete({ current }, { signal: signalArg });
431
+ if (deleted.status !== 'deleted' && await InferenceSpendModel.exact.findStoredOne({ id }, { signal: signalArg })) {
432
+ throw new Error('A spend answer could not be removed from the proxy\'s store.');
433
+ }
434
+ }
435
+ }
436
+ }
437
+
438
+ /**
439
+ * Something the outbox says in the proxy's journal: an answer that could not be priced (`unpriced`, an accounting
440
+ * error), a report the accounts service refused (`refused`), a key whose reports keep failing (`unreported`, said once
441
+ * until one goes through), answers the store could not keep (`unkept`) or acknowledged answers it could not remove
442
+ * (`unremoved`), each said once until the store takes writes again, an answer recorded with no login of its key
443
+ * (`unattributed`), and each answer that ended without its provider stating its usage, which counts nowhere (`uncounted`).
444
+ * Keys are named by their account's digest.
445
+ */
446
+ export type TInferenceSpendJournalEntry =
447
+ | { kind: 'unpriced'; account: string; model: string; reason: string }
448
+ | { kind: 'refused'; account: string; answers: number; error: string; retryAt: string }
449
+ | { kind: 'unreported'; account: string; answers: number; error: string }
450
+ | { kind: 'unkept'; answers: number; error: string }
451
+ | { kind: 'unremoved'; answers: number; error: string }
452
+ | { kind: 'unattributed'; account: string; model: string }
453
+ | { kind: 'uncounted'; account: string; model: string };
454
+
455
+ /** A count of answers in words: `1 answer`, `2 answers`. */
456
+ const answersText = (countArg: number): string => `${countArg} ${countArg === 1 ? 'answer' : 'answers'}`;
457
+
458
+ const writeJournal = (entryArg: TInferenceSpendJournalEntry): void => {
459
+ process.stderr.write(entryArg.kind === 'unpriced'
460
+ ? `AGL proxy could not price an answer on API key ${entryArg.account} (${entryArg.model}): ${entryArg.reason}. It counts `
461
+ + 'as an answer that could not be priced, which holds a key with a monthly spend limit until its month ends or the '
462
+ + 'limit is removed.\n'
463
+ : entryArg.kind === 'refused'
464
+ ? `AGL proxy's spend report of API key ${entryArg.account} (${answersText(entryArg.answers)}) was refused by the accounts `
465
+ + `service: ${entryArg.error} The proxy keeps the answers and counts them, sends them again at ${entryArg.retryAt} `
466
+ + 'and at every start, and places nothing on the key while it has a monthly spend limit.\n'
467
+ : entryArg.kind === 'unreported'
468
+ ? `AGL proxy could not report the spend of API key ${entryArg.account} (${answersText(entryArg.answers)}) to the `
469
+ + `accounts service: ${entryArg.error} -- it keeps the answers, counts them against the key's limit and tries again.\n`
470
+ : entryArg.kind === 'unkept'
471
+ ? `AGL proxy could not keep the spend of ${answersText(entryArg.answers)} in its store: ${entryArg.error} -- it counts `
472
+ + 'them and tries again.\n'
473
+ : entryArg.kind === 'unremoved'
474
+ ? `AGL proxy could not remove ${answersText(entryArg.answers)}, acknowledged, from its store: ${entryArg.error} -- it `
475
+ + 'tries again; a start before then reports them again, which the accounts service records once.\n'
476
+ : entryArg.kind === 'unattributed'
477
+ ? `AGL proxy holds an answer on API key ${entryArg.account} (${entryArg.model}) with no login of the key to report `
478
+ + 'it on: it counts, and holds the key while it has a monthly spend limit, until the account service names a login.\n'
479
+ : `AGL proxy did not count an answer on API key ${entryArg.account} (${entryArg.model}): it ended without the provider `
480
+ + 'stating its usage, so nothing of it counts against the key\'s monthly spend limit, though the provider may bill it.\n');
481
+ };
482
+
483
+ export interface IInferenceSpendOutboxOptions {
484
+ store: IInferenceSpendStore;
485
+ /**
486
+ * Sends one report of answers a key served on one of its logins (`authswitch.authority.usage.spend`, one to 64 answers,
487
+ * each id once) and resolves with the key's usage the accounts service answered once the report committed.
488
+ */
489
+ report: (accountIdArg: string, loginIdArg: string, answersArg: TSpendAnswer[], signalArg: AbortSignal) => Promise<TUsageSnapshot>;
490
+ /**
491
+ * Names a login of the key for answers recorded without one -- the account service's live or removed API-key login of
492
+ * it -- or `undefined` while it names none; such answers count and hold the key until then.
493
+ */
494
+ attribute: (accountIdArg: string) => string | undefined;
495
+ /**
496
+ * Holds the usage an acknowledged report answered, which counts the report's answers: called in the same synchronous
497
+ * step that drops them from what is pending, so no reading is judged without them or with them twice.
498
+ */
499
+ acknowledged: (accountIdArg: string, usageArg: TUsageSnapshot) => void;
500
+ /** Where the outbox's journal entries go; one line on standard error, the proxy's journal, when unset. */
501
+ journal?: (entryArg: TInferenceSpendJournalEntry) => void;
502
+ now?: () => number;
503
+ /** The first wait after a report or a store write failed for a passing reason; it doubles up to `maxRetryMs`. */
504
+ retryMs?: number;
505
+ maxRetryMs?: number;
506
+ /** How long answers whose report the accounts service refused wait before they are sent again. */
507
+ heldRetryMs?: number;
508
+ }
509
+
510
+ interface IEntry {
511
+ answer: TSpendAnswer;
512
+ /** The key's login it was served on, which its report names; `null` until one is named (`attribute`). */
513
+ loginId: string | null;
514
+ /**
515
+ * The store keeps it: written as it is recorded, on a chain of store writes of its own that never waits for a report.
516
+ * An acknowledged answer is removed from the store, once it was kept.
517
+ */
518
+ kept: boolean;
519
+ /** A store write of it failed: until it is kept or acknowledged it is held in neither, and holds a limited key. */
520
+ keepFailed: boolean;
521
+ /** Its report was refused (`invalid_input`, or a malformed request): sent again at this time, or `null` while it is not. */
522
+ heldUntil: number | null;
523
+ }
524
+
525
+ interface IAccountState {
526
+ /** Reports of the key that failed in a row for a passing reason. */
527
+ failures: number;
528
+ /** No report of the key is sent before this time. */
529
+ retryAt: number;
530
+ /** The failures were said in the journal: once until a report goes through. */
531
+ said: boolean;
532
+ }
533
+
534
+ /**
535
+ * The spend of the answers the proxy relayed on API keys, kept until the accounts service acknowledged it: durable in
536
+ * the proxy's database (`agl_inference_spend`), never expired, and counted against each key's limit while unacknowledged
537
+ * (`judged`), across restarts.
538
+ *
539
+ * `record` counts an answer at once and writes it to the store at once, on a chain of store writes that never waits for
540
+ * a report, so an answer recorded while a report is in flight is durable before that report settles. Answers are
541
+ * reported whether the store kept them yet or not -- the ledger records an answer once, so either makes it durable --
542
+ * per key and login in batches of at most 64 unique answers, one report in flight at a time, the first as soon as an
543
+ * answer is recorded and every one at start. An answer held in neither after a failed store write holds its key, with
544
+ * a spend limit, like a refused report (`held`). The in-memory index is complete because the proxy store has one owner
545
+ * (`ProxyStore` refuses a second engine); a second writer of `agl_inference_spend` would break it.
546
+ * An acknowledgement drops the report's answers and holds the usage the service answered in one step (`acknowledged`).
547
+ * A refusal of the report -- `invalid_input`, or a malformed request (`isAuthSwitchMalformedRequest`) -- holds its
548
+ * answers: they keep counting, the key with a spend limit takes no request while any is held (`held`), the refusal goes
549
+ * to the journal, and the answers are sent again after `heldRetryMs` and at every start, never dropped. Every other
550
+ * failure, `Internal server error` included, is passing: the key's reports wait, doubling from `retryMs` to
551
+ * `maxRetryMs`. Answers of months before the previous one are acknowledged by the service and counted nowhere, so they
552
+ * leave as every acknowledged answer does.
553
+ */
554
+ export class InferenceSpendOutbox {
555
+ /** The answers not acknowledged yet, by key, then answer id: an id is held once, so no report names one twice. */
556
+ private readonly entries = new Map<string, Map<string, IEntry>>();
557
+ private readonly accounts = new Map<string, IAccountState>();
558
+ /** Acknowledged answers still to be removed from the store. */
559
+ private readonly acknowledgedIds = new Set<string>();
560
+ private readonly abort = new AbortController();
561
+ private readonly now: () => number;
562
+ private readonly retryMs: number;
563
+ private readonly maxRetryMs: number;
564
+ private readonly heldRetryMs: number;
565
+ /** The store writes of recorded answers, one at a time, apart from the reports. */
566
+ private keepTail: Promise<void> = Promise.resolve();
567
+ private keepFailures = 0;
568
+ private keepRetryAt = 0;
569
+ private keepSaid = false;
570
+ private keepTimer?: ReturnType<typeof setTimeout>;
571
+ private forgetFailures = 0;
572
+ private forgetRetryAt = 0;
573
+ private forgetSaid = false;
574
+ private started = false;
575
+ private stopped = false;
576
+ private running?: Promise<void>;
577
+ private again = false;
578
+ private timer?: ReturnType<typeof setTimeout>;
579
+
580
+ constructor(private readonly options: IInferenceSpendOutboxOptions) {
581
+ this.now = options.now ?? Date.now;
582
+ this.retryMs = options.retryMs ?? 5_000;
583
+ this.maxRetryMs = options.maxRetryMs ?? 300_000;
584
+ this.heldRetryMs = options.heldRetryMs ?? 600_000;
585
+ }
586
+
587
+ /** Loads the answers a run before left unacknowledged and sends them all now. */
588
+ public async start(signalArg?: AbortSignal): Promise<void> {
589
+ if (this.started || this.stopped) throw new Error('The spend outbox cannot be started twice.');
590
+ for (const entry of await this.options.store.load(signalArg ?? this.abort.signal)) {
591
+ this.add(entry.accountId, { answer: entry.answer, loginId: entry.loginId, kept: true, keepFailed: false, heldUntil: null });
592
+ }
593
+ this.started = true;
594
+ this.kick();
595
+ }
596
+
597
+ /**
598
+ * Counts a relayed answer's spend at once, writes it to the store and reports it, naming the key's login it was served
599
+ * on (`loginIdArg`), until the accounts service acknowledged it; an answer that could not be priced goes to the journal
600
+ * as well. One recorded without a login (`null`) counts all the same, holds the key (`held`) and waits for a login to
601
+ * be named (`attribute`), which the journal says. Throws for an answer no report could carry, a defect of its caller.
602
+ */
603
+ public record(accountIdArg: string, loginIdArg: string | null, recordArg: IInferenceSpendRecord): void {
604
+ if (!hex64.test(accountIdArg) || (loginIdArg !== null && !hex64.test(loginIdArg))
605
+ || !plugins.authswitchAuthority.isAuthSwitchSpendAnswer(recordArg.answer)) {
606
+ throw new Error('A spend answer the accounts service would refuse was recorded.');
607
+ }
608
+ this.add(accountIdArg, { answer: { ...recordArg.answer }, loginId: loginIdArg, kept: false, keepFailed: false,
609
+ heldUntil: null });
610
+ if (recordArg.unpriced !== undefined) {
611
+ this.say({ kind: 'unpriced', account: accountDigest(accountIdArg), model: recordArg.answer.model, reason: recordArg.unpriced });
612
+ }
613
+ if (loginIdArg === null) this.say({ kind: 'unattributed', account: accountDigest(accountIdArg), model: recordArg.answer.model });
614
+ this.keep();
615
+ this.kick();
616
+ }
617
+
618
+ /**
619
+ * Says in the journal that an answer relayed on the key ended without its provider stating its usage -- the client left,
620
+ * the stream was cut, or the response failed stating none -- so nothing of it is counted; no usage is estimated for it
621
+ * (`inferenceOpenAiSpend`).
622
+ */
623
+ public uncounted(accountIdArg: string, modelArg: string): void {
624
+ this.say({ kind: 'uncounted', account: accountDigest(accountIdArg), model: modelArg });
625
+ }
626
+
627
+ /** The answers relayed on a key whose spend the accounts service has not acknowledged yet. */
628
+ public pending(accountIdArg: string): plugins.authswitchSpend.IAuthSwitchSpendPending[] {
629
+ return [...this.entries.get(accountIdArg)?.values() ?? []]
630
+ .map((entryArg) => ({ endedAt: entryArg.answer.endedAt, microUsd: entryArg.answer.microUsd }));
631
+ }
632
+
633
+ /**
634
+ * Whether spend of the key is not recorded and might never be: the accounts service refused a report of its answers
635
+ * that is still held, an answer is held in neither the store, whose write failed, nor the accounts service's ledger,
636
+ * or an answer has no login of the key to be reported on yet.
637
+ */
638
+ public held(accountIdArg: string): boolean {
639
+ return [...this.entries.get(accountIdArg)?.values() ?? []]
640
+ .some((entryArg) => entryArg.heldUntil !== null || (entryArg.keepFailed && !entryArg.kept) || entryArg.loginId === null);
641
+ }
642
+
643
+ /**
644
+ * A key's usage as the proxy judges it now: its spend reading with the answers not acknowledged yet added, in the
645
+ * month now falls in (`authSwitchSpendWithPending`). Any other usage is answered as it is.
646
+ */
647
+ public judged(accountIdArg: string, usageArg: TUsageSnapshot): TUsageSnapshot {
648
+ if (usageArg.reading?.kind !== 'spend') return usageArg;
649
+ return { ...usageArg, reading: { kind: 'spend', spend: plugins.authswitchSpend.authSwitchSpendWithPending(
650
+ usageArg.reading.spend, this.pending(accountIdArg), this.now()) } };
651
+ }
652
+
653
+ /**
654
+ * Stops sending; what is held stays in the store for the next start. The store writes in progress finish, and what is
655
+ * still not kept or not removed is written once more.
656
+ */
657
+ public async stop(): Promise<void> {
658
+ if (this.stopped) return;
659
+ this.stopped = true;
660
+ clearTimeout(this.timer);
661
+ this.timer = undefined;
662
+ clearTimeout(this.keepTimer);
663
+ this.keepTimer = undefined;
664
+ this.abort.abort(new Error('The spend outbox stopped.'));
665
+ await this.running;
666
+ await this.keepTail;
667
+ await this.keepOnce(true);
668
+ if (this.started) await this.forget(AbortSignal.timeout(storeWriteTimeoutMs), true);
669
+ }
670
+
671
+ private add(accountIdArg: string, entryArg: IEntry): void {
672
+ let entries = this.entries.get(accountIdArg);
673
+ if (!entries) { entries = new Map(); this.entries.set(accountIdArg, entries); }
674
+ entries.set(entryArg.answer.answerId, entryArg);
675
+ }
676
+
677
+ private say(entryArg: TInferenceSpendJournalEntry): void {
678
+ try { (this.options.journal ?? writeJournal)(entryArg); } catch { /* a journal line that fails changes nothing */ }
679
+ }
680
+
681
+ /** Runs the outbox's work now, or once more after the run in progress. */
682
+ private kick(): void {
683
+ if (!this.started || this.stopped) return;
684
+ if (this.running) { this.again = true; return; }
685
+ clearTimeout(this.timer);
686
+ this.timer = undefined;
687
+ this.running = (async () => {
688
+ do {
689
+ this.again = false;
690
+ await this.cycle();
691
+ } while (this.again && !this.stopped);
692
+ })().catch(() => undefined).finally(() => {
693
+ this.running = undefined;
694
+ this.arm();
695
+ });
696
+ }
697
+
698
+ /** Removes what was acknowledged from the store, then sends every report that is due. */
699
+ private async cycle(): Promise<void> {
700
+ const signal = this.abort.signal;
701
+ if (this.now() >= this.forgetRetryAt) await this.forget(signal, false);
702
+ for (let batch = this.nextBatch(); batch && !this.stopped; batch = this.nextBatch()) {
703
+ await this.send(batch.accountId, batch.loginId, batch.entries, signal);
704
+ }
705
+ }
706
+
707
+ /**
708
+ * Writes the recorded answers not kept yet to the store, on the outbox's chain of store writes, which runs apart from
709
+ * the reports and never waits for one; once they are kept the reports run. After a failed write the next waits,
710
+ * doubling from `retryMs`, unless an answer recorded meanwhile, or the stop, tries again sooner.
711
+ */
712
+ private keep(): void {
713
+ if (this.stopped) return;
714
+ clearTimeout(this.keepTimer);
715
+ this.keepTimer = undefined;
716
+ this.keepTail = this.keepTail.then(() => this.keepOnce(false));
717
+ }
718
+
719
+ /**
720
+ * One pass of the store writes: every answer not kept yet, one at a time. A write that fails is said once until the
721
+ * store takes writes again; the answers count meanwhile all the same.
722
+ */
723
+ private async keepOnce(finalArg: boolean): Promise<void> {
724
+ if (this.stopped && !finalArg) return;
725
+ const unkept = [...this.entries].flatMap(([accountId, entries]) => [...entries.values()]
726
+ .filter((entryArg) => !entryArg.kept).map((entryArg) => ({ accountId, entry: entryArg })));
727
+ if (!unkept.length) return;
728
+ try {
729
+ for (const { accountId, entry } of unkept) {
730
+ await this.options.store.insert({ accountId, loginId: entry.loginId, answer: entry.answer },
731
+ AbortSignal.timeout(storeWriteTimeoutMs));
732
+ entry.kept = true;
733
+ // Acknowledged while it was written: it leaves the store again.
734
+ if (this.entries.get(accountId)?.get(entry.answer.answerId) !== entry) this.acknowledgedIds.add(entry.answer.answerId);
735
+ }
736
+ this.keepFailures = 0;
737
+ this.keepRetryAt = 0;
738
+ this.keepSaid = false;
739
+ this.kick();
740
+ } catch (errorArg) {
741
+ for (const { entry } of unkept) if (!entry.kept) entry.keepFailed = true;
742
+ this.keepFailures += 1;
743
+ this.keepRetryAt = this.now() + this.backoff(this.keepFailures);
744
+ if (!this.keepSaid) {
745
+ this.keepSaid = true;
746
+ this.say({ kind: 'unkept', answers: unkept.filter((itemArg) => !itemArg.entry.kept).length, error: oneLine(errorArg) });
747
+ }
748
+ if (this.stopped) return;
749
+ clearTimeout(this.keepTimer);
750
+ this.keepTimer = setTimeout(() => this.keep(), Math.max(0, this.keepRetryAt - this.now()));
751
+ this.keepTimer.unref?.();
752
+ }
753
+ }
754
+
755
+ /** Removes the acknowledged answers from the store; a failed removal waits, doubling, said once. */
756
+ private async forget(signalArg: AbortSignal, finalArg: boolean): Promise<void> {
757
+ if (!this.acknowledgedIds.size) return;
758
+ const ids = [...this.acknowledgedIds];
759
+ try {
760
+ await this.options.store.delete(ids, signalArg);
761
+ for (const id of ids) this.acknowledgedIds.delete(id);
762
+ this.forgetFailures = 0;
763
+ this.forgetRetryAt = 0;
764
+ this.forgetSaid = false;
765
+ } catch (errorArg) {
766
+ if (this.stopped && !finalArg) return;
767
+ this.forgetFailures += 1;
768
+ this.forgetRetryAt = this.now() + this.backoff(this.forgetFailures);
769
+ if (!this.forgetSaid) {
770
+ this.forgetSaid = true;
771
+ this.say({ kind: 'unremoved', answers: ids.length, error: oneLine(errorArg) });
772
+ }
773
+ }
774
+ }
775
+
776
+ /**
777
+ * The next report due: a key's held answers once their wait ended, as a report of their own, else its other answers,
778
+ * kept or not; of one login, oldest first, at most 64, each id once. Answers recorded without a login take the one
779
+ * `attribute` names now; while it names none they wait, the key's reports backing off as after a passing failure.
780
+ */
781
+ private nextBatch(): { accountId: string; loginId: string; entries: IEntry[] } | undefined {
782
+ const now = this.now();
783
+ for (const [accountId, entries] of this.entries) {
784
+ if ((this.accounts.get(accountId)?.retryAt ?? 0) > now) continue;
785
+ const all = [...entries.values()];
786
+ if (all.some((entryArg) => entryArg.loginId === null)) {
787
+ const login = this.options.attribute(accountId);
788
+ if (login !== undefined && hex64.test(login)) for (const entry of all) entry.loginId ??= login;
789
+ }
790
+ const named = all.filter((entryArg) => entryArg.loginId !== null);
791
+ const due = named.filter((entryArg) => entryArg.heldUntil !== null && entryArg.heldUntil <= now);
792
+ const fresh = named.filter((entryArg) => entryArg.heldUntil === null);
793
+ const oldest = (due.length ? due : fresh)
794
+ .sort((leftArg, rightArg) => Date.parse(leftArg.answer.endedAt) - Date.parse(rightArg.answer.endedAt));
795
+ const loginId = oldest[0]?.loginId;
796
+ if (loginId) {
797
+ return { accountId, loginId, entries: oldest.filter((entryArg) => entryArg.loginId === loginId)
798
+ .slice(0, plugins.authswitchAuthority.authSwitchSpendAnswerBatchLimit) };
799
+ }
800
+ if (named.length < all.length) {
801
+ // Nothing of the key can be reported until a login is named: ask again later.
802
+ const account = this.accounts.get(accountId) ?? { failures: 0, retryAt: 0, said: true };
803
+ account.failures += 1;
804
+ account.retryAt = now + this.backoff(account.failures);
805
+ this.accounts.set(accountId, account);
806
+ }
807
+ }
808
+ return undefined;
809
+ }
810
+
811
+ private async send(accountIdArg: string, loginIdArg: string, entriesArg: IEntry[], signalArg: AbortSignal): Promise<void> {
812
+ const account = this.accounts.get(accountIdArg) ?? { failures: 0, retryAt: 0, said: false };
813
+ this.accounts.set(accountIdArg, account);
814
+ let usage: TUsageSnapshot;
815
+ try {
816
+ usage = await this.options.report(accountIdArg, loginIdArg, entriesArg.map((entryArg) => ({ ...entryArg.answer })),
817
+ signalArg);
818
+ } catch (errorArg) {
819
+ if (this.stopped) return;
820
+ const refusal = plugins.authswitchAuthority.asAuthSwitchRefusal(errorArg);
821
+ if (refusal?.code === 'invalid_input' || plugins.authswitchAuthority.isAuthSwitchMalformedRequest(errorArg)) {
822
+ const retryAt = this.now() + this.heldRetryMs;
823
+ for (const entry of entriesArg) entry.heldUntil = retryAt;
824
+ this.say({ kind: 'refused', account: accountDigest(accountIdArg), answers: entriesArg.length,
825
+ error: refusal?.instruction ?? plugins.authswitchAuthority.authSwitchMalformedRequestText,
826
+ retryAt: new Date(retryAt).toISOString() });
827
+ return;
828
+ }
829
+ account.failures += 1;
830
+ account.retryAt = this.now() + this.backoff(account.failures);
831
+ if (!account.said) {
832
+ account.said = true;
833
+ this.say({ kind: 'unreported', account: accountDigest(accountIdArg),
834
+ answers: this.entries.get(accountIdArg)?.size ?? entriesArg.length, error: oneLine(errorArg) });
835
+ }
836
+ return;
837
+ }
838
+ // The one acknowledgement step: the service's answer, which counts the report's answers, replaces the held reading
839
+ // in the same synchronous step that drops them from what is pending.
840
+ this.options.acknowledged(accountIdArg, usage);
841
+ const entries = this.entries.get(accountIdArg);
842
+ for (const entry of entriesArg) {
843
+ entries?.delete(entry.answer.answerId);
844
+ // One the store has not kept yet leaves it once its write lands (`keepOnce`).
845
+ if (entry.kept) this.acknowledgedIds.add(entry.answer.answerId);
846
+ }
847
+ if (entries?.size === 0) this.entries.delete(accountIdArg);
848
+ this.accounts.delete(accountIdArg);
849
+ }
850
+
851
+ private backoff(failuresArg: number): number {
852
+ return Math.min(this.maxRetryMs, this.retryMs * 2 ** Math.max(0, failuresArg - 1));
853
+ }
854
+
855
+ /** Wakes the reports when the next held or waiting answer, or a removal to try again, is due. */
856
+ private arm(): void {
857
+ clearTimeout(this.timer);
858
+ this.timer = undefined;
859
+ if (this.stopped || !this.started) return;
860
+ let due = Number.POSITIVE_INFINITY;
861
+ if (this.acknowledgedIds.size) due = Math.min(due, this.forgetRetryAt);
862
+ for (const [accountId, entries] of this.entries) {
863
+ const retryAt = this.accounts.get(accountId)?.retryAt ?? 0;
864
+ for (const entry of entries.values()) due = Math.min(due, Math.max(retryAt, entry.heldUntil ?? 0));
865
+ }
866
+ if (!Number.isFinite(due)) return;
867
+ this.timer = setTimeout(() => this.kick(), Math.max(0, due - this.now()));
868
+ this.timer.unref?.();
869
+ }
870
+ }