@acentric/llm 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/README.md +128 -2
  2. package/dist/catalog/anthropic.d.ts +77 -0
  3. package/dist/catalog/anthropic.d.ts.map +1 -0
  4. package/dist/catalog/anthropic.js +73 -0
  5. package/dist/catalog/anthropic.js.map +1 -0
  6. package/dist/catalog/fireworks.d.ts +93 -0
  7. package/dist/catalog/fireworks.d.ts.map +1 -0
  8. package/dist/catalog/fireworks.js +117 -0
  9. package/dist/catalog/fireworks.js.map +1 -0
  10. package/dist/catalog/openai.d.ts +103 -0
  11. package/dist/catalog/openai.d.ts.map +1 -0
  12. package/dist/catalog/openai.js +102 -0
  13. package/dist/catalog/openai.js.map +1 -0
  14. package/dist/client/anthropic/errors.d.ts +4 -0
  15. package/dist/client/anthropic/errors.d.ts.map +1 -0
  16. package/dist/client/anthropic/errors.js +80 -0
  17. package/dist/client/anthropic/errors.js.map +1 -0
  18. package/dist/client/anthropic/request.d.ts +9 -0
  19. package/dist/client/anthropic/request.d.ts.map +1 -0
  20. package/dist/client/anthropic/request.js +232 -0
  21. package/dist/client/anthropic/request.js.map +1 -0
  22. package/dist/client/anthropic/response.d.ts +13 -0
  23. package/dist/client/anthropic/response.d.ts.map +1 -0
  24. package/dist/client/anthropic/response.js +207 -0
  25. package/dist/client/anthropic/response.js.map +1 -0
  26. package/dist/client/direct.d.ts +18 -0
  27. package/dist/client/direct.d.ts.map +1 -0
  28. package/dist/client/direct.js +104 -0
  29. package/dist/client/direct.js.map +1 -0
  30. package/dist/client/gateway.d.ts +43 -0
  31. package/dist/client/gateway.d.ts.map +1 -0
  32. package/dist/client/gateway.js +72 -0
  33. package/dist/client/gateway.js.map +1 -0
  34. package/dist/client/http.d.ts +41 -0
  35. package/dist/client/http.d.ts.map +1 -0
  36. package/dist/client/http.js +190 -0
  37. package/dist/client/http.js.map +1 -0
  38. package/dist/client/llm.d.ts +41 -0
  39. package/dist/client/llm.d.ts.map +1 -0
  40. package/dist/client/llm.js +100 -0
  41. package/dist/client/llm.js.map +1 -0
  42. package/dist/client/responses/errors.d.ts +13 -0
  43. package/dist/client/responses/errors.d.ts.map +1 -0
  44. package/dist/client/responses/errors.js +144 -0
  45. package/dist/client/responses/errors.js.map +1 -0
  46. package/dist/client/responses/profile.d.ts +50 -0
  47. package/dist/client/responses/profile.d.ts.map +1 -0
  48. package/dist/client/responses/profile.js +4 -0
  49. package/dist/client/responses/profile.js.map +1 -0
  50. package/dist/client/responses/providers.d.ts +4 -0
  51. package/dist/client/responses/providers.d.ts.map +1 -0
  52. package/dist/client/responses/providers.js +75 -0
  53. package/dist/client/responses/providers.js.map +1 -0
  54. package/dist/client/responses/request.d.ts +5 -0
  55. package/dist/client/responses/request.d.ts.map +1 -0
  56. package/dist/client/responses/request.js +248 -0
  57. package/dist/client/responses/request.js.map +1 -0
  58. package/dist/client/responses/response.d.ts +10 -0
  59. package/dist/client/responses/response.d.ts.map +1 -0
  60. package/dist/client/responses/response.js +176 -0
  61. package/dist/client/responses/response.js.map +1 -0
  62. package/dist/client/util.d.ts +21 -0
  63. package/dist/client/util.d.ts.map +1 -0
  64. package/dist/client/util.js +63 -0
  65. package/dist/client/util.js.map +1 -0
  66. package/dist/client.d.ts +9 -0
  67. package/dist/client.d.ts.map +1 -0
  68. package/dist/client.js +10 -0
  69. package/dist/client.js.map +1 -0
  70. package/dist/content.d.ts +38 -0
  71. package/dist/content.d.ts.map +1 -0
  72. package/dist/content.js +5 -0
  73. package/dist/content.js.map +1 -0
  74. package/dist/convert/anthropic.d.ts +3 -0
  75. package/dist/convert/anthropic.d.ts.map +1 -0
  76. package/dist/convert/anthropic.js +114 -0
  77. package/dist/convert/anthropic.js.map +1 -0
  78. package/dist/convert/index.d.ts +12 -0
  79. package/dist/convert/index.d.ts.map +1 -0
  80. package/dist/convert/index.js +221 -0
  81. package/dist/convert/index.js.map +1 -0
  82. package/dist/convert/responses.d.ts +3 -0
  83. package/dist/convert/responses.d.ts.map +1 -0
  84. package/dist/convert/responses.js +197 -0
  85. package/dist/convert/responses.js.map +1 -0
  86. package/dist/convert/types.d.ts +88 -0
  87. package/dist/convert/types.d.ts.map +1 -0
  88. package/dist/convert/types.js +3 -0
  89. package/dist/convert/types.js.map +1 -0
  90. package/dist/cost.d.ts +18 -0
  91. package/dist/cost.d.ts.map +1 -0
  92. package/dist/cost.js +49 -0
  93. package/dist/cost.js.map +1 -0
  94. package/dist/failure.d.ts +90 -0
  95. package/dist/failure.d.ts.map +1 -0
  96. package/dist/failure.js +111 -0
  97. package/dist/failure.js.map +1 -0
  98. package/dist/fields.d.ts +14 -0
  99. package/dist/fields.d.ts.map +1 -0
  100. package/dist/fields.js +3 -0
  101. package/dist/fields.js.map +1 -0
  102. package/dist/formats/anthropic.d.ts +152 -0
  103. package/dist/formats/anthropic.d.ts.map +1 -0
  104. package/dist/formats/anthropic.js +7 -0
  105. package/dist/formats/anthropic.js.map +1 -0
  106. package/dist/formats/responses.d.ts +179 -0
  107. package/dist/formats/responses.d.ts.map +1 -0
  108. package/dist/formats/responses.js +9 -0
  109. package/dist/formats/responses.js.map +1 -0
  110. package/dist/index.d.ts +26 -0
  111. package/dist/index.d.ts.map +1 -0
  112. package/dist/index.js +11 -0
  113. package/dist/index.js.map +1 -0
  114. package/dist/json.d.ts +6 -0
  115. package/dist/json.d.ts.map +1 -0
  116. package/dist/json.js +10 -0
  117. package/dist/json.js.map +1 -0
  118. package/dist/messages.d.ts +82 -0
  119. package/dist/messages.d.ts.map +1 -0
  120. package/dist/messages.js +4 -0
  121. package/dist/messages.js.map +1 -0
  122. package/dist/model.d.ts +36 -0
  123. package/dist/model.d.ts.map +1 -0
  124. package/dist/model.js +4 -0
  125. package/dist/model.js.map +1 -0
  126. package/dist/models.d.ts +9 -0
  127. package/dist/models.d.ts.map +1 -0
  128. package/dist/models.js +19 -0
  129. package/dist/models.js.map +1 -0
  130. package/dist/providers/anthropic.d.ts +130 -0
  131. package/dist/providers/anthropic.d.ts.map +1 -0
  132. package/dist/providers/anthropic.js +5 -0
  133. package/dist/providers/anthropic.js.map +1 -0
  134. package/dist/providers/fireworks.d.ts +108 -0
  135. package/dist/providers/fireworks.d.ts.map +1 -0
  136. package/dist/providers/fireworks.js +6 -0
  137. package/dist/providers/fireworks.js.map +1 -0
  138. package/dist/providers/openai.d.ts +193 -0
  139. package/dist/providers/openai.d.ts.map +1 -0
  140. package/dist/providers/openai.js +4 -0
  141. package/dist/providers/openai.js.map +1 -0
  142. package/dist/providers.d.ts +50 -0
  143. package/dist/providers.d.ts.map +1 -0
  144. package/dist/providers.js +5 -0
  145. package/dist/providers.js.map +1 -0
  146. package/dist/read/anthropic.d.ts +15 -0
  147. package/dist/read/anthropic.d.ts.map +1 -0
  148. package/dist/read/anthropic.js +56 -0
  149. package/dist/read/anthropic.js.map +1 -0
  150. package/dist/read/responses.d.ts +26 -0
  151. package/dist/read/responses.d.ts.map +1 -0
  152. package/dist/read/responses.js +121 -0
  153. package/dist/read/responses.js.map +1 -0
  154. package/dist/read.d.ts +32 -0
  155. package/dist/read.d.ts.map +1 -0
  156. package/dist/read.js +73 -0
  157. package/dist/read.js.map +1 -0
  158. package/dist/request.d.ts +23 -0
  159. package/dist/request.d.ts.map +1 -0
  160. package/dist/request.js +4 -0
  161. package/dist/request.js.map +1 -0
  162. package/dist/response.d.ts +38 -0
  163. package/dist/response.d.ts.map +1 -0
  164. package/dist/response.js +4 -0
  165. package/dist/response.js.map +1 -0
  166. package/dist/secrets.d.ts +22 -0
  167. package/dist/secrets.d.ts.map +1 -0
  168. package/dist/secrets.js +35 -0
  169. package/dist/secrets.js.map +1 -0
  170. package/dist/tool-calls.d.ts +28 -0
  171. package/dist/tool-calls.d.ts.map +1 -0
  172. package/dist/tool-calls.js +4 -0
  173. package/dist/tool-calls.js.map +1 -0
  174. package/dist/tools.d.ts +33 -0
  175. package/dist/tools.d.ts.map +1 -0
  176. package/dist/tools.js +4 -0
  177. package/dist/tools.js.map +1 -0
  178. package/dist/usage.d.ts +31 -0
  179. package/dist/usage.d.ts.map +1 -0
  180. package/dist/usage.js +3 -0
  181. package/dist/usage.js.map +1 -0
  182. package/dist/version.d.ts +2 -0
  183. package/dist/version.d.ts.map +1 -0
  184. package/dist/version.js +6 -0
  185. package/dist/version.js.map +1 -0
  186. package/package.json +33 -3
  187. package/src/catalog/anthropic.ts +83 -0
  188. package/src/catalog/fireworks.ts +121 -0
  189. package/src/catalog/openai.ts +133 -0
  190. package/src/client/anthropic/errors.ts +88 -0
  191. package/src/client/anthropic/request.ts +272 -0
  192. package/src/client/anthropic/response.ts +236 -0
  193. package/src/client/direct.ts +157 -0
  194. package/src/client/gateway.ts +120 -0
  195. package/src/client/http.ts +231 -0
  196. package/src/client/llm.ts +151 -0
  197. package/src/client/responses/errors.ts +166 -0
  198. package/src/client/responses/profile.ts +55 -0
  199. package/src/client/responses/providers.ts +87 -0
  200. package/src/client/responses/request.ts +326 -0
  201. package/src/client/responses/response.ts +203 -0
  202. package/src/client/util.ts +81 -0
  203. package/src/client.ts +20 -0
  204. package/src/content.ts +46 -0
  205. package/src/convert/anthropic.ts +115 -0
  206. package/src/convert/index.ts +275 -0
  207. package/src/convert/responses.ts +203 -0
  208. package/src/convert/types.ts +104 -0
  209. package/src/cost.ts +71 -0
  210. package/src/failure.ts +164 -0
  211. package/src/fields.ts +13 -0
  212. package/src/formats/anthropic.ts +191 -0
  213. package/src/formats/responses.ts +207 -0
  214. package/src/index.ts +44 -0
  215. package/src/json.ts +13 -0
  216. package/src/messages.ts +99 -0
  217. package/src/model.ts +41 -0
  218. package/src/models.ts +23 -0
  219. package/src/providers/anthropic.ts +137 -0
  220. package/src/providers/fireworks.ts +140 -0
  221. package/src/providers/openai.ts +275 -0
  222. package/src/providers.ts +59 -0
  223. package/src/read/anthropic.ts +64 -0
  224. package/src/read/responses.ts +159 -0
  225. package/src/read.ts +108 -0
  226. package/src/request.ts +28 -0
  227. package/src/response.ts +44 -0
  228. package/src/secrets.ts +40 -0
  229. package/src/tool-calls.ts +33 -0
  230. package/src/tools.ts +38 -0
  231. package/src/usage.ts +33 -0
  232. package/src/version.ts +6 -0
@@ -0,0 +1,120 @@
1
+ // One call through the acentric gateway, which acentric-cloud runs with
2
+ // provider keys of its own. The request goes as it is, with the token and the
3
+ // client's version in headers and the effect that made the call in the body;
4
+ // the response comes back as it is.
5
+
6
+ import { isLlmFailure, LlmError } from '../failure.js';
7
+ import type { LlmFailure } from '../failure.js';
8
+ import { isObject } from '../json.js';
9
+ import type { Provider } from '../providers.js';
10
+ import type { LlmRequest } from '../request.js';
11
+ import type { LlmResponse } from '../response.js';
12
+ import { LLM_VERSION } from '../version.js';
13
+ import type { CallOptions } from './direct.js';
14
+ import { postJson, retryAfter } from './http.js';
15
+ import type { HttpAnswer } from './http.js';
16
+ import { kindForStatus } from './responses/errors.js';
17
+ import { cut, invalidResponse } from './util.js';
18
+
19
+ /** Where a gateway takes completions, under its URL. */
20
+ export const GATEWAY_ROUTE = '/v1/complete';
21
+
22
+ /** The header that carries the client's LLM_VERSION to the gateway. */
23
+ export const VERSION_HEADER = 'acentric-llm-version';
24
+
25
+ /**
26
+ * The effect a call is made for, as its handler's context names it. A gateway
27
+ * shows each call under it, but it is the caller's word: never a credential,
28
+ * and never what a call is charged to.
29
+ */
30
+ export type GatewayEffect = {
31
+ readonly key?: string;
32
+ readonly kind?: string;
33
+ readonly attempt?: number;
34
+ };
35
+
36
+ /**
37
+ * The body of a call to the gateway: the request, and the effect that made
38
+ * it. It carries no credentials, since the gateway calls providers with its
39
+ * own keys.
40
+ */
41
+ export type GatewayRequest<P extends Provider = Provider> = P extends Provider
42
+ ? LlmRequest<P> & { readonly effect?: GatewayEffect }
43
+ : never;
44
+
45
+ /** The body of a gateway's answer that is not 2xx. */
46
+ export type GatewayFailure = { readonly error: LlmFailure };
47
+
48
+ export type Gateway = {
49
+ /** The gateway's address, such as `https://llm.cloud.acentric.dev`. */
50
+ readonly url: string;
51
+ /** Sent as `Authorization: Bearer`. */
52
+ readonly token: string;
53
+ };
54
+
55
+ export type GatewayCallOptions = CallOptions & {
56
+ /** The effect the call is made for, sent for the gateway to show. */
57
+ readonly effect?: GatewayEffect;
58
+ };
59
+
60
+ export async function callGateway<P extends Provider>(
61
+ request: LlmRequest<P>,
62
+ gateway: Gateway,
63
+ { signal, fetch, effect }: GatewayCallOptions = {},
64
+ ): Promise<LlmResponse<P>> {
65
+ const provider = request.provider;
66
+ const answer = await postJson({
67
+ provider,
68
+ label: 'the acentric gateway',
69
+ url: `${base(provider, gateway.url)}${GATEWAY_ROUTE}`,
70
+ headers: { authorization: `Bearer ${gateway.token}`, [VERSION_HEADER]: LLM_VERSION },
71
+ body: effect === undefined ? request : { ...request, effect },
72
+ ...(request.timeoutMs === undefined ? {} : { timeoutMs: request.timeoutMs }),
73
+ ...(signal === undefined ? {} : { signal }),
74
+ ...(fetch === undefined ? {} : { fetch }),
75
+ failed: (http) => gatewayFailure(provider, http),
76
+ // The gateway's request ID is not the provider's; its failures carry that.
77
+ providerAnswers: false,
78
+ });
79
+ const body = answer.body;
80
+ const message = isObject(body) ? body['message'] : undefined;
81
+ if (!isObject(body) || body['provider'] !== provider || !isObject(message)) {
82
+ throw invalidResponse(provider, 'the acentric gateway did not answer with a response', body);
83
+ }
84
+ // The gateway answers what callProvider returned, for this provider.
85
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion
86
+ return body as unknown as LlmResponse<P>;
87
+ }
88
+
89
+ /** The gateway's own failure as it sent it, or one read from the HTTP status. */
90
+ function gatewayFailure(provider: Provider, answer: HttpAnswer): LlmError {
91
+ const error = isObject(answer.body) ? answer.body['error'] : undefined;
92
+ if (isLlmFailure(error)) return LlmError.fromJSON(error);
93
+ const after = retryAfter(answer.headers);
94
+ return new LlmError(`the acentric gateway answered HTTP ${answer.status}`, {
95
+ kind: kindForStatus(answer.status),
96
+ provider,
97
+ httpStatus: answer.status,
98
+ providerError: cut(answer.body),
99
+ ...(after === undefined ? {} : { retryAfterMs: after }),
100
+ });
101
+ }
102
+
103
+ function base(provider: Provider, value: string): string {
104
+ let url: URL;
105
+ try {
106
+ url = new URL(value);
107
+ } catch {
108
+ throw new LlmError(`the gateway's URL is not a URL: ${value}`, {
109
+ kind: 'invalid_request',
110
+ provider,
111
+ });
112
+ }
113
+ if (url.protocol !== 'https:' && url.protocol !== 'http:') {
114
+ throw new LlmError('the gateway must be reached over https, or http for local development', {
115
+ kind: 'invalid_request',
116
+ provider,
117
+ });
118
+ }
119
+ return url.href.replace(/\/+$/, '');
120
+ }
@@ -0,0 +1,231 @@
1
+ // One HTTP exchange with a provider or the gateway: a JSON body out, a JSON
2
+ // body back, stopped by the handler's signal or a time limit. Never retried:
3
+ // whether to try again is the agent's decision.
4
+
5
+ import type { Json } from '@acentric/types';
6
+
7
+ import { LlmError } from '../failure.js';
8
+ import { cut } from './util.js';
9
+
10
+ /** The longest one call may take, and the default, in milliseconds. */
11
+ export const MAX_TIMEOUT_MS = 15 * 60 * 1000;
12
+
13
+ const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
14
+
15
+ export type Post = {
16
+ readonly provider: string;
17
+ /** Names the other end in messages, such as "Anthropic". */
18
+ readonly label: string;
19
+ readonly url: string;
20
+ readonly headers: { readonly [name: string]: string };
21
+ readonly body: Json;
22
+ readonly timeoutMs?: number;
23
+ readonly signal?: AbortSignal;
24
+ readonly fetch?: typeof fetch;
25
+ /** Turns an answer that is not 2xx into an error. */
26
+ readonly failed: (answer: HttpAnswer) => LlmError;
27
+ /** Whether the answer's request ID is the provider's, for its failures to name. Defaults to true. */
28
+ readonly providerAnswers?: boolean;
29
+ };
30
+
31
+ /** An answer that is not 2xx. */
32
+ export type HttpAnswer = {
33
+ readonly status: number;
34
+ readonly headers: Headers;
35
+ /** The JSON body, or the text body cut short. */
36
+ readonly body: Json;
37
+ };
38
+
39
+ export type JsonAnswer = {
40
+ readonly body: Json;
41
+ readonly durationMs: number;
42
+ /** When the answer arrived, in milliseconds since the Unix epoch. */
43
+ readonly timestamp: number;
44
+ readonly requestId?: string;
45
+ };
46
+
47
+ export async function postJson(post: Post): Promise<JsonAnswer> {
48
+ const { provider, label } = post;
49
+ const timeoutMs = Math.min(post.timeoutMs ?? MAX_TIMEOUT_MS, MAX_TIMEOUT_MS);
50
+ if (!(Number.isSafeInteger(timeoutMs) && timeoutMs > 0)) {
51
+ throw new LlmError('timeoutMs must be a positive whole number', {
52
+ kind: 'invalid_request',
53
+ provider,
54
+ });
55
+ }
56
+ const controller = new AbortController();
57
+ const onAbort = () =>
58
+ controller.abort(
59
+ new LlmError(`the ${label} call was cancelled`, { kind: 'cancelled', provider }),
60
+ );
61
+ post.signal?.addEventListener('abort', onAbort, { once: true });
62
+ if (post.signal?.aborted === true) onAbort();
63
+ const timer = setTimeout(
64
+ () =>
65
+ controller.abort(
66
+ new LlmError(`the ${label} call took longer than ${timeoutMs} ms`, {
67
+ kind: 'timeout',
68
+ provider,
69
+ }),
70
+ ),
71
+ timeoutMs,
72
+ );
73
+ const started = performance.now();
74
+ let requestId: string | undefined;
75
+ try {
76
+ controller.signal.throwIfAborted();
77
+ const send = post.fetch ?? globalThis.fetch;
78
+ // Raced against the signal, so a fetch that does not watch it cannot outlast a timeout.
79
+ const response = await untilAborted(
80
+ send(post.url, {
81
+ method: 'POST',
82
+ headers: requestHeaders(provider, post.headers),
83
+ body: JSON.stringify(post.body),
84
+ signal: controller.signal,
85
+ redirect: 'manual',
86
+ }),
87
+ controller.signal,
88
+ );
89
+ if (post.providerAnswers !== false) requestId = readRequestId(response.headers);
90
+ if (response.status >= 300 && response.status < 400) {
91
+ void response.body?.cancel().catch(() => undefined);
92
+ throw new LlmError(`${label} answered with a redirect, which is not followed`, {
93
+ kind: 'network_error',
94
+ provider,
95
+ });
96
+ }
97
+ const text = await readText(response, provider, label);
98
+ controller.signal.throwIfAborted();
99
+ if (!response.ok) {
100
+ throw post.failed({
101
+ status: response.status,
102
+ headers: response.headers,
103
+ body: parseLoosely(text),
104
+ });
105
+ }
106
+ let body: Json;
107
+ try {
108
+ body = JSON.parse(text);
109
+ } catch {
110
+ throw new LlmError(`${label} answered with something that is not JSON`, {
111
+ kind: 'invalid_response',
112
+ provider,
113
+ httpStatus: response.status,
114
+ providerError: cut(text),
115
+ });
116
+ }
117
+ return {
118
+ body,
119
+ durationMs: Math.round(performance.now() - started),
120
+ timestamp: Date.now(),
121
+ ...(requestId === undefined ? {} : { requestId }),
122
+ };
123
+ } catch (error) {
124
+ const reason: unknown = controller.signal.aborted ? controller.signal.reason : error;
125
+ if (reason instanceof LlmError) throw withRequestId(reason, requestId);
126
+ throw new LlmError(`${label} could not be reached`, {
127
+ kind: 'network_error',
128
+ provider,
129
+ ...(requestId === undefined ? {} : { requestId }),
130
+ });
131
+ } finally {
132
+ clearTimeout(timer);
133
+ post.signal?.removeEventListener('abort', onAbort);
134
+ }
135
+ }
136
+
137
+ /** `Retry-After` in milliseconds, given in seconds or as an HTTP date. */
138
+ export function retryAfter(headers: Headers): number | undefined {
139
+ const value = headers.get('retry-after')?.trim();
140
+ if (value === undefined || value === '') return undefined;
141
+ const seconds = Number(value);
142
+ if (Number.isFinite(seconds)) return seconds >= 0 ? Math.round(seconds * 1000) : undefined;
143
+ const date = Date.parse(value);
144
+ return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now());
145
+ }
146
+
147
+ /** A failure from after the provider answered names its request ID, for its support. */
148
+ export function withRequestId(error: LlmError, requestId: string | undefined): LlmError {
149
+ if (requestId === undefined || error.failure.requestId !== undefined) return error;
150
+ return LlmError.fromJSON({ ...error.failure, requestId });
151
+ }
152
+
153
+ /** The promise's value, or the signal's reason as soon as it fires. */
154
+ function untilAborted<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
155
+ return new Promise<T>((resolve, reject) => {
156
+ const abort = () => reject(signal.reason);
157
+ signal.addEventListener('abort', abort, { once: true });
158
+ promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
159
+ });
160
+ }
161
+
162
+ function requestHeaders(provider: string, entries: { readonly [name: string]: string }): Headers {
163
+ try {
164
+ return new Headers({
165
+ accept: 'application/json',
166
+ 'content-type': 'application/json',
167
+ ...entries,
168
+ });
169
+ } catch {
170
+ throw new LlmError('the credentials hold characters a header cannot', {
171
+ kind: 'invalid_request',
172
+ provider,
173
+ });
174
+ }
175
+ }
176
+
177
+ function readRequestId(answer: Headers): string | undefined {
178
+ const id =
179
+ answer.get('x-request-id') ?? answer.get('request-id') ?? answer.get('x-oai-request-id');
180
+ return id?.trim() || undefined;
181
+ }
182
+
183
+ function parseLoosely(text: string): Json {
184
+ try {
185
+ return JSON.parse(text);
186
+ } catch {
187
+ return cut(text);
188
+ }
189
+ }
190
+
191
+ async function readText(response: Response, provider: string, label: string): Promise<string> {
192
+ if (response.body === null) return '';
193
+ const reader = response.body.getReader();
194
+ const decoder = new TextDecoder('utf-8', { fatal: true });
195
+ const decode = (value?: Uint8Array): string => {
196
+ try {
197
+ return decoder.decode(value, { stream: value !== undefined });
198
+ } catch {
199
+ throw new LlmError(`${label} answered with text that is not UTF-8`, {
200
+ kind: 'invalid_response',
201
+ provider,
202
+ httpStatus: response.status,
203
+ });
204
+ }
205
+ };
206
+ let bytes = 0;
207
+ let text = '';
208
+ try {
209
+ for (;;) {
210
+ // Reading a stream is one chunk at a time.
211
+ // oxlint-disable-next-line no-await-in-loop
212
+ const { done, value } = await reader.read();
213
+ if (done) break;
214
+ bytes += value.byteLength;
215
+ if (bytes > MAX_RESPONSE_BYTES) {
216
+ throw new LlmError(`${label} answered with more than ${MAX_RESPONSE_BYTES} bytes`, {
217
+ kind: 'invalid_response',
218
+ provider,
219
+ httpStatus: response.status,
220
+ });
221
+ }
222
+ text += decode(value);
223
+ }
224
+ return text + decode();
225
+ } catch (error) {
226
+ void reader.cancel().catch(() => undefined);
227
+ throw error;
228
+ } finally {
229
+ reader.releaseLock();
230
+ }
231
+ }
@@ -0,0 +1,151 @@
1
+ // The client a handler uses: it finds the provider's key or the gateway in
2
+ // the handler's secrets, never in the environment, so the same code runs on
3
+ // every runner.
4
+ //
5
+ // A call goes straight to its provider when that provider's key is set, so an
6
+ // application with its own account uses it. Otherwise, with ACENTRIC_LLM_URL
7
+ // and ACENTRIC_LLM_TOKEN set, it goes through the gateway, which calls the
8
+ // provider with keys of its own. Options override all of this.
9
+
10
+ import type { Secrets } from '@acentric/types';
11
+
12
+ import { LlmError } from '../failure.js';
13
+ import type { Credentials, Provider } from '../providers.js';
14
+ import { requestMessages } from '../read.js';
15
+ import type { LlmRequest } from '../request.js';
16
+ import type { LlmResponse } from '../response.js';
17
+ import { GATEWAY_SECRETS, PROVIDER_SECRETS } from '../secrets.js';
18
+ import { callProvider } from './direct.js';
19
+ import { callGateway } from './gateway.js';
20
+ import type { Gateway, GatewayEffect } from './gateway.js';
21
+
22
+ /** What the client needs of a handler's context. A HandlerContext is one. */
23
+ export type LlmContext = {
24
+ readonly secrets: Secrets;
25
+ /** Stops every call, unless a call passes its own. */
26
+ readonly signal?: AbortSignal;
27
+ /** The effect's key, kind and attempt, which the gateway shows each call under. */
28
+ readonly key?: string;
29
+ readonly kind?: string;
30
+ readonly attempt?: number;
31
+ };
32
+
33
+ export type LlmOptions = {
34
+ /** Credentials for a provider, used instead of its secret or a gateway. */
35
+ readonly credentials?: { readonly [P in Provider]?: Credentials<P> };
36
+ /**
37
+ * A gateway to call instead of the provider's key or the gateway in the
38
+ * secrets, or `false` to call providers directly even when the secrets name
39
+ * one.
40
+ */
41
+ readonly gateway?: Gateway | false;
42
+ /** A fetch to use instead of the global one, such as in tests. */
43
+ readonly fetch?: typeof fetch;
44
+ };
45
+
46
+ export type LlmClient = {
47
+ /**
48
+ * One completion, from the request's provider. The application's `custom`
49
+ * messages are left out. Never retried: on failure it throws an LlmError,
50
+ * which step reads back with `llmFailure`.
51
+ */
52
+ complete<P extends Provider>(
53
+ request: LlmRequest<P>,
54
+ options?: { readonly signal?: AbortSignal },
55
+ ): Promise<LlmResponse<P>>;
56
+ };
57
+
58
+ export function createLlm(context: LlmContext, options: LlmOptions = {}): LlmClient {
59
+ return {
60
+ async complete<P extends Provider>(
61
+ request: LlmRequest<P>,
62
+ call: { readonly signal?: AbortSignal } = {},
63
+ ): Promise<LlmResponse<P>> {
64
+ const signal = call.signal ?? context.signal;
65
+ const callOptions = {
66
+ ...(signal === undefined ? {} : { signal }),
67
+ ...(options.fetch === undefined ? {} : { fetch: options.fetch }),
68
+ };
69
+ const sent = withoutCustom(request);
70
+ const route = await routeFor(request.provider, context.secrets, options);
71
+ if (route === undefined) {
72
+ throw new LlmError(
73
+ `no credentials for ${request.provider}: set the secret ${PROVIDER_SECRETS[request.provider]}, ` +
74
+ `or ${GATEWAY_SECRETS.url} and ${GATEWAY_SECRETS.token} for the gateway`,
75
+ { kind: 'auth', provider: request.provider },
76
+ );
77
+ }
78
+ if ('credentials' in route) return callProvider(sent, route.credentials, callOptions);
79
+ return callGateway(sent, route.gateway, { ...callOptions, ...effectOf(context) });
80
+ },
81
+ };
82
+ }
83
+
84
+ /** The request without the application's own messages, which never reach a provider. */
85
+ function withoutCustom<P extends Provider>(request: LlmRequest<P>): LlmRequest<P> {
86
+ return { ...request, messages: requestMessages(request.messages) };
87
+ }
88
+
89
+ async function gatewayIn(secrets: Secrets): Promise<Gateway | undefined> {
90
+ const url = await secret(secrets, GATEWAY_SECRETS.url);
91
+ const token = await secret(secrets, GATEWAY_SECRETS.token);
92
+ if (url === undefined && token === undefined) return undefined;
93
+ if (url === undefined || token === undefined) {
94
+ const [set, missing] =
95
+ url === undefined
96
+ ? [GATEWAY_SECRETS.token, GATEWAY_SECRETS.url]
97
+ : [GATEWAY_SECRETS.url, GATEWAY_SECRETS.token];
98
+ throw new LlmError(`${set} is set but ${missing} is not: the gateway needs both`, {
99
+ kind: 'auth',
100
+ });
101
+ }
102
+ return { url, token };
103
+ }
104
+
105
+ /**
106
+ * Where a call goes: credentials or a gateway given as options, then the
107
+ * provider's key, then the gateway in the secrets. Undefined when there is
108
+ * none of these.
109
+ */
110
+ async function routeFor<P extends Provider>(
111
+ provider: P,
112
+ secrets: Secrets,
113
+ options: LlmOptions,
114
+ ): Promise<{ readonly credentials: Credentials<P> } | { readonly gateway: Gateway } | undefined> {
115
+ const given: Credentials<P> | undefined = options.credentials?.[provider];
116
+ if (given !== undefined) return { credentials: given };
117
+ if (options.gateway !== undefined && options.gateway !== false) {
118
+ return { gateway: options.gateway };
119
+ }
120
+ const apiKey = await secret(secrets, PROVIDER_SECRETS[provider]);
121
+ if (apiKey !== undefined) return { credentials: { apiKey } };
122
+ if (options.gateway === false) return undefined;
123
+ const gateway = await gatewayIn(secrets);
124
+ return gateway === undefined ? undefined : { gateway };
125
+ }
126
+
127
+ /** The effect the context names, for the gateway, when it names one. */
128
+ function effectOf(context: LlmContext): { readonly effect?: GatewayEffect } {
129
+ const { key, kind, attempt } = context;
130
+ if (key === undefined && kind === undefined && attempt === undefined) return {};
131
+ return {
132
+ effect: {
133
+ ...(key === undefined ? {} : { key }),
134
+ ...(kind === undefined ? {} : { kind }),
135
+ ...(attempt === undefined ? {} : { attempt }),
136
+ },
137
+ };
138
+ }
139
+
140
+ /**
141
+ * A secret's value, or undefined when it is not set. Any other failure to
142
+ * read it is thrown: it must not quietly send the call another way.
143
+ */
144
+ async function secret(secrets: Secrets, name: string): Promise<string | undefined> {
145
+ try {
146
+ return await secrets.get(name);
147
+ } catch (error) {
148
+ if (error instanceof Error && error.name === 'SecretNotSet') return undefined;
149
+ throw error;
150
+ }
151
+ }
@@ -0,0 +1,166 @@
1
+ // Responses failures read into LlmErrors: from an HTTP error, or from a
2
+ // failed response that came with HTTP 200.
3
+
4
+ import type { Json, JsonObject } from '@acentric/types';
5
+
6
+ import { LlmError } from '../../failure.js';
7
+ import type { LlmFailureKind } from '../../failure.js';
8
+ import { isArray, isObject } from '../../json.js';
9
+ import type { Usage } from '../../usage.js';
10
+ import { retryAfter } from '../http.js';
11
+ import type { HttpAnswer } from '../http.js';
12
+ import { cut, show } from '../util.js';
13
+ import type { ErrorInput, ResponsesProfile, ResponsesProvider } from './profile.js';
14
+
15
+ const POLICY_CODES: ReadonlySet<string> = new Set([
16
+ 'bio_policy',
17
+ 'cyber_policy',
18
+ 'misalignment_policy_violation',
19
+ 'image_content_policy_violation',
20
+ ]);
21
+
22
+ const QUOTA_CODES: ReadonlySet<string> = new Set([
23
+ 'insufficient_quota',
24
+ 'credit_balance_exhausted',
25
+ 'organization_spend_limit_exceeded',
26
+ 'project_spend_limit_exceeded',
27
+ 'organization_usage_limit_exceeded',
28
+ ]);
29
+
30
+ const KIND_BY_CODE: { readonly [code: string]: LlmFailureKind } = {
31
+ context_length_exceeded: 'context_overflow',
32
+ previous_response_not_found: 'not_found',
33
+ rate_limit_exceeded: 'rate_limited',
34
+ slow_down: 'rate_limited',
35
+ server_is_overloaded: 'overloaded',
36
+ server_error: 'provider_error',
37
+ vector_store_timeout: 'timeout',
38
+ };
39
+
40
+ /** The kind an HTTP status means when nothing more is known. */
41
+ export function kindForStatus(status: number): LlmFailureKind {
42
+ if (status === 401 || status === 403) return 'auth';
43
+ if (status === 402) return 'quota_exceeded';
44
+ if (status === 404) return 'not_found';
45
+ if (status === 408) return 'timeout';
46
+ if (status === 429) return 'rate_limited';
47
+ if (status === 503 || status === 529) return 'overloaded';
48
+ if (status >= 500) return 'provider_error';
49
+ return 'invalid_request';
50
+ }
51
+
52
+ type Fields = {
53
+ readonly message?: string;
54
+ readonly code?: string;
55
+ readonly type?: string;
56
+ readonly param?: string;
57
+ };
58
+
59
+ /** A provider's answer that is not 2xx, as an LlmError. */
60
+ export function httpFailure<P extends ResponsesProvider>(
61
+ profile: ResponsesProfile<P>,
62
+ answer: HttpAnswer,
63
+ ): LlmError {
64
+ const { fields, detail } = readError(answer.body);
65
+ const input: ErrorInput = { ...fields, status: answer.status };
66
+ const reading = profile.classifyError?.(input);
67
+ const message =
68
+ fields.message ??
69
+ (typeof answer.body === 'string' && answer.body !== '' ? answer.body : undefined) ??
70
+ `the ${profile.label} call failed with HTTP ${answer.status}`;
71
+ const param = fields.param ?? reading?.param;
72
+ const after = retryAfter(answer.headers);
73
+ return new LlmError(message, {
74
+ kind: reading?.kind ?? kindForCode(fields) ?? kindForStatus(answer.status),
75
+ provider: profile.provider,
76
+ httpStatus: answer.status,
77
+ providerError: cut(detail),
78
+ ...(fields.code === undefined ? {} : { providerCode: fields.code }),
79
+ ...(fields.type === undefined ? {} : { providerType: fields.type }),
80
+ ...(param === undefined ? {} : { providerParam: param }),
81
+ ...(after === undefined ? {} : { retryAfterMs: after }),
82
+ });
83
+ }
84
+
85
+ /** A response whose status is `failed`, or an error that came with HTTP 200. */
86
+ export function failedResponse<P extends ResponsesProvider>(
87
+ profile: ResponsesProfile<P>,
88
+ error: Json,
89
+ responseId: string | undefined,
90
+ usage: Usage | undefined,
91
+ ): LlmError {
92
+ const { fields, detail } = readError({ error });
93
+ const reading = profile.classifyError?.(fields);
94
+ const param = fields.param ?? reading?.param;
95
+ return new LlmError(fields.message ?? `the ${profile.label} response failed`, {
96
+ kind:
97
+ reading?.kind ??
98
+ kindForCode(fields) ??
99
+ (fields.code === undefined ? 'provider_error' : 'invalid_request'),
100
+ provider: profile.provider,
101
+ providerError: cut(detail),
102
+ ...(fields.code === undefined ? {} : { providerCode: fields.code }),
103
+ ...(param === undefined ? {} : { providerParam: param }),
104
+ ...(responseId === undefined ? {} : { responseId }),
105
+ ...(usage === undefined ? {} : { usage }),
106
+ });
107
+ }
108
+
109
+ function kindForCode({ code, type }: Fields): LlmFailureKind | undefined {
110
+ if (code !== undefined && POLICY_CODES.has(code)) return 'policy_violation';
111
+ if ((code !== undefined && QUOTA_CODES.has(code)) || type === 'insufficient_quota') {
112
+ return 'quota_exceeded';
113
+ }
114
+ return code === undefined ? undefined : KIND_BY_CODE[code];
115
+ }
116
+
117
+ /**
118
+ * The error in an answer. Besides the format's `{ error: {...} }`, Fireworks
119
+ * also answers FastAPI's `{ detail }`: a message, an error, or a list of
120
+ * validation errors.
121
+ */
122
+ function readError(body: Json): { fields: Fields; detail: Json } {
123
+ if (isObject(body) && body['error'] === undefined && body['detail'] !== undefined) {
124
+ const detail = body['detail'];
125
+ if (typeof detail === 'string') return { fields: { message: detail }, detail };
126
+ if (isArray(detail)) return { fields: validationErrors(detail), detail };
127
+ return { fields: errorFields(detail), detail };
128
+ }
129
+ const detail = isObject(body) && body['error'] !== undefined ? body['error'] : body;
130
+ return { fields: errorFields(detail), detail };
131
+ }
132
+
133
+ function errorFields(detail: Json): Fields {
134
+ if (!isObject(detail)) return {};
135
+ const { message, code, type, param } = detail;
136
+ return {
137
+ ...(typeof message === 'string' ? { message } : {}),
138
+ ...(typeof code === 'string' || typeof code === 'number' ? { code: String(code) } : {}),
139
+ ...(typeof type === 'string' ? { type } : {}),
140
+ ...(typeof param === 'string' ? { param } : {}),
141
+ };
142
+ }
143
+
144
+ /** FastAPI's validation errors: `[{ loc: ["body", "temperature"], msg }]`. */
145
+ function validationErrors(list: readonly Json[]): Fields {
146
+ const entries = list.filter(isObject).map((entry: JsonObject) => {
147
+ const loc = entry['loc'];
148
+ const msg = entry['msg'];
149
+ const path = isArray(loc)
150
+ ? loc
151
+ .filter((part) => part !== 'body')
152
+ .map((part) => show(part))
153
+ .join('.')
154
+ : '';
155
+ return { param: path, message: typeof msg === 'string' ? msg : '' };
156
+ });
157
+ const message = entries
158
+ .map((entry) => (entry.param === '' ? entry.message : `${entry.param}: ${entry.message}`))
159
+ .filter((text) => text !== '')
160
+ .join('; ');
161
+ const only = entries.length === 1 ? entries[0]?.param : undefined;
162
+ return {
163
+ ...(message === '' ? {} : { message }),
164
+ ...(only === undefined || only === '' ? {} : { param: only }),
165
+ };
166
+ }