@codilore/llm 1.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +321 -0
- package/README.md +131 -0
- package/example/call-sites.md +591 -0
- package/example/tutorial.ts +255 -0
- package/package.json +50 -0
- package/script/recording-cost-report.ts +250 -0
- package/script/setup-recording-env.ts +542 -0
- package/src/cache-policy.ts +111 -0
- package/src/index.ts +32 -0
- package/src/llm.ts +186 -0
- package/src/protocols/anthropic-messages.ts +841 -0
- package/src/protocols/bedrock-converse.ts +649 -0
- package/src/protocols/bedrock-event-stream.ts +87 -0
- package/src/protocols/gemini.ts +465 -0
- package/src/protocols/index.ts +6 -0
- package/src/protocols/openai-chat.ts +431 -0
- package/src/protocols/openai-compatible-chat.ts +24 -0
- package/src/protocols/openai-responses.ts +987 -0
- package/src/protocols/shared.ts +283 -0
- package/src/protocols/utils/bedrock-auth.ts +70 -0
- package/src/protocols/utils/bedrock-cache.ts +37 -0
- package/src/protocols/utils/bedrock-media.ts +80 -0
- package/src/protocols/utils/cache.ts +16 -0
- package/src/protocols/utils/gemini-tool-schema.ts +101 -0
- package/src/protocols/utils/lifecycle.ts +102 -0
- package/src/protocols/utils/openai-options.ts +84 -0
- package/src/protocols/utils/tool-stream.ts +218 -0
- package/src/provider.ts +37 -0
- package/src/providers/amazon-bedrock.ts +43 -0
- package/src/providers/anthropic.ts +35 -0
- package/src/providers/azure.ts +110 -0
- package/src/providers/cloudflare.ts +127 -0
- package/src/providers/github-copilot.ts +66 -0
- package/src/providers/google.ts +35 -0
- package/src/providers/index.ts +11 -0
- package/src/providers/openai-compatible-profile.ts +20 -0
- package/src/providers/openai-compatible.ts +65 -0
- package/src/providers/openai-options.ts +81 -0
- package/src/providers/openai.ts +63 -0
- package/src/providers/openrouter.ts +98 -0
- package/src/providers/xai.ts +56 -0
- package/src/route/auth-options.ts +57 -0
- package/src/route/auth.ts +156 -0
- package/src/route/client.ts +434 -0
- package/src/route/endpoint.ts +53 -0
- package/src/route/executor.ts +374 -0
- package/src/route/framing.ts +27 -0
- package/src/route/index.ts +25 -0
- package/src/route/protocol.ts +84 -0
- package/src/route/transport/http.ts +108 -0
- package/src/route/transport/index.ts +33 -0
- package/src/route/transport/websocket.ts +280 -0
- package/src/schema/errors.ts +203 -0
- package/src/schema/events.ts +370 -0
- package/src/schema/ids.ts +43 -0
- package/src/schema/index.ts +5 -0
- package/src/schema/messages.ts +404 -0
- package/src/schema/options.ts +221 -0
- package/src/tool-runtime.ts +78 -0
- package/src/tool.ts +241 -0
- package/src/utils/record.ts +3 -0
- package/sst-env.d.ts +10 -0
- package/test/adapter.test.ts +164 -0
- package/test/auth-options.types.ts +168 -0
- package/test/auth.test.ts +103 -0
- package/test/cache-policy.test.ts +262 -0
- package/test/continuation-scenarios.ts +104 -0
- package/test/endpoint.test.ts +58 -0
- package/test/executor.test.ts +418 -0
- package/test/exports.test.ts +62 -0
- package/test/fixtures/media/restroom.png +0 -0
- package/test/fixtures/recordings/anthropic-messages/accepts-malformed-assistant-tool-order-with-default-patch.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-image-tool-result.json +43 -0
- package/test/fixtures/recordings/anthropic-messages/claude-opus-4-7-drives-a-tool-loop.json +56 -0
- package/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order-without-patch.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/streams-text.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/streams-tool-call.json +29 -0
- package/test/fixtures/recordings/anthropic-messages-cache/writes-then-reads-cache-control-on-identical-second-call.json +48 -0
- package/test/fixtures/recordings/bedrock-converse/drives-a-tool-loop.json +55 -0
- package/test/fixtures/recordings/bedrock-converse/streams-a-tool-call.json +29 -0
- package/test/fixtures/recordings/bedrock-converse/streams-text.json +29 -0
- package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
- package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-llama-3-1-8b-text.json +32 -0
- package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
- package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-llama-3-1-8b-text.json +32 -0
- package/test/fixtures/recordings/gemini/gemini-2-5-flash-image.json +32 -0
- package/test/fixtures/recordings/gemini/streams-text.json +28 -0
- package/test/fixtures/recordings/gemini/streams-tool-call.json +28 -0
- package/test/fixtures/recordings/gemini-cache/reports-cachedcontenttokencount-on-identical-second-call.json +46 -0
- package/test/fixtures/recordings/openai-chat/continues-after-tool-result.json +28 -0
- package/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json +46 -0
- package/test/fixtures/recordings/openai-chat/streams-text.json +28 -0
- package/test/fixtures/recordings/openai-chat/streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/deepseek-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-llama-3-3-70b-drives-a-tool-loop.json +53 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json +53 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-image-tool-result.json +42 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning-continuation.json +58 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json +32 -0
- package/test/fixtures/recordings/openai-responses-cache/reports-cached-tokens-on-identical-second-call.json +46 -0
- package/test/generate-object.test.ts +184 -0
- package/test/lib/effect.ts +50 -0
- package/test/lib/http.ts +98 -0
- package/test/lib/openai-chunks.ts +27 -0
- package/test/lib/sse.ts +17 -0
- package/test/lib/tool-runtime.ts +146 -0
- package/test/llm.test.ts +167 -0
- package/test/provider/anthropic-messages-cache.recorded.test.ts +54 -0
- package/test/provider/anthropic-messages.recorded.test.ts +46 -0
- package/test/provider/anthropic-messages.test.ts +829 -0
- package/test/provider/bedrock-converse-cache.recorded.test.ts +54 -0
- package/test/provider/bedrock-converse.test.ts +707 -0
- package/test/provider/cloudflare.test.ts +230 -0
- package/test/provider/gemini-cache.recorded.test.ts +48 -0
- package/test/provider/gemini.test.ts +476 -0
- package/test/provider/golden.recorded.test.ts +219 -0
- package/test/provider/openai-chat.test.ts +446 -0
- package/test/provider/openai-compatible-chat.test.ts +238 -0
- package/test/provider/openai-responses-cache.recorded.test.ts +46 -0
- package/test/provider/openai-responses.test.ts +1322 -0
- package/test/provider/openrouter.test.ts +56 -0
- package/test/provider.types.ts +41 -0
- package/test/recorded-golden.ts +97 -0
- package/test/recorded-runner.ts +100 -0
- package/test/recorded-scenarios.ts +531 -0
- package/test/recorded-test.ts +74 -0
- package/test/recorded-utils.ts +56 -0
- package/test/recorded-websocket.ts +26 -0
- package/test/route.test.ts +43 -0
- package/test/schema.test.ts +97 -0
- package/test/tool-runtime.test.ts +802 -0
- package/test/tool-stream.test.ts +99 -0
- package/test/tool.types.ts +40 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { Effect, Schema, Stream } from "effect"
|
|
3
|
+
import * as Sse from "effect/unstable/encoding/Sse"
|
|
4
|
+
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
|
5
|
+
import {
|
|
6
|
+
InvalidProviderOutputReason,
|
|
7
|
+
InvalidRequestReason,
|
|
8
|
+
LLMError,
|
|
9
|
+
type ContentPart,
|
|
10
|
+
type LLMRequest,
|
|
11
|
+
type MediaPart,
|
|
12
|
+
type TextPart,
|
|
13
|
+
type ToolResultPart,
|
|
14
|
+
} from "../schema"
|
|
15
|
+
export { isRecord } from "../utils/record"
|
|
16
|
+
|
|
17
|
+
export const Json = Schema.fromJsonString(Schema.Unknown)
|
|
18
|
+
export const decodeJson = Schema.decodeUnknownSync(Json)
|
|
19
|
+
export const encodeJson = Schema.encodeSync(Json)
|
|
20
|
+
export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
|
|
21
|
+
export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
|
|
22
|
+
export const optionalNull = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.NullOr(schema))
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Streaming tool-call accumulator. Adapters that build a tool call across
|
|
26
|
+
* multiple `tool-input-delta` chunks store the partial JSON input string here
|
|
27
|
+
* and finalize it with `parseToolInput` once the call completes.
|
|
28
|
+
*/
|
|
29
|
+
export interface ToolAccumulator {
|
|
30
|
+
readonly id: string
|
|
31
|
+
readonly name: string
|
|
32
|
+
readonly input: string
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* `Usage.totalTokens` policy shared by every route. Honors a provider-
|
|
37
|
+
* supplied total; otherwise falls back to `inputTokens + outputTokens` only
|
|
38
|
+
* when at least one is defined. Returns `undefined` when neither input nor
|
|
39
|
+
* output is known so routes don't publish a misleading `0`.
|
|
40
|
+
*
|
|
41
|
+
* Under the additive `LLM.Usage` contract, `inputTokens` and `outputTokens`
|
|
42
|
+
* are the non-cached input and visible output only. The provider-supplied
|
|
43
|
+
* `total` is the source of truth when present; the computed fallback
|
|
44
|
+
* under-counts cache and reasoning by design and exists mainly so
|
|
45
|
+
* Anthropic-style providers (which don't surface a total) still get a
|
|
46
|
+
* sensible aggregate on the input + output axes.
|
|
47
|
+
*/
|
|
48
|
+
export const totalTokens = (
|
|
49
|
+
inputTokens: number | undefined,
|
|
50
|
+
outputTokens: number | undefined,
|
|
51
|
+
total: number | undefined,
|
|
52
|
+
) => {
|
|
53
|
+
if (total !== undefined) return total
|
|
54
|
+
if (inputTokens === undefined && outputTokens === undefined) return undefined
|
|
55
|
+
return (inputTokens ?? 0) + (outputTokens ?? 0)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Subtract `subtrahend` from `total`, clamping to zero if the provider
|
|
60
|
+
* reports a non-sensical breakdown (e.g. `cached_tokens > prompt_tokens`).
|
|
61
|
+
* Used by protocol mappers when deriving a non-overlapping breakdown field
|
|
62
|
+
* from a provider's inclusive total — `nonCachedInputTokens` from
|
|
63
|
+
* `inputTokens - cacheReadInputTokens - cacheWriteInputTokens`.
|
|
64
|
+
*
|
|
65
|
+
* If `total` is `undefined`, returns `undefined` (we don't fabricate
|
|
66
|
+
* counts). If `subtrahend` is `undefined`, returns `total` unchanged. The
|
|
67
|
+
* provider-native breakdown stays available on `Usage.native` for debugging.
|
|
68
|
+
*/
|
|
69
|
+
export const subtractTokens = (total: number | undefined, subtrahend: number | undefined): number | undefined => {
|
|
70
|
+
if (total === undefined) return undefined
|
|
71
|
+
if (subtrahend === undefined) return total
|
|
72
|
+
return Math.max(0, total - subtrahend)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Sum a list of optional token counts, returning `undefined` only when
|
|
77
|
+
* every value is `undefined` (so we don't fabricate a `0`). Used by
|
|
78
|
+
* protocol mappers to derive the inclusive `inputTokens` total from a
|
|
79
|
+
* provider that natively reports a non-overlapping breakdown
|
|
80
|
+
* (e.g. Anthropic, whose `input_tokens` is already non-cached only).
|
|
81
|
+
*/
|
|
82
|
+
export const sumTokens = (...values: ReadonlyArray<number | undefined>): number | undefined => {
|
|
83
|
+
if (values.every((value) => value === undefined)) return undefined
|
|
84
|
+
return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export const eventError = (route: string, message: string, raw?: string) =>
|
|
88
|
+
new LLMError({
|
|
89
|
+
module: "ProviderShared",
|
|
90
|
+
method: "stream",
|
|
91
|
+
reason: new InvalidProviderOutputReason({ route, message, raw }),
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
export const parseJson = (route: string, input: string, message: string) =>
|
|
95
|
+
Effect.try({
|
|
96
|
+
try: () => decodeJson(input),
|
|
97
|
+
catch: () => eventError(route, message, input),
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Join the `text` field of a list of parts with newlines. Used by routes
|
|
102
|
+
* that flatten system / message content arrays into a single provider string
|
|
103
|
+
* (OpenAI Chat `system` content, OpenAI Responses `system` content, Gemini
|
|
104
|
+
* `systemInstruction.parts[].text`).
|
|
105
|
+
*/
|
|
106
|
+
export const joinText = (parts: ReadonlyArray<{ readonly text: string }>) => parts.map((part) => part.text).join("\n")
|
|
107
|
+
|
|
108
|
+
const escapeSystemUpdateText = (text: string) =>
|
|
109
|
+
text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">")
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Stable fallback representation for chronological `Message.system(...)`
|
|
113
|
+
* updates on routes that do not support that privileged role natively. The
|
|
114
|
+
* wrapper remains visibly lower-authority user text, preserves the original
|
|
115
|
+
* temporal position, and XML-escapes content so it cannot close the wrapper.
|
|
116
|
+
*/
|
|
117
|
+
export const wrapSystemUpdate = (parts: ReadonlyArray<{ readonly text: string }>) =>
|
|
118
|
+
`<system-update>\n${escapeSystemUpdateText(joinText(parts))}\n</system-update>`
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Chronological system updates deliberately accept text only. Do not insert
|
|
122
|
+
* raw retrieved, tool, or web content into privileged updates: keep untrusted
|
|
123
|
+
* data in ordinary user/tool messages instead.
|
|
124
|
+
*/
|
|
125
|
+
export const systemUpdateText = Effect.fn("ProviderShared.systemUpdateText")(function* (
|
|
126
|
+
route: string,
|
|
127
|
+
message: LLMRequest["messages"][number],
|
|
128
|
+
) {
|
|
129
|
+
const content: TextPart[] = []
|
|
130
|
+
for (const part of message.content) {
|
|
131
|
+
if (!supportsContent(part, ["text"])) return yield* unsupportedContent(route, "system", ["text"])
|
|
132
|
+
content.push(part)
|
|
133
|
+
}
|
|
134
|
+
return content
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
/** Lower an unsupported privileged update into visible, in-order user text. */
|
|
138
|
+
export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate")(function* (
|
|
139
|
+
route: string,
|
|
140
|
+
message: LLMRequest["messages"][number],
|
|
141
|
+
) {
|
|
142
|
+
const content = yield* systemUpdateText(route, message)
|
|
143
|
+
return { type: "text" as const, text: wrapSystemUpdate(content), cache: content.at(-1)?.cache }
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Parse the streamed JSON input of a tool call. Treats an empty string as
|
|
148
|
+
* `"{}"` — providers occasionally finish a tool call without ever emitting
|
|
149
|
+
* input deltas (e.g. zero-arg tools). The error message is uniform across
|
|
150
|
+
* routes: `Invalid JSON input for <route> tool call <name>`.
|
|
151
|
+
*/
|
|
152
|
+
export const parseToolInput = (route: string, name: string, raw: string) =>
|
|
153
|
+
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
|
|
157
|
+
* `data: string` is assumed to already be base64 (matches caller convention
|
|
158
|
+
* across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
|
|
159
|
+
* by every route that supports image / document inputs.
|
|
160
|
+
*/
|
|
161
|
+
export const mediaBytes = (part: MediaPart) =>
|
|
162
|
+
typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
|
|
163
|
+
|
|
164
|
+
export const mediaBase64 = (part: MediaPart) => {
|
|
165
|
+
if (typeof part.data !== "string" || !part.data.startsWith("data:")) return mediaBytes(part)
|
|
166
|
+
return part.data.slice(part.data.indexOf(",") + 1)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export const mediaDataUrl = (part: MediaPart) =>
|
|
170
|
+
typeof part.data === "string" && part.data.startsWith("data:")
|
|
171
|
+
? part.data
|
|
172
|
+
: `data:${part.mediaType};base64,${mediaBytes(part)}`
|
|
173
|
+
|
|
174
|
+
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
|
|
175
|
+
|
|
176
|
+
export const toolResultText = (part: ToolResultPart) => {
|
|
177
|
+
if (part.result.type === "text" || part.result.type === "error") return String(part.result.value)
|
|
178
|
+
if (part.result.type === "content") return encodeJson(part.result.value)
|
|
179
|
+
return encodeJson(part.result.value)
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export const errorText = (error: unknown) => {
|
|
183
|
+
if (error instanceof Error) return error.message
|
|
184
|
+
if (typeof error === "string") return error
|
|
185
|
+
if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") return String(error)
|
|
186
|
+
if (error === null) return "null"
|
|
187
|
+
if (error === undefined) return "undefined"
|
|
188
|
+
return "Unknown stream error"
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
|
|
193
|
+
* decoder, and drops empty / `[DONE]` keep-alive events so the downstream
|
|
194
|
+
* `decodeChunk` sees one JSON string per element. The SSE channel emits a
|
|
195
|
+
* `Retry` control event on its error channel; we drop it here (we don't
|
|
196
|
+
* implement client-driven retries) so the public error channel stays
|
|
197
|
+
* `LLMError`.
|
|
198
|
+
*/
|
|
199
|
+
export const sseFraming = (bytes: Stream.Stream<Uint8Array, LLMError>): Stream.Stream<string, LLMError> =>
|
|
200
|
+
bytes.pipe(
|
|
201
|
+
Stream.decodeText(),
|
|
202
|
+
Stream.pipeThroughChannel(Sse.decode()),
|
|
203
|
+
Stream.catchTag("Retry", () => Stream.empty),
|
|
204
|
+
Stream.filter((event) => event.data.length > 0 && event.data !== "[DONE]"),
|
|
205
|
+
Stream.map((event) => event.data),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Canonical invalid-request constructor. Lift one-line `const invalid =
|
|
210
|
+
* (message) => invalidRequest(message)` aliases out of every
|
|
211
|
+
* route so the error constructor lives in one place. If we ever extend
|
|
212
|
+
* `InvalidRequestReason` with route context or trace metadata, the change
|
|
213
|
+
* lands here.
|
|
214
|
+
*/
|
|
215
|
+
export const invalidRequest = (message: string) =>
|
|
216
|
+
new LLMError({
|
|
217
|
+
module: "ProviderShared",
|
|
218
|
+
method: "request",
|
|
219
|
+
reason: new InvalidRequestReason({ message }),
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
export const matchToolChoice = <Auto, None, Required, Tool>(
|
|
223
|
+
route: string,
|
|
224
|
+
toolChoice: NonNullable<LLMRequest["toolChoice"]>,
|
|
225
|
+
cases: {
|
|
226
|
+
readonly auto: () => Auto
|
|
227
|
+
readonly none: () => None
|
|
228
|
+
readonly required: () => Required
|
|
229
|
+
readonly tool: (name: string) => Tool
|
|
230
|
+
},
|
|
231
|
+
) =>
|
|
232
|
+
Effect.gen(function* () {
|
|
233
|
+
if (toolChoice.type === "auto") return cases.auto()
|
|
234
|
+
if (toolChoice.type === "none") return cases.none()
|
|
235
|
+
if (toolChoice.type === "required") return cases.required()
|
|
236
|
+
if (!toolChoice.name) return yield* invalidRequest(`${route} tool choice requires a tool name`)
|
|
237
|
+
return cases.tool(toolChoice.name)
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
type ContentType = ContentPart["type"]
|
|
241
|
+
|
|
242
|
+
const formatContentTypes = (types: ReadonlyArray<ContentType>) => {
|
|
243
|
+
if (types.length <= 1) return types[0] ?? ""
|
|
244
|
+
if (types.length === 2) return `${types[0]} and ${types[1]}`
|
|
245
|
+
return `${types.slice(0, -1).join(", ")}, and ${types.at(-1)}`
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export const supportsContent = <const Type extends ContentType>(
|
|
249
|
+
part: ContentPart,
|
|
250
|
+
types: ReadonlyArray<Type>,
|
|
251
|
+
): part is Extract<ContentPart, { readonly type: Type }> => (types as ReadonlyArray<ContentType>).includes(part.type)
|
|
252
|
+
|
|
253
|
+
export const unsupportedContent = (
|
|
254
|
+
route: string,
|
|
255
|
+
role: LLMRequest["messages"][number]["role"],
|
|
256
|
+
types: ReadonlyArray<ContentType>,
|
|
257
|
+
) => invalidRequest(`${route} ${role} messages only support ${formatContentTypes(types)} content for now`)
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Build a `validate` step from a Schema decoder. Replaces the per-route
|
|
261
|
+
* lambda body `(payload) => decode(payload).pipe(Effect.mapError((e) =>
|
|
262
|
+
* invalid(e.message)))`. Any decode error is translated into
|
|
263
|
+
* `LLMError` carrying the original parse-error message.
|
|
264
|
+
*/
|
|
265
|
+
export const validateWith =
|
|
266
|
+
<A, I, E extends { readonly message: string }>(decode: (input: I) => Effect.Effect<A, E>) =>
|
|
267
|
+
(payload: I) =>
|
|
268
|
+
decode(payload).pipe(Effect.mapError((error) => invalidRequest(error.message)))
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Build an HTTP POST with a JSON body. Sets `content-type: application/json`
|
|
272
|
+
* automatically after caller-supplied headers so routes cannot accidentally
|
|
273
|
+
* send JSON with a stale content type. The body is passed pre-encoded so
|
|
274
|
+
* routes can choose between
|
|
275
|
+
* `Schema.encodeSync(payload)` and `ProviderShared.encodeJson(payload)`.
|
|
276
|
+
*/
|
|
277
|
+
export const jsonPost = (input: { readonly url: string; readonly body: string; readonly headers?: Headers.Input }) =>
|
|
278
|
+
HttpClientRequest.post(input.url).pipe(
|
|
279
|
+
HttpClientRequest.setHeaders(Headers.set(Headers.fromInput(input.headers), "content-type", "application/json")),
|
|
280
|
+
HttpClientRequest.bodyText(input.body, "application/json"),
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
export * as ProviderShared from "./shared"
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { AwsV4Signer } from "aws4fetch"
|
|
2
|
+
import { Effect } from "effect"
|
|
3
|
+
import { Headers } from "effect/unstable/http"
|
|
4
|
+
import { Auth, type AuthInput } from "../../route/auth"
|
|
5
|
+
import { ProviderShared } from "../shared"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* AWS credentials for SigV4 signing. Bedrock also supports Bearer API key auth,
|
|
9
|
+
* which provider facades configure as route auth instead of SigV4. STS-vended
|
|
10
|
+
* credentials should be refreshed by the consumer (rebuild the model) before
|
|
11
|
+
* they expire; the route does not refresh.
|
|
12
|
+
*/
|
|
13
|
+
export interface Credentials {
|
|
14
|
+
readonly region: string
|
|
15
|
+
readonly accessKeyId: string
|
|
16
|
+
readonly secretAccessKey: string
|
|
17
|
+
readonly sessionToken?: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const signRequest = (input: {
|
|
21
|
+
readonly url: string
|
|
22
|
+
readonly body: string
|
|
23
|
+
readonly headers: Headers.Headers
|
|
24
|
+
readonly credentials: Credentials
|
|
25
|
+
}) =>
|
|
26
|
+
Effect.tryPromise({
|
|
27
|
+
try: async () => {
|
|
28
|
+
const signed = await new AwsV4Signer({
|
|
29
|
+
url: input.url,
|
|
30
|
+
method: "POST",
|
|
31
|
+
headers: Object.entries(input.headers),
|
|
32
|
+
body: input.body,
|
|
33
|
+
region: input.credentials.region,
|
|
34
|
+
accessKeyId: input.credentials.accessKeyId,
|
|
35
|
+
secretAccessKey: input.credentials.secretAccessKey,
|
|
36
|
+
sessionToken: input.credentials.sessionToken,
|
|
37
|
+
service: "bedrock",
|
|
38
|
+
}).sign()
|
|
39
|
+
return Object.fromEntries(signed.headers.entries())
|
|
40
|
+
},
|
|
41
|
+
catch: (error) =>
|
|
42
|
+
ProviderShared.invalidRequest(
|
|
43
|
+
`Bedrock Converse SigV4 signing failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
44
|
+
),
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
/** Sign the exact JSON bytes with SigV4 using credentials configured on the route. */
|
|
48
|
+
export const sigV4 = (credentials: Credentials | undefined) =>
|
|
49
|
+
Auth.custom((input: AuthInput) => {
|
|
50
|
+
return Effect.gen(function* () {
|
|
51
|
+
if (!credentials) {
|
|
52
|
+
return yield* ProviderShared.invalidRequest(
|
|
53
|
+
"Bedrock Converse requires either route bearer auth or AWS credentials configured on the route",
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
const headersForSigning = Headers.set(input.headers, "content-type", "application/json")
|
|
57
|
+
const signed = yield* signRequest({
|
|
58
|
+
url: input.url,
|
|
59
|
+
body: input.body,
|
|
60
|
+
headers: headersForSigning,
|
|
61
|
+
credentials,
|
|
62
|
+
})
|
|
63
|
+
return Headers.setAll(headersForSigning, signed)
|
|
64
|
+
})
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
/** Bedrock route auth defaults to SigV4 and expects credentials from route configuration. */
|
|
68
|
+
export const auth = sigV4(undefined)
|
|
69
|
+
|
|
70
|
+
export * as BedrockAuth from "./bedrock-auth"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { Schema } from "effect"
|
|
2
|
+
import type { CacheHint } from "../../schema"
|
|
3
|
+
import { newBreakpoints, ttlBucket, type Breakpoints } from "./cache"
|
|
4
|
+
|
|
5
|
+
// Bedrock cache markers are positional: emit a `cachePoint` block immediately
|
|
6
|
+
// after the content the caller wants treated as a cacheable prefix. Bedrock
|
|
7
|
+
// accepts optional `ttl: "5m" | "1h"` on cachePoint, mirroring Anthropic.
|
|
8
|
+
export const CachePointBlock = Schema.Struct({
|
|
9
|
+
cachePoint: Schema.Struct({
|
|
10
|
+
type: Schema.tag("default"),
|
|
11
|
+
ttl: Schema.optional(Schema.Literals(["5m", "1h"])),
|
|
12
|
+
}),
|
|
13
|
+
})
|
|
14
|
+
export type CachePointBlock = Schema.Schema.Type<typeof CachePointBlock>
|
|
15
|
+
|
|
16
|
+
// Bedrock-Claude enforces the same 4-breakpoint cap as the Anthropic Messages
|
|
17
|
+
// API. Callers pass a shared counter through every `block()` call site so the
|
|
18
|
+
// budget is respected across `system`, `messages`, and `tools`.
|
|
19
|
+
export const BEDROCK_BREAKPOINT_CAP = 4
|
|
20
|
+
|
|
21
|
+
export type { Breakpoints } from "./cache"
|
|
22
|
+
export const breakpoints = () => newBreakpoints(BEDROCK_BREAKPOINT_CAP)
|
|
23
|
+
|
|
24
|
+
const DEFAULT_5M: CachePointBlock = { cachePoint: { type: "default" } }
|
|
25
|
+
const DEFAULT_1H: CachePointBlock = { cachePoint: { type: "default", ttl: "1h" } }
|
|
26
|
+
|
|
27
|
+
export const block = (breakpoints: Breakpoints, cache: CacheHint | undefined): CachePointBlock | undefined => {
|
|
28
|
+
if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined
|
|
29
|
+
if (breakpoints.remaining <= 0) {
|
|
30
|
+
breakpoints.dropped += 1
|
|
31
|
+
return undefined
|
|
32
|
+
}
|
|
33
|
+
breakpoints.remaining -= 1
|
|
34
|
+
return ttlBucket(cache.ttlSeconds) === "1h" ? DEFAULT_1H : DEFAULT_5M
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export * as BedrockCache from "./bedrock-cache"
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect"
|
|
2
|
+
import type { MediaPart } from "../../schema"
|
|
3
|
+
import { ProviderShared } from "../shared"
|
|
4
|
+
|
|
5
|
+
// Bedrock Converse accepts image `format` as the file extension and
|
|
6
|
+
// `source.bytes` as base64 in the JSON wire format.
|
|
7
|
+
export const ImageFormat = Schema.Literals(["png", "jpeg", "gif", "webp"])
|
|
8
|
+
export type ImageFormat = Schema.Schema.Type<typeof ImageFormat>
|
|
9
|
+
|
|
10
|
+
export const ImageBlock = Schema.Struct({
|
|
11
|
+
image: Schema.Struct({
|
|
12
|
+
format: ImageFormat,
|
|
13
|
+
source: Schema.Struct({ bytes: Schema.String }),
|
|
14
|
+
}),
|
|
15
|
+
})
|
|
16
|
+
export type ImageBlock = Schema.Schema.Type<typeof ImageBlock>
|
|
17
|
+
|
|
18
|
+
// Bedrock document blocks require a user-facing name so the model can refer to
|
|
19
|
+
// the uploaded document.
|
|
20
|
+
export const DocumentFormat = Schema.Literals(["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"])
|
|
21
|
+
export type DocumentFormat = Schema.Schema.Type<typeof DocumentFormat>
|
|
22
|
+
|
|
23
|
+
export const DocumentBlock = Schema.Struct({
|
|
24
|
+
document: Schema.Struct({
|
|
25
|
+
format: DocumentFormat,
|
|
26
|
+
name: Schema.String,
|
|
27
|
+
source: Schema.Struct({ bytes: Schema.String }),
|
|
28
|
+
}),
|
|
29
|
+
})
|
|
30
|
+
export type DocumentBlock = Schema.Schema.Type<typeof DocumentBlock>
|
|
31
|
+
|
|
32
|
+
const IMAGE_FORMATS = {
|
|
33
|
+
"image/png": "png",
|
|
34
|
+
"image/jpeg": "jpeg",
|
|
35
|
+
"image/jpg": "jpeg",
|
|
36
|
+
"image/gif": "gif",
|
|
37
|
+
"image/webp": "webp",
|
|
38
|
+
} as const satisfies Record<string, ImageFormat>
|
|
39
|
+
|
|
40
|
+
const DOCUMENT_FORMATS = {
|
|
41
|
+
"application/pdf": "pdf",
|
|
42
|
+
"text/csv": "csv",
|
|
43
|
+
"application/msword": "doc",
|
|
44
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
|
|
45
|
+
"application/vnd.ms-excel": "xls",
|
|
46
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
|
47
|
+
"text/html": "html",
|
|
48
|
+
"text/plain": "txt",
|
|
49
|
+
"text/markdown": "md",
|
|
50
|
+
} as const satisfies Record<string, DocumentFormat>
|
|
51
|
+
|
|
52
|
+
const imageBlock = (part: MediaPart, format: ImageFormat): ImageBlock => ({
|
|
53
|
+
image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock => ({
|
|
57
|
+
document: {
|
|
58
|
+
format,
|
|
59
|
+
name: part.filename ?? `document.${format}`,
|
|
60
|
+
source: { bytes: ProviderShared.mediaBytes(part) },
|
|
61
|
+
},
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
// Route by MIME. Known image/document formats lower into a typed block; anything
|
|
65
|
+
// else fails with a clear error instead of silently degrading to a malformed
|
|
66
|
+
// document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
|
|
67
|
+
// get an image-specific error so the caller knows it's a format-support issue,
|
|
68
|
+
// not a kind-detection issue.
|
|
69
|
+
export const lower = (part: MediaPart) => {
|
|
70
|
+
const mime = part.mediaType.toLowerCase()
|
|
71
|
+
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
|
|
72
|
+
if (imageFormat) return Effect.succeed(imageBlock(part, imageFormat))
|
|
73
|
+
if (mime.startsWith("image/"))
|
|
74
|
+
return ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
|
|
75
|
+
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
|
|
76
|
+
if (documentFormat) return Effect.succeed(documentBlock(part, documentFormat))
|
|
77
|
+
return ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export * as BedrockMedia from "./bedrock-media"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Shared helpers for provider cache-marker lowering. Anthropic and Bedrock
|
|
2
|
+
// both enforce a 4-breakpoint cap per request and accept the same `5m`/`1h`
|
|
3
|
+
// TTL buckets, so the counter and TTL mapping live here.
|
|
4
|
+
|
|
5
|
+
export interface Breakpoints {
|
|
6
|
+
remaining: number
|
|
7
|
+
dropped: number
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const newBreakpoints = (cap: number): Breakpoints => ({ remaining: cap, dropped: 0 })
|
|
11
|
+
|
|
12
|
+
// Returns `"1h"` for any `ttlSeconds >= 3600`, otherwise `undefined` (the
|
|
13
|
+
// provider default 5m). Anthropic & Bedrock both treat anything shorter than
|
|
14
|
+
// an hour as 5m.
|
|
15
|
+
export const ttlBucket = (ttlSeconds: number | undefined): "1h" | undefined =>
|
|
16
|
+
ttlSeconds !== undefined && ttlSeconds >= 3600 ? "1h" : undefined
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { ProviderShared } from "../shared"
|
|
2
|
+
|
|
3
|
+
// Gemini accepts a JSON Schema-like dialect for tool parameters, but rejects a
|
|
4
|
+
// handful of common JSON Schema shapes. Keep this projection isolated so the
|
|
5
|
+
// Gemini protocol file still reads like the other protocol modules.
|
|
6
|
+
const SCHEMA_INTENT_KEYS = [
|
|
7
|
+
"type",
|
|
8
|
+
"properties",
|
|
9
|
+
"items",
|
|
10
|
+
"prefixItems",
|
|
11
|
+
"enum",
|
|
12
|
+
"const",
|
|
13
|
+
"$ref",
|
|
14
|
+
"additionalProperties",
|
|
15
|
+
"patternProperties",
|
|
16
|
+
"required",
|
|
17
|
+
"not",
|
|
18
|
+
"if",
|
|
19
|
+
"then",
|
|
20
|
+
"else",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
const isRecord = ProviderShared.isRecord
|
|
24
|
+
|
|
25
|
+
const hasCombiner = (schema: unknown) =>
|
|
26
|
+
isRecord(schema) && (Array.isArray(schema.anyOf) || Array.isArray(schema.oneOf) || Array.isArray(schema.allOf))
|
|
27
|
+
|
|
28
|
+
const hasSchemaIntent = (schema: unknown) =>
|
|
29
|
+
isRecord(schema) && (hasCombiner(schema) || SCHEMA_INTENT_KEYS.some((key) => key in schema))
|
|
30
|
+
|
|
31
|
+
const sanitizeNode = (schema: unknown): unknown => {
|
|
32
|
+
if (!isRecord(schema)) return Array.isArray(schema) ? schema.map(sanitizeNode) : schema
|
|
33
|
+
|
|
34
|
+
const result: Record<string, unknown> = Object.fromEntries(
|
|
35
|
+
Object.entries(schema).map(([key, value]) => [
|
|
36
|
+
key,
|
|
37
|
+
key === "enum" && Array.isArray(value) ? value.map(String) : sanitizeNode(value),
|
|
38
|
+
]),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
if (Array.isArray(result.enum) && (result.type === "integer" || result.type === "number")) result.type = "string"
|
|
42
|
+
|
|
43
|
+
const properties = result.properties
|
|
44
|
+
if (result.type === "object" && isRecord(properties) && Array.isArray(result.required)) {
|
|
45
|
+
result.required = result.required.filter((field) => typeof field === "string" && field in properties)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (result.type === "array" && !hasCombiner(result)) {
|
|
49
|
+
result.items = result.items ?? {}
|
|
50
|
+
if (isRecord(result.items) && !hasSchemaIntent(result.items)) result.items = { ...result.items, type: "string" }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (typeof result.type === "string" && result.type !== "object" && !hasCombiner(result)) {
|
|
54
|
+
delete result.properties
|
|
55
|
+
delete result.required
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return result
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const emptyObjectSchema = (schema: Record<string, unknown>) =>
|
|
62
|
+
schema.type === "object" &&
|
|
63
|
+
(!isRecord(schema.properties) || Object.keys(schema.properties).length === 0) &&
|
|
64
|
+
!schema.additionalProperties
|
|
65
|
+
|
|
66
|
+
const projectNode = (schema: unknown): Record<string, unknown> | undefined => {
|
|
67
|
+
if (!isRecord(schema)) return undefined
|
|
68
|
+
if (emptyObjectSchema(schema)) return undefined
|
|
69
|
+
return Object.fromEntries(
|
|
70
|
+
[
|
|
71
|
+
["description", schema.description],
|
|
72
|
+
["required", schema.required],
|
|
73
|
+
["format", schema.format],
|
|
74
|
+
["type", Array.isArray(schema.type) ? schema.type.filter((type) => type !== "null")[0] : schema.type],
|
|
75
|
+
["nullable", Array.isArray(schema.type) && schema.type.includes("null") ? true : undefined],
|
|
76
|
+
["enum", schema.const !== undefined ? [schema.const] : schema.enum],
|
|
77
|
+
[
|
|
78
|
+
"properties",
|
|
79
|
+
isRecord(schema.properties)
|
|
80
|
+
? Object.fromEntries(Object.entries(schema.properties).map(([key, value]) => [key, projectNode(value)]))
|
|
81
|
+
: undefined,
|
|
82
|
+
],
|
|
83
|
+
[
|
|
84
|
+
"items",
|
|
85
|
+
Array.isArray(schema.items)
|
|
86
|
+
? schema.items.map(projectNode)
|
|
87
|
+
: schema.items === undefined
|
|
88
|
+
? undefined
|
|
89
|
+
: projectNode(schema.items),
|
|
90
|
+
],
|
|
91
|
+
["allOf", Array.isArray(schema.allOf) ? schema.allOf.map(projectNode) : undefined],
|
|
92
|
+
["anyOf", Array.isArray(schema.anyOf) ? schema.anyOf.map(projectNode) : undefined],
|
|
93
|
+
["oneOf", Array.isArray(schema.oneOf) ? schema.oneOf.map(projectNode) : undefined],
|
|
94
|
+
["minLength", schema.minLength],
|
|
95
|
+
].filter((entry) => entry[1] !== undefined),
|
|
96
|
+
)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export const convert = (schema: unknown) => projectNode(sanitizeNode(schema))
|
|
100
|
+
|
|
101
|
+
export * as GeminiToolSchema from "./gemini-tool-schema"
|