@codilore/llm 1.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +321 -0
- package/README.md +131 -0
- package/example/call-sites.md +591 -0
- package/example/tutorial.ts +255 -0
- package/package.json +50 -0
- package/script/recording-cost-report.ts +250 -0
- package/script/setup-recording-env.ts +542 -0
- package/src/cache-policy.ts +111 -0
- package/src/index.ts +32 -0
- package/src/llm.ts +186 -0
- package/src/protocols/anthropic-messages.ts +841 -0
- package/src/protocols/bedrock-converse.ts +649 -0
- package/src/protocols/bedrock-event-stream.ts +87 -0
- package/src/protocols/gemini.ts +465 -0
- package/src/protocols/index.ts +6 -0
- package/src/protocols/openai-chat.ts +431 -0
- package/src/protocols/openai-compatible-chat.ts +24 -0
- package/src/protocols/openai-responses.ts +987 -0
- package/src/protocols/shared.ts +283 -0
- package/src/protocols/utils/bedrock-auth.ts +70 -0
- package/src/protocols/utils/bedrock-cache.ts +37 -0
- package/src/protocols/utils/bedrock-media.ts +80 -0
- package/src/protocols/utils/cache.ts +16 -0
- package/src/protocols/utils/gemini-tool-schema.ts +101 -0
- package/src/protocols/utils/lifecycle.ts +102 -0
- package/src/protocols/utils/openai-options.ts +84 -0
- package/src/protocols/utils/tool-stream.ts +218 -0
- package/src/provider.ts +37 -0
- package/src/providers/amazon-bedrock.ts +43 -0
- package/src/providers/anthropic.ts +35 -0
- package/src/providers/azure.ts +110 -0
- package/src/providers/cloudflare.ts +127 -0
- package/src/providers/github-copilot.ts +66 -0
- package/src/providers/google.ts +35 -0
- package/src/providers/index.ts +11 -0
- package/src/providers/openai-compatible-profile.ts +20 -0
- package/src/providers/openai-compatible.ts +65 -0
- package/src/providers/openai-options.ts +81 -0
- package/src/providers/openai.ts +63 -0
- package/src/providers/openrouter.ts +98 -0
- package/src/providers/xai.ts +56 -0
- package/src/route/auth-options.ts +57 -0
- package/src/route/auth.ts +156 -0
- package/src/route/client.ts +434 -0
- package/src/route/endpoint.ts +53 -0
- package/src/route/executor.ts +374 -0
- package/src/route/framing.ts +27 -0
- package/src/route/index.ts +25 -0
- package/src/route/protocol.ts +84 -0
- package/src/route/transport/http.ts +108 -0
- package/src/route/transport/index.ts +33 -0
- package/src/route/transport/websocket.ts +280 -0
- package/src/schema/errors.ts +203 -0
- package/src/schema/events.ts +370 -0
- package/src/schema/ids.ts +43 -0
- package/src/schema/index.ts +5 -0
- package/src/schema/messages.ts +404 -0
- package/src/schema/options.ts +221 -0
- package/src/tool-runtime.ts +78 -0
- package/src/tool.ts +241 -0
- package/src/utils/record.ts +3 -0
- package/sst-env.d.ts +10 -0
- package/test/adapter.test.ts +164 -0
- package/test/auth-options.types.ts +168 -0
- package/test/auth.test.ts +103 -0
- package/test/cache-policy.test.ts +262 -0
- package/test/continuation-scenarios.ts +104 -0
- package/test/endpoint.test.ts +58 -0
- package/test/executor.test.ts +418 -0
- package/test/exports.test.ts +62 -0
- package/test/fixtures/media/restroom.png +0 -0
- package/test/fixtures/recordings/anthropic-messages/accepts-malformed-assistant-tool-order-with-default-patch.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-image-tool-result.json +43 -0
- package/test/fixtures/recordings/anthropic-messages/claude-opus-4-7-drives-a-tool-loop.json +56 -0
- package/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order-without-patch.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/streams-text.json +29 -0
- package/test/fixtures/recordings/anthropic-messages/streams-tool-call.json +29 -0
- package/test/fixtures/recordings/anthropic-messages-cache/writes-then-reads-cache-control-on-identical-second-call.json +48 -0
- package/test/fixtures/recordings/bedrock-converse/drives-a-tool-loop.json +55 -0
- package/test/fixtures/recordings/bedrock-converse/streams-a-tool-call.json +29 -0
- package/test/fixtures/recordings/bedrock-converse/streams-text.json +29 -0
- package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
- package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-llama-3-1-8b-text.json +32 -0
- package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
- package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-llama-3-1-8b-text.json +32 -0
- package/test/fixtures/recordings/gemini/gemini-2-5-flash-image.json +32 -0
- package/test/fixtures/recordings/gemini/streams-text.json +28 -0
- package/test/fixtures/recordings/gemini/streams-tool-call.json +28 -0
- package/test/fixtures/recordings/gemini-cache/reports-cachedcontenttokencount-on-identical-second-call.json +46 -0
- package/test/fixtures/recordings/openai-chat/continues-after-tool-result.json +28 -0
- package/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json +46 -0
- package/test/fixtures/recordings/openai-chat/streams-text.json +28 -0
- package/test/fixtures/recordings/openai-chat/streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/deepseek-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-llama-3-3-70b-drives-a-tool-loop.json +53 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/groq-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json +53 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-drives-a-tool-loop.json +54 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-text.json +28 -0
- package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-tool-call.json +28 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-image-tool-result.json +42 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning-continuation.json +58 -0
- package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json +32 -0
- package/test/fixtures/recordings/openai-responses-cache/reports-cached-tokens-on-identical-second-call.json +46 -0
- package/test/generate-object.test.ts +184 -0
- package/test/lib/effect.ts +50 -0
- package/test/lib/http.ts +98 -0
- package/test/lib/openai-chunks.ts +27 -0
- package/test/lib/sse.ts +17 -0
- package/test/lib/tool-runtime.ts +146 -0
- package/test/llm.test.ts +167 -0
- package/test/provider/anthropic-messages-cache.recorded.test.ts +54 -0
- package/test/provider/anthropic-messages.recorded.test.ts +46 -0
- package/test/provider/anthropic-messages.test.ts +829 -0
- package/test/provider/bedrock-converse-cache.recorded.test.ts +54 -0
- package/test/provider/bedrock-converse.test.ts +707 -0
- package/test/provider/cloudflare.test.ts +230 -0
- package/test/provider/gemini-cache.recorded.test.ts +48 -0
- package/test/provider/gemini.test.ts +476 -0
- package/test/provider/golden.recorded.test.ts +219 -0
- package/test/provider/openai-chat.test.ts +446 -0
- package/test/provider/openai-compatible-chat.test.ts +238 -0
- package/test/provider/openai-responses-cache.recorded.test.ts +46 -0
- package/test/provider/openai-responses.test.ts +1322 -0
- package/test/provider/openrouter.test.ts +56 -0
- package/test/provider.types.ts +41 -0
- package/test/recorded-golden.ts +97 -0
- package/test/recorded-runner.ts +100 -0
- package/test/recorded-scenarios.ts +531 -0
- package/test/recorded-test.ts +74 -0
- package/test/recorded-utils.ts +56 -0
- package/test/recorded-websocket.ts +26 -0
- package/test/route.test.ts +43 -0
- package/test/schema.test.ts +97 -0
- package/test/tool-runtime.test.ts +802 -0
- package/test/tool-stream.test.ts +99 -0
- package/test/tool.types.ts +40 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,649 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect"
|
|
2
|
+
import { Route } from "../route/client"
|
|
3
|
+
import { Endpoint } from "../route/endpoint"
|
|
4
|
+
import { Protocol } from "../route/protocol"
|
|
5
|
+
import {
|
|
6
|
+
LLMEvent,
|
|
7
|
+
Usage,
|
|
8
|
+
type CacheHint,
|
|
9
|
+
type FinishReason,
|
|
10
|
+
type LLMRequest,
|
|
11
|
+
type ProviderMetadata,
|
|
12
|
+
type ReasoningPart,
|
|
13
|
+
type ToolCallPart,
|
|
14
|
+
type ToolDefinition,
|
|
15
|
+
type ToolResultPart,
|
|
16
|
+
} from "../schema"
|
|
17
|
+
import { BedrockEventStream } from "./bedrock-event-stream"
|
|
18
|
+
import { JsonObject, optionalArray, ProviderShared } from "./shared"
|
|
19
|
+
import { BedrockAuth } from "./utils/bedrock-auth"
|
|
20
|
+
import { BedrockCache } from "./utils/bedrock-cache"
|
|
21
|
+
import { BedrockMedia } from "./utils/bedrock-media"
|
|
22
|
+
import { Lifecycle } from "./utils/lifecycle"
|
|
23
|
+
import { ToolStream } from "./utils/tool-stream"
|
|
24
|
+
|
|
25
|
+
const ADAPTER = "bedrock-converse"
|
|
26
|
+
|
|
27
|
+
export type { Credentials as BedrockCredentials } from "./utils/bedrock-auth"
|
|
28
|
+
|
|
29
|
+
// =============================================================================
|
|
30
|
+
// Request Body Schema
|
|
31
|
+
// =============================================================================
|
|
32
|
+
const BedrockTextBlock = Schema.Struct({
|
|
33
|
+
text: Schema.String,
|
|
34
|
+
})
|
|
35
|
+
type BedrockTextBlock = Schema.Schema.Type<typeof BedrockTextBlock>
|
|
36
|
+
|
|
37
|
+
const BedrockToolUseBlock = Schema.Struct({
|
|
38
|
+
toolUse: Schema.Struct({
|
|
39
|
+
toolUseId: Schema.String,
|
|
40
|
+
name: Schema.String,
|
|
41
|
+
input: Schema.Unknown,
|
|
42
|
+
}),
|
|
43
|
+
})
|
|
44
|
+
type BedrockToolUseBlock = Schema.Schema.Type<typeof BedrockToolUseBlock>
|
|
45
|
+
|
|
46
|
+
const BedrockToolResultContentItem = Schema.Union([
|
|
47
|
+
Schema.Struct({ text: Schema.String }),
|
|
48
|
+
Schema.Struct({ json: Schema.Unknown }),
|
|
49
|
+
BedrockMedia.ImageBlock,
|
|
50
|
+
])
|
|
51
|
+
|
|
52
|
+
const BedrockToolResultBlock = Schema.Struct({
|
|
53
|
+
toolResult: Schema.Struct({
|
|
54
|
+
toolUseId: Schema.String,
|
|
55
|
+
content: Schema.Array(BedrockToolResultContentItem),
|
|
56
|
+
status: Schema.optional(Schema.Literals(["success", "error"])),
|
|
57
|
+
}),
|
|
58
|
+
})
|
|
59
|
+
type BedrockToolResultBlock = Schema.Schema.Type<typeof BedrockToolResultBlock>
|
|
60
|
+
|
|
61
|
+
const BedrockReasoningBlock = Schema.Struct({
|
|
62
|
+
reasoningContent: Schema.Struct({
|
|
63
|
+
reasoningText: Schema.optional(
|
|
64
|
+
Schema.Struct({
|
|
65
|
+
text: Schema.String,
|
|
66
|
+
signature: Schema.optional(Schema.String),
|
|
67
|
+
}),
|
|
68
|
+
),
|
|
69
|
+
}),
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
const BedrockUserBlock = Schema.Union([
|
|
73
|
+
BedrockTextBlock,
|
|
74
|
+
BedrockMedia.ImageBlock,
|
|
75
|
+
BedrockMedia.DocumentBlock,
|
|
76
|
+
BedrockToolResultBlock,
|
|
77
|
+
BedrockCache.CachePointBlock,
|
|
78
|
+
])
|
|
79
|
+
type BedrockUserBlock = Schema.Schema.Type<typeof BedrockUserBlock>
|
|
80
|
+
|
|
81
|
+
const BedrockAssistantBlock = Schema.Union([
|
|
82
|
+
BedrockTextBlock,
|
|
83
|
+
BedrockReasoningBlock,
|
|
84
|
+
BedrockToolUseBlock,
|
|
85
|
+
BedrockCache.CachePointBlock,
|
|
86
|
+
])
|
|
87
|
+
type BedrockAssistantBlock = Schema.Schema.Type<typeof BedrockAssistantBlock>
|
|
88
|
+
|
|
89
|
+
const BedrockMessage = Schema.Union([
|
|
90
|
+
Schema.Struct({ role: Schema.Literal("user"), content: Schema.Array(BedrockUserBlock) }),
|
|
91
|
+
Schema.Struct({ role: Schema.Literal("assistant"), content: Schema.Array(BedrockAssistantBlock) }),
|
|
92
|
+
]).pipe(Schema.toTaggedUnion("role"))
|
|
93
|
+
type BedrockMessage = Schema.Schema.Type<typeof BedrockMessage>
|
|
94
|
+
|
|
95
|
+
const BedrockSystemBlock = Schema.Union([BedrockTextBlock, BedrockCache.CachePointBlock])
|
|
96
|
+
type BedrockSystemBlock = Schema.Schema.Type<typeof BedrockSystemBlock>
|
|
97
|
+
|
|
98
|
+
const BedrockToolSpec = Schema.Struct({
|
|
99
|
+
toolSpec: Schema.Struct({
|
|
100
|
+
name: Schema.String,
|
|
101
|
+
description: Schema.String,
|
|
102
|
+
inputSchema: Schema.Struct({
|
|
103
|
+
json: JsonObject,
|
|
104
|
+
}),
|
|
105
|
+
}),
|
|
106
|
+
})
|
|
107
|
+
type BedrockToolSpec = Schema.Schema.Type<typeof BedrockToolSpec>
|
|
108
|
+
|
|
109
|
+
const BedrockTool = Schema.Union([BedrockToolSpec, BedrockCache.CachePointBlock])
|
|
110
|
+
type BedrockTool = Schema.Schema.Type<typeof BedrockTool>
|
|
111
|
+
|
|
112
|
+
const BedrockToolChoice = Schema.Union([
|
|
113
|
+
Schema.Struct({ auto: Schema.Struct({}) }),
|
|
114
|
+
Schema.Struct({ any: Schema.Struct({}) }),
|
|
115
|
+
Schema.Struct({ tool: Schema.Struct({ name: Schema.String }) }),
|
|
116
|
+
])
|
|
117
|
+
|
|
118
|
+
const BedrockBodyFields = {
|
|
119
|
+
modelId: Schema.String,
|
|
120
|
+
messages: Schema.Array(BedrockMessage),
|
|
121
|
+
system: optionalArray(BedrockSystemBlock),
|
|
122
|
+
inferenceConfig: Schema.optional(
|
|
123
|
+
Schema.Struct({
|
|
124
|
+
maxTokens: Schema.optional(Schema.Number),
|
|
125
|
+
temperature: Schema.optional(Schema.Number),
|
|
126
|
+
topP: Schema.optional(Schema.Number),
|
|
127
|
+
stopSequences: optionalArray(Schema.String),
|
|
128
|
+
}),
|
|
129
|
+
),
|
|
130
|
+
toolConfig: Schema.optional(
|
|
131
|
+
Schema.Struct({
|
|
132
|
+
tools: Schema.Array(BedrockTool),
|
|
133
|
+
toolChoice: Schema.optional(BedrockToolChoice),
|
|
134
|
+
}),
|
|
135
|
+
),
|
|
136
|
+
additionalModelRequestFields: Schema.optional(JsonObject),
|
|
137
|
+
}
|
|
138
|
+
const BedrockConverseBody = Schema.Struct(BedrockBodyFields)
|
|
139
|
+
export type BedrockConverseBody = Schema.Schema.Type<typeof BedrockConverseBody>
|
|
140
|
+
|
|
141
|
+
const BedrockUsageSchema = Schema.Struct({
|
|
142
|
+
inputTokens: Schema.optional(Schema.Number),
|
|
143
|
+
outputTokens: Schema.optional(Schema.Number),
|
|
144
|
+
totalTokens: Schema.optional(Schema.Number),
|
|
145
|
+
cacheReadInputTokens: Schema.optional(Schema.Number),
|
|
146
|
+
cacheWriteInputTokens: Schema.optional(Schema.Number),
|
|
147
|
+
})
|
|
148
|
+
type BedrockUsageSchema = Schema.Schema.Type<typeof BedrockUsageSchema>
|
|
149
|
+
|
|
150
|
+
// Streaming event shape — the AWS event stream wraps each JSON payload by its
|
|
151
|
+
// `:event-type` header (e.g. `messageStart`, `contentBlockDelta`). We
|
|
152
|
+
// reconstruct that wrapping in `decodeFrames` below so the event schema can
|
|
153
|
+
// stay a plain discriminated record.
|
|
154
|
+
const BedrockEvent = Schema.Struct({
|
|
155
|
+
messageStart: Schema.optional(Schema.Struct({ role: Schema.String })),
|
|
156
|
+
contentBlockStart: Schema.optional(
|
|
157
|
+
Schema.Struct({
|
|
158
|
+
contentBlockIndex: Schema.Number,
|
|
159
|
+
start: Schema.optional(
|
|
160
|
+
Schema.Struct({
|
|
161
|
+
toolUse: Schema.optional(Schema.Struct({ toolUseId: Schema.String, name: Schema.String })),
|
|
162
|
+
}),
|
|
163
|
+
),
|
|
164
|
+
}),
|
|
165
|
+
),
|
|
166
|
+
contentBlockDelta: Schema.optional(
|
|
167
|
+
Schema.Struct({
|
|
168
|
+
contentBlockIndex: Schema.Number,
|
|
169
|
+
delta: Schema.optional(
|
|
170
|
+
Schema.Struct({
|
|
171
|
+
text: Schema.optional(Schema.String),
|
|
172
|
+
toolUse: Schema.optional(Schema.Struct({ input: Schema.String })),
|
|
173
|
+
reasoningContent: Schema.optional(
|
|
174
|
+
Schema.Struct({
|
|
175
|
+
text: Schema.optional(Schema.String),
|
|
176
|
+
signature: Schema.optional(Schema.String),
|
|
177
|
+
}),
|
|
178
|
+
),
|
|
179
|
+
}),
|
|
180
|
+
),
|
|
181
|
+
}),
|
|
182
|
+
),
|
|
183
|
+
contentBlockStop: Schema.optional(Schema.Struct({ contentBlockIndex: Schema.Number })),
|
|
184
|
+
messageStop: Schema.optional(
|
|
185
|
+
Schema.Struct({
|
|
186
|
+
stopReason: Schema.String,
|
|
187
|
+
additionalModelResponseFields: Schema.optional(Schema.Unknown),
|
|
188
|
+
}),
|
|
189
|
+
),
|
|
190
|
+
metadata: Schema.optional(
|
|
191
|
+
Schema.Struct({
|
|
192
|
+
usage: Schema.optional(BedrockUsageSchema),
|
|
193
|
+
metrics: Schema.optional(Schema.Unknown),
|
|
194
|
+
}),
|
|
195
|
+
),
|
|
196
|
+
internalServerException: Schema.optional(Schema.Struct({ message: Schema.String })),
|
|
197
|
+
modelStreamErrorException: Schema.optional(Schema.Struct({ message: Schema.String })),
|
|
198
|
+
validationException: Schema.optional(Schema.Struct({ message: Schema.String })),
|
|
199
|
+
throttlingException: Schema.optional(Schema.Struct({ message: Schema.String })),
|
|
200
|
+
serviceUnavailableException: Schema.optional(Schema.Struct({ message: Schema.String })),
|
|
201
|
+
})
|
|
202
|
+
type BedrockEvent = Schema.Schema.Type<typeof BedrockEvent>
|
|
203
|
+
|
|
204
|
+
// =============================================================================
|
|
205
|
+
// Request Lowering
|
|
206
|
+
// =============================================================================
|
|
207
|
+
const lowerToolSpec = (tool: ToolDefinition): BedrockToolSpec => ({
|
|
208
|
+
toolSpec: {
|
|
209
|
+
name: tool.name,
|
|
210
|
+
description: tool.description,
|
|
211
|
+
inputSchema: { json: tool.inputSchema },
|
|
212
|
+
},
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
const lowerTools = (breakpoints: BedrockCache.Breakpoints, tools: ReadonlyArray<ToolDefinition>): BedrockTool[] => {
|
|
216
|
+
const result: BedrockTool[] = []
|
|
217
|
+
for (const tool of tools) {
|
|
218
|
+
result.push(lowerToolSpec(tool))
|
|
219
|
+
const cachePoint = BedrockCache.block(breakpoints, tool.cache)
|
|
220
|
+
if (cachePoint) result.push(cachePoint)
|
|
221
|
+
}
|
|
222
|
+
return result
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const textWithCache = (
|
|
226
|
+
breakpoints: BedrockCache.Breakpoints,
|
|
227
|
+
text: string,
|
|
228
|
+
cache: CacheHint | undefined,
|
|
229
|
+
): Array<BedrockTextBlock | BedrockCache.CachePointBlock> => {
|
|
230
|
+
const cachePoint = BedrockCache.block(breakpoints, cache)
|
|
231
|
+
return cachePoint ? [{ text }, cachePoint] : [{ text }]
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
|
235
|
+
ProviderShared.matchToolChoice("Bedrock Converse", toolChoice, {
|
|
236
|
+
auto: () => ({ auto: {} }) as const,
|
|
237
|
+
none: () => undefined,
|
|
238
|
+
required: () => ({ any: {} }) as const,
|
|
239
|
+
tool: (name) => ({ tool: { name } }) as const,
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
const bedrockMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ bedrock: metadata })
|
|
243
|
+
|
|
244
|
+
const reasoningSignature = (part: ReasoningPart) => {
|
|
245
|
+
const bedrock = part.providerMetadata?.bedrock
|
|
246
|
+
return (
|
|
247
|
+
part.encrypted ??
|
|
248
|
+
(ProviderShared.isRecord(bedrock) && typeof bedrock.signature === "string" ? bedrock.signature : undefined)
|
|
249
|
+
)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const lowerToolCall = (part: ToolCallPart): BedrockToolUseBlock => ({
|
|
253
|
+
toolUse: {
|
|
254
|
+
toolUseId: part.id,
|
|
255
|
+
name: part.name,
|
|
256
|
+
input: part.input,
|
|
257
|
+
},
|
|
258
|
+
})
|
|
259
|
+
|
|
260
|
+
const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent")(function* (part: ToolResultPart) {
|
|
261
|
+
if (part.result.type === "text" || part.result.type === "error")
|
|
262
|
+
return [{ text: ProviderShared.toolResultText(part) }]
|
|
263
|
+
if (part.result.type === "json") return [{ json: part.result.value }]
|
|
264
|
+
|
|
265
|
+
const content: Array<Schema.Schema.Type<typeof BedrockToolResultContentItem>> = []
|
|
266
|
+
for (const item of part.result.value) {
|
|
267
|
+
if (item.type === "text") {
|
|
268
|
+
content.push({ text: item.text })
|
|
269
|
+
continue
|
|
270
|
+
}
|
|
271
|
+
const media = yield* BedrockMedia.lower(item)
|
|
272
|
+
if (!("image" in media))
|
|
273
|
+
return yield* ProviderShared.invalidRequest("Bedrock Converse only supports image media in tool results")
|
|
274
|
+
content.push(media)
|
|
275
|
+
}
|
|
276
|
+
return content
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
const lowerToolResult = Effect.fn("BedrockConverse.lowerToolResult")(function* (part: ToolResultPart) {
|
|
280
|
+
return {
|
|
281
|
+
toolResult: {
|
|
282
|
+
toolUseId: part.id,
|
|
283
|
+
content: yield* lowerToolResultContent(part),
|
|
284
|
+
status: part.result.type === "error" ? "error" : "success",
|
|
285
|
+
},
|
|
286
|
+
} satisfies BedrockToolResultBlock
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
|
290
|
+
request: LLMRequest,
|
|
291
|
+
breakpoints: BedrockCache.Breakpoints,
|
|
292
|
+
) {
|
|
293
|
+
const messages: BedrockMessage[] = []
|
|
294
|
+
|
|
295
|
+
for (const message of request.messages) {
|
|
296
|
+
if (message.role === "system") {
|
|
297
|
+
const part = yield* ProviderShared.wrappedSystemUpdate("Bedrock Converse", message)
|
|
298
|
+
const content = textWithCache(breakpoints, part.text, part.cache)
|
|
299
|
+
const previous = messages.at(-1)
|
|
300
|
+
if (previous?.role === "user")
|
|
301
|
+
messages[messages.length - 1] = { role: "user", content: [...previous.content, ...content] }
|
|
302
|
+
else messages.push({ role: "user", content })
|
|
303
|
+
continue
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
if (message.role === "user") {
|
|
307
|
+
const content: BedrockUserBlock[] = []
|
|
308
|
+
for (const part of message.content) {
|
|
309
|
+
if (!ProviderShared.supportsContent(part, ["text", "media"]))
|
|
310
|
+
return yield* ProviderShared.unsupportedContent("Bedrock Converse", "user", ["text", "media"])
|
|
311
|
+
if (part.type === "text") {
|
|
312
|
+
content.push(...textWithCache(breakpoints, part.text, part.cache))
|
|
313
|
+
continue
|
|
314
|
+
}
|
|
315
|
+
if (part.type === "media") {
|
|
316
|
+
content.push(yield* BedrockMedia.lower(part))
|
|
317
|
+
continue
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
messages.push({ role: "user", content })
|
|
321
|
+
continue
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
if (message.role === "assistant") {
|
|
325
|
+
const content: BedrockAssistantBlock[] = []
|
|
326
|
+
for (const part of message.content) {
|
|
327
|
+
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
|
|
328
|
+
return yield* ProviderShared.unsupportedContent("Bedrock Converse", "assistant", [
|
|
329
|
+
"text",
|
|
330
|
+
"reasoning",
|
|
331
|
+
"tool-call",
|
|
332
|
+
])
|
|
333
|
+
if (part.type === "text") {
|
|
334
|
+
content.push(...textWithCache(breakpoints, part.text, part.cache))
|
|
335
|
+
continue
|
|
336
|
+
}
|
|
337
|
+
if (part.type === "reasoning") {
|
|
338
|
+
content.push({
|
|
339
|
+
reasoningContent: {
|
|
340
|
+
reasoningText: { text: part.text, signature: reasoningSignature(part) },
|
|
341
|
+
},
|
|
342
|
+
})
|
|
343
|
+
continue
|
|
344
|
+
}
|
|
345
|
+
if (part.type === "tool-call") {
|
|
346
|
+
content.push(lowerToolCall(part))
|
|
347
|
+
continue
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
messages.push({ role: "assistant", content })
|
|
351
|
+
continue
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
const content: BedrockUserBlock[] = []
|
|
355
|
+
for (const part of message.content) {
|
|
356
|
+
if (!ProviderShared.supportsContent(part, ["tool-result"]))
|
|
357
|
+
return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"])
|
|
358
|
+
content.push(yield* lowerToolResult(part))
|
|
359
|
+
const cachePoint = BedrockCache.block(breakpoints, part.cache)
|
|
360
|
+
if (cachePoint) content.push(cachePoint)
|
|
361
|
+
}
|
|
362
|
+
messages.push({ role: "user", content })
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
return messages
|
|
366
|
+
})
|
|
367
|
+
|
|
368
|
+
// System prompts share the cache-point convention: emit the text block, then
|
|
369
|
+
// optionally a positional `cachePoint` marker.
|
|
370
|
+
const lowerSystem = (
|
|
371
|
+
breakpoints: BedrockCache.Breakpoints,
|
|
372
|
+
system: ReadonlyArray<LLMRequest["system"][number]>,
|
|
373
|
+
): BedrockSystemBlock[] => system.flatMap((part) => textWithCache(breakpoints, part.text, part.cache))
|
|
374
|
+
|
|
375
|
+
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
|
|
376
|
+
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
|
|
377
|
+
const generation = request.generation
|
|
378
|
+
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
|
|
379
|
+
// tools → system → messages order to favour the highest-impact prefixes.
|
|
380
|
+
const breakpoints = BedrockCache.breakpoints()
|
|
381
|
+
const toolConfig =
|
|
382
|
+
request.tools.length > 0 && request.toolChoice?.type !== "none"
|
|
383
|
+
? { tools: lowerTools(breakpoints, request.tools), toolChoice }
|
|
384
|
+
: undefined
|
|
385
|
+
const system = request.system.length === 0 ? undefined : lowerSystem(breakpoints, request.system)
|
|
386
|
+
const messages = yield* lowerMessages(request, breakpoints)
|
|
387
|
+
if (breakpoints.dropped > 0) {
|
|
388
|
+
yield* Effect.logWarning(
|
|
389
|
+
`Bedrock Converse: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${BedrockCache.BEDROCK_BREAKPOINT_CAP} per request.`,
|
|
390
|
+
)
|
|
391
|
+
}
|
|
392
|
+
return {
|
|
393
|
+
modelId: request.model.id,
|
|
394
|
+
messages,
|
|
395
|
+
system,
|
|
396
|
+
inferenceConfig:
|
|
397
|
+
generation?.maxTokens === undefined &&
|
|
398
|
+
generation?.temperature === undefined &&
|
|
399
|
+
generation?.topP === undefined &&
|
|
400
|
+
(generation?.stop === undefined || generation.stop.length === 0)
|
|
401
|
+
? undefined
|
|
402
|
+
: {
|
|
403
|
+
maxTokens: generation?.maxTokens,
|
|
404
|
+
temperature: generation?.temperature,
|
|
405
|
+
topP: generation?.topP,
|
|
406
|
+
stopSequences: generation?.stop,
|
|
407
|
+
},
|
|
408
|
+
toolConfig,
|
|
409
|
+
}
|
|
410
|
+
})
|
|
411
|
+
|
|
412
|
+
// =============================================================================
|
|
413
|
+
// Stream Parsing
|
|
414
|
+
// =============================================================================
|
|
415
|
+
const mapFinishReason = (reason: string): FinishReason => {
|
|
416
|
+
if (reason === "end_turn" || reason === "stop_sequence") return "stop"
|
|
417
|
+
if (reason === "max_tokens") return "length"
|
|
418
|
+
if (reason === "tool_use") return "tool-calls"
|
|
419
|
+
if (reason === "content_filtered" || reason === "guardrail_intervened") return "content-filter"
|
|
420
|
+
return "unknown"
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// AWS Bedrock Converse reports `inputTokens` (inclusive total) with
|
|
424
|
+
// `cacheReadInputTokens` and `cacheWriteInputTokens` as subsets. Pass
|
|
425
|
+
// the total through and derive the non-cached breakdown. Bedrock does
|
|
426
|
+
// not break reasoning out of `outputTokens` for any current model.
|
|
427
|
+
const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
|
|
428
|
+
if (!usage) return undefined
|
|
429
|
+
const cacheTotal = (usage.cacheReadInputTokens ?? 0) + (usage.cacheWriteInputTokens ?? 0)
|
|
430
|
+
const nonCached = ProviderShared.subtractTokens(usage.inputTokens, cacheTotal)
|
|
431
|
+
return new Usage({
|
|
432
|
+
inputTokens: usage.inputTokens,
|
|
433
|
+
outputTokens: usage.outputTokens,
|
|
434
|
+
nonCachedInputTokens: nonCached,
|
|
435
|
+
cacheReadInputTokens: usage.cacheReadInputTokens,
|
|
436
|
+
cacheWriteInputTokens: usage.cacheWriteInputTokens,
|
|
437
|
+
totalTokens: ProviderShared.totalTokens(usage.inputTokens, usage.outputTokens, usage.totalTokens),
|
|
438
|
+
providerMetadata: { bedrock: usage },
|
|
439
|
+
})
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
interface ParserState {
|
|
443
|
+
readonly tools: ToolStream.State<number>
|
|
444
|
+
// Bedrock splits the finish into `messageStop` (carries `stopReason`) and
|
|
445
|
+
// `metadata` (carries usage). Hold the terminal event in state so `onHalt`
|
|
446
|
+
// can emit exactly one finish after both chunks have had a chance to arrive.
|
|
447
|
+
readonly pendingFinish: { readonly reason: FinishReason; readonly usage?: Usage } | undefined
|
|
448
|
+
readonly hasToolCalls: boolean
|
|
449
|
+
readonly lifecycle: Lifecycle.State
|
|
450
|
+
readonly reasoningSignatures: Readonly<Record<number, string>>
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const step = (state: ParserState, event: BedrockEvent) =>
|
|
454
|
+
Effect.gen(function* () {
|
|
455
|
+
if (event.contentBlockStart?.start?.toolUse) {
|
|
456
|
+
const index = event.contentBlockStart.contentBlockIndex
|
|
457
|
+
const events: LLMEvent[] = []
|
|
458
|
+
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
|
459
|
+
return [
|
|
460
|
+
{
|
|
461
|
+
...state,
|
|
462
|
+
lifecycle,
|
|
463
|
+
tools: ToolStream.start(state.tools, index, {
|
|
464
|
+
id: event.contentBlockStart.start.toolUse.toolUseId,
|
|
465
|
+
name: event.contentBlockStart.start.toolUse.name,
|
|
466
|
+
}),
|
|
467
|
+
},
|
|
468
|
+
[
|
|
469
|
+
...events,
|
|
470
|
+
LLMEvent.toolInputStart({
|
|
471
|
+
id: event.contentBlockStart.start.toolUse.toolUseId,
|
|
472
|
+
name: event.contentBlockStart.start.toolUse.name,
|
|
473
|
+
}),
|
|
474
|
+
],
|
|
475
|
+
] as const
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (event.contentBlockDelta?.delta?.text) {
|
|
479
|
+
const events: LLMEvent[] = []
|
|
480
|
+
return [
|
|
481
|
+
{
|
|
482
|
+
...state,
|
|
483
|
+
lifecycle: Lifecycle.textDelta(
|
|
484
|
+
state.lifecycle,
|
|
485
|
+
events,
|
|
486
|
+
`text-${event.contentBlockDelta.contentBlockIndex}`,
|
|
487
|
+
event.contentBlockDelta.delta.text,
|
|
488
|
+
),
|
|
489
|
+
},
|
|
490
|
+
events,
|
|
491
|
+
] as const
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
if (event.contentBlockDelta?.delta?.reasoningContent) {
|
|
495
|
+
const index = event.contentBlockDelta.contentBlockIndex
|
|
496
|
+
const reasoning = event.contentBlockDelta.delta.reasoningContent
|
|
497
|
+
const events: LLMEvent[] = []
|
|
498
|
+
return [
|
|
499
|
+
{
|
|
500
|
+
...state,
|
|
501
|
+
lifecycle: reasoning.text
|
|
502
|
+
? Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, reasoning.text)
|
|
503
|
+
: state.lifecycle,
|
|
504
|
+
reasoningSignatures: reasoning.signature
|
|
505
|
+
? { ...state.reasoningSignatures, [index]: reasoning.signature }
|
|
506
|
+
: state.reasoningSignatures,
|
|
507
|
+
},
|
|
508
|
+
events,
|
|
509
|
+
] as const
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
if (event.contentBlockDelta?.delta?.toolUse) {
|
|
513
|
+
const index = event.contentBlockDelta.contentBlockIndex
|
|
514
|
+
const result = ToolStream.appendExisting(
|
|
515
|
+
ADAPTER,
|
|
516
|
+
state.tools,
|
|
517
|
+
index,
|
|
518
|
+
event.contentBlockDelta.delta.toolUse.input,
|
|
519
|
+
"Bedrock Converse tool delta is missing its tool call",
|
|
520
|
+
)
|
|
521
|
+
if (ToolStream.isError(result)) return yield* result
|
|
522
|
+
const events: LLMEvent[] = []
|
|
523
|
+
const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
|
|
524
|
+
events.push(...result.events)
|
|
525
|
+
return [{ ...state, lifecycle, tools: result.tools }, events] as const
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
if (event.contentBlockStop) {
|
|
529
|
+
const index = event.contentBlockStop.contentBlockIndex
|
|
530
|
+
const result = yield* ToolStream.finish(ADAPTER, state.tools, index)
|
|
531
|
+
const events: LLMEvent[] = []
|
|
532
|
+
const resultEvents = result.events ?? []
|
|
533
|
+
const lifecycle = resultEvents.length
|
|
534
|
+
? Lifecycle.stepStart(state.lifecycle, events)
|
|
535
|
+
: Lifecycle.reasoningEnd(
|
|
536
|
+
Lifecycle.textEnd(state.lifecycle, events, `text-${index}`),
|
|
537
|
+
events,
|
|
538
|
+
`reasoning-${index}`,
|
|
539
|
+
state.reasoningSignatures[index]
|
|
540
|
+
? bedrockMetadata({ signature: state.reasoningSignatures[index] })
|
|
541
|
+
: undefined,
|
|
542
|
+
)
|
|
543
|
+
events.push(...resultEvents)
|
|
544
|
+
return [
|
|
545
|
+
{
|
|
546
|
+
...state,
|
|
547
|
+
hasToolCalls: resultEvents.some(LLMEvent.is.toolCall) ? true : state.hasToolCalls,
|
|
548
|
+
lifecycle,
|
|
549
|
+
tools: result.tools,
|
|
550
|
+
reasoningSignatures: Object.fromEntries(
|
|
551
|
+
Object.entries(state.reasoningSignatures).filter(([key]) => key !== String(index)),
|
|
552
|
+
),
|
|
553
|
+
},
|
|
554
|
+
events,
|
|
555
|
+
] as const
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
if (event.messageStop) {
|
|
559
|
+
return [
|
|
560
|
+
{
|
|
561
|
+
...state,
|
|
562
|
+
pendingFinish: { reason: mapFinishReason(event.messageStop.stopReason), usage: state.pendingFinish?.usage },
|
|
563
|
+
},
|
|
564
|
+
[],
|
|
565
|
+
] as const
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
if (event.metadata) {
|
|
569
|
+
const usage = mapUsage(event.metadata.usage)
|
|
570
|
+
return [{ ...state, pendingFinish: { reason: state.pendingFinish?.reason ?? "stop", usage } }, []] as const
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
if (event.internalServerException || event.modelStreamErrorException || event.serviceUnavailableException) {
|
|
574
|
+
const message =
|
|
575
|
+
event.internalServerException?.message ??
|
|
576
|
+
event.modelStreamErrorException?.message ??
|
|
577
|
+
event.serviceUnavailableException?.message ??
|
|
578
|
+
"Bedrock Converse stream error"
|
|
579
|
+
return [state, [LLMEvent.providerError({ message, retryable: true })]] as const
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
if (event.validationException || event.throttlingException) {
|
|
583
|
+
const message =
|
|
584
|
+
event.validationException?.message ?? event.throttlingException?.message ?? "Bedrock Converse error"
|
|
585
|
+
return [state, [LLMEvent.providerError({ message, retryable: event.throttlingException !== undefined })]] as const
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
return [state, []] as const
|
|
589
|
+
})
|
|
590
|
+
|
|
591
|
+
const framing = BedrockEventStream.framing(ADAPTER)
|
|
592
|
+
|
|
593
|
+
const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> =>
|
|
594
|
+
state.pendingFinish
|
|
595
|
+
? (() => {
|
|
596
|
+
const events: LLMEvent[] = []
|
|
597
|
+
Lifecycle.finish(state.lifecycle, events, {
|
|
598
|
+
reason:
|
|
599
|
+
state.pendingFinish.reason === "stop" && state.hasToolCalls ? "tool-calls" : state.pendingFinish.reason,
|
|
600
|
+
usage: state.pendingFinish.usage,
|
|
601
|
+
})
|
|
602
|
+
return events
|
|
603
|
+
})()
|
|
604
|
+
: []
|
|
605
|
+
|
|
606
|
+
// =============================================================================
|
|
607
|
+
// Protocol And Bedrock Route
|
|
608
|
+
// =============================================================================
|
|
609
|
+
/**
|
|
610
|
+
* The Bedrock Converse protocol — request body construction, body schema, and
|
|
611
|
+
* the streaming-event state machine.
|
|
612
|
+
*/
|
|
613
|
+
export const protocol = Protocol.make({
|
|
614
|
+
id: ADAPTER,
|
|
615
|
+
body: {
|
|
616
|
+
schema: BedrockConverseBody,
|
|
617
|
+
from: fromRequest,
|
|
618
|
+
},
|
|
619
|
+
stream: {
|
|
620
|
+
event: BedrockEvent,
|
|
621
|
+
initial: () => ({
|
|
622
|
+
tools: ToolStream.empty<number>(),
|
|
623
|
+
pendingFinish: undefined,
|
|
624
|
+
hasToolCalls: false,
|
|
625
|
+
lifecycle: Lifecycle.initial(),
|
|
626
|
+
reasoningSignatures: {},
|
|
627
|
+
}),
|
|
628
|
+
step,
|
|
629
|
+
onHalt,
|
|
630
|
+
},
|
|
631
|
+
})
|
|
632
|
+
|
|
633
|
+
export const route = Route.make({
|
|
634
|
+
id: ADAPTER,
|
|
635
|
+
provider: "bedrock",
|
|
636
|
+
protocol,
|
|
637
|
+
// Bedrock's URL embeds the region in the route endpoint host and the
|
|
638
|
+
// validated modelId in the path. We read the validated body so the URL
|
|
639
|
+
// matches the body that gets signed.
|
|
640
|
+
endpoint: Endpoint.path<BedrockConverseBody>(
|
|
641
|
+
({ body }) => `/model/${encodeURIComponent(body.modelId)}/converse-stream`,
|
|
642
|
+
),
|
|
643
|
+
auth: BedrockAuth.auth,
|
|
644
|
+
framing,
|
|
645
|
+
})
|
|
646
|
+
|
|
647
|
+
export const sigV4Auth = BedrockAuth.sigV4
|
|
648
|
+
|
|
649
|
+
export * as BedrockConverse from "./bedrock-converse"
|