@codilore/llm 1.15.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/AGENTS.md +321 -0
  2. package/README.md +131 -0
  3. package/example/call-sites.md +591 -0
  4. package/example/tutorial.ts +255 -0
  5. package/package.json +50 -0
  6. package/script/recording-cost-report.ts +250 -0
  7. package/script/setup-recording-env.ts +542 -0
  8. package/src/cache-policy.ts +111 -0
  9. package/src/index.ts +32 -0
  10. package/src/llm.ts +186 -0
  11. package/src/protocols/anthropic-messages.ts +841 -0
  12. package/src/protocols/bedrock-converse.ts +649 -0
  13. package/src/protocols/bedrock-event-stream.ts +87 -0
  14. package/src/protocols/gemini.ts +465 -0
  15. package/src/protocols/index.ts +6 -0
  16. package/src/protocols/openai-chat.ts +431 -0
  17. package/src/protocols/openai-compatible-chat.ts +24 -0
  18. package/src/protocols/openai-responses.ts +987 -0
  19. package/src/protocols/shared.ts +283 -0
  20. package/src/protocols/utils/bedrock-auth.ts +70 -0
  21. package/src/protocols/utils/bedrock-cache.ts +37 -0
  22. package/src/protocols/utils/bedrock-media.ts +80 -0
  23. package/src/protocols/utils/cache.ts +16 -0
  24. package/src/protocols/utils/gemini-tool-schema.ts +101 -0
  25. package/src/protocols/utils/lifecycle.ts +102 -0
  26. package/src/protocols/utils/openai-options.ts +84 -0
  27. package/src/protocols/utils/tool-stream.ts +218 -0
  28. package/src/provider.ts +37 -0
  29. package/src/providers/amazon-bedrock.ts +43 -0
  30. package/src/providers/anthropic.ts +35 -0
  31. package/src/providers/azure.ts +110 -0
  32. package/src/providers/cloudflare.ts +127 -0
  33. package/src/providers/github-copilot.ts +66 -0
  34. package/src/providers/google.ts +35 -0
  35. package/src/providers/index.ts +11 -0
  36. package/src/providers/openai-compatible-profile.ts +20 -0
  37. package/src/providers/openai-compatible.ts +65 -0
  38. package/src/providers/openai-options.ts +81 -0
  39. package/src/providers/openai.ts +63 -0
  40. package/src/providers/openrouter.ts +98 -0
  41. package/src/providers/xai.ts +56 -0
  42. package/src/route/auth-options.ts +57 -0
  43. package/src/route/auth.ts +156 -0
  44. package/src/route/client.ts +434 -0
  45. package/src/route/endpoint.ts +53 -0
  46. package/src/route/executor.ts +374 -0
  47. package/src/route/framing.ts +27 -0
  48. package/src/route/index.ts +25 -0
  49. package/src/route/protocol.ts +84 -0
  50. package/src/route/transport/http.ts +108 -0
  51. package/src/route/transport/index.ts +33 -0
  52. package/src/route/transport/websocket.ts +280 -0
  53. package/src/schema/errors.ts +203 -0
  54. package/src/schema/events.ts +370 -0
  55. package/src/schema/ids.ts +43 -0
  56. package/src/schema/index.ts +5 -0
  57. package/src/schema/messages.ts +404 -0
  58. package/src/schema/options.ts +221 -0
  59. package/src/tool-runtime.ts +78 -0
  60. package/src/tool.ts +241 -0
  61. package/src/utils/record.ts +3 -0
  62. package/sst-env.d.ts +10 -0
  63. package/test/adapter.test.ts +164 -0
  64. package/test/auth-options.types.ts +168 -0
  65. package/test/auth.test.ts +103 -0
  66. package/test/cache-policy.test.ts +262 -0
  67. package/test/continuation-scenarios.ts +104 -0
  68. package/test/endpoint.test.ts +58 -0
  69. package/test/executor.test.ts +418 -0
  70. package/test/exports.test.ts +62 -0
  71. package/test/fixtures/media/restroom.png +0 -0
  72. package/test/fixtures/recordings/anthropic-messages/accepts-malformed-assistant-tool-order-with-default-patch.json +29 -0
  73. package/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-image-tool-result.json +43 -0
  74. package/test/fixtures/recordings/anthropic-messages/claude-opus-4-7-drives-a-tool-loop.json +56 -0
  75. package/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order-without-patch.json +29 -0
  76. package/test/fixtures/recordings/anthropic-messages/streams-text.json +29 -0
  77. package/test/fixtures/recordings/anthropic-messages/streams-tool-call.json +29 -0
  78. package/test/fixtures/recordings/anthropic-messages-cache/writes-then-reads-cache-control-on-identical-second-call.json +48 -0
  79. package/test/fixtures/recordings/bedrock-converse/drives-a-tool-loop.json +55 -0
  80. package/test/fixtures/recordings/bedrock-converse/streams-a-tool-call.json +29 -0
  81. package/test/fixtures/recordings/bedrock-converse/streams-text.json +29 -0
  82. package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
  83. package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-llama-3-1-8b-text.json +32 -0
  84. package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
  85. package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-llama-3-1-8b-text.json +32 -0
  86. package/test/fixtures/recordings/gemini/gemini-2-5-flash-image.json +32 -0
  87. package/test/fixtures/recordings/gemini/streams-text.json +28 -0
  88. package/test/fixtures/recordings/gemini/streams-tool-call.json +28 -0
  89. package/test/fixtures/recordings/gemini-cache/reports-cachedcontenttokencount-on-identical-second-call.json +46 -0
  90. package/test/fixtures/recordings/openai-chat/continues-after-tool-result.json +28 -0
  91. package/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json +46 -0
  92. package/test/fixtures/recordings/openai-chat/streams-text.json +28 -0
  93. package/test/fixtures/recordings/openai-chat/streams-tool-call.json +28 -0
  94. package/test/fixtures/recordings/openai-compatible-chat/deepseek-streams-text.json +28 -0
  95. package/test/fixtures/recordings/openai-compatible-chat/groq-llama-3-3-70b-drives-a-tool-loop.json +53 -0
  96. package/test/fixtures/recordings/openai-compatible-chat/groq-streams-text.json +28 -0
  97. package/test/fixtures/recordings/openai-compatible-chat/groq-streams-tool-call.json +28 -0
  98. package/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json +54 -0
  99. package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json +53 -0
  100. package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json +54 -0
  101. package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json +28 -0
  102. package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json +28 -0
  103. package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-text.json +28 -0
  104. package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-tool-call.json +28 -0
  105. package/test/fixtures/recordings/openai-responses/gpt-5-5-drives-a-tool-loop.json +54 -0
  106. package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-text.json +28 -0
  107. package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-tool-call.json +28 -0
  108. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-image-tool-result.json +42 -0
  109. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning-continuation.json +58 -0
  110. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json +32 -0
  111. package/test/fixtures/recordings/openai-responses-cache/reports-cached-tokens-on-identical-second-call.json +46 -0
  112. package/test/generate-object.test.ts +184 -0
  113. package/test/lib/effect.ts +50 -0
  114. package/test/lib/http.ts +98 -0
  115. package/test/lib/openai-chunks.ts +27 -0
  116. package/test/lib/sse.ts +17 -0
  117. package/test/lib/tool-runtime.ts +146 -0
  118. package/test/llm.test.ts +167 -0
  119. package/test/provider/anthropic-messages-cache.recorded.test.ts +54 -0
  120. package/test/provider/anthropic-messages.recorded.test.ts +46 -0
  121. package/test/provider/anthropic-messages.test.ts +829 -0
  122. package/test/provider/bedrock-converse-cache.recorded.test.ts +54 -0
  123. package/test/provider/bedrock-converse.test.ts +707 -0
  124. package/test/provider/cloudflare.test.ts +230 -0
  125. package/test/provider/gemini-cache.recorded.test.ts +48 -0
  126. package/test/provider/gemini.test.ts +476 -0
  127. package/test/provider/golden.recorded.test.ts +219 -0
  128. package/test/provider/openai-chat.test.ts +446 -0
  129. package/test/provider/openai-compatible-chat.test.ts +238 -0
  130. package/test/provider/openai-responses-cache.recorded.test.ts +46 -0
  131. package/test/provider/openai-responses.test.ts +1322 -0
  132. package/test/provider/openrouter.test.ts +56 -0
  133. package/test/provider.types.ts +41 -0
  134. package/test/recorded-golden.ts +97 -0
  135. package/test/recorded-runner.ts +100 -0
  136. package/test/recorded-scenarios.ts +531 -0
  137. package/test/recorded-test.ts +74 -0
  138. package/test/recorded-utils.ts +56 -0
  139. package/test/recorded-websocket.ts +26 -0
  140. package/test/route.test.ts +43 -0
  141. package/test/schema.test.ts +97 -0
  142. package/test/tool-runtime.test.ts +802 -0
  143. package/test/tool-stream.test.ts +99 -0
  144. package/test/tool.types.ts +40 -0
  145. package/tsconfig.json +15 -0
@@ -0,0 +1,78 @@
1
+ import { Effect } from "effect"
2
+ import {
3
+ LLMEvent,
4
+ type ToolCallPart,
5
+ ToolFailure,
6
+ ToolOutput,
7
+ ToolResultValue,
8
+ type ToolOutput as ToolOutputType,
9
+ type ToolResultValue as ToolResultValueType,
10
+ } from "./schema"
11
+ import { type AnyTool, type Tools } from "./tool"
12
+
13
+ export interface ToolSettlement {
14
+ readonly result: ToolResultValueType
15
+ readonly output?: ToolOutputType
16
+ }
17
+
18
+ export interface DispatchResult extends ToolSettlement {
19
+ readonly events: ReadonlyArray<LLMEvent>
20
+ }
21
+
22
+ /** Execute one canonical tool call without owning provider IO or continuation. */
23
+ export const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect<DispatchResult> => {
24
+ const tool = tools[call.name]
25
+ if (!tool) return Effect.succeed(result(call, { type: "error", value: `Unknown tool: ${call.name}` }))
26
+ if (!tool.execute)
27
+ return Effect.succeed(result(call, { type: "error", value: `Tool has no execute handler: ${call.name}` }))
28
+
29
+ return decodeAndExecute(tool, call).pipe(
30
+ Effect.map((value) => result(call, value)),
31
+ Effect.catchTag("LLM.ToolFailure", (failure) =>
32
+ Effect.succeed(result(call, { type: "error", value: failure.message }, failure.error)),
33
+ ),
34
+ )
35
+ }
36
+
37
+ const decodeAndExecute = (tool: AnyTool, call: ToolCallPart): Effect.Effect<ToolSettlement, ToolFailure> =>
38
+ tool._decode(call.input).pipe(
39
+ Effect.mapError((error) => new ToolFailure({ message: `Invalid tool input: ${error.message}` })),
40
+ Effect.flatMap((decoded) =>
41
+ tool.execute!(decoded, { id: call.id, name: call.name }).pipe(
42
+ Effect.flatMap((value) =>
43
+ tool._encode(value).pipe(
44
+ Effect.mapError(
45
+ (error) =>
46
+ new ToolFailure({
47
+ message: `Tool returned an invalid value for its success schema: ${error.message}`,
48
+ }),
49
+ ),
50
+ ),
51
+ ),
52
+ Effect.map((encoded) => {
53
+ if (tool._legacyResult && ToolResultValue.is(encoded))
54
+ return { result: encoded, output: ToolOutput.fromResultValue(encoded) }
55
+ const output = tool._project(decoded, call.id, encoded)
56
+ const result = ToolOutput.toResultValue(output)
57
+ return result.type === "error" ? { result } : { result, output }
58
+ }),
59
+ ),
60
+ ),
61
+ )
62
+
63
+ const result = (call: ToolCallPart, value: ToolResultValueType | ToolSettlement, error?: unknown): DispatchResult => {
64
+ const settlement = ToolResultValue.is(value) ? { result: value } : value
65
+ return {
66
+ result: settlement.result,
67
+ output: settlement.output,
68
+ events:
69
+ settlement.result.type === "error"
70
+ ? [
71
+ LLMEvent.toolError({ id: call.id, name: call.name, message: String(settlement.result.value), error }),
72
+ LLMEvent.toolResult({ id: call.id, name: call.name, result: settlement.result }),
73
+ ]
74
+ : [LLMEvent.toolResult({ id: call.id, name: call.name, result: settlement.result, output: settlement.output })],
75
+ }
76
+ }
77
+
78
+ export const ToolRuntime = { dispatch } as const
package/src/tool.ts ADDED
@@ -0,0 +1,241 @@
1
+ import { Effect, JsonSchema, Schema } from "effect"
2
+ import type {
3
+ ToolCallPart,
4
+ ToolContent,
5
+ ToolDefinition as ToolDefinitionClass,
6
+ ToolOutput as ToolOutputType,
7
+ } from "./schema"
8
+ import { ToolDefinition, ToolFailure, ToolOutput, toolText } from "./schema"
9
+
10
+ /**
11
+ * Schema constraint for tool parameters / success values: no decoding or
12
+ * encoding services are allowed. Tools should be self-contained — anything
13
+ * beyond pure data conversion belongs in the handler closure.
14
+ */
15
+ export type ToolSchema<T> = Schema.Codec<T, any, never, never>
16
+ export interface ToolExecuteContext {
17
+ readonly id: ToolCallPart["id"]
18
+ readonly name: ToolCallPart["name"]
19
+ }
20
+
21
+ export type ToolExecute<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = (
22
+ params: Schema.Schema.Type<Parameters>,
23
+ context?: ToolExecuteContext,
24
+ ) => Effect.Effect<Schema.Schema.Type<Success>, ToolFailure>
25
+
26
+ export interface ToolModelOutputInput<Parameters, Output> {
27
+ readonly callID: ToolCallPart["id"]
28
+ readonly parameters: Parameters
29
+ readonly output: Output
30
+ }
31
+
32
+ export type ToolToModelOutput<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = (
33
+ input: ToolModelOutputInput<Schema.Schema.Type<Parameters>, Success["Encoded"]>,
34
+ ) => ReadonlyArray<ToolContent>
35
+
36
+ /**
37
+ * A type-safe LLM tool. Each tool bundles its own description, parameter
38
+ * Schema and success Schema. The execute handler is optional: omit it when you
39
+ * only want to expose a tool schema to the model and handle tool calls outside
40
+ * this package.
41
+ *
42
+ * Errors must be expressed as `ToolFailure`. Unmapped errors and defects fail
43
+ * the stream.
44
+ *
45
+ * Internally each tool also carries memoized codecs and a precomputed
46
+ * `ToolDefinition` so callers do not rebuild them per invocation.
47
+ */
48
+ export interface Tool<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> {
49
+ readonly description: string
50
+ readonly parameters: Parameters
51
+ readonly success: Success
52
+ readonly execute?: ToolExecute<Parameters, Success>
53
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
54
+ /** @internal */
55
+ readonly _decode: (input: unknown) => Effect.Effect<Schema.Schema.Type<Parameters>, Schema.SchemaError>
56
+ /** @internal */
57
+ readonly _encode: (value: Schema.Schema.Type<Success>) => Effect.Effect<unknown, Schema.SchemaError>
58
+ /** @internal */
59
+ readonly _project: (
60
+ parameters: Schema.Schema.Type<Parameters>,
61
+ callID: ToolCallPart["id"],
62
+ output: unknown,
63
+ ) => ToolOutputType
64
+ /** @internal */
65
+ readonly _legacyResult: boolean
66
+ /** @internal */
67
+ readonly _definition: ToolDefinitionClass
68
+ }
69
+
70
+ export type AnyTool = Tool<any, any>
71
+
72
+ export type ExecutableTool<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = Tool<
73
+ Parameters,
74
+ Success
75
+ > & {
76
+ readonly execute: ToolExecute<Parameters, Success>
77
+ }
78
+
79
+ export type AnyExecutableTool = ExecutableTool<any, any>
80
+
81
+ export type ExecutableTools = Record<string, AnyExecutableTool>
82
+
83
+ type TypedToolConfig = {
84
+ readonly description: string
85
+ readonly parameters: ToolSchema<any>
86
+ readonly success: ToolSchema<any>
87
+ readonly execute?: ToolExecute<ToolSchema<any>, ToolSchema<any>>
88
+ readonly toModelOutput?: ToolToModelOutput<ToolSchema<any>, ToolSchema<any>>
89
+ }
90
+
91
+ type DynamicToolConfig = {
92
+ readonly description: string
93
+ readonly jsonSchema: JsonSchema.JsonSchema
94
+ readonly outputSchema?: JsonSchema.JsonSchema
95
+ readonly execute?: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
96
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
97
+ }
98
+
99
+ /**
100
+ * Constructs a tool. Two input modes:
101
+ *
102
+ * 1. **Typed** — pass Effect `parameters` and `success` Schemas; inputs and
103
+ * outputs are statically typed and decoded/encoded automatically.
104
+ *
105
+ * ```ts
106
+ * Tool.make({
107
+ * description: "Get current weather",
108
+ * parameters: Schema.Struct({ city: Schema.String }),
109
+ * success: Schema.Struct({ temperature: Schema.Number }),
110
+ * execute: ({ city }) => Effect.succeed({ temperature: 22 }),
111
+ * })
112
+ * ```
113
+ *
114
+ * 2. **Dynamic** — pass raw JSON Schema as `jsonSchema`. Use this when the
115
+ * schema comes from an external source (MCP server, plugin manifest,
116
+ * dynamic config) and is not known at compile time. Inputs are typed as
117
+ * `unknown`; the handler is responsible for any validation it needs.
118
+ *
119
+ * ```ts
120
+ * Tool.make({
121
+ * description: "Look something up",
122
+ * jsonSchema: { type: "object", properties: { ... } },
123
+ * execute: (params) => Effect.succeed(...),
124
+ * })
125
+ * ```
126
+ *
127
+ * In both modes the produced tool flows through `toDefinitions(...)`
128
+ * identically.
129
+ */
130
+ export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
131
+ readonly description: string
132
+ readonly parameters: Parameters
133
+ readonly success: Success
134
+ readonly execute: ToolExecute<Parameters, Success>
135
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
136
+ }): ExecutableTool<Parameters, Success>
137
+ export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
138
+ readonly description: string
139
+ readonly parameters: Parameters
140
+ readonly success: Success
141
+ readonly execute?: undefined
142
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
143
+ }): Tool<Parameters, Success>
144
+ export function make(config: {
145
+ readonly description: string
146
+ readonly jsonSchema: JsonSchema.JsonSchema
147
+ readonly outputSchema?: JsonSchema.JsonSchema
148
+ readonly execute: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
149
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
150
+ }): AnyExecutableTool
151
+ export function make(config: {
152
+ readonly description: string
153
+ readonly jsonSchema: JsonSchema.JsonSchema
154
+ readonly outputSchema?: JsonSchema.JsonSchema
155
+ readonly execute?: undefined
156
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
157
+ }): AnyTool
158
+ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
159
+ if ("jsonSchema" in config) {
160
+ return {
161
+ description: config.description,
162
+ parameters: Schema.Unknown as ToolSchema<unknown>,
163
+ success: Schema.Unknown as ToolSchema<unknown>,
164
+ execute: config.execute,
165
+ toModelOutput: config.toModelOutput,
166
+ _decode: Effect.succeed,
167
+ _encode: Effect.succeed,
168
+ _project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
169
+ _legacyResult: config.toModelOutput === undefined,
170
+ _definition: new ToolDefinition({
171
+ name: "",
172
+ description: config.description,
173
+ inputSchema: config.jsonSchema,
174
+ outputSchema: config.outputSchema,
175
+ }),
176
+ }
177
+ }
178
+ return {
179
+ description: config.description,
180
+ parameters: config.parameters,
181
+ success: config.success,
182
+ execute: config.execute,
183
+ toModelOutput: config.toModelOutput,
184
+ _decode: Schema.decodeUnknownEffect(config.parameters),
185
+ _encode: Schema.encodeEffect(config.success),
186
+ _project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
187
+ _legacyResult: false,
188
+ _definition: new ToolDefinition({
189
+ name: "",
190
+ description: config.description,
191
+ inputSchema: toJsonSchema(config.parameters),
192
+ outputSchema: toJsonSchema(config.success),
193
+ }),
194
+ }
195
+ }
196
+
197
+ /**
198
+ * A record of named tools. The record key becomes the tool name on the wire.
199
+ */
200
+ export type Tools = Record<string, AnyTool>
201
+
202
+ /**
203
+ * Convert a tools record into the `ToolDefinition[]` shape that
204
+ * `LLMRequest.tools` expects.
205
+ *
206
+ * Tool names come from the record keys, so the per-tool cached
207
+ * `_definition` is rebuilt with the correct name here. The JSON Schema body
208
+ * is reused.
209
+ */
210
+ export const toDefinitions = (tools: Tools): ReadonlyArray<ToolDefinitionClass> =>
211
+ Object.entries(tools).map(
212
+ ([name, item]) =>
213
+ new ToolDefinition({
214
+ name,
215
+ description: item._definition.description,
216
+ inputSchema: item._definition.inputSchema,
217
+ outputSchema: item._definition.outputSchema,
218
+ }),
219
+ )
220
+
221
+ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => {
222
+ const document = Schema.toJsonSchemaDocument(schema)
223
+ if (Object.keys(document.definitions).length === 0) return document.schema
224
+ return { ...document.schema, $defs: document.definitions }
225
+ }
226
+
227
+ const project = (
228
+ toModelOutput: ((input: ToolModelOutputInput<any, any>) => ReadonlyArray<ToolContent>) | undefined,
229
+ parameters: unknown,
230
+ callID: ToolCallPart["id"],
231
+ output: unknown,
232
+ ): ToolOutputType =>
233
+ ToolOutput.make(
234
+ output,
235
+ toModelOutput?.({ callID, parameters, output }) ??
236
+ (typeof output === "string" ? [toolText({ type: "text", text: output })] : []),
237
+ )
238
+
239
+ export { ToolFailure }
240
+
241
+ export * as Tool from "./tool"
@@ -0,0 +1,3 @@
1
+ /** Plain-record narrowing. Excludes arrays so JSON object checks don't accept tuples as key/value bags. */
2
+ export const isRecord = (value: unknown): value is Record<string, unknown> =>
3
+ typeof value === "object" && value !== null && !Array.isArray(value)
package/sst-env.d.ts ADDED
@@ -0,0 +1,10 @@
1
+ /* This file is auto-generated by SST. Do not edit. */
2
+ /* tslint:disable */
3
+ /* eslint-disable */
4
+ /* deno-fmt-ignore-file */
5
+ /* biome-ignore-all lint: auto-generated */
6
+
7
+ /// <reference path="../../sst-env.d.ts" />
8
+
9
+ import "sst"
10
+ export {}
@@ -0,0 +1,164 @@
1
+ import { describe, expect } from "bun:test"
2
+ import { Effect, Schema, Stream } from "effect"
3
+ import { LLM } from "../src"
4
+ import { Route, Endpoint, LLMClient, Protocol, type FramingDef } from "../src/route"
5
+ import { Model } from "../src/schema"
6
+ import { testEffect } from "./lib/effect"
7
+ import { dynamicResponse } from "./lib/http"
8
+
9
+ const updateModel = (model: Model, patch: Partial<Model.Input>) => Model.update(model, patch)
10
+
11
+ const Json = Schema.fromJsonString(Schema.Unknown)
12
+ const encodeJson = Schema.encodeSync(Json)
13
+
14
+ type FakeBody = {
15
+ readonly body: string
16
+ }
17
+
18
+ const FakeEvent = Schema.Union([
19
+ Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
20
+ Schema.Struct({ type: Schema.Literal("finish"), reason: Schema.Literal("stop") }),
21
+ ])
22
+ type FakeEvent = Schema.Schema.Type<typeof FakeEvent>
23
+ const decodeFakeEvents = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(FakeEvent)))
24
+
25
+ const fakeFraming: FramingDef<FakeEvent> = {
26
+ id: "fake-json-array",
27
+ frame: (bytes) =>
28
+ Stream.fromEffect(
29
+ bytes.pipe(
30
+ Stream.decodeText(),
31
+ Stream.runFold(
32
+ () => "",
33
+ (text, event) => text + event,
34
+ ),
35
+ Effect.flatMap(decodeFakeEvents),
36
+ Effect.orDie,
37
+ ),
38
+ ).pipe(Stream.flatMap(Stream.fromIterable)),
39
+ }
40
+
41
+ const raiseEvent = (event: FakeEvent): import("../src/schema").LLMEvent =>
42
+ event.type === "finish"
43
+ ? { type: "finish", reason: event.reason }
44
+ : { type: "text-delta", id: "text-0", text: event.text }
45
+
46
+ const fakeProtocol = Protocol.make<FakeBody, FakeEvent, FakeEvent, void>({
47
+ id: "fake",
48
+ body: {
49
+ schema: Schema.Struct({
50
+ body: Schema.String,
51
+ }),
52
+ from: (request) =>
53
+ Effect.succeed({
54
+ body: [
55
+ ...request.messages
56
+ .flatMap((message) => message.content)
57
+ .filter((part) => part.type === "text")
58
+ .map((part) => part.text),
59
+ ...request.tools.map((tool) => `tool:${tool.name}:${tool.description}`),
60
+ ].join("\n"),
61
+ }),
62
+ },
63
+ stream: {
64
+ event: FakeEvent,
65
+ initial: () => undefined,
66
+ step: (state, event) => Effect.succeed([state, [raiseEvent(event)]] as const),
67
+ },
68
+ })
69
+
70
+ const fake = Route.make({
71
+ id: "fake",
72
+ protocol: fakeProtocol,
73
+ endpoint: Endpoint.path("/chat"),
74
+ framing: fakeFraming,
75
+ })
76
+ const configuredFake = fake.with({ endpoint: { baseURL: "https://fake.local" } })
77
+
78
+ const gemini = Route.make({
79
+ id: "gemini-fake",
80
+ protocol: fakeProtocol,
81
+ endpoint: Endpoint.path("/chat"),
82
+ framing: fakeFraming,
83
+ })
84
+ const configuredGemini = gemini.with({ endpoint: { baseURL: "https://fake.local" } })
85
+
86
+ const request = LLM.request({
87
+ id: "req_1",
88
+ model: Model.make({
89
+ id: "fake-model",
90
+ provider: "fake-provider",
91
+ route: configuredFake,
92
+ }),
93
+ prompt: "hello",
94
+ })
95
+
96
+ const echoLayer = dynamicResponse(({ text, respond }) =>
97
+ Effect.succeed(
98
+ respond(
99
+ encodeJson([
100
+ { type: "text", text: `echo:${text}` },
101
+ { type: "finish", reason: "stop" },
102
+ ]),
103
+ ),
104
+ ),
105
+ )
106
+
107
+ const it = testEffect(echoLayer)
108
+
109
+ describe("llm route", () => {
110
+ it.effect("stream and generate use the route pipeline", () =>
111
+ Effect.gen(function* () {
112
+ const llm = yield* LLMClient.Service
113
+ const events = Array.from(yield* llm.stream(request).pipe(Stream.runCollect))
114
+ const response = yield* llm.generate(request)
115
+
116
+ expect(events.map((event) => event.type)).toEqual(["text-delta", "finish"])
117
+ expect(response.events.map((event) => event.type)).toEqual(["text-delta", "finish"])
118
+ }),
119
+ )
120
+
121
+ it.effect("selects routes by model route value", () =>
122
+ Effect.gen(function* () {
123
+ const llm = yield* LLMClient.Service
124
+ const prepared = yield* llm.prepare(
125
+ LLM.updateRequest(request, { model: updateModel(request.model, { route: configuredGemini }) }),
126
+ )
127
+
128
+ expect(prepared.route).toBe("gemini-fake")
129
+ }),
130
+ )
131
+
132
+ it.effect("builds models from configured routes", () =>
133
+ Effect.gen(function* () {
134
+ const configured = fake.with({ provider: "fake-provider", endpoint: { baseURL: "https://fake.local" } })
135
+
136
+ expect(configured.model({ id: "fake-model" })).toMatchObject({
137
+ provider: "fake-provider",
138
+ })
139
+ }),
140
+ )
141
+
142
+ it.effect("does not register duplicate route ids globally", () =>
143
+ Effect.gen(function* () {
144
+ const duplicate = Route.make({
145
+ id: "fake",
146
+ protocol: Protocol.make({
147
+ ...fakeProtocol,
148
+ body: {
149
+ ...fakeProtocol.body,
150
+ from: () => Effect.succeed({ body: "late-default" }),
151
+ },
152
+ }),
153
+ endpoint: Endpoint.path("/chat", { baseURL: "https://fake.local" }),
154
+ framing: fakeFraming,
155
+ })
156
+
157
+ const prepared = yield* (yield* LLMClient.Service).prepare(
158
+ LLM.updateRequest(request, { model: updateModel(request.model, { route: duplicate }) }),
159
+ )
160
+
161
+ expect(prepared.body).toEqual({ body: "late-default" })
162
+ }),
163
+ )
164
+ })
@@ -0,0 +1,168 @@
1
+ import { Config } from "effect"
2
+ import type { Auth } from "../src/route/auth"
3
+ import type { ModelFactory } from "../src/route/auth-options"
4
+ import { Auth as RuntimeAuth } from "../src/route/auth"
5
+ import * as OpenAIChat from "../src/protocols/openai-chat"
6
+ import * as AmazonBedrock from "../src/providers/amazon-bedrock"
7
+ import * as Anthropic from "../src/providers/anthropic"
8
+ import * as Azure from "../src/providers/azure"
9
+ import * as Cloudflare from "../src/providers/cloudflare"
10
+ import * as GitHubCopilot from "../src/providers/github-copilot"
11
+ import * as Google from "../src/providers/google"
12
+ import * as OpenAI from "../src/providers/openai"
13
+ import * as OpenAICompatible from "../src/providers/openai-compatible"
14
+ import * as OpenRouter from "../src/providers/openrouter"
15
+ import * as XAI from "../src/providers/xai"
16
+
17
+ type BaseOptions = {
18
+ readonly baseURL?: string
19
+ readonly headers?: Record<string, string>
20
+ }
21
+
22
+ type Model = {
23
+ readonly id: string
24
+ }
25
+
26
+ declare const auth: Auth
27
+ declare const optionalAuthModel: ModelFactory<BaseOptions, "optional", Model>
28
+ declare const requiredAuthModel: ModelFactory<BaseOptions, "required", Model>
29
+ const configApiKey = Config.redacted("OPENAI_API_KEY")
30
+
31
+ OpenAIChat.route.model({ id: "gpt-4.1-mini" })
32
+
33
+ // @ts-expect-error route model selection does not configure endpoints.
34
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", baseURL: "https://gateway.example.com/v1" })
35
+
36
+ // @ts-expect-error route model selection does not configure query params.
37
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", queryParams: { debug: "1" } })
38
+
39
+ // @ts-expect-error route model selection does not configure auth.
40
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", auth })
41
+
42
+ // @ts-expect-error route model selection does not configure api keys.
43
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", apiKey: "sk-test" })
44
+
45
+ optionalAuthModel("gpt-4.1-mini")
46
+ optionalAuthModel("gpt-4.1-mini", {})
47
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test" })
48
+ optionalAuthModel("gpt-4.1-mini", { apiKey: configApiKey })
49
+ optionalAuthModel("gpt-4.1-mini", { auth })
50
+ optionalAuthModel("gpt-4.1-mini", { auth, baseURL: "https://gateway.example.com/v1" })
51
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test", headers: { "x-source": "test" } })
52
+
53
+ // @ts-expect-error auth is an override, so apiKey cannot be supplied with it.
54
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test", auth })
55
+
56
+ requiredAuthModel("custom-model", { apiKey: "key" })
57
+ requiredAuthModel("custom-model", { apiKey: configApiKey })
58
+ requiredAuthModel("custom-model", { auth })
59
+ requiredAuthModel("custom-model", { auth, headers: { "x-tenant-id": "tenant" } })
60
+
61
+ // @ts-expect-error providers without config fallback need apiKey or auth.
62
+ requiredAuthModel("custom-model")
63
+
64
+ // @ts-expect-error providers without config fallback need apiKey or auth.
65
+ requiredAuthModel("custom-model", {})
66
+
67
+ // @ts-expect-error auth is an override, so apiKey cannot be supplied with it.
68
+ requiredAuthModel("custom-model", { apiKey: "key", auth })
69
+
70
+ OpenAI.responses("gpt-4.1-mini")
71
+ OpenAI.configure({}).responses("gpt-4.1-mini")
72
+ OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini")
73
+ OpenAI.configure({ apiKey: configApiKey }).responses("gpt-4.1-mini")
74
+ OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).responses("gpt-4.1-mini")
75
+ OpenAI.configure({
76
+ auth: RuntimeAuth.headers({ authorization: "Bearer gateway" }),
77
+ baseURL: "https://gateway.example.com/v1",
78
+ }).responses("gpt-4.1-mini")
79
+ OpenAI.configure({
80
+ generation: { maxTokens: 100 },
81
+ providerOptions: { openai: { store: false } },
82
+ }).responses("gpt-4.1-mini")
83
+
84
+ // @ts-expect-error OpenAI model selectors only accept model ids.
85
+ OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini", {})
86
+
87
+ // @ts-expect-error apiKey only accepts string, Redacted<string>, or Config<string | Redacted<string>>.
88
+ OpenAI.configure({ apiKey: 123 })
89
+
90
+ // @ts-expect-error provider helpers reject unknown top-level options.
91
+ OpenAI.configure({ bogus: true })
92
+
93
+ // @ts-expect-error common generation options remain typed.
94
+ OpenAI.configure({ generation: { maxTokens: "many" } })
95
+
96
+ // @ts-expect-error provider-native options remain typed.
97
+ OpenAI.configure({ providerOptions: { openai: { store: "false" } } })
98
+
99
+ // @ts-expect-error auth is an override, so OpenAI rejects apiKey with auth.
100
+ OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
101
+
102
+ OpenAI.chat("gpt-4.1-mini")
103
+ OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini")
104
+ OpenAI.configure({ apiKey: configApiKey }).chat("gpt-4.1-mini")
105
+ OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).chat("gpt-4.1-mini")
106
+
107
+ // @ts-expect-error OpenAI chat selectors only accept model ids.
108
+ OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini", {})
109
+
110
+ // @ts-expect-error auth is an override, so OpenAI Chat rejects apiKey with auth.
111
+ OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
112
+
113
+ // @ts-expect-error Azure requires at least one of `resourceName` or `baseURL`.
114
+ Azure.configure()
115
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment")
116
+ Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).responses("deployment")
117
+ Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).responses("deployment")
118
+
119
+ // @ts-expect-error Azure model selectors only accept deployment ids.
120
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment", {})
121
+
122
+ // @ts-expect-error auth is an override, so Azure rejects apiKey with auth.
123
+ Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
124
+
125
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment")
126
+ Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).chat("deployment")
127
+ Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).chat("deployment")
128
+
129
+ // @ts-expect-error Azure chat model selectors only accept deployment ids.
130
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment", {})
131
+
132
+ // @ts-expect-error auth is an override, so Azure Chat rejects apiKey with auth.
133
+ Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
134
+
135
+ Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku")
136
+ // @ts-expect-error Anthropic model selectors only accept model ids.
137
+ Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku", {})
138
+
139
+ Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash")
140
+ // @ts-expect-error Google model selectors only accept model ids.
141
+ Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash", {})
142
+
143
+ AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude")
144
+ // @ts-expect-error Bedrock model selectors only accept model ids.
145
+ AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude", {})
146
+
147
+ OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini")
148
+ // @ts-expect-error OpenRouter model selectors only accept model ids.
149
+ OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini", {})
150
+
151
+ XAI.configure({ apiKey: "xai-key" }).responses("grok-4")
152
+ XAI.configure({ apiKey: "xai-key" }).chat("grok-4")
153
+ // @ts-expect-error xAI Responses selectors only accept model ids.
154
+ XAI.configure({ apiKey: "xai-key" }).responses("grok-4", {})
155
+ // @ts-expect-error xAI Chat selectors only accept model ids.
156
+ XAI.configure({ apiKey: "xai-key" }).chat("grok-4", {})
157
+
158
+ OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat")
159
+ // @ts-expect-error OpenAI-compatible family selectors only accept model ids.
160
+ OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat", {})
161
+
162
+ Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama")
163
+ // @ts-expect-error Cloudflare Workers AI model selectors only accept model ids.
164
+ Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama", {})
165
+
166
+ GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1")
167
+ // @ts-expect-error GitHub Copilot model selectors only accept model ids.
168
+ GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1", {})