@neurocode-ai/llm 1.18.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/AGENTS.md +321 -0
  2. package/DESIGN.md +1114 -0
  3. package/README.md +131 -0
  4. package/example/call-sites.md +591 -0
  5. package/example/tutorial.ts +255 -0
  6. package/package.json +51 -0
  7. package/script/recording-cost-report.ts +250 -0
  8. package/script/setup-recording-env.ts +542 -0
  9. package/src/cache-policy.ts +111 -0
  10. package/src/index.ts +33 -0
  11. package/src/llm.ts +186 -0
  12. package/src/protocols/anthropic-messages.ts +855 -0
  13. package/src/protocols/bedrock-converse.ts +674 -0
  14. package/src/protocols/bedrock-event-stream.ts +87 -0
  15. package/src/protocols/gemini.ts +512 -0
  16. package/src/protocols/index.ts +6 -0
  17. package/src/protocols/openai-chat.ts +506 -0
  18. package/src/protocols/openai-compatible-chat.ts +24 -0
  19. package/src/protocols/openai-responses.ts +1022 -0
  20. package/src/protocols/shared.ts +326 -0
  21. package/src/protocols/utils/bedrock-auth.ts +70 -0
  22. package/src/protocols/utils/bedrock-cache.ts +37 -0
  23. package/src/protocols/utils/bedrock-media.ts +90 -0
  24. package/src/protocols/utils/cache.ts +16 -0
  25. package/src/protocols/utils/gemini-tool-schema.ts +99 -0
  26. package/src/protocols/utils/lifecycle.ts +102 -0
  27. package/src/protocols/utils/openai-options.ts +93 -0
  28. package/src/protocols/utils/tool-schema.ts +86 -0
  29. package/src/protocols/utils/tool-stream.ts +218 -0
  30. package/src/provider-error.ts +43 -0
  31. package/src/provider.ts +36 -0
  32. package/src/providers/amazon-bedrock.ts +43 -0
  33. package/src/providers/anthropic.ts +35 -0
  34. package/src/providers/azure.ts +110 -0
  35. package/src/providers/cloudflare.ts +127 -0
  36. package/src/providers/github-copilot.ts +69 -0
  37. package/src/providers/google.ts +35 -0
  38. package/src/providers/index.ts +11 -0
  39. package/src/providers/openai-compatible-profile.ts +20 -0
  40. package/src/providers/openai-compatible.ts +65 -0
  41. package/src/providers/openai-options.ts +83 -0
  42. package/src/providers/openai.ts +63 -0
  43. package/src/providers/openrouter.ts +98 -0
  44. package/src/providers/xai.ts +56 -0
  45. package/src/route/auth-options.ts +57 -0
  46. package/src/route/auth.ts +156 -0
  47. package/src/route/client.ts +436 -0
  48. package/src/route/endpoint.ts +53 -0
  49. package/src/route/executor.ts +385 -0
  50. package/src/route/framing.ts +27 -0
  51. package/src/route/index.ts +25 -0
  52. package/src/route/protocol.ts +84 -0
  53. package/src/route/transport/http.ts +155 -0
  54. package/src/route/transport/index.ts +33 -0
  55. package/src/route/transport/websocket.ts +280 -0
  56. package/src/schema/errors.ts +207 -0
  57. package/src/schema/events.ts +618 -0
  58. package/src/schema/ids.ts +43 -0
  59. package/src/schema/index.ts +5 -0
  60. package/src/schema/messages.ts +312 -0
  61. package/src/schema/options.ts +276 -0
  62. package/src/tool-runtime.ts +78 -0
  63. package/src/tool.ts +253 -0
  64. package/src/utils/record.ts +3 -0
  65. package/sst-env.d.ts +10 -0
  66. package/test/adapter.test.ts +171 -0
  67. package/test/auth-options.types.ts +168 -0
  68. package/test/auth.test.ts +103 -0
  69. package/test/cache-policy.test.ts +262 -0
  70. package/test/continuation-scenarios.ts +104 -0
  71. package/test/endpoint.test.ts +58 -0
  72. package/test/executor.test.ts +458 -0
  73. package/test/exports.test.ts +76 -0
  74. package/test/fixtures/media/restroom.png +0 -0
  75. package/test/fixtures/recordings/anthropic-messages/accepts-malformed-assistant-tool-order-with-default-patch.json +29 -0
  76. package/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-image-tool-result.json +43 -0
  77. package/test/fixtures/recordings/anthropic-messages/claude-opus-4-7-drives-a-tool-loop.json +56 -0
  78. package/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order-without-patch.json +29 -0
  79. package/test/fixtures/recordings/anthropic-messages/streams-text.json +29 -0
  80. package/test/fixtures/recordings/anthropic-messages/streams-tool-call.json +29 -0
  81. package/test/fixtures/recordings/anthropic-messages-cache/writes-then-reads-cache-control-on-identical-second-call.json +48 -0
  82. package/test/fixtures/recordings/bedrock-converse/drives-a-tool-loop.json +55 -0
  83. package/test/fixtures/recordings/bedrock-converse/streams-a-tool-call.json +29 -0
  84. package/test/fixtures/recordings/bedrock-converse/streams-text.json +29 -0
  85. package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
  86. package/test/fixtures/recordings/cloudflare-ai-gateway/cloudflare-ai-gateway-workers-ai-llama-3-1-8b-text.json +32 -0
  87. package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-gpt-oss-20b-tools-tool-call.json +32 -0
  88. package/test/fixtures/recordings/cloudflare-workers-ai/cloudflare-workers-ai-llama-3-1-8b-text.json +32 -0
  89. package/test/fixtures/recordings/gemini/gemini-2-5-flash-image.json +32 -0
  90. package/test/fixtures/recordings/gemini/streams-text.json +28 -0
  91. package/test/fixtures/recordings/gemini/streams-tool-call.json +28 -0
  92. package/test/fixtures/recordings/gemini-cache/reports-cachedcontenttokencount-on-identical-second-call.json +46 -0
  93. package/test/fixtures/recordings/openai-chat/continues-after-tool-result.json +28 -0
  94. package/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json +46 -0
  95. package/test/fixtures/recordings/openai-chat/streams-text.json +28 -0
  96. package/test/fixtures/recordings/openai-chat/streams-tool-call.json +28 -0
  97. package/test/fixtures/recordings/openai-compatible-chat/deepseek-streams-text.json +28 -0
  98. package/test/fixtures/recordings/openai-compatible-chat/groq-llama-3-3-70b-drives-a-tool-loop.json +53 -0
  99. package/test/fixtures/recordings/openai-compatible-chat/groq-streams-text.json +28 -0
  100. package/test/fixtures/recordings/openai-compatible-chat/groq-streams-tool-call.json +28 -0
  101. package/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json +54 -0
  102. package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json +53 -0
  103. package/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json +54 -0
  104. package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json +28 -0
  105. package/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json +28 -0
  106. package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-text.json +28 -0
  107. package/test/fixtures/recordings/openai-compatible-chat/togetherai-streams-tool-call.json +28 -0
  108. package/test/fixtures/recordings/openai-responses/gpt-5-5-drives-a-tool-loop.json +54 -0
  109. package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-text.json +28 -0
  110. package/test/fixtures/recordings/openai-responses/gpt-5-5-streams-tool-call.json +28 -0
  111. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-image-tool-result.json +42 -0
  112. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning-continuation.json +58 -0
  113. package/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json +32 -0
  114. package/test/fixtures/recordings/openai-responses-cache/reports-cached-tokens-on-identical-second-call.json +46 -0
  115. package/test/generate-object.test.ts +184 -0
  116. package/test/lib/effect.ts +50 -0
  117. package/test/lib/http.ts +98 -0
  118. package/test/lib/openai-chunks.ts +27 -0
  119. package/test/lib/sse.ts +17 -0
  120. package/test/lib/tool-runtime.ts +146 -0
  121. package/test/llm.test.ts +199 -0
  122. package/test/prepare.test.ts +178 -0
  123. package/test/provider/anthropic-messages-cache.recorded.test.ts +53 -0
  124. package/test/provider/anthropic-messages.recorded.test.ts +45 -0
  125. package/test/provider/anthropic-messages.test.ts +895 -0
  126. package/test/provider/bedrock-converse-cache.recorded.test.ts +54 -0
  127. package/test/provider/bedrock-converse.test.ts +744 -0
  128. package/test/provider/cloudflare.test.ts +230 -0
  129. package/test/provider/gemini-cache.recorded.test.ts +48 -0
  130. package/test/provider/gemini.test.ts +584 -0
  131. package/test/provider/golden.recorded.test.ts +223 -0
  132. package/test/provider/openai-chat.test.ts +674 -0
  133. package/test/provider/openai-compatible-chat.test.ts +238 -0
  134. package/test/provider/openai-responses-cache.recorded.test.ts +46 -0
  135. package/test/provider/openai-responses.test.ts +1472 -0
  136. package/test/provider/openrouter.test.ts +56 -0
  137. package/test/provider-error.test.ts +30 -0
  138. package/test/provider.types.ts +41 -0
  139. package/test/recorded-golden.ts +97 -0
  140. package/test/recorded-runner.ts +100 -0
  141. package/test/recorded-scenarios.ts +531 -0
  142. package/test/recorded-test.ts +94 -0
  143. package/test/recorded-utils.ts +56 -0
  144. package/test/recorded-websocket.ts +26 -0
  145. package/test/response.test.ts +98 -0
  146. package/test/route.test.ts +43 -0
  147. package/test/schema.test.ts +86 -0
  148. package/test/tool-runtime.test.ts +818 -0
  149. package/test/tool-schema-projection.test.ts +117 -0
  150. package/test/tool-stream.test.ts +99 -0
  151. package/test/tool.types.ts +40 -0
  152. package/tsconfig.json +15 -0
package/src/tool.ts ADDED
@@ -0,0 +1,253 @@
1
+ import { Effect, JsonSchema, Schema } from "effect"
2
+ import type {
3
+ ToolCallPart,
4
+ ToolContent,
5
+ ToolDefinition as ToolDefinitionClass,
6
+ ToolOutput as ToolOutputType,
7
+ } from "./schema"
8
+ import { ToolDefinition, ToolFailure, ToolOutput } from "./schema"
9
+
10
+ /**
11
+ * Schema constraint for tool parameters / success values: no decoding or
12
+ * encoding services are allowed. Tools should be self-contained — anything
13
+ * beyond pure data conversion belongs in the handler closure.
14
+ */
15
+ export type ToolSchema<T> = Schema.Codec<T, any, never, never>
16
+ export interface ToolExecuteContext {
17
+ readonly id: ToolCallPart["id"]
18
+ readonly name: ToolCallPart["name"]
19
+ }
20
+
21
+ export type ToolExecute<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = (
22
+ params: Schema.Schema.Type<Parameters>,
23
+ context?: ToolExecuteContext,
24
+ ) => Effect.Effect<Schema.Schema.Type<Success>, ToolFailure>
25
+
26
+ export interface ToolModelOutputInput<Parameters, Output> {
27
+ readonly callID: ToolCallPart["id"]
28
+ readonly parameters: Parameters
29
+ readonly output: Output
30
+ }
31
+
32
+ export type ToolToModelOutput<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = (
33
+ input: ToolModelOutputInput<Schema.Schema.Type<Parameters>, Success["Encoded"]>,
34
+ ) => ReadonlyArray<ToolContent>
35
+
36
+ /**
37
+ * A type-safe LLM tool. Each tool bundles its own description, parameter
38
+ * Schema and success Schema. The execute handler is optional: omit it when you
39
+ * only want to expose a tool schema to the model and handle tool calls outside
40
+ * this package.
41
+ *
42
+ * Errors must be expressed as `ToolFailure`. Unmapped errors and defects fail
43
+ * the stream.
44
+ *
45
+ * Internally each tool also carries memoized codecs and a precomputed
46
+ * `ToolDefinition` so callers do not rebuild them per invocation.
47
+ */
48
+ export interface Tool<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> {
49
+ readonly description: string
50
+ readonly parameters: Parameters
51
+ readonly success: Success
52
+ readonly execute?: ToolExecute<Parameters, Success>
53
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
54
+ readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
55
+ /** @internal */
56
+ readonly _decode: (input: unknown) => Effect.Effect<Schema.Schema.Type<Parameters>, Schema.SchemaError>
57
+ /** @internal */
58
+ readonly _encode: (value: Schema.Schema.Type<Success>) => Effect.Effect<unknown, Schema.SchemaError>
59
+ /** @internal */
60
+ readonly _project: (
61
+ parameters: Schema.Schema.Type<Parameters>,
62
+ callID: ToolCallPart["id"],
63
+ output: unknown,
64
+ ) => ToolOutputType
65
+ /** @internal */
66
+ readonly _legacyResult: boolean
67
+ /** @internal */
68
+ readonly _definition: ToolDefinitionClass
69
+ }
70
+
71
+ export type AnyTool = Tool<any, any>
72
+
73
+ export type ExecutableTool<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = Tool<
74
+ Parameters,
75
+ Success
76
+ > & {
77
+ readonly execute: ToolExecute<Parameters, Success>
78
+ }
79
+
80
+ export type AnyExecutableTool = ExecutableTool<any, any>
81
+
82
+ export type ExecutableTools = Record<string, AnyExecutableTool>
83
+
84
+ type TypedToolConfig = {
85
+ readonly description: string
86
+ readonly parameters: ToolSchema<any>
87
+ readonly success: ToolSchema<any>
88
+ readonly execute?: ToolExecute<ToolSchema<any>, ToolSchema<any>>
89
+ readonly toModelOutput?: ToolToModelOutput<ToolSchema<any>, ToolSchema<any>>
90
+ readonly toStructuredOutput?: (output: unknown) => unknown
91
+ }
92
+
93
+ type DynamicToolConfig = {
94
+ readonly description: string
95
+ readonly jsonSchema: JsonSchema.JsonSchema
96
+ readonly outputSchema?: JsonSchema.JsonSchema
97
+ readonly execute?: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
98
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
99
+ readonly toStructuredOutput?: (output: unknown) => unknown
100
+ }
101
+
102
+ /**
103
+ * Constructs a tool. Two input modes:
104
+ *
105
+ * 1. **Typed** — pass Effect `parameters` and `success` Schemas; inputs and
106
+ * outputs are statically typed and decoded/encoded automatically.
107
+ *
108
+ * ```ts
109
+ * Tool.make({
110
+ * description: "Get current weather",
111
+ * parameters: Schema.Struct({ city: Schema.String }),
112
+ * success: Schema.Struct({ temperature: Schema.Number }),
113
+ * execute: ({ city }) => Effect.succeed({ temperature: 22 }),
114
+ * })
115
+ * ```
116
+ *
117
+ * 2. **Dynamic** — pass raw JSON Schema as `jsonSchema`. Use this when the
118
+ * schema comes from an external source (MCP server, plugin manifest,
119
+ * dynamic config) and is not known at compile time. Inputs are typed as
120
+ * `unknown`; the handler is responsible for any validation it needs.
121
+ *
122
+ * ```ts
123
+ * Tool.make({
124
+ * description: "Look something up",
125
+ * jsonSchema: { type: "object", properties: { ... } },
126
+ * execute: (params) => Effect.succeed(...),
127
+ * })
128
+ * ```
129
+ *
130
+ * In both modes the produced tool flows through `toDefinitions(...)`
131
+ * identically.
132
+ */
133
+ export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
134
+ readonly description: string
135
+ readonly parameters: Parameters
136
+ readonly success: Success
137
+ readonly execute: ToolExecute<Parameters, Success>
138
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
139
+ readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
140
+ }): ExecutableTool<Parameters, Success>
141
+ export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
142
+ readonly description: string
143
+ readonly parameters: Parameters
144
+ readonly success: Success
145
+ readonly execute?: undefined
146
+ readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
147
+ readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
148
+ }): Tool<Parameters, Success>
149
+ export function make(config: {
150
+ readonly description: string
151
+ readonly jsonSchema: JsonSchema.JsonSchema
152
+ readonly outputSchema?: JsonSchema.JsonSchema
153
+ readonly execute: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
154
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
155
+ readonly toStructuredOutput?: (output: unknown) => unknown
156
+ }): AnyExecutableTool
157
+ export function make(config: {
158
+ readonly description: string
159
+ readonly jsonSchema: JsonSchema.JsonSchema
160
+ readonly outputSchema?: JsonSchema.JsonSchema
161
+ readonly execute?: undefined
162
+ readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
163
+ readonly toStructuredOutput?: (output: unknown) => unknown
164
+ }): AnyTool
165
+ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
166
+ if ("jsonSchema" in config) {
167
+ return {
168
+ description: config.description,
169
+ parameters: Schema.Unknown as ToolSchema<unknown>,
170
+ success: Schema.Unknown as ToolSchema<unknown>,
171
+ execute: config.execute,
172
+ toModelOutput: config.toModelOutput,
173
+ toStructuredOutput: config.toStructuredOutput,
174
+ _decode: Effect.succeed,
175
+ _encode: Effect.succeed,
176
+ _project: (parameters, callID, output) =>
177
+ project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
178
+ _legacyResult: config.toModelOutput === undefined && config.toStructuredOutput === undefined,
179
+ _definition: new ToolDefinition({
180
+ name: "",
181
+ description: config.description,
182
+ inputSchema: config.jsonSchema,
183
+ outputSchema: config.outputSchema,
184
+ }),
185
+ }
186
+ }
187
+ return {
188
+ description: config.description,
189
+ parameters: config.parameters,
190
+ success: config.success,
191
+ execute: config.execute,
192
+ toModelOutput: config.toModelOutput,
193
+ toStructuredOutput: config.toStructuredOutput,
194
+ _decode: Schema.decodeUnknownEffect(config.parameters),
195
+ _encode: Schema.encodeEffect(config.success),
196
+ _project: (parameters, callID, output) =>
197
+ project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
198
+ _legacyResult: false,
199
+ _definition: new ToolDefinition({
200
+ name: "",
201
+ description: config.description,
202
+ inputSchema: toJsonSchema(config.parameters),
203
+ outputSchema: toJsonSchema(config.success),
204
+ }),
205
+ }
206
+ }
207
+
208
+ /**
209
+ * A record of named tools. The record key becomes the tool name on the wire.
210
+ */
211
+ export type Tools = Record<string, AnyTool>
212
+
213
+ /**
214
+ * Convert a tools record into the `ToolDefinition[]` shape that
215
+ * `LLMRequest.tools` expects.
216
+ *
217
+ * Tool names come from the record keys, so the per-tool cached
218
+ * `_definition` is rebuilt with the correct name here. The JSON Schema body
219
+ * is reused.
220
+ */
221
+ export const toDefinitions = (tools: Tools): ReadonlyArray<ToolDefinitionClass> =>
222
+ Object.entries(tools).map(
223
+ ([name, item]) =>
224
+ new ToolDefinition({
225
+ name,
226
+ description: item._definition.description,
227
+ inputSchema: item._definition.inputSchema,
228
+ outputSchema: item._definition.outputSchema,
229
+ }),
230
+ )
231
+
232
+ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => {
233
+ const document = Schema.toJsonSchemaDocument(schema)
234
+ if (Object.keys(document.definitions).length === 0) return document.schema
235
+ return { ...document.schema, $defs: document.definitions }
236
+ }
237
+
238
+ const project = (
239
+ toModelOutput: ((input: ToolModelOutputInput<any, any>) => ReadonlyArray<ToolContent>) | undefined,
240
+ toStructuredOutput: ((output: unknown) => unknown) | undefined,
241
+ parameters: unknown,
242
+ callID: ToolCallPart["id"],
243
+ output: unknown,
244
+ ): ToolOutputType =>
245
+ ToolOutput.make(
246
+ toStructuredOutput?.(output) ?? output,
247
+ toModelOutput?.({ callID, parameters, output }) ??
248
+ (typeof output === "string" ? [{ type: "text", text: output }] : []),
249
+ )
250
+
251
+ export { ToolFailure }
252
+
253
+ export * as Tool from "./tool"
@@ -0,0 +1,3 @@
1
+ /** Plain-record narrowing. Excludes arrays so JSON object checks don't accept tuples as key/value bags. */
2
+ export const isRecord = (value: unknown): value is Record<string, unknown> =>
3
+ typeof value === "object" && value !== null && !Array.isArray(value)
package/sst-env.d.ts ADDED
@@ -0,0 +1,10 @@
1
+ /* This file is auto-generated by SST. Do not edit. */
2
+ /* tslint:disable */
3
+ /* eslint-disable */
4
+ /* deno-fmt-ignore-file */
5
+ /* biome-ignore-all lint: auto-generated */
6
+
7
+ /// <reference path="../../sst-env.d.ts" />
8
+
9
+ import "sst"
10
+ export {}
@@ -0,0 +1,171 @@
1
+ import { describe, expect } from "bun:test"
2
+ import { Effect, Schema, Stream } from "effect"
3
+ import { LLM, LLMResponse } from "../src"
4
+ import { Route, Endpoint, LLMClient, Protocol, type FramingDef } from "../src/route"
5
+ import { Model } from "../src/schema"
6
+ import { testEffect } from "./lib/effect"
7
+ import { dynamicResponse } from "./lib/http"
8
+
9
+ const updateModel = (model: Model, patch: Partial<Model.Input>) => Model.update(model, patch)
10
+
11
+ const Json = Schema.fromJsonString(Schema.Unknown)
12
+ const encodeJson = Schema.encodeSync(Json)
13
+
14
+ type FakeBody = {
15
+ readonly body: string
16
+ }
17
+
18
+ const FakeEvent = Schema.Union([
19
+ Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
20
+ Schema.Struct({ type: Schema.Literal("finish"), reason: Schema.Literal("stop") }),
21
+ ])
22
+ type FakeEvent = Schema.Schema.Type<typeof FakeEvent>
23
+ const decodeFakeEvents = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(FakeEvent)))
24
+
25
+ const fakeFraming: FramingDef<FakeEvent> = {
26
+ id: "fake-json-array",
27
+ frame: (bytes) =>
28
+ Stream.fromEffect(
29
+ bytes.pipe(
30
+ Stream.decodeText(),
31
+ Stream.runFold(
32
+ () => "",
33
+ (text, event) => text + event,
34
+ ),
35
+ Effect.flatMap(decodeFakeEvents),
36
+ Effect.orDie,
37
+ ),
38
+ ).pipe(Stream.flatMap(Stream.fromIterable)),
39
+ }
40
+
41
+ const raiseEvent = (event: FakeEvent): import("../src/schema").LLMEvent =>
42
+ event.type === "finish"
43
+ ? { type: "finish", reason: event.reason }
44
+ : { type: "text-delta", id: "text-0", text: event.text }
45
+
46
+ const fakeProtocol = Protocol.make<FakeBody, FakeEvent, FakeEvent, void>({
47
+ id: "fake",
48
+ body: {
49
+ schema: Schema.Struct({
50
+ body: Schema.String,
51
+ }),
52
+ from: (request) =>
53
+ Effect.succeed({
54
+ body: [
55
+ ...request.messages
56
+ .flatMap((message) => message.content)
57
+ .filter((part) => part.type === "text")
58
+ .map((part) => part.text),
59
+ ...request.tools.map((tool) => `tool:${tool.name}:${tool.description}`),
60
+ ].join("\n"),
61
+ }),
62
+ },
63
+ stream: {
64
+ event: FakeEvent,
65
+ initial: () => undefined,
66
+ step: (state, event) => Effect.succeed([state, [raiseEvent(event)]] as const),
67
+ },
68
+ })
69
+
70
+ const fake = Route.make({
71
+ id: "fake",
72
+ protocol: fakeProtocol,
73
+ endpoint: Endpoint.path("/chat"),
74
+ framing: fakeFraming,
75
+ })
76
+ const configuredFake = fake.with({ endpoint: { baseURL: "https://fake.local" } })
77
+
78
+ const gemini = Route.make({
79
+ id: "gemini-fake",
80
+ protocol: fakeProtocol,
81
+ endpoint: Endpoint.path("/chat"),
82
+ framing: fakeFraming,
83
+ })
84
+ const configuredGemini = gemini.with({ endpoint: { baseURL: "https://fake.local" } })
85
+
86
+ const request = LLM.request({
87
+ id: "req_1",
88
+ model: Model.make({
89
+ id: "fake-model",
90
+ provider: "fake-provider",
91
+ route: configuredFake,
92
+ }),
93
+ prompt: "hello",
94
+ })
95
+
96
+ const echoLayer = dynamicResponse(({ text, respond }) =>
97
+ Effect.succeed(
98
+ respond(
99
+ encodeJson([
100
+ { type: "text", text: `echo:${text}` },
101
+ { type: "finish", reason: "stop" },
102
+ ]),
103
+ ),
104
+ ),
105
+ )
106
+
107
+ const it = testEffect(echoLayer)
108
+
109
+ describe("llm route", () => {
110
+ it.effect("stream and generate use the route pipeline", () =>
111
+ Effect.gen(function* () {
112
+ const llm = yield* LLMClient.Service
113
+ const events = Array.from(yield* llm.stream(request).pipe(Stream.runCollect))
114
+ const response = yield* llm.generate(request)
115
+ const reduced = LLMResponse.fromEvents(events)
116
+
117
+ expect(events.map((event) => event.type)).toEqual(["text-delta", "finish"])
118
+ expect(reduced).toBeDefined()
119
+ if (!reduced) throw new Error("stream reducer did not produce a completed response")
120
+ expect(response.events).toEqual(events)
121
+ expect(response.message).toEqual(reduced.message)
122
+ expect(response.usage).toEqual(reduced.usage)
123
+ expect(response.finishReason).toEqual(reduced.finishReason)
124
+ expect(response.message.content).toEqual([{ type: "text", text: 'echo:{"body":"hello"}' }])
125
+ }),
126
+ )
127
+
128
+ it.effect("selects routes by model route value", () =>
129
+ Effect.gen(function* () {
130
+ const llm = yield* LLMClient.Service
131
+ const prepared = yield* llm.prepare(
132
+ LLM.updateRequest(request, { model: updateModel(request.model, { route: configuredGemini }) }),
133
+ )
134
+
135
+ expect(prepared.route).toBe("gemini-fake")
136
+ }),
137
+ )
138
+
139
+ it.effect("builds models from configured routes", () =>
140
+ Effect.gen(function* () {
141
+ const configured = fake.with({ provider: "fake-provider", endpoint: { baseURL: "https://fake.local" } })
142
+
143
+ expect(configured.model({ id: "fake-model" })).toMatchObject({
144
+ provider: "fake-provider",
145
+ })
146
+ }),
147
+ )
148
+
149
+ it.effect("does not register duplicate route ids globally", () =>
150
+ Effect.gen(function* () {
151
+ const duplicate = Route.make({
152
+ id: "fake",
153
+ protocol: Protocol.make({
154
+ ...fakeProtocol,
155
+ body: {
156
+ ...fakeProtocol.body,
157
+ from: () => Effect.succeed({ body: "late-default" }),
158
+ },
159
+ }),
160
+ endpoint: Endpoint.path("/chat", { baseURL: "https://fake.local" }),
161
+ framing: fakeFraming,
162
+ })
163
+
164
+ const prepared = yield* (yield* LLMClient.Service).prepare(
165
+ LLM.updateRequest(request, { model: updateModel(request.model, { route: duplicate }) }),
166
+ )
167
+
168
+ expect(prepared.body).toEqual({ body: "late-default" })
169
+ }),
170
+ )
171
+ })
@@ -0,0 +1,168 @@
1
+ import { Config } from "effect"
2
+ import type { Auth } from "../src/route/auth"
3
+ import type { ModelFactory } from "../src/route/auth-options"
4
+ import { Auth as RuntimeAuth } from "../src/route/auth"
5
+ import * as OpenAIChat from "../src/protocols/openai-chat"
6
+ import * as AmazonBedrock from "../src/providers/amazon-bedrock"
7
+ import * as Anthropic from "../src/providers/anthropic"
8
+ import * as Azure from "../src/providers/azure"
9
+ import * as Cloudflare from "../src/providers/cloudflare"
10
+ import * as GitHubCopilot from "../src/providers/github-copilot"
11
+ import * as Google from "../src/providers/google"
12
+ import * as OpenAI from "../src/providers/openai"
13
+ import * as OpenAICompatible from "../src/providers/openai-compatible"
14
+ import * as OpenRouter from "../src/providers/openrouter"
15
+ import * as XAI from "../src/providers/xai"
16
+
17
+ type BaseOptions = {
18
+ readonly baseURL?: string
19
+ readonly headers?: Record<string, string>
20
+ }
21
+
22
+ type Model = {
23
+ readonly id: string
24
+ }
25
+
26
+ declare const auth: Auth
27
+ declare const optionalAuthModel: ModelFactory<BaseOptions, "optional", Model>
28
+ declare const requiredAuthModel: ModelFactory<BaseOptions, "required", Model>
29
+ const configApiKey = Config.redacted("OPENAI_API_KEY")
30
+
31
+ OpenAIChat.route.model({ id: "gpt-4.1-mini" })
32
+
33
+ // @ts-expect-error route model selection does not configure endpoints.
34
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", baseURL: "https://gateway.example.com/v1" })
35
+
36
+ // @ts-expect-error route model selection does not configure query params.
37
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", queryParams: { debug: "1" } })
38
+
39
+ // @ts-expect-error route model selection does not configure auth.
40
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", auth })
41
+
42
+ // @ts-expect-error route model selection does not configure api keys.
43
+ OpenAIChat.route.model({ id: "gpt-4.1-mini", apiKey: "sk-test" })
44
+
45
+ optionalAuthModel("gpt-4.1-mini")
46
+ optionalAuthModel("gpt-4.1-mini", {})
47
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test" })
48
+ optionalAuthModel("gpt-4.1-mini", { apiKey: configApiKey })
49
+ optionalAuthModel("gpt-4.1-mini", { auth })
50
+ optionalAuthModel("gpt-4.1-mini", { auth, baseURL: "https://gateway.example.com/v1" })
51
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test", headers: { "x-source": "test" } })
52
+
53
+ // @ts-expect-error auth is an override, so apiKey cannot be supplied with it.
54
+ optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test", auth })
55
+
56
+ requiredAuthModel("custom-model", { apiKey: "key" })
57
+ requiredAuthModel("custom-model", { apiKey: configApiKey })
58
+ requiredAuthModel("custom-model", { auth })
59
+ requiredAuthModel("custom-model", { auth, headers: { "x-tenant-id": "tenant" } })
60
+
61
+ // @ts-expect-error providers without config fallback need apiKey or auth.
62
+ requiredAuthModel("custom-model")
63
+
64
+ // @ts-expect-error providers without config fallback need apiKey or auth.
65
+ requiredAuthModel("custom-model", {})
66
+
67
+ // @ts-expect-error auth is an override, so apiKey cannot be supplied with it.
68
+ requiredAuthModel("custom-model", { apiKey: "key", auth })
69
+
70
+ OpenAI.responses("gpt-4.1-mini")
71
+ OpenAI.configure({}).responses("gpt-4.1-mini")
72
+ OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini")
73
+ OpenAI.configure({ apiKey: configApiKey }).responses("gpt-4.1-mini")
74
+ OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).responses("gpt-4.1-mini")
75
+ OpenAI.configure({
76
+ auth: RuntimeAuth.headers({ authorization: "Bearer gateway" }),
77
+ baseURL: "https://gateway.example.com/v1",
78
+ }).responses("gpt-4.1-mini")
79
+ OpenAI.configure({
80
+ generation: { maxTokens: 100 },
81
+ providerOptions: { openai: { store: false } },
82
+ }).responses("gpt-4.1-mini")
83
+
84
+ // @ts-expect-error OpenAI model selectors only accept model ids.
85
+ OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini", {})
86
+
87
+ // @ts-expect-error apiKey only accepts string, Redacted<string>, or Config<string | Redacted<string>>.
88
+ OpenAI.configure({ apiKey: 123 })
89
+
90
+ // @ts-expect-error provider helpers reject unknown top-level options.
91
+ OpenAI.configure({ bogus: true })
92
+
93
+ // @ts-expect-error common generation options remain typed.
94
+ OpenAI.configure({ generation: { maxTokens: "many" } })
95
+
96
+ // @ts-expect-error provider-native options remain typed.
97
+ OpenAI.configure({ providerOptions: { openai: { store: "false" } } })
98
+
99
+ // @ts-expect-error auth is an override, so OpenAI rejects apiKey with auth.
100
+ OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
101
+
102
+ OpenAI.chat("gpt-4.1-mini")
103
+ OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini")
104
+ OpenAI.configure({ apiKey: configApiKey }).chat("gpt-4.1-mini")
105
+ OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).chat("gpt-4.1-mini")
106
+
107
+ // @ts-expect-error OpenAI chat selectors only accept model ids.
108
+ OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini", {})
109
+
110
+ // @ts-expect-error auth is an override, so OpenAI Chat rejects apiKey with auth.
111
+ OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
112
+
113
+ // @ts-expect-error Azure requires at least one of `resourceName` or `baseURL`.
114
+ Azure.configure()
115
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment")
116
+ Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).responses("deployment")
117
+ Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).responses("deployment")
118
+
119
+ // @ts-expect-error Azure model selectors only accept deployment ids.
120
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment", {})
121
+
122
+ // @ts-expect-error auth is an override, so Azure rejects apiKey with auth.
123
+ Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
124
+
125
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment")
126
+ Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).chat("deployment")
127
+ Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).chat("deployment")
128
+
129
+ // @ts-expect-error Azure chat model selectors only accept deployment ids.
130
+ Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment", {})
131
+
132
+ // @ts-expect-error auth is an override, so Azure Chat rejects apiKey with auth.
133
+ Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
134
+
135
+ Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku")
136
+ // @ts-expect-error Anthropic model selectors only accept model ids.
137
+ Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku", {})
138
+
139
+ Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash")
140
+ // @ts-expect-error Google model selectors only accept model ids.
141
+ Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash", {})
142
+
143
+ AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude")
144
+ // @ts-expect-error Bedrock model selectors only accept model ids.
145
+ AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude", {})
146
+
147
+ OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini")
148
+ // @ts-expect-error OpenRouter model selectors only accept model ids.
149
+ OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini", {})
150
+
151
+ XAI.configure({ apiKey: "xai-key" }).responses("grok-4")
152
+ XAI.configure({ apiKey: "xai-key" }).chat("grok-4")
153
+ // @ts-expect-error xAI Responses selectors only accept model ids.
154
+ XAI.configure({ apiKey: "xai-key" }).responses("grok-4", {})
155
+ // @ts-expect-error xAI Chat selectors only accept model ids.
156
+ XAI.configure({ apiKey: "xai-key" }).chat("grok-4", {})
157
+
158
+ OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat")
159
+ // @ts-expect-error OpenAI-compatible family selectors only accept model ids.
160
+ OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat", {})
161
+
162
+ Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama")
163
+ // @ts-expect-error Cloudflare Workers AI model selectors only accept model ids.
164
+ Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama", {})
165
+
166
+ GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1")
167
+ // @ts-expect-error GitHub Copilot model selectors only accept model ids.
168
+ GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1", {})