@opencode/ai 2.0.15 → 2.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. package/README.md +441 -99
  2. package/dist/ai-client.d.ts +8 -0
  3. package/dist/ai-client.js +12 -0
  4. package/dist/experimental/evaluation-client.d.ts +3 -3
  5. package/dist/experimental/evaluation-client.js +1 -1
  6. package/dist/experimental/evaluation.js +1 -1
  7. package/dist/generation.d.ts +39 -29
  8. package/dist/generation.js +62 -36
  9. package/dist/image-client.d.ts +59 -17
  10. package/dist/image-client.js +16 -24
  11. package/dist/image.d.ts +394 -55
  12. package/dist/image.js +48 -57
  13. package/dist/index.d.ts +15 -2
  14. package/dist/index.js +10 -0
  15. package/dist/llm.d.ts +7 -5
  16. package/dist/llm.js +10 -4
  17. package/dist/media-client.d.ts +30 -0
  18. package/dist/media-client.js +51 -0
  19. package/dist/media-model.d.ts +43 -0
  20. package/dist/media-model.js +47 -0
  21. package/dist/media.d.ts +10 -9
  22. package/dist/media.js +11 -12
  23. package/dist/promise.d.ts +473 -15
  24. package/dist/promise.js +71 -13
  25. package/dist/protocols/alibaba-chat.d.ts +12 -0
  26. package/dist/protocols/alibaba-chat.js +4 -1
  27. package/dist/protocols/alibaba-messages.d.ts +1 -1
  28. package/dist/protocols/alibaba-messages.js +6 -4
  29. package/dist/protocols/alibaba-responses.d.ts +2 -2
  30. package/dist/protocols/anthropic-messages.d.ts +34 -34
  31. package/dist/protocols/anthropic-messages.js +14 -9
  32. package/dist/protocols/assemblyai-transcription.d.ts +41 -0
  33. package/dist/protocols/assemblyai-transcription.js +136 -0
  34. package/dist/protocols/bedrock-converse.d.ts +7 -0
  35. package/dist/protocols/bedrock-converse.js +39 -19
  36. package/dist/protocols/bfl-images.d.ts +38 -0
  37. package/dist/protocols/bfl-images.js +153 -0
  38. package/dist/protocols/cartesia-speech.d.ts +143 -0
  39. package/dist/protocols/cartesia-speech.js +120 -0
  40. package/dist/protocols/deepgram-speech.d.ts +135 -0
  41. package/dist/protocols/deepgram-speech.js +91 -0
  42. package/dist/protocols/deepgram-transcription.d.ts +26 -0
  43. package/dist/protocols/deepgram-transcription.js +125 -0
  44. package/dist/protocols/elevenlabs-speech.d.ts +138 -0
  45. package/dist/protocols/elevenlabs-speech.js +111 -0
  46. package/dist/protocols/fal-images.d.ts +25 -0
  47. package/dist/protocols/fal-images.js +104 -0
  48. package/dist/protocols/fal-video.d.ts +29 -0
  49. package/dist/protocols/fal-video.js +73 -0
  50. package/dist/protocols/gemini.d.ts +22 -22
  51. package/dist/protocols/gemini.js +19 -36
  52. package/dist/protocols/google-images.d.ts +3 -3
  53. package/dist/protocols/google-images.js +12 -34
  54. package/dist/protocols/google-speech.d.ts +146 -0
  55. package/dist/protocols/google-speech.js +88 -0
  56. package/dist/protocols/google-transcription.d.ts +174 -0
  57. package/dist/protocols/google-transcription.js +126 -0
  58. package/dist/protocols/google-video.d.ts +26 -0
  59. package/dist/protocols/google-video.js +142 -0
  60. package/dist/protocols/meta-images.d.ts +3 -4
  61. package/dist/protocols/meta-images.js +13 -29
  62. package/dist/protocols/meta-messages.d.ts +5 -5
  63. package/dist/protocols/meta-responses.d.ts +4 -4
  64. package/dist/protocols/meta-responses.js +6 -4
  65. package/dist/protocols/open-responses.d.ts +10 -9
  66. package/dist/protocols/open-responses.js +4 -9
  67. package/dist/protocols/openai-chat.d.ts +84 -0
  68. package/dist/protocols/openai-chat.js +28 -17
  69. package/dist/protocols/openai-compatible-chat.d.ts +12 -0
  70. package/dist/protocols/openai-compatible-responses.d.ts +2 -2
  71. package/dist/protocols/openai-images.d.ts +130 -7
  72. package/dist/protocols/openai-images.js +143 -91
  73. package/dist/protocols/openai-responses.d.ts +20 -20
  74. package/dist/protocols/openai-responses.js +31 -31
  75. package/dist/protocols/openai-speech.d.ts +136 -0
  76. package/dist/protocols/openai-speech.js +97 -0
  77. package/dist/protocols/openai-transcription.d.ts +211 -0
  78. package/dist/protocols/openai-transcription.js +202 -0
  79. package/dist/protocols/replicate-images.d.ts +28 -0
  80. package/dist/protocols/replicate-images.js +127 -0
  81. package/dist/protocols/runway-video.d.ts +38 -0
  82. package/dist/protocols/runway-video.js +140 -0
  83. package/dist/protocols/shared.d.ts +21 -17
  84. package/dist/protocols/shared.js +27 -36
  85. package/dist/protocols/stability-images.d.ts +39 -0
  86. package/dist/protocols/stability-images.js +136 -0
  87. package/dist/protocols/utils/fal-queue.d.ts +26 -0
  88. package/dist/protocols/utils/fal-queue.js +67 -0
  89. package/dist/protocols/utils/gemini-generate-content.d.ts +65 -0
  90. package/dist/protocols/utils/gemini-generate-content.js +65 -0
  91. package/dist/protocols/utils/gemini-json-schema.d.ts +3 -0
  92. package/dist/protocols/utils/gemini-json-schema.js +76 -0
  93. package/dist/protocols/utils/media-input.d.ts +23 -1
  94. package/dist/protocols/utils/media-input.js +40 -0
  95. package/dist/protocols/utils/responses-checkpoint.js +3 -7
  96. package/dist/protocols/utils/responses-compaction.d.ts +3 -1
  97. package/dist/protocols/utils/responses-compaction.js +16 -3
  98. package/dist/protocols/utils/speech-stream.d.ts +49 -0
  99. package/dist/protocols/utils/speech-stream.js +64 -0
  100. package/dist/protocols/utils/tool-schema.d.ts +2 -2
  101. package/dist/protocols/utils/tool-schema.js +62 -19
  102. package/dist/protocols/xai-images.d.ts +4 -4
  103. package/dist/protocols/xai-images.js +14 -40
  104. package/dist/protocols/xai-responses.d.ts +2 -2
  105. package/dist/protocols/xai-responses.js +1 -1
  106. package/dist/protocols/xai-video.d.ts +34 -0
  107. package/dist/protocols/xai-video.js +141 -0
  108. package/dist/protocols/zai-chat.d.ts +13 -1
  109. package/dist/protocols/zai-images.d.ts +2 -2
  110. package/dist/protocols/zai-images.js +11 -14
  111. package/dist/protocols/zai-messages.d.ts +1 -1
  112. package/dist/provider-error.js +10 -1
  113. package/dist/providers/alibaba.d.ts +15 -3
  114. package/dist/providers/amazon-bedrock-mantle.d.ts +14 -2
  115. package/dist/providers/amazon-bedrock.d.ts +2 -0
  116. package/dist/providers/amazon-bedrock.js +1 -0
  117. package/dist/providers/anthropic-compatible.d.ts +5 -5
  118. package/dist/providers/anthropic.d.ts +5 -5
  119. package/dist/providers/assemblyai.d.ts +25 -0
  120. package/dist/providers/assemblyai.js +24 -0
  121. package/dist/providers/azure.d.ts +20 -8
  122. package/dist/providers/azure.js +2 -2
  123. package/dist/providers/baseten.d.ts +24 -0
  124. package/dist/providers/black-forest-labs.d.ts +25 -0
  125. package/dist/providers/black-forest-labs.js +23 -0
  126. package/dist/providers/cartesia.d.ts +24 -0
  127. package/dist/providers/cartesia.js +17 -0
  128. package/dist/providers/cerebras.d.ts +24 -0
  129. package/dist/providers/cloudflare-ai-gateway.d.ts +42 -18
  130. package/dist/providers/cloudflare-workers-ai.d.ts +24 -0
  131. package/dist/providers/deepgram.d.ts +29 -0
  132. package/dist/providers/deepgram.js +25 -0
  133. package/dist/providers/deepinfra.d.ts +24 -0
  134. package/dist/providers/deepseek.d.ts +24 -0
  135. package/dist/providers/elevenlabs.d.ts +24 -0
  136. package/dist/providers/elevenlabs.js +23 -0
  137. package/dist/providers/fal.d.ts +29 -0
  138. package/dist/providers/fal.js +26 -0
  139. package/dist/providers/fireworks.d.ts +24 -0
  140. package/dist/providers/google-vertex-chat.d.ts +12 -0
  141. package/dist/providers/google-vertex-messages.d.ts +5 -5
  142. package/dist/providers/google-vertex-responses.d.ts +2 -2
  143. package/dist/providers/google-vertex.d.ts +6 -6
  144. package/dist/providers/google.d.ts +21 -6
  145. package/dist/providers/google.js +13 -9
  146. package/dist/providers/groq.d.ts +24 -0
  147. package/dist/providers/index.d.ts +9 -0
  148. package/dist/providers/index.js +9 -0
  149. package/dist/providers/meta.d.ts +22 -10
  150. package/dist/providers/meta.js +4 -8
  151. package/dist/providers/minimax.d.ts +19 -7
  152. package/dist/providers/moonshot.d.ts +19 -7
  153. package/dist/providers/moonshot.js +3 -3
  154. package/dist/providers/openai-compatible-responses.d.ts +2 -2
  155. package/dist/providers/openai-compatible.d.ts +12 -0
  156. package/dist/providers/openai-options.d.ts +3 -9
  157. package/dist/providers/openai-options.js +4 -7
  158. package/dist/providers/openai.d.ts +33 -12
  159. package/dist/providers/openai.js +17 -9
  160. package/dist/providers/opencode-zen.js +1 -1
  161. package/dist/providers/openrouter.d.ts +54 -7
  162. package/dist/providers/openrouter.js +10 -5
  163. package/dist/providers/replicate.d.ts +25 -0
  164. package/dist/providers/replicate.js +17 -0
  165. package/dist/providers/runway.d.ts +24 -0
  166. package/dist/providers/runway.js +17 -0
  167. package/dist/providers/stability.d.ts +28 -0
  168. package/dist/providers/stability.js +18 -0
  169. package/dist/providers/togetherai.d.ts +24 -0
  170. package/dist/providers/typesafe-ai.js +1 -1
  171. package/dist/providers/vercel-ai-gateway.js +1 -1
  172. package/dist/providers/xai.d.ts +17 -0
  173. package/dist/providers/xai.js +7 -9
  174. package/dist/providers/zai-coding-plan.d.ts +16 -4
  175. package/dist/providers/zai.d.ts +13 -1
  176. package/dist/providers/zai.js +4 -8
  177. package/dist/route/auth.d.ts +5 -2
  178. package/dist/route/auth.js +20 -13
  179. package/dist/route/client.d.ts +9 -7
  180. package/dist/route/client.js +6 -8
  181. package/dist/route/endpoint.d.ts +1 -0
  182. package/dist/route/endpoint.js +2 -2
  183. package/dist/route/executor-service.d.ts +4 -2
  184. package/dist/route/executor-service.js +2 -1
  185. package/dist/route/executor.d.ts +3 -1
  186. package/dist/route/executor.js +7 -0
  187. package/dist/route/framing.d.ts +15 -2
  188. package/dist/route/framing.js +53 -4
  189. package/dist/route/index.d.ts +1 -1
  190. package/dist/route/media-protocol.d.ts +145 -18
  191. package/dist/route/media-protocol.js +96 -28
  192. package/dist/route/media.d.ts +54 -9
  193. package/dist/route/media.js +194 -38
  194. package/dist/route/protocol.d.ts +3 -1
  195. package/dist/schema/events.d.ts +0 -6
  196. package/dist/schema/messages.d.ts +0 -3
  197. package/dist/schema/options.d.ts +10 -6
  198. package/dist/schema/options.js +10 -5
  199. package/dist/speech-client.d.ts +66 -0
  200. package/dist/speech-client.js +21 -0
  201. package/dist/speech.d.ts +1307 -0
  202. package/dist/speech.js +124 -0
  203. package/dist/testing.d.ts +2 -2
  204. package/dist/transcription-client.d.ts +82 -0
  205. package/dist/transcription-client.js +13 -0
  206. package/dist/transcription.d.ts +1492 -0
  207. package/dist/transcription.js +127 -0
  208. package/dist/utils/bytes.d.ts +1 -0
  209. package/dist/utils/bytes.js +10 -0
  210. package/dist/utils/json.d.ts +4 -0
  211. package/dist/utils/json.js +4 -0
  212. package/dist/utils/media-type.d.ts +3 -1
  213. package/dist/utils/media-type.js +25 -2
  214. package/dist/video-client.d.ts +59 -0
  215. package/dist/video-client.js +20 -0
  216. package/dist/video.d.ts +1351 -0
  217. package/dist/video.js +118 -0
  218. package/package.json +3 -3
  219. package/dist/protocols/utils/gemini-tool-schema.d.ts +0 -2
  220. package/dist/protocols/utils/gemini-tool-schema.js +0 -103
  221. package/dist/protocols/utils/meta-image.d.ts +0 -2
  222. package/dist/protocols/utils/meta-image.js +0 -13
  223. package/dist/protocols/utils/openai-image.d.ts +0 -5
  224. package/dist/protocols/utils/openai-image.js +0 -18
@@ -11,7 +11,6 @@ import { classifyProviderFailure } from "../provider-error.js";
11
11
  import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js";
12
12
  import { OpenAIOptions } from "./utils/openai-options.js";
13
13
  import { Lifecycle } from "./utils/lifecycle.js";
14
- import { ToolSchemaProjection } from "./utils/tool-schema.js";
15
14
  import { ToolStream } from "./utils/tool-stream.js";
16
15
  const ADAPTER = "openai-chat";
17
16
  const RESERVED_REASONING_FIELDS = new Set(["role", "content", "refusal", "tool_calls"]);
@@ -91,6 +90,10 @@ const OpenAIChatUserContent = Schema.Union([
91
90
  type: Schema.Literal("image_url"),
92
91
  image_url: Schema.Struct({ url: Schema.String }),
93
92
  }),
93
+ Schema.Struct({
94
+ type: Schema.Literal("file"),
95
+ file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
96
+ }),
94
97
  ]);
95
98
  const OpenAIChatMessage = Schema.Union([
96
99
  Schema.Struct({
@@ -205,12 +208,12 @@ export const OpenAIChatEvent = Schema.StructWithRest(Schema.Struct({
205
208
  }), [Schema.Record(Schema.String, Schema.Unknown)]);
206
209
  const DONE = "[DONE]";
207
210
  const OpenAIChatStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(OpenAIChatEvent)]);
208
- const lowerTool = (tool, inputSchema, options, supportsStrictMode) => ({
211
+ const lowerTool = (tool, options, supportsStrictMode) => ({
209
212
  type: "function",
210
213
  function: {
211
214
  name: tool.name,
212
215
  description: tool.description,
213
- parameters: inputSchema,
216
+ parameters: tool.inputSchema,
214
217
  ...(supportsStrictMode ? { strict: false } : {}),
215
218
  },
216
219
  cache_control: options.cacheControl?.(tool.cache),
@@ -230,6 +233,15 @@ const lowerToolCall = (part, options) => ({
230
233
  },
231
234
  });
232
235
  const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part) {
236
+ // Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported.
237
+ if (part.media.mediaType.toLowerCase() === "application/pdf")
238
+ return {
239
+ type: "file",
240
+ file: {
241
+ filename: part.filename ?? "document.pdf",
242
+ file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
243
+ },
244
+ };
233
245
  if (part.media.kind !== "image")
234
246
  return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`);
235
247
  const url = ProviderShared.mediaUrl(part.media) ?? (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl;
@@ -345,7 +357,7 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
345
357
  });
346
358
  const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (message, options) {
347
359
  const messages = [];
348
- const images = [];
360
+ const attachments = [];
349
361
  for (const part of message.content) {
350
362
  if (!ProviderShared.supportsContent(part, ["tool-result"]))
351
363
  return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"]);
@@ -367,9 +379,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (m
367
379
  cache_control: options.cacheControl?.(part.cache),
368
380
  });
369
381
  const files = content.filter((item) => item.type === "file");
370
- images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))));
382
+ attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))));
371
383
  }
372
- return { messages, images };
384
+ return { messages, attachments };
373
385
  });
374
386
  const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message, reasoningField, requireReasoning, options) {
375
387
  if (message.role === "user")
@@ -423,23 +435,23 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request,
423
435
  if (requireAssistantAfterTool && messages.at(-1)?.role === "tool")
424
436
  messages.push({ role: "assistant", content: "Done." });
425
437
  };
426
- const pendingImages = [];
427
- const flushImages = () => {
428
- if (pendingImages.length === 0)
438
+ const pendingAttachments = [];
439
+ const flushAttachments = () => {
440
+ if (pendingAttachments.length === 0)
429
441
  return;
430
442
  bridgeTools();
431
- messages.push({ role: "user", content: pendingImages.splice(0) });
443
+ messages.push({ role: "user", content: pendingAttachments.splice(0) });
432
444
  };
433
445
  for (const message of request.messages) {
434
446
  if (message.role === "user")
435
447
  bridgeTools();
436
448
  if (message.role === "system") {
437
449
  const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message);
438
- if (pendingImages.length > 0) {
450
+ if (pendingAttachments.length > 0) {
439
451
  messages.push({
440
452
  role: "user",
441
453
  content: [
442
- ...pendingImages.splice(0),
454
+ ...pendingAttachments.splice(0),
443
455
  { type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) },
444
456
  ],
445
457
  });
@@ -479,13 +491,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request,
479
491
  if (message.role === "tool") {
480
492
  const lowered = yield* lowerToolMessages(message, lowering);
481
493
  messages.push(...lowered.messages);
482
- pendingImages.push(...lowered.images);
494
+ pendingAttachments.push(...lowered.attachments);
483
495
  continue;
484
496
  }
485
- flushImages();
497
+ flushAttachments();
486
498
  messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering)));
487
499
  }
488
- flushImages();
500
+ flushAttachments();
489
501
  return messages;
490
502
  });
491
503
  // Anthropic via LiteLLM and Amazon Bedrock require `tools` to be present
@@ -618,7 +630,6 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (reques
618
630
  if (reasoningField && RESERVED_REASONING_FIELDS.has(reasoningField))
619
631
  return yield* ProviderShared.invalidRequest(`OpenAI Chat reasoning field conflicts with reserved field ${reasoningField}`);
620
632
  const generation = request.generation;
621
- const toolSchemaCompatibility = request.model.compatibility?.toolSchema;
622
633
  const flattened = ProviderShared.flattenToolRequest(request);
623
634
  const provider = String(request.model.provider);
624
635
  const baseURL = request.model.route.endpoint.baseURL;
@@ -637,7 +648,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (reques
637
648
  ? hasHistory
638
649
  ? []
639
650
  : undefined
640
- : flattened.tools.map((tool) => lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility), options, supportsStrictMode)),
651
+ : flattened.tools.map((tool) => lowerTool(tool, options, supportsStrictMode)),
641
652
  tool_choice: hasActiveTools && request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
642
653
  stream: true,
643
654
  ...(supportsUsageInStreaming ? { stream_options: { include_usage: true } } : {}),
@@ -23,6 +23,12 @@ export declare const route: Route<{
23
23
  readonly image_url: {
24
24
  readonly url: string;
25
25
  };
26
+ } | {
27
+ readonly type: "file";
28
+ readonly file: {
29
+ readonly filename: string;
30
+ readonly file_data: string;
31
+ };
26
32
  })[];
27
33
  } | {
28
34
  readonly role: "user";
@@ -38,6 +44,12 @@ export declare const route: Route<{
38
44
  readonly image_url: {
39
45
  readonly url: string;
40
46
  };
47
+ } | {
48
+ readonly type: "file";
49
+ readonly file: {
50
+ readonly filename: string;
51
+ readonly file_data: string;
52
+ };
41
53
  })[];
42
54
  } | {
43
55
  readonly [x: string]: unknown;
@@ -72,8 +72,8 @@ export declare const route: Route<{
72
72
  } | {
73
73
  readonly type: "input_file";
74
74
  readonly filename: string;
75
- readonly detail?: string | undefined;
76
75
  readonly file_data?: string | undefined;
76
+ readonly detail?: string | undefined;
77
77
  readonly file_url?: string | undefined;
78
78
  } | {
79
79
  readonly type: "input_text";
@@ -109,8 +109,8 @@ export declare const route: Route<{
109
109
  } | {
110
110
  readonly type: "input_file";
111
111
  readonly filename: string;
112
- readonly detail?: string | undefined;
113
112
  readonly file_data?: string | undefined;
113
+ readonly detail?: string | undefined;
114
114
  readonly file_url?: string | undefined;
115
115
  } | {
116
116
  readonly type: "input_text";
@@ -1,21 +1,144 @@
1
- import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js";
1
+ import { ImageModel, type ImageRequestFor } from "../image.js";
2
+ import { Media } from "../media.js";
2
3
  import { MediaProtocol } from "../route/media-protocol.js";
3
4
  import { MediaRoute } from "../route/media.js";
5
+ import { type MediaUsage, type OpenString } from "../schema/index.js";
4
6
  export declare const DEFAULT_BASE_URL = "https://api.openai.com/v1";
5
7
  export declare const PATH = "/images/generations";
6
8
  export declare const EDIT_PATH = "/images/edits";
7
- export type OpenAIImageString<Known extends string> = Known | (string & {});
8
9
  /** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
9
10
  export type OpenAIImageOptions = {
10
- readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">;
11
- readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">;
12
- readonly moderation?: OpenAIImageString<"auto" | "low">;
11
+ readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">;
12
+ readonly background?: OpenString<"auto" | "opaque" | "transparent">;
13
+ readonly moderation?: OpenString<"auto" | "low">;
13
14
  readonly outputCompression?: number;
15
+ /** Previews sent before the final image when streaming (default 2); ignored by `Image.generate`. */
16
+ readonly partialImages?: number;
14
17
  } & Record<string, unknown>;
15
18
  export type Request = ImageRequestFor<OpenAIImageOptions>;
16
- export declare const protocol: MediaProtocol.Inline<Request, ImageResponse>;
19
+ /** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
20
+ type Frame = string | {
21
+ readonly document: string;
22
+ readonly requested: string | undefined;
23
+ };
24
+ interface State {
25
+ readonly completed: number;
26
+ readonly format?: string;
27
+ readonly size?: string;
28
+ readonly quality?: string;
29
+ readonly background?: string;
30
+ readonly usage?: MediaUsage;
31
+ }
32
+ export declare const protocol: MediaProtocol.Streamed<Request, {
33
+ readonly id: string;
34
+ readonly type: "generation-queued";
35
+ readonly position?: number | undefined;
36
+ } | {
37
+ readonly id: string;
38
+ readonly type: "generation-progress";
39
+ readonly progress?: number | undefined;
40
+ } | {
41
+ readonly type: "image-partial";
42
+ readonly index: number;
43
+ readonly image: Media.Asset;
44
+ } | {
45
+ readonly type: "image";
46
+ readonly index: number;
47
+ readonly image: Media.Asset;
48
+ } | {
49
+ readonly type: "finish";
50
+ readonly providerMetadata?: {
51
+ readonly [x: string]: {
52
+ readonly [x: string]: unknown;
53
+ };
54
+ } | undefined;
55
+ readonly usage?: {
56
+ readonly type: "tokens";
57
+ readonly input?: number | undefined;
58
+ readonly output?: number | undefined;
59
+ readonly total?: number | undefined;
60
+ readonly details?: {
61
+ readonly [x: string]: unknown;
62
+ } | undefined;
63
+ } | {
64
+ readonly type: "seconds";
65
+ readonly seconds: number;
66
+ } | {
67
+ readonly type: "characters";
68
+ readonly characters: number;
69
+ } | {
70
+ readonly type: "credits";
71
+ readonly credits: number;
72
+ } | {
73
+ readonly type: "compute";
74
+ readonly seconds: number;
75
+ } | undefined;
76
+ readonly notices?: readonly {
77
+ readonly type: "other" | "moderated" | "filtered";
78
+ readonly message: string;
79
+ readonly providerMetadata?: {
80
+ readonly [x: string]: {
81
+ readonly [x: string]: unknown;
82
+ };
83
+ } | undefined;
84
+ }[] | undefined;
85
+ }, Frame, State>;
17
86
  export declare const model: (input: MediaRoute.ModelInput) => ImageModel<OpenAIImageOptions>;
18
87
  export declare const OpenAIImages: {
19
- readonly protocol: MediaProtocol.Inline<Request, ImageResponse>;
88
+ readonly protocol: MediaProtocol.Streamed<Request, {
89
+ readonly id: string;
90
+ readonly type: "generation-queued";
91
+ readonly position?: number | undefined;
92
+ } | {
93
+ readonly id: string;
94
+ readonly type: "generation-progress";
95
+ readonly progress?: number | undefined;
96
+ } | {
97
+ readonly type: "image-partial";
98
+ readonly index: number;
99
+ readonly image: Media.Asset;
100
+ } | {
101
+ readonly type: "image";
102
+ readonly index: number;
103
+ readonly image: Media.Asset;
104
+ } | {
105
+ readonly type: "finish";
106
+ readonly providerMetadata?: {
107
+ readonly [x: string]: {
108
+ readonly [x: string]: unknown;
109
+ };
110
+ } | undefined;
111
+ readonly usage?: {
112
+ readonly type: "tokens";
113
+ readonly input?: number | undefined;
114
+ readonly output?: number | undefined;
115
+ readonly total?: number | undefined;
116
+ readonly details?: {
117
+ readonly [x: string]: unknown;
118
+ } | undefined;
119
+ } | {
120
+ readonly type: "seconds";
121
+ readonly seconds: number;
122
+ } | {
123
+ readonly type: "characters";
124
+ readonly characters: number;
125
+ } | {
126
+ readonly type: "credits";
127
+ readonly credits: number;
128
+ } | {
129
+ readonly type: "compute";
130
+ readonly seconds: number;
131
+ } | undefined;
132
+ readonly notices?: readonly {
133
+ readonly type: "other" | "moderated" | "filtered";
134
+ readonly message: string;
135
+ readonly providerMetadata?: {
136
+ readonly [x: string]: {
137
+ readonly [x: string]: unknown;
138
+ };
139
+ } | undefined;
140
+ }[] | undefined;
141
+ }, Frame, State>;
20
142
  readonly model: (input: MediaRoute.ModelInput) => ImageModel<OpenAIImageOptions>;
21
143
  };
144
+ export {};
@@ -1,36 +1,60 @@
1
- import { Effect, Schema } from "effect";
2
- import { ImageModel, ImageResponse } from "../image.js";
1
+ import { Effect, Schema, Stream } from "effect";
2
+ import { ImageFinishEvent, ImageModel, ImageOutputEvent, ImagePartialEvent, } from "../image.js";
3
3
  import { Media } from "../media.js";
4
+ import { Framing } from "../route/framing.js";
4
5
  import { MediaProtocol } from "../route/media-protocol.js";
5
6
  import { MediaRoute } from "../route/media.js";
6
- import { ProviderID, mergeJsonRecords } from "../schema/index.js";
7
+ import { mergeJsonRecords } from "../schema/index.js";
7
8
  import { ProviderShared } from "./shared.js";
8
9
  import { MediaInput } from "./utils/media-input.js";
9
- const ADAPTER = "openai-images";
10
- const NAME = "OpenAI Images";
11
- const PROVIDER = ProviderID.make("openai");
10
+ const route = MediaProtocol.identity({ id: "openai-images", name: "OpenAI Images", provider: "openai" });
12
11
  export const DEFAULT_BASE_URL = "https://api.openai.com/v1";
13
12
  export const PATH = "/images/generations";
14
13
  export const EDIT_PATH = "/images/edits";
15
14
  // ---------------------------------------------------------------------------
16
15
  // 2. Response schema
17
16
  // ---------------------------------------------------------------------------
18
- const OpenAIImageResponse = Schema.Struct({
19
- data: Schema.Array(Schema.Struct({
20
- b64_json: Schema.optional(Schema.String),
21
- url: Schema.optional(Schema.String),
22
- revised_prompt: Schema.optional(Schema.String),
23
- })),
17
+ const Usage = Schema.Struct({
18
+ input_tokens: Schema.optional(Schema.Number),
19
+ output_tokens: Schema.optional(Schema.Number),
20
+ total_tokens: Schema.optional(Schema.Number),
21
+ input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
22
+ output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
23
+ });
24
+ /** What the provider actually rendered; it can differ from the request when `auto` or a default applied. */
25
+ const Settings = {
24
26
  output_format: Schema.optional(Schema.String),
25
- usage: Schema.optional(Schema.Struct({
26
- input_tokens: Schema.optional(Schema.Number),
27
- output_tokens: Schema.optional(Schema.Number),
28
- total_tokens: Schema.optional(Schema.Number),
29
- input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
30
- output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
31
- })),
27
+ size: Schema.optional(Schema.String),
28
+ quality: Schema.optional(Schema.String),
29
+ background: Schema.optional(Schema.String),
30
+ };
31
+ const OpenAIImageResponse = Schema.Struct({
32
+ data: Schema.Array(Schema.Struct({ b64_json: Schema.String })),
33
+ ...Settings,
34
+ usage: Schema.optional(Usage),
32
35
  });
33
36
  // ---------------------------------------------------------------------------
37
+ // 3. Streaming event schema
38
+ // ---------------------------------------------------------------------------
39
+ const StreamEvent = Schema.Union([
40
+ Schema.Struct({
41
+ type: Schema.Literals(["image_generation.partial_image", "image_edit.partial_image"]),
42
+ b64_json: Schema.String,
43
+ partial_image_index: Schema.Number,
44
+ ...Settings,
45
+ output_format: Schema.String,
46
+ }),
47
+ Schema.Struct({
48
+ type: Schema.Literals(["image_generation.completed", "image_edit.completed"]),
49
+ b64_json: Schema.String,
50
+ ...Settings,
51
+ output_format: Schema.String,
52
+ usage: Schema.optional(Usage),
53
+ }),
54
+ ]);
55
+ const decodeEvent = route.decodeFrame(StreamEvent);
56
+ const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse));
57
+ // ---------------------------------------------------------------------------
34
58
  // 5. Request body construction
35
59
  // ---------------------------------------------------------------------------
36
60
  /** Multipart field names the route owns; `http.body` overlays cannot smuggle replacements for them. */
@@ -38,48 +62,33 @@ const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "im
38
62
  const nativeOptions = (options) => {
39
63
  if (!options)
40
64
  return undefined;
41
- const { outputCompression, ...native } = options;
65
+ const { outputCompression, partialImages: _, ...native } = options;
42
66
  return { output_compression: outputCompression, ...native };
43
67
  };
44
- const isEdit = (request) => (request.images?.length ?? 0) > 0;
45
- const isInline = (asset) => asset.inline() !== undefined;
46
- const blob = (data, mediaType) => {
47
- const buffer = new ArrayBuffer(data.byteLength);
48
- new Uint8Array(buffer).set(data);
49
- return new Blob([buffer], { type: mediaType });
50
- };
51
- const reference = (asset) => {
52
- const inline = asset.inline();
53
- if (inline)
54
- return Effect.succeed({ image_url: inline.dataUrl });
55
- const url = ProviderShared.mediaUrl(asset);
56
- if (url)
57
- return Effect.succeed({ image_url: url });
58
- const id = MediaInput.refID(asset, PROVIDER);
59
- if (id)
60
- return Effect.succeed({ file_id: id });
61
- return Effect.fail(ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"));
68
+ const streamOptions = (request) => {
69
+ if (request.mode !== "stream")
70
+ return Effect.succeed(undefined);
71
+ if (request.n !== undefined && request.n > 1)
72
+ return Effect.fail(route.unsupported("media.n", `${route.name} streams one image; use Image.generate for n=${request.n}`));
73
+ return Effect.succeed({ stream: true, partial_images: request.providerOptions?.partialImages ?? 2 });
62
74
  };
75
+ const isEdit = (request) => (request.images?.length ?? 0) > 0;
76
+ const isInline = (asset) => asset.source.type === "bytes" || asset.source.type === "base64";
77
+ const reference = (asset) => ProviderShared.mediaReference(asset, route.provider, route.name).pipe(Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })));
63
78
  const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request) {
64
79
  const images = request.images ?? [];
65
80
  const mask = request.mask;
66
81
  if (mask !== undefined && images.length === 0)
67
82
  return yield* ProviderShared.invalidRequest("An OpenAI image mask requires at least one input image");
68
- const fields = mergeJsonRecords({ n: request.n, size: request.size, output_format: request.format }, nativeOptions(request.providerOptions), request.http?.body);
83
+ const fields = mergeJsonRecords({ n: request.n, size: request.size, output_format: request.format, ...(yield* streamOptions(request)) }, nativeOptions(request.providerOptions), request.http?.body);
69
84
  // Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
70
85
  if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
71
86
  const form = new FormData();
72
- form.append("model", request.model.id);
73
- form.append("prompt", request.prompt);
74
- Object.entries(fields ?? {}).forEach(([key, value]) => {
75
- if (RESERVED_FORM_FIELDS.has(key))
76
- return;
77
- form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value));
78
- });
79
- const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image));
80
- uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`));
87
+ MediaInput.appendFields(form, { model: request.model.id, prompt: request.prompt }, { overlay: fields, reserved: RESERVED_FORM_FIELDS });
88
+ const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(route.id, image));
89
+ uploads.forEach((data, index) => form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`));
81
90
  if (mask !== undefined)
82
- form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask");
91
+ form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(route.id, mask), mask.mediaType), "mask");
83
92
  return MediaProtocol.multipart(form);
84
93
  }
85
94
  const references = yield* Effect.forEach(images, reference);
@@ -92,61 +101,104 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request) {
92
101
  }, fields) ?? {});
93
102
  });
94
103
  // ---------------------------------------------------------------------------
95
- // 6. Response decoding
104
+ // 6. Stream parsing
96
105
  // ---------------------------------------------------------------------------
97
106
  const requestedFormat = (body) => {
107
+ if (body.type === "binary")
108
+ return undefined;
98
109
  const value = body.type === "json" ? body.value.output_format : body.value.get("output_format");
99
110
  return typeof value === "string" ? value : undefined;
100
111
  };
101
- const decodeResponse = Effect.fn("OpenAIImages.decodeResponse")(function* (response, context) {
102
- const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, OpenAIImageResponse)(response);
103
- const decoded = output.value;
104
- const format = decoded.output_format ?? requestedFormat(context.body) ?? "png";
105
- const mediaType = `image/${format}`;
106
- const images = yield* Effect.forEach(decoded.data, (item, index) => {
107
- const providerMetadata = item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } };
108
- if (item.b64_json)
109
- return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
110
- info: { format },
111
- providerMetadata,
112
- });
113
- if (item.url)
114
- return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }));
115
- return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`));
116
- });
112
+ const usage = (value) => value === undefined
113
+ ? undefined
114
+ : {
115
+ type: "tokens",
116
+ input: value.input_tokens,
117
+ output: value.output_tokens,
118
+ total: value.total_tokens,
119
+ details: { openai: value },
120
+ };
121
+ /** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
122
+ const info = (format, size) => {
123
+ const match = size?.match(/^(\d+)x(\d+)$/);
124
+ return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format };
125
+ };
126
+ const eventImage = (frame, label, data, format, size) => MediaInput.decodedAsset((message, cause) => route.frameError(message, frame, cause), label, data, `image/${format}`, {
127
+ info: info(format, size),
128
+ });
129
+ const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state, frame) {
130
+ const event = yield* decodeEvent(frame);
131
+ const format = event.output_format;
132
+ if ("partial_image_index" in event) {
133
+ const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format, event.size);
134
+ return [state, [ImagePartialEvent.make({ index: event.partial_image_index, image })]];
135
+ }
136
+ const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format, event.size);
137
+ return [
138
+ {
139
+ completed: state.completed + 1,
140
+ format,
141
+ size: event.size,
142
+ quality: event.quality,
143
+ background: event.background,
144
+ usage: usage(event.usage),
145
+ },
146
+ [ImageOutputEvent.make({ index: state.completed, image })],
147
+ ];
148
+ });
149
+ const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame) {
150
+ const invalid = (message, cause) => route.frameError(message, frame.document, cause);
151
+ const decoded = yield* decodeDocument(frame.document).pipe(Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)));
152
+ const format = decoded.output_format ?? frame.requested ?? "png";
153
+ const images = yield* Effect.forEach(decoded.data, (item, index) => MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, {
154
+ info: info(format, decoded.size),
155
+ }));
117
156
  if (images.length === 0)
118
- return yield* output.invalid(`${NAME} returned no images`);
119
- return new ImageResponse({
120
- images,
121
- usage: decoded.usage === undefined
122
- ? undefined
123
- : {
124
- type: "tokens",
125
- input: decoded.usage.input_tokens,
126
- output: decoded.usage.output_tokens,
127
- total: decoded.usage.total_tokens,
128
- details: { openai: decoded.usage },
129
- },
130
- providerMetadata: { openai: { outputFormat: format } },
131
- });
157
+ return yield* invalid(`${route.name} returned no images`);
158
+ const state = {
159
+ completed: images.length,
160
+ format,
161
+ size: decoded.size,
162
+ quality: decoded.quality,
163
+ background: decoded.background,
164
+ usage: usage(decoded.usage),
165
+ };
166
+ return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))];
132
167
  });
168
+ const step = (state, frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame));
169
+ const finish = (state) => {
170
+ if (state.completed === 0)
171
+ return Effect.fail(route.incomplete());
172
+ return Effect.succeed([
173
+ ImageFinishEvent.make({
174
+ usage: state.usage,
175
+ providerMetadata: {
176
+ openai: {
177
+ outputFormat: state.format,
178
+ size: state.size,
179
+ quality: state.quality,
180
+ background: state.background,
181
+ },
182
+ },
183
+ }),
184
+ ]);
185
+ };
133
186
  // ---------------------------------------------------------------------------
134
187
  // 7. Protocol and route
135
188
  // ---------------------------------------------------------------------------
136
- export const protocol = MediaProtocol.inline({
137
- id: ADAPTER,
138
- name: NAME,
189
+ export const protocol = MediaProtocol.stream(route, {
139
190
  unsupported: ["aspectRatio", "seed"],
140
191
  body: { from: fromRequest },
141
- response: { decode: decodeResponse },
192
+ frames: (bytes, context) => context.request.mode === "stream"
193
+ ? Framing.sse.frame(bytes)
194
+ : Framing.document
195
+ .frame(bytes)
196
+ .pipe(Stream.map((document) => ({ document, requested: requestedFormat(context.body) }))),
197
+ initial: () => ({ completed: 0 }),
198
+ step,
199
+ finish,
142
200
  });
143
- export const model = (input) => ImageModel.fromRoute({
144
- id: ADAPTER,
145
- provider: PROVIDER,
146
- protocol,
147
- baseURL: DEFAULT_BASE_URL,
148
- path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
149
- }, input);
201
+ export const model = (input) => ImageModel.fromRoute({ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) }, input);
150
202
  export const OpenAIImages = {
151
203
  protocol,
152
204
  model,