@effect-uai/core 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/{Audio-60lKKHxS.d.mts → Audio-_nGR4V0X.d.mts} +2 -2
  2. package/dist/{Audio-60lKKHxS.d.mts.map → Audio-_nGR4V0X.d.mts.map} +1 -1
  3. package/dist/Image-Cv6iBLTy.d.mts +99 -0
  4. package/dist/Image-Cv6iBLTy.d.mts.map +1 -0
  5. package/dist/{Items-SfUabNuB.d.mts → Items-C7MyrYPX.d.mts} +35 -4
  6. package/dist/Items-C7MyrYPX.d.mts.map +1 -0
  7. package/dist/Media-BbX7BAaO.d.mts +60 -0
  8. package/dist/Media-BbX7BAaO.d.mts.map +1 -0
  9. package/dist/{Tool-DE-rnvkw.d.mts → Tool-BF78QWtn.d.mts} +2 -2
  10. package/dist/{Tool-DE-rnvkw.d.mts.map → Tool-BF78QWtn.d.mts.map} +1 -1
  11. package/dist/{ToolEvent-D99MGQYH.d.mts → ToolEvent-Di4a723B.d.mts} +2 -2
  12. package/dist/{ToolEvent-D99MGQYH.d.mts.map → ToolEvent-Di4a723B.d.mts.map} +1 -1
  13. package/dist/{ToolResult-Cj8ChodL.d.mts → ToolResult-BsYQRjmW.d.mts} +2 -2
  14. package/dist/{ToolResult-Cj8ChodL.d.mts.map → ToolResult-BsYQRjmW.d.mts.map} +1 -1
  15. package/dist/{Toolkit-BEZrNPTI.d.mts → Toolkit-DAjJ9Tvl.d.mts} +5 -5
  16. package/dist/{Toolkit-BEZrNPTI.d.mts.map → Toolkit-DAjJ9Tvl.d.mts.map} +1 -1
  17. package/dist/browser/BrowserTool.d.mts +1 -1
  18. package/dist/capabilities/Capabilities.d.mts +26 -2
  19. package/dist/capabilities/Capabilities.d.mts.map +1 -1
  20. package/dist/capabilities/Capabilities.mjs +32 -1
  21. package/dist/capabilities/Capabilities.mjs.map +1 -1
  22. package/dist/domain/Audio.d.mts +1 -1
  23. package/dist/domain/Image.d.mts +3 -2
  24. package/dist/domain/Image.mjs +29 -3
  25. package/dist/domain/Image.mjs.map +1 -1
  26. package/dist/domain/Items.d.mts +2 -2
  27. package/dist/domain/Items.mjs +19 -1
  28. package/dist/domain/Items.mjs.map +1 -1
  29. package/dist/domain/Media.d.mts +2 -2
  30. package/dist/domain/Media.mjs +51 -2
  31. package/dist/domain/Media.mjs.map +1 -1
  32. package/dist/domain/Music.d.mts +3 -8
  33. package/dist/domain/Music.d.mts.map +1 -1
  34. package/dist/domain/Music.mjs.map +1 -1
  35. package/dist/domain/Research.d.mts +1 -1
  36. package/dist/domain/Turn.d.mts +73 -9
  37. package/dist/domain/Turn.d.mts.map +1 -1
  38. package/dist/domain/Turn.mjs +38 -2
  39. package/dist/domain/Turn.mjs.map +1 -1
  40. package/dist/embedding-model/Embedding.d.mts +1 -1
  41. package/dist/image-generator/ImageGenerator.d.mts +188 -0
  42. package/dist/image-generator/ImageGenerator.d.mts.map +1 -0
  43. package/dist/image-generator/ImageGenerator.mjs +50 -0
  44. package/dist/image-generator/ImageGenerator.mjs.map +1 -0
  45. package/dist/index.d.mts +11 -9
  46. package/dist/index.mjs +4 -2
  47. package/dist/language-model/LanguageModel.d.mts +2 -2
  48. package/dist/music-generator/MusicGenerator.d.mts +3 -2
  49. package/dist/music-generator/MusicGenerator.d.mts.map +1 -1
  50. package/dist/observability/Metrics.d.mts +1 -1
  51. package/dist/speech-synthesizer/SpeechSynthesizer.d.mts +1 -1
  52. package/dist/streaming/Multipart.d.mts +21 -0
  53. package/dist/streaming/Multipart.d.mts.map +1 -0
  54. package/dist/streaming/Multipart.mjs +26 -0
  55. package/dist/streaming/Multipart.mjs.map +1 -0
  56. package/dist/testing/MockMusicGenerator.d.mts +1 -1
  57. package/dist/testing/MockProvider.d.mts +1 -1
  58. package/dist/testing/MockSpeechSynthesizer.d.mts +1 -1
  59. package/dist/tool/Approval.d.mts +3 -3
  60. package/dist/tool/HistoryCheck.d.mts +2 -2
  61. package/dist/tool/Tool.d.mts +1 -1
  62. package/dist/tool/ToolEvent.d.mts +1 -1
  63. package/dist/tool/ToolResult.d.mts +1 -1
  64. package/dist/tool/Toolkit.d.mts +1 -1
  65. package/dist/transcriber/Transcriber.d.mts +1 -1
  66. package/dist/web-read/WebReadTool.d.mts +1 -1
  67. package/dist/web-search/WebSearchTool.d.mts +1 -1
  68. package/package.json +9 -1
  69. package/src/capabilities/Capabilities.ts +38 -0
  70. package/src/domain/Image.ts +68 -2
  71. package/src/domain/Items.ts +18 -1
  72. package/src/domain/Media.ts +66 -0
  73. package/src/domain/Music.ts +4 -3
  74. package/src/domain/Turn.ts +56 -0
  75. package/src/image-generator/ImageGenerator.ts +165 -0
  76. package/src/index.ts +2 -0
  77. package/src/streaming/Multipart.ts +29 -0
  78. package/dist/Image-BK8-4H6s.d.mts +0 -59
  79. package/dist/Image-BK8-4H6s.d.mts.map +0 -1
  80. package/dist/Items-SfUabNuB.d.mts.map +0 -1
  81. package/dist/Media-DwgpifPD.d.mts +0 -57
  82. package/dist/Media-DwgpifPD.d.mts.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@effect-uai/core",
3
- "version": "0.13.0",
3
+ "version": "0.14.0",
4
4
  "description": "Low-level primitives (loop, conversation, items, tools, streaming codecs) for building AI agents with Effect.",
5
5
  "keywords": [
6
6
  "agents",
@@ -106,6 +106,10 @@
106
106
  "types": "./dist/embedding-model/EmbeddingModel.d.mts",
107
107
  "import": "./dist/embedding-model/EmbeddingModel.mjs"
108
108
  },
109
+ "./ImageGenerator": {
110
+ "types": "./dist/image-generator/ImageGenerator.d.mts",
111
+ "import": "./dist/image-generator/ImageGenerator.mjs"
112
+ },
109
113
  "./LanguageModel": {
110
114
  "types": "./dist/language-model/LanguageModel.d.mts",
111
115
  "import": "./dist/language-model/LanguageModel.mjs"
@@ -214,6 +218,10 @@
214
218
  "types": "./dist/streaming/Lines.d.mts",
215
219
  "import": "./dist/streaming/Lines.mjs"
216
220
  },
221
+ "./Multipart": {
222
+ "types": "./dist/streaming/Multipart.d.mts",
223
+ "import": "./dist/streaming/Multipart.mjs"
224
+ },
217
225
  "./SSE": {
218
226
  "types": "./dist/streaming/SSE.d.mts",
219
227
  "import": "./dist/streaming/SSE.mjs"
@@ -1,4 +1,5 @@
1
1
  import { Effect } from "effect"
2
+ import type { ContentBlock, HistoryItem } from "../domain/Items.js"
2
3
 
3
4
  /**
4
5
  * Structured event emitted when a provider adapter drops a Common
@@ -79,3 +80,40 @@ export const warnDroppedWhen = <T>(
79
80
  value: T | undefined,
80
81
  warning: Omit<CapabilityWarning, "_tag" | "value">,
81
82
  ): Effect.Effect<void> => (value === undefined ? Effect.void : warnDropped({ ...warning, value }))
83
+
84
+ /**
85
+ * Warn when history carries a content block this provider's wire has no
86
+ * place for, which its encoder therefore omits.
87
+ *
88
+ * History is provider-agnostic, so a turn one provider produced can be
89
+ * replayed into another. A block only one provider can express
90
+ * (`output_image`, which needs an assistant-role image part) then reaches
91
+ * an encoder with nothing to map it to. Omitting is right, since failing
92
+ * would break replaying a mixed history just to continue the text, but it
93
+ * should not be silent.
94
+ *
95
+ * Call once from the effectful request path; the block encoders stay pure.
96
+ *
97
+ * @example
98
+ * ```ts
99
+ * yield* warnDroppedBlocks(history, "output_image", {
100
+ * provider: "anthropic",
101
+ * capability: "output_image",
102
+ * reason: "Assistant messages carry no image here. Convert with `imagesAsInput` to send it as user content.",
103
+ * })
104
+ * ```
105
+ */
106
+ export const warnDroppedBlocks = (
107
+ history: ReadonlyArray<HistoryItem>,
108
+ type: ContentBlock["type"],
109
+ warning: Omit<CapabilityWarning, "_tag" | "field" | "value">,
110
+ ): Effect.Effect<void> => {
111
+ const count = history.reduce(
112
+ (n, item) =>
113
+ n + (item.type === "message" ? item.content.filter((b) => b.type === type).length : 0),
114
+ 0,
115
+ )
116
+ return count === 0
117
+ ? Effect.void
118
+ : warnDropped({ ...warning, field: type, value: `${count} block(s)` })
119
+ }
@@ -1,5 +1,14 @@
1
- import { Schema } from "effect"
2
- import type { MediaBase64, MediaBytes, MediaSource, MediaUrl } from "./Media.js"
1
+ import { Result, Schema } from "effect"
2
+ import type {
3
+ AspectRatio,
4
+ Dimensions,
5
+ MediaBase64,
6
+ MediaBytes,
7
+ MediaSource,
8
+ MediaUrl,
9
+ Watermark,
10
+ } from "./Media.js"
11
+ import { parseAspectRatio, round16 } from "./Media.js"
3
12
 
4
13
  /**
5
14
  * Image MIME types AI providers typically accept. The first four are the
@@ -70,6 +79,63 @@ export const imageBytes = (bytes: Uint8Array, mimeType: ImageMimeType): ImageByt
70
79
  mimeType,
71
80
  })
72
81
 
82
+ /** Cross-modality; lives in `Media.ts` because video generation shares it. */
83
+ export type { AspectRatio, Dimensions } from "./Media.js"
84
+
85
+ /**
86
+ * Resolution tier, roughly the short edge in pixels. A tier rather than
87
+ * a pixel pair: adapters whose wire takes exact dimensions derive them
88
+ * from the tier and the aspect ratio. Image-typed on purpose, video
89
+ * models tier by scan height instead.
90
+ */
91
+ export type ImageResolution = "1K" | "2K" | "4K"
92
+
93
+ const SHORT_EDGE: Record<ImageResolution, number> = { "1K": 1024, "2K": 2048, "4K": 4096 }
94
+
95
+ /**
96
+ * Turn the portable ratio and tier into pixels: the tier is the short
97
+ * edge, the long edge follows the ratio. `undefined` when neither was
98
+ * asked for, which the caller sends as "no size" so the endpoint picks.
99
+ *
100
+ * Adapters whose wire wants `"WxH"` format the pair; those wanting an
101
+ * object send it as is. Whether the result is in range for a given model
102
+ * is the server's call: per-model limits churn and we do not table them.
103
+ */
104
+ export const pixelsOf = (request: {
105
+ readonly aspectRatio?: AspectRatio
106
+ readonly resolution?: ImageResolution
107
+ }): Result.Result<Dimensions | undefined, string> =>
108
+ request.aspectRatio === undefined && request.resolution === undefined
109
+ ? Result.succeed(undefined)
110
+ : Result.map(parseAspectRatio(request.aspectRatio ?? "1:1"), ([w, h]) => {
111
+ const short = SHORT_EDGE[request.resolution ?? "1K"]
112
+ return w >= h
113
+ ? { width: round16((short * w) / h), height: short }
114
+ : { width: short, height: round16((short * h) / w) }
115
+ })
116
+
117
+ /**
118
+ * Per-image extras a provider reported and this type has no field for:
119
+ * pixel dimensions, a file name, a revised prompt. Opaque here, and a
120
+ * shared slot, so a provider keys its data under its own name
121
+ * (`{ fal: … }`, `{ openai: … }`) and reads only that key. Each provider
122
+ * package ships a typed reader; the framework never interprets it.
123
+ *
124
+ * Not promoted to real fields because the providers disagree on what
125
+ * they report. Only some fal endpoints return dimensions, and neither
126
+ * OpenAI's Images API nor Gemini returns them at all.
127
+ */
128
+ export type ProviderData = unknown
129
+
130
+ /** One image off a generation call, plus what the provider stamped into it. */
131
+ export type GeneratedImage = {
132
+ readonly image: ImageSource
133
+ /** Set only when the provider applies one. */
134
+ readonly watermark?: Watermark
135
+ /** {@link ProviderData}: per-image extras, keyed by provider name. */
136
+ readonly providerData?: ProviderData
137
+ }
138
+
73
139
  export const isImageUrl = Schema.is(ImageUrlSource)
74
140
  export const isImageBase64 = Schema.is(ImageBase64Source)
75
141
  export const isImageBytes = Schema.is(ImageBytesSource)
@@ -88,6 +88,22 @@ export const OutputText = Schema.Struct({
88
88
  })
89
89
  export type OutputText = typeof OutputText.Type
90
90
 
91
+ /**
92
+ * Model-emitted image. The counterpart to {@link InputImage}: same
93
+ * `ImageSource`, so an image the model drew goes straight back into the
94
+ * next turn with no conversion.
95
+ *
96
+ * How the pixels were produced is not modelled here. A provider may draw
97
+ * them itself, as an image part of the turn, or reach a hosted tool that
98
+ * draws them; either way the fact is the same, and enabling it is
99
+ * provider-specific.
100
+ */
101
+ export const OutputImage = Schema.Struct({
102
+ type: Schema.Literal("output_image"),
103
+ source: ImageSource,
104
+ })
105
+ export type OutputImage = typeof OutputImage.Type
106
+
91
107
  /**
92
108
  * Model-emitted refusal. Distinct from `output_text`: the model declined
93
109
  * to answer rather than producing normal output. Pair with
@@ -100,7 +116,7 @@ export const Refusal = Schema.Struct({
100
116
  })
101
117
  export type Refusal = typeof Refusal.Type
102
118
 
103
- export const ContentBlock = Schema.Union([InputText, InputImage, OutputText, Refusal])
119
+ export const ContentBlock = Schema.Union([InputText, InputImage, OutputText, OutputImage, Refusal])
104
120
  export type ContentBlock = typeof ContentBlock.Type
105
121
 
106
122
  export const Role = Schema.Literals(["user", "assistant", "system"])
@@ -176,6 +192,7 @@ export type HistoryItem = typeof HistoryItem.Type
176
192
  export const isInputText = Schema.is(InputText)
177
193
  export const isInputImage = Schema.is(InputImage)
178
194
  export const isOutputText = Schema.is(OutputText)
195
+ export const isOutputImage = Schema.is(OutputImage)
179
196
  export const isRefusal = Schema.is(Refusal)
180
197
 
181
198
  export const isMessage = Schema.is(Message)
@@ -31,6 +31,72 @@
31
31
  * they're a separate union (`FileRef`) added when needed.
32
32
  */
33
33
 
34
+ import { Result } from "effect"
35
+
36
+ /**
37
+ * Watermark kind a provider embedded in generated media. Shared across
38
+ * modalities (audio, image). No provider exposes metadata beyond the
39
+ * kind, so a bare literal union is enough.
40
+ */
41
+ export type Watermark = "synthid" | "c2pa" | (string & {})
42
+
43
+ /**
44
+ * Requested shape of generated visual media, width to height. Shared by
45
+ * image and video generation: the listed ratios are the ones providers
46
+ * accept as literals, and the `(string & {})` tail keeps a
47
+ * newly-supported ratio working without an SDK update. Which ratios a
48
+ * given model accepts is not checked here; the provider rejects what it
49
+ * cannot render and the adapter translates that error.
50
+ *
51
+ * Resolution is NOT shared: image models tier by short edge
52
+ * (`ImageResolution`), video models by scan height. Each modality types
53
+ * its own.
54
+ */
55
+ export type AspectRatio =
56
+ | "1:1"
57
+ | "2:3"
58
+ | "3:2"
59
+ | "3:4"
60
+ | "4:3"
61
+ | "4:5"
62
+ | "5:4"
63
+ | "9:16"
64
+ | "16:9"
65
+ | "21:9"
66
+ | "1:4"
67
+ | "4:1"
68
+ | "1:8"
69
+ | "8:1"
70
+ | (string & {})
71
+
72
+ /** Exact pixels, for adapters whose wire takes dimensions rather than a ratio. */
73
+ export type Dimensions = {
74
+ readonly width: number
75
+ readonly height: number
76
+ }
77
+
78
+ const positive = (n: number | undefined): n is number =>
79
+ n !== undefined && Number.isFinite(n) && n > 0
80
+
81
+ /**
82
+ * Read `"W:H"` as a pair. Fails when the ratio is not arithmetic, which
83
+ * the `(string & {})` tail on {@link AspectRatio} lets through: a caller
84
+ * asking for `"cinematic"` gets a message rather than NaN pixels. The
85
+ * failure is a plain string so this file stays free of provider
86
+ * vocabulary; each adapter wraps it in its own error.
87
+ */
88
+ export const parseAspectRatio = (
89
+ ratio: AspectRatio,
90
+ ): Result.Result<readonly [number, number], string> => {
91
+ const [w, h, ...rest] = ratio.split(":").map(Number)
92
+ return positive(w) && positive(h) && rest.length === 0
93
+ ? Result.succeed([w, h] as const)
94
+ : Result.fail(`aspectRatio "${ratio}" must read "W:H" with positive numbers.`)
95
+ }
96
+
97
+ /** Several image endpoints require both edges to be multiples of 16. */
98
+ export const round16 = (n: number): number => Math.max(16, Math.round(n / 16) * 16)
99
+
34
100
  export type MediaUrl<M extends string = string> = {
35
101
  readonly _tag: "url"
36
102
  readonly url: string
@@ -1,5 +1,6 @@
1
1
  import type { Duration } from "effect"
2
2
  import type { AudioBlob, AudioChunk, AudioFormat } from "./Audio.js"
3
+ import type { Watermark } from "./Media.js"
3
4
 
4
5
  /**
5
6
  * Prompt fragment with a relative weight. Native to Lyria RealTime
@@ -95,10 +96,10 @@ export const controlInput = (action: MusicSessionControl): MusicSessionInput =>
95
96
 
96
97
  /**
97
98
  * Watermark kind on the response. SynthID is mandatory on Lyria; C2PA
98
- * is opt-in on ElevenLabs. No provider exposes additional metadata
99
- * about the watermark itself, so a bare literal-union is enough.
99
+ * is opt-in on ElevenLabs. Lives in `Media.ts` since image providers
100
+ * report the same kinds; re-exported here unchanged.
100
101
  */
101
- export type Watermark = "synthid" | "c2pa" | (string & {})
102
+ export type { Watermark } from "./Media.js"
102
103
 
103
104
  /**
104
105
  * Labelled section with absolute timing. Populated when the provider
@@ -1,5 +1,6 @@
1
1
  import { Data, Effect, Result, Schema, Stream, pipe } from "effect"
2
2
  import * as StructuredFormat from "../structured-format/StructuredFormat.js"
3
+ import type { ImageSource } from "./Image.js"
3
4
  import {
4
5
  type Annotation,
5
6
  HistoryItem,
@@ -9,6 +10,7 @@ import {
9
10
  ToolCall,
10
11
  ToolCallOutput,
11
12
  Usage,
13
+ isOutputImage,
12
14
  isOutputText,
13
15
  isReasoning,
14
16
  isRefusal,
@@ -49,6 +51,12 @@ export type Turn = typeof Turn.Type
49
51
  * `WebSearchCall`: lifecycle of a provider-executed web search within the
50
52
  * turn. Fires only when the turn grounds against search; absent otherwise.
51
53
  *
54
+ * `ImageOutput`: an image the model produced, whether it drew it as part
55
+ * of its turn or reached a hosted tool for it. `partialIndex` marks a
56
+ * preview frame from providers that stream them; the finished image
57
+ * arrives without one and is also on `TurnComplete.turn` as an
58
+ * `OutputImage` block.
59
+ *
52
60
  * `CitationAdded`: a citation attached to the answer, emitted incrementally
53
61
  * where the provider streams citations. Providers that bundle citations emit
54
62
  * none of these; their citations still arrive on `TurnComplete.turn`
@@ -66,6 +74,11 @@ export type TurnEvent = Data.TaggedEnum<{
66
74
  readonly query?: string
67
75
  readonly action?: "search" | "open_page" | "find_in_page"
68
76
  }
77
+ ImageOutput: {
78
+ readonly image: ImageSource
79
+ /** Set only on a preview frame, counting from 0. Absent means finished. */
80
+ readonly partialIndex?: number
81
+ }
69
82
  CitationAdded: { readonly annotation: Annotation }
70
83
  TurnComplete: { readonly turn: Turn }
71
84
  }>
@@ -109,6 +122,49 @@ export const assistantTexts = (turn: Turn): ReadonlyArray<string> =>
109
122
  */
110
123
  export const assistantText = (turn: Turn): string => assistantTexts(turn).join("")
111
124
 
125
+ /**
126
+ * Every image the model produced this turn, in order. The counterpart to
127
+ * {@link assistantTexts}: each is an `ImageSource`, so it can be written
128
+ * to disk or passed straight back as an `input_image` on the next turn.
129
+ */
130
+ export const assistantImages = (turn: Turn): ReadonlyArray<ImageSource> =>
131
+ assistantMessages(turn)
132
+ .flatMap((m) => m.content)
133
+ .filter(isOutputImage)
134
+ .map((b) => b.source)
135
+
136
+ /**
137
+ * Rewrite assistant `output_image` blocks into a user message carrying
138
+ * the same pictures as `input_image`, so another provider can look at
139
+ * what this one drew.
140
+ *
141
+ * Only Gemini's wire has an assistant-role image part; every other
142
+ * adapter omits an `output_image` on replay and says so. Moving it to a
143
+ * user message is the conversion that makes it portable, and it is
144
+ * yours to make rather than the adapter's: "the assistant drew this" and
145
+ * "here is an image, look at it" are different claims, and silently
146
+ * swapping one for the other behind your back would be the wrong kind of
147
+ * helpful.
148
+ *
149
+ * Items with no images pass through untouched, and the new user message
150
+ * follows the assistant message it came from.
151
+ */
152
+ export const imagesAsInput = (history: ReadonlyArray<HistoryItem>): ReadonlyArray<HistoryItem> =>
153
+ history.flatMap((item) => {
154
+ if (item.type !== "message" || item.role !== "assistant") return [item]
155
+ const images = item.content.filter(isOutputImage)
156
+ return images.length === 0
157
+ ? [item]
158
+ : [
159
+ item,
160
+ {
161
+ type: "message" as const,
162
+ role: "user" as const,
163
+ content: images.map((b) => ({ type: "input_image" as const, source: b.source })),
164
+ },
165
+ ]
166
+ })
167
+
112
168
  /**
113
169
  * Every citation annotation attached to the turn's assistant `output_text`
114
170
  * blocks, in order. The flat view of a grounded turn's sources, whether the
@@ -0,0 +1,165 @@
1
+ import { Context, Data, Effect, Stream } from "effect"
2
+ import type * as AiError from "../domain/AiError.js"
3
+ import type {
4
+ AspectRatio,
5
+ GeneratedImage,
6
+ ImageResolution,
7
+ ImageSource,
8
+ ProviderData,
9
+ } from "../domain/Image.js"
10
+
11
+ export type { AspectRatio, GeneratedImage, ImageResolution, ProviderData } from "../domain/Image.js"
12
+
13
+ /**
14
+ * Cross-provider text-to-image request. Shape and resolution are given
15
+ * as a ratio plus a tier rather than exact pixels: every provider takes
16
+ * one or derives the other. Exact dimensions, quality tiers, output
17
+ * encodings, and moderation knobs are vendor-specific and live on each
18
+ * provider's typed request.
19
+ */
20
+ export type CommonImageGenerateRequest = {
21
+ readonly prompt: string
22
+ /** Each provider narrows this to its typed literal union. */
23
+ readonly model: string
24
+ readonly aspectRatio?: AspectRatio
25
+ readonly resolution?: ImageResolution
26
+ /** Default 1. Implementors without multi-image output fail `Unsupported` for more. */
27
+ readonly n?: number
28
+ }
29
+
30
+ /**
31
+ * Edit request: the same prompt-driven generation, conditioned on
32
+ * reference images. Separate from `generate` because the reference set
33
+ * is required here and absent there, and providers expose the two as
34
+ * distinct wire shapes. Per-provider caps on the array length apply.
35
+ */
36
+ export type CommonImageEditRequest = CommonImageGenerateRequest & {
37
+ readonly images: ReadonlyArray<ImageSource>
38
+ }
39
+
40
+ /** Gated by {@link ImageStreaming}; `partialImages` is the preview-frame count. */
41
+ export type CommonStreamImageRequest = CommonImageGenerateRequest & {
42
+ readonly partialImages: 1 | 2 | 3
43
+ }
44
+
45
+ /** The same previews, for an edit. Gated by {@link ImageStreaming}. */
46
+ export type CommonStreamImageEditRequest = CommonImageEditRequest & {
47
+ readonly partialImages: 1 | 2 | 3
48
+ }
49
+
50
+ /** Optional throughout: not every provider bills or reports per token. */
51
+ export type ImageUsage = {
52
+ readonly inputTokens?: number
53
+ readonly outputTokens?: number
54
+ readonly totalTokens?: number
55
+ }
56
+
57
+ export type ImageResponse = {
58
+ readonly images: ReadonlyArray<GeneratedImage>
59
+ readonly usage: ImageUsage
60
+ /**
61
+ * Response-level extras, keyed by provider name: the seed a request
62
+ * resolved to, timings, safety flags. Per-image extras live on each
63
+ * {@link GeneratedImage} instead. See `Image.ProviderData`.
64
+ */
65
+ readonly providerData?: ProviderData
66
+ }
67
+
68
+ /**
69
+ * Zero or more `PartialImage` previews, then exactly one `Complete`.
70
+ * `Complete` carries the `ImageResponse` fields flat, so it is
71
+ * assignable to `ImageResponse` and needs no unwrapping.
72
+ */
73
+ export type ImageStreamEvent = Data.TaggedEnum<{
74
+ PartialImage: {
75
+ readonly image: ImageSource
76
+ /** Counts from 0 in emission order. */
77
+ readonly index: number
78
+ }
79
+ Complete: {
80
+ readonly images: ReadonlyArray<GeneratedImage>
81
+ readonly usage: ImageUsage
82
+ }
83
+ }>
84
+
85
+ /**
86
+ * Namespace of constructors, type guards, and matchers for
87
+ * `ImageStreamEvent`, provided by `Data.taggedEnum`.
88
+ */
89
+ export const ImageStreamEvent = Data.taggedEnum<ImageStreamEvent>()
90
+
91
+ export const isPartialImage = ImageStreamEvent.$is("PartialImage")
92
+ export const isComplete = ImageStreamEvent.$is("Complete")
93
+
94
+ export type ImageGeneratorService = {
95
+ /** Prompt in, images out. Universally supported. */
96
+ readonly generate: (
97
+ request: CommonImageGenerateRequest,
98
+ ) => Effect.Effect<ImageResponse, AiError.AiError>
99
+ /** Prompt plus reference images in, images out. Universally supported. */
100
+ readonly edit: (request: CommonImageEditRequest) => Effect.Effect<ImageResponse, AiError.AiError>
101
+ /**
102
+ * Preview frames as the image resolves, then the final response.
103
+ * Implementors without a partial-image wire return `Unsupported` and
104
+ * do NOT register {@link ImageStreaming}, so the top-level helper is
105
+ * a compile-time error against their Layer.
106
+ */
107
+ readonly streamGeneration: (
108
+ request: CommonStreamImageRequest,
109
+ ) => Stream.Stream<ImageStreamEvent, AiError.AiError>
110
+ /** {@link streamGeneration} conditioned on reference images. */
111
+ readonly streamEdit: (
112
+ request: CommonStreamImageEditRequest,
113
+ ) => Stream.Stream<ImageStreamEvent, AiError.AiError>
114
+ }
115
+
116
+ export class ImageGenerator extends Context.Service<ImageGenerator, ImageGeneratorService>()(
117
+ "@betalyra/effect-uai/ImageGenerator",
118
+ ) {}
119
+
120
+ /**
121
+ * Capability marker for partial-image streaming. Phantom: the value is
122
+ * `void`, providers register with
123
+ * `Layer.succeed(ImageStreaming, undefined)`. Calling
124
+ * {@link streamGeneration} while only a non-streaming Layer is in scope
125
+ * fails at `Effect.provide` with a type error.
126
+ */
127
+ export class ImageStreaming extends Context.Service<ImageStreaming, void>()(
128
+ "@betalyra/effect-uai/capability/ImageStreaming",
129
+ ) {}
130
+
131
+ /** Text to image. */
132
+ export const generate = (
133
+ request: CommonImageGenerateRequest,
134
+ ): Effect.Effect<ImageResponse, AiError.AiError, ImageGenerator> =>
135
+ Effect.flatMap(ImageGenerator, (g) => g.generate(request))
136
+
137
+ /** Text plus reference images to image. */
138
+ export const edit = (
139
+ request: CommonImageEditRequest,
140
+ ): Effect.Effect<ImageResponse, AiError.AiError, ImageGenerator> =>
141
+ Effect.flatMap(ImageGenerator, (g) => g.edit(request))
142
+
143
+ /** Preview frames then the final response. Requires {@link ImageStreaming} in R. */
144
+ export const streamGeneration = (
145
+ request: CommonStreamImageRequest,
146
+ ): Stream.Stream<ImageStreamEvent, AiError.AiError, ImageGenerator | ImageStreaming> =>
147
+ Stream.unwrap(
148
+ Effect.gen(function* () {
149
+ const g = yield* ImageGenerator
150
+ yield* ImageStreaming
151
+ return g.streamGeneration(request)
152
+ }),
153
+ )
154
+
155
+ /** The same, conditioned on reference images. Requires {@link ImageStreaming} in R. */
156
+ export const streamEdit = (
157
+ request: CommonStreamImageEditRequest,
158
+ ): Stream.Stream<ImageStreamEvent, AiError.AiError, ImageGenerator | ImageStreaming> =>
159
+ Stream.unwrap(
160
+ Effect.gen(function* () {
161
+ const g = yield* ImageGenerator
162
+ yield* ImageStreaming
163
+ return g.streamEdit(request)
164
+ }),
165
+ )
package/src/index.ts CHANGED
@@ -17,6 +17,7 @@ export * as Transcript from "./domain/Transcript.js"
17
17
  export * as Turn from "./domain/Turn.js"
18
18
  export * as Embedding from "./embedding-model/Embedding.js"
19
19
  export * as EmbeddingModel from "./embedding-model/EmbeddingModel.js"
20
+ export * as ImageGenerator from "./image-generator/ImageGenerator.js"
20
21
  export * as LanguageModel from "./language-model/LanguageModel.js"
21
22
  export * as MusicGenerator from "./music-generator/MusicGenerator.js"
22
23
  export * as Reranker from "./reranker/Reranker.js"
@@ -43,6 +44,7 @@ export * as Approval from "./tool/Approval.js"
43
44
  export * as HistoryCheck from "./tool/HistoryCheck.js"
44
45
  export * as JSONL from "./streaming/JSONL.js"
45
46
  export * as Lines from "./streaming/Lines.js"
47
+ export * as Multipart from "./streaming/Multipart.js"
46
48
  export * as SSE from "./streaming/SSE.js"
47
49
  export * as StructuredFormat from "./structured-format/StructuredFormat.js"
48
50
  export * as Metrics from "./observability/Metrics.js"
@@ -0,0 +1,29 @@
1
+ import { Effect } from "effect"
2
+ import { HttpClientRequest } from "effect/unstable/http"
3
+
4
+ /**
5
+ * Multipart body for an `HttpClientRequest`, encoded to bytes.
6
+ *
7
+ * Use this over `HttpClientRequest.bodyFormData`: the Undici client
8
+ * passes the `FormData` object straight to `dispatcher.request`, which
9
+ * cannot serialise it, so the request is never sent and the effect
10
+ * hangs with no error. Encoding here produces the same bytes every
11
+ * other client derives internally, so it works on all of them.
12
+ *
13
+ * The failure is whatever `Response.arrayBuffer` rejects with; map it
14
+ * to your provider's transport error at the call site.
15
+ */
16
+ export const bodyMultipart = (
17
+ form: FormData,
18
+ ): Effect.Effect<
19
+ (request: HttpClientRequest.HttpClientRequest) => HttpClientRequest.HttpClientRequest,
20
+ unknown
21
+ > =>
22
+ Effect.gen(function* () {
23
+ const encoded = new Response(form)
24
+ const buffer = yield* Effect.tryPromise(() => encoded.arrayBuffer())
25
+ return HttpClientRequest.bodyUint8Array(
26
+ new Uint8Array(buffer),
27
+ encoded.headers.get("content-type") ?? "multipart/form-data",
28
+ )
29
+ })
@@ -1,59 +0,0 @@
1
- import { i as MediaUrl, n as MediaBytes, r as MediaSource, t as MediaBase64 } from "./Media-DwgpifPD.mjs";
2
- import { Schema } from "effect";
3
- declare namespace Image_d_exports {
4
- export { ImageBase64Source, ImageBytesSource, ImageMimeType, ImageSource, ImageUrlSource, imageBase64, imageBytes, imageUrl, isImageBase64, isImageBytes, isImageUrl };
5
- }
6
- /**
7
- * Image MIME types AI providers typically accept. The first four are the
8
- * universal subset (Cohere v4, Voyage multimodal, Jina v4, Google
9
- * `gemini-embedding-2`); HEIC / HEIF are Google-specific. The
10
- * `(string & {})` tail keeps autocomplete on the literals while still
11
- * accepting any string, so a newly-supported format works without an
12
- * SDK update.
13
- */
14
- type ImageMimeType = "image/png" | "image/jpeg" | "image/webp" | "image/gif" | "image/heic" | "image/heif" | (string & {});
15
- type ImageUrlSource = MediaUrl<ImageMimeType>;
16
- type ImageBase64Source = MediaBase64<ImageMimeType>;
17
- type ImageBytesSource = MediaBytes<ImageMimeType>;
18
- /**
19
- * Where an image lives. Provider layers normalize across these:
20
- * `bytes` becomes a base64 data URI for OpenAI / Anthropic, an
21
- * `inlineData` part for Gemini, and a separate field for Cohere /
22
- * Voyage. URL constraints (must be HTTPS, must be public, …) are
23
- * provider-specific and validated at the layer, not in the type.
24
- */
25
- type ImageSource = MediaSource<ImageMimeType>;
26
- declare const ImageUrlSource: Schema.TaggedStruct<"url", {
27
- readonly url: Schema.String;
28
- readonly mimeType: Schema.optional<Schema.Schema<ImageMimeType>>;
29
- }>;
30
- declare const ImageBase64Source: Schema.TaggedStruct<"base64", {
31
- readonly base64: Schema.String;
32
- readonly mimeType: Schema.Schema<ImageMimeType>;
33
- }>;
34
- declare const ImageBytesSource: Schema.TaggedStruct<"bytes", {
35
- readonly bytes: Schema.Uint8Array;
36
- readonly mimeType: Schema.Schema<ImageMimeType>;
37
- }>;
38
- declare const ImageSource: Schema.Schema<ImageSource>;
39
- declare const imageUrl: (url: string, mimeType?: ImageMimeType) => ImageUrlSource;
40
- declare const imageBase64: (base64: string, mimeType: ImageMimeType) => ImageBase64Source;
41
- declare const imageBytes: (bytes: Uint8Array, mimeType: ImageMimeType) => ImageBytesSource;
42
- declare const isImageUrl: <I>(input: I) => input is I & {
43
- readonly _tag: "url";
44
- readonly url: string;
45
- readonly mimeType?: ImageMimeType | undefined;
46
- };
47
- declare const isImageBase64: <I>(input: I) => input is I & {
48
- readonly _tag: "base64";
49
- readonly base64: string;
50
- readonly mimeType: ImageMimeType;
51
- };
52
- declare const isImageBytes: <I>(input: I) => input is I & {
53
- readonly _tag: "bytes";
54
- readonly bytes: Uint8Array<ArrayBufferLike>;
55
- readonly mimeType: ImageMimeType;
56
- };
57
- //#endregion
58
- export { ImageUrlSource as a, imageBytes as c, isImageBytes as d, isImageUrl as f, ImageSource as i, imageUrl as l, ImageBytesSource as n, Image_d_exports as o, ImageMimeType as r, imageBase64 as s, ImageBase64Source as t, isImageBase64 as u };
59
- //# sourceMappingURL=Image-BK8-4H6s.d.mts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"Image-BK8-4H6s.d.mts","names":[],"sources":["../src/domain/Image.ts"],"mappings":";;;;;;;;;;;;;KAWY;KAYA,iBAAiB,SAAS;KAC1B,oBAAoB,YAAY;KAChC,mBAAmB,WAAW;;;;;;;;KAS9B,cAAc,YAAY;cAEzB,gBAAc,OAAA;;;;cAKd,mBAAiB,OAAA;;;;cAKjB,kBAAgB,OAAA;;;;cAKhB,aAAa,OAAO,OAAO;cAM3B,WAAQ,aAAe,WAAa,kBAAgB;cAGpD,cAAW,gBAAkB,UAAY,kBAAgB;cAMzD,aAAU,OAAW,YAAU,UAAY,kBAAgB;cAM3D,aAAU,GAAA,OAAA,MAAA,SAAA;;;;;cACV,gBAAa,GAAA,OAAA,MAAA,SAAA;;;;;cACb,eAAY,GAAA,OAAA,MAAA,SAAA"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"Items-SfUabNuB.d.mts","names":[],"sources":["../src/domain/Items.ts"],"mappings":";;;;;cAOa,WAAS,OAAA;;;;KAIV,mBAAmB,UAAU;;;;;;;;;cAU5B,YAAU,OAAA;;;;KAIX,oBAAoB,WAAW;;;;;;;;;cAgB9B,aAAW,OAAA;;;;;;;;;KASZ,qBAAqB,YAAY;cAEhC,cAAY,OAAA;;;;;KAKb,sBAAsB,aAAa;cAElC,uBAAqB,OAAA;;;;;;;KAOtB,+BAA+B,sBAAsB;cAEpD,UAAQ,OAAA;;;;;KAKT,kBAAkB,SAAS;cAE1B,YAAU,OAAA,gBAAA,OAAA;;;;;;;;;;;;;;;;;;;;;;;KACX,oBAAoB,WAAW;cAE9B,gBAAa,GAAA,OAAA,MAAA,SAAA;;;;;;;;;cACb,iBAAc,GAAA,OAAA,MAAA,SAAA;;;;;cACd,0BAAuB,GAAA,OAAA,MAAA,SAAA;;;;;;;cACvB,aAAU,GAAA,OAAA,MAAA,SAAA;;;;;cAEV,YAAU,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;KAKX,oBAAoB,WAAW;;;;;;;cAQ9B,SAAO,OAAA;;;;KAIR,iBAAiB,QAAQ;cAExB,cAAY,OAAA,gBAAA,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KACb,sBAAsB,aAAa;cAElC,MAAI,OAAA;KACL,cAAc,KAAK;cAqBlB,SAAO,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAMR,iBAAiB,QAAQ;cAExB,UAAQ,OAAA;;;;;;;KAQT,kBAAkB,SAAS;cAE1B,gBAAc,OAAA;;;;;;KAMf,wBAAwB,eAAe;;;;;;;cAQtC,WAAS,OAAA;;;;;;;KAOV,mBAAmB,UAAU;cAE5B,aAAW,OAAA,gBAAA,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KACZ,qBAAqB,YAAY;cAMhC,cAAW,GAAA,OAAA,MAAA,SAAA;;;;cACX,eAAY,GAAA,OAAA,MAAA,SAAA;;;;cACZ,eAAY,GAAA,OAAA,MAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;cACZ,YAAS,GAAA,OAAA,MAAA,SAAA;;;;cAET,YAAS,GAAA,OAAA,MAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cACT,aAAU,GAAA,OAAA,MAAA,SAAA;;;;;;;cACV,mBAAgB,GAAA,OAAA,MAAA,SAAA;;;;;;cAChB,cAAW,GAAA,OAAA,MAAA,SAAA;;;;;;;;;;;;;;;;;;cAiBX,oBAAkB,OAAA;;;;KAInB,4BAA4B,mBAAmB;cAE9C,qBAAmB,OAAA;;;KAGpB,6BAA6B,oBAAoB;cAEhD,OAAK,OAAA;;;;;;;;;;;;KAON,eAAe,MAAM;cAEpB,YAAU,OAAA;KAUX,oBAAoB,WAAW;cAM9B,WAAQ,iBAAmB;cAM3B,aAAU,iBAAmB;cAM7B,gBAAa,iBAAmB;cAMhC,iBAAc,iBAAmB,mBAAmB"}