@opencode/ai 2.0.14 → 2.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +115 -56
  2. package/dist/experimental/evaluation-client.d.ts +1 -1
  3. package/dist/experimental/evaluation-client.js +39 -3
  4. package/dist/experimental/evaluation.d.ts +4 -4
  5. package/dist/experimental/evaluation.js +2 -2
  6. package/dist/experimental/system-one.d.ts +3 -3
  7. package/dist/experimental/system-one.js +40 -51
  8. package/dist/generation.d.ts +69 -0
  9. package/dist/generation.js +84 -0
  10. package/dist/image-client.d.ts +6 -4
  11. package/dist/image-client.js +11 -11
  12. package/dist/image.d.ts +1040 -62
  13. package/dist/image.js +80 -52
  14. package/dist/index.d.ts +5 -2
  15. package/dist/index.js +3 -1
  16. package/dist/llm.d.ts +9 -1
  17. package/dist/media.d.ts +212 -0
  18. package/dist/media.js +228 -0
  19. package/dist/promise.d.ts +554 -0
  20. package/dist/promise.js +44 -0
  21. package/dist/protocols/anthropic-messages.js +7 -17
  22. package/dist/protocols/bedrock-converse.d.ts +4 -4
  23. package/dist/protocols/bedrock-converse.js +1 -6
  24. package/dist/protocols/gemini.d.ts +21 -0
  25. package/dist/protocols/gemini.js +42 -6
  26. package/dist/protocols/google-images.d.ts +9 -21
  27. package/dist/protocols/google-images.js +169 -132
  28. package/dist/protocols/meta-images.d.ts +7 -12
  29. package/dist/protocols/meta-images.js +92 -66
  30. package/dist/protocols/mistral-chat.js +7 -6
  31. package/dist/protocols/open-responses.d.ts +11 -3
  32. package/dist/protocols/open-responses.js +23 -12
  33. package/dist/protocols/openai-chat.d.ts +34 -1
  34. package/dist/protocols/openai-chat.js +100 -31
  35. package/dist/protocols/openai-images.d.ts +9 -20
  36. package/dist/protocols/openai-images.js +112 -142
  37. package/dist/protocols/shared.d.ts +17 -17
  38. package/dist/protocols/shared.js +32 -35
  39. package/dist/protocols/utils/bedrock-media.d.ts +2 -3
  40. package/dist/protocols/utils/bedrock-media.js +4 -4
  41. package/dist/protocols/utils/media-input.d.ts +10 -0
  42. package/dist/protocols/utils/media-input.js +17 -0
  43. package/dist/protocols/utils/responses-compaction.js +6 -5
  44. package/dist/protocols/utils/tool-stream.d.ts +27 -3
  45. package/dist/protocols/xai-images.d.ts +9 -15
  46. package/dist/protocols/xai-images.js +85 -83
  47. package/dist/protocols/zai-images.d.ts +9 -13
  48. package/dist/protocols/zai-images.js +59 -57
  49. package/dist/providers/amazon-bedrock.d.ts +2 -2
  50. package/dist/providers/cerebras.js +6 -1
  51. package/dist/providers/deepinfra.js +6 -1
  52. package/dist/providers/google-vertex.d.ts +7 -0
  53. package/dist/providers/google.d.ts +7 -0
  54. package/dist/providers/index.d.ts +1 -0
  55. package/dist/providers/index.js +1 -0
  56. package/dist/providers/openrouter.d.ts +19 -0
  57. package/dist/providers/openrouter.js +13 -1
  58. package/dist/providers/vercel-ai-gateway.d.ts +41 -0
  59. package/dist/providers/vercel-ai-gateway.js +85 -0
  60. package/dist/route/client.d.ts +9 -1
  61. package/dist/route/endpoint.d.ts +10 -10
  62. package/dist/route/executor-service.d.ts +12 -0
  63. package/dist/route/executor-service.js +3 -0
  64. package/dist/route/executor.d.ts +4 -9
  65. package/dist/route/executor.js +3 -3
  66. package/dist/route/index.d.ts +2 -0
  67. package/dist/route/index.js +2 -0
  68. package/dist/route/media-protocol.d.ts +45 -0
  69. package/dist/route/media-protocol.js +40 -0
  70. package/dist/route/media.d.ts +46 -0
  71. package/dist/route/media.js +64 -0
  72. package/dist/schema/errors.d.ts +13 -3
  73. package/dist/schema/errors.js +7 -0
  74. package/dist/schema/events.d.ts +563 -40
  75. package/dist/schema/events.js +35 -2
  76. package/dist/schema/messages.d.ts +98 -8
  77. package/dist/schema/messages.js +8 -6
  78. package/dist/schema/options.d.ts +2 -0
  79. package/dist/schema/options.js +3 -0
  80. package/dist/testing.d.ts +72 -8
  81. package/dist/utils/media-type.d.ts +6 -0
  82. package/dist/utils/media-type.js +49 -0
  83. package/dist/utils/sanitize.js +3 -1
  84. package/package.json +7 -3
  85. package/dist/protocols/utils/image-input.d.ts +0 -21
  86. package/dist/protocols/utils/image-input.js +0 -20
package/dist/image.js CHANGED
@@ -1,5 +1,8 @@
1
- import { Effect, Schema } from "effect";
2
- import { HttpOptions, InvalidRequestError, AIError, ModelID, ProviderID, ProviderMetadata, Usage, } from "./schema/index.js";
1
+ import { Effect, Schema, Stream } from "effect";
2
+ import { Media } from "./media.js";
3
+ import { Endpoint } from "./route/endpoint.js";
4
+ import { MediaRoute } from "./route/media.js";
5
+ import { AIError, HttpOptions, InvalidRequestError, MediaUsage, ModelID, ProviderID, ProviderMetadata, } from "./schema/index.js";
3
6
  import { ImageClient, Service } from "./image-client.js";
4
7
  export class ImageModel {
5
8
  id;
@@ -20,84 +23,109 @@ export class ImageModel {
20
23
  http: input.http,
21
24
  });
22
25
  }
26
+ /** Compose an inline image protocol with its canonical path into a model for one deployment. */
27
+ static fromRoute(route, input) {
28
+ return ImageModel.make({
29
+ id: input.id,
30
+ provider: route.provider,
31
+ http: input.http,
32
+ route: MediaRoute.make({
33
+ id: route.id,
34
+ provider: route.provider,
35
+ protocol: route.protocol,
36
+ endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
37
+ auth: input.auth,
38
+ headers: input.headers,
39
+ }),
40
+ });
41
+ }
23
42
  }
24
43
  export const ImageModelSchema = Schema.declare((value) => value instanceof ImageModel, {
25
44
  expected: "Image.Model",
26
45
  });
27
- const ImageBytesInput = Schema.Struct({
28
- type: Schema.Literal("bytes"),
29
- data: Schema.Uint8Array,
30
- mediaType: Schema.String,
31
- });
32
- const ImageUrlInput = Schema.Struct({
33
- type: Schema.Literal("url"),
34
- url: Schema.String,
35
- });
36
- const ImageFileIDInput = Schema.Struct({
37
- type: Schema.Literal("file-id"),
38
- id: Schema.String,
39
- });
40
- const ImageFileURIInput = Schema.Struct({
41
- type: Schema.Literal("file-uri"),
42
- uri: Schema.String,
43
- mediaType: Schema.String,
44
- });
45
- export const ImageInputSchema = Schema.Union([
46
- ImageBytesInput,
47
- ImageUrlInput,
48
- ImageFileIDInput,
49
- ImageFileURIInput,
50
- ]).pipe(Schema.toTaggedUnion("type"));
51
- export const ImageInput = {
52
- bytes: (data, mediaType) => ({ type: "bytes", data, mediaType }),
53
- url: (url) => ({ type: "url", url }),
54
- file: (id) => ({ type: "file-id", id }),
55
- fileUri: (uri, mediaType) => ({ type: "file-uri", uri, mediaType }),
56
- };
46
+ export const ImageSize = Schema.declare((value) => typeof value === "string" && /^\d+x\d+$/.test(value), { title: "ImageSize" });
47
+ export const ImageAspectRatio = Schema.declare((value) => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value), { title: "ImageAspectRatio" });
57
48
  export class ImageRequest extends Schema.Class("Image.Request")({
58
49
  model: ImageModelSchema,
59
50
  prompt: Schema.String,
60
- images: Schema.optional(Schema.Array(ImageInputSchema)),
61
- options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
51
+ /** Edit sources or style/subject references, in order. */
52
+ images: Schema.optional(Schema.Array(Media.AssetSchema)),
53
+ /** Inpainting mask; routes that cannot honor it fail with `UnsupportedOperation`. */
54
+ mask: Schema.optional(Media.AssetSchema),
55
+ n: Schema.optional(Schema.Int),
56
+ size: Schema.optional(ImageSize),
57
+ aspectRatio: Schema.optional(ImageAspectRatio),
58
+ seed: Schema.optional(Schema.Number),
59
+ format: Schema.optional(Schema.String),
60
+ providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
62
61
  http: Schema.optional(HttpOptions),
63
62
  }) {
64
63
  }
65
- export class GeneratedImage extends Schema.Class("Image.Generated")({
66
- mediaType: Schema.String,
67
- data: Schema.Union([Schema.String, Schema.Uint8Array]),
68
- providerMetadata: Schema.optional(ProviderMetadata),
69
- }) {
70
- }
64
+ // ---------------------------------------------------------------------------
65
+ // Response and events
66
+ // ---------------------------------------------------------------------------
71
67
  export class ImageResponse extends Schema.Class("Image.Response")({
72
- images: Schema.Array(GeneratedImage),
73
- usage: Schema.optional(Usage),
68
+ images: Schema.Array(Media.AssetSchema),
69
+ usage: Schema.optional(MediaUsage),
70
+ notices: Schema.optional(Schema.Array(Media.Notice)),
74
71
  providerMetadata: Schema.optional(ProviderMetadata),
75
72
  }) {
76
73
  get image() {
77
74
  return this.images[0];
78
75
  }
79
76
  }
77
+ export const ImageOutputEvent = Schema.Struct({
78
+ type: Schema.tag("image"),
79
+ index: Schema.Number,
80
+ image: Media.AssetSchema,
81
+ }).annotate({ identifier: "Image.Event.Image" });
82
+ export const ImageFinishEvent = Schema.Struct({
83
+ type: Schema.tag("finish"),
84
+ usage: Schema.optional(MediaUsage),
85
+ notices: Schema.optional(Schema.Array(Media.Notice)),
86
+ providerMetadata: Schema.optional(ProviderMetadata),
87
+ }).annotate({ identifier: "Image.Event.Finish" });
88
+ const imageEventTagged = Schema.Union([ImageOutputEvent, ImageFinishEvent]).pipe(Schema.toTaggedUnion("type"));
89
+ export const ImageEvent = Object.assign(imageEventTagged, {
90
+ is: {
91
+ image: imageEventTagged.guards.image,
92
+ finish: imageEventTagged.guards.finish,
93
+ },
94
+ });
95
+ /** Inline routes produce every image at once; expand the response into the streaming event shape. */
96
+ export const responseEvents = (response) => [
97
+ ...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
98
+ ImageFinishEvent.make({
99
+ usage: response.usage,
100
+ notices: response.notices,
101
+ providerMetadata: response.providerMetadata,
102
+ }),
103
+ ];
80
104
  export function request(input) {
81
105
  if (input instanceof ImageRequest)
82
106
  return input;
83
107
  return new ImageRequest({
84
108
  ...input,
85
- model: input.model,
86
109
  http: input.http === undefined ? undefined : HttpOptions.make(input.http),
87
110
  });
88
111
  }
89
- export function generate(input) {
90
- return Effect.try({
91
- try: () => (input instanceof ImageRequest ? input : request(input)),
92
- catch: (error) => new AIError({
93
- reason: new InvalidRequestError({
94
- message: error instanceof Error ? error.message : String(error),
95
- cause: error,
96
- }),
112
+ const requestEffect = (input) => Effect.try({
113
+ try: () => request(input),
114
+ catch: (error) => new AIError({
115
+ reason: new InvalidRequestError({
116
+ message: error instanceof Error ? error.message : String(error),
117
+ cause: error,
97
118
  }),
98
- }).pipe(Effect.flatMap((request) => ImageClient.generate(request)));
119
+ }),
120
+ });
121
+ export function generate(input) {
122
+ return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request)));
123
+ }
124
+ export function stream(input) {
125
+ return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request))));
99
126
  }
100
127
  export const Image = {
101
128
  request,
102
129
  generate,
130
+ stream,
103
131
  };
package/dist/index.d.ts CHANGED
@@ -6,9 +6,12 @@ export { ProviderPackage } from "./provider-package.js";
6
6
  export { isContextOverflow, isContextOverflowFailure } from "./provider-error.js";
7
7
  export type { RouteLanguageModelInput, RouteRoutedLanguageModelInput, Interface as LLMClientShape, Service as LLMClientService, } from "./route/client.js";
8
8
  export * from "./schema/index.js";
9
- export { GeneratedImage, ImageInput, ImageInputSchema, ImageModel, ImageRequest, ImageResponse } from "./image.js";
10
- export type { ImageModelOptions, ImageOptions, ImageRequestFor, ImageRequestInput, ImageRoute } from "./image.js";
9
+ export { ImageAspectRatio, ImageEvent, ImageModel, ImageModelSchema, ImageRequest, ImageResponse, ImageSize, } from "./image.js";
10
+ export type { ImageFormat, ImageModelOptions, ImageOptions, ImageRequestFor, ImageRequestInput, ImageRoute, } from "./image.js";
11
11
  export { Image } from "./image.js";
12
+ export { Media } from "./media.js";
13
+ export { Generation } from "./generation.js";
14
+ export type { Event as GenerationEvent, Poll, Route as GenerationRoute, Snapshot as GenerationSnapshot, Status as GenerationStatus } from "./generation.js";
12
15
  export { Tool, ToolFailure, toDefinitions } from "./tool.js";
13
16
  export { ToolRuntime } from "./tool-runtime.js";
14
17
  export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js";
package/dist/index.js CHANGED
@@ -5,8 +5,10 @@ export { Provider } from "./provider.js";
5
5
  export { ProviderPackage } from "./provider-package.js";
6
6
  export { isContextOverflow, isContextOverflowFailure } from "./provider-error.js";
7
7
  export * from "./schema/index.js";
8
- export { GeneratedImage, ImageInput, ImageInputSchema, ImageModel, ImageRequest, ImageResponse } from "./image.js";
8
+ export { ImageAspectRatio, ImageEvent, ImageModel, ImageModelSchema, ImageRequest, ImageResponse, ImageSize, } from "./image.js";
9
9
  export { Image } from "./image.js";
10
+ export { Media } from "./media.js";
11
+ export { Generation } from "./generation.js";
10
12
  export { Tool, ToolFailure, toDefinitions } from "./tool.js";
11
13
  export { ToolRuntime } from "./tool-runtime.js";
12
14
  export * as LLM from "./llm.js";
package/dist/llm.d.ts CHANGED
@@ -191,6 +191,14 @@ export declare class GenerateObjectResponse<T> {
191
191
  };
192
192
  } | undefined;
193
193
  readonly namespace?: string | undefined;
194
+ } | {
195
+ readonly type: "media";
196
+ readonly media: import("./media.js").Asset;
197
+ readonly providerMetadata?: {
198
+ readonly [x: string]: {
199
+ readonly [x: string]: unknown;
200
+ };
201
+ } | undefined;
194
202
  } | {
195
203
  readonly type: "step-finish";
196
204
  readonly reason: {
@@ -219,12 +227,12 @@ export declare class GenerateObjectResponse<T> {
219
227
  } | {
220
228
  readonly type: "provider-error";
221
229
  readonly message: string;
230
+ readonly classification?: "context-overflow" | "payload-too-large" | undefined;
222
231
  readonly providerMetadata?: {
223
232
  readonly [x: string]: {
224
233
  readonly [x: string]: unknown;
225
234
  };
226
235
  } | undefined;
227
- readonly classification?: "context-overflow" | "payload-too-large" | undefined;
228
236
  })[];
229
237
  get usage(): import("./schema/events.js").Usage | undefined;
230
238
  }
@@ -0,0 +1,212 @@
1
+ export * as Media from "./media.js";
2
+ import { Effect, FileSystem, Schema } from "effect";
3
+ import { ProviderID } from "./schema/ids.js";
4
+ import { AIError } from "./schema/errors.js";
5
+ import { ProviderMetadata } from "./schema/options.js";
6
+ import { Service } from "./route/executor-service.js";
7
+ export { detectMediaType } from "./utils/media-type.js";
8
+ export declare const Source: Schema.Union<readonly [Schema.Struct<{
9
+ readonly type: Schema.Literal<"bytes">;
10
+ readonly data: Schema.Uint8Array;
11
+ readonly mediaType: Schema.String;
12
+ }>, Schema.Struct<{
13
+ readonly type: Schema.Literal<"base64">;
14
+ readonly data: Schema.String;
15
+ readonly mediaType: Schema.String;
16
+ }>, Schema.Struct<{
17
+ readonly type: Schema.Literal<"url">;
18
+ readonly url: Schema.String;
19
+ readonly mediaType: Schema.optional<Schema.String>;
20
+ /** Epoch milliseconds after which the provider no longer serves the URL. */
21
+ readonly expiresAt: Schema.optional<Schema.Number>;
22
+ /** Headers required to fetch the URL, such as provider auth for Veo downloads. */
23
+ readonly headers: Schema.optional<Schema.$Record<Schema.String, Schema.String>>;
24
+ }>, Schema.Struct<{
25
+ readonly type: Schema.Literal<"ref">;
26
+ readonly provider: Schema.brand<Schema.String, "AI.ProviderID">;
27
+ readonly id: Schema.String;
28
+ readonly mediaType: Schema.optional<Schema.String>;
29
+ }>]>;
30
+ export type Source = Schema.Schema.Type<typeof Source>;
31
+ export declare const Kind: Schema.Literals<readonly ["image", "video", "audio", "document", "other"]>;
32
+ export type Kind = Schema.Schema.Type<typeof Kind>;
33
+ export declare const kindOf: (mediaType: string) => Kind;
34
+ /** Container-independent facts about the payload; raw PCM audio relies on these because it has no header. */
35
+ export declare const Info: Schema.Struct<{
36
+ readonly width: Schema.optional<Schema.Number>;
37
+ readonly height: Schema.optional<Schema.Number>;
38
+ readonly durationSeconds: Schema.optional<Schema.Number>;
39
+ readonly sampleRate: Schema.optional<Schema.Number>;
40
+ readonly channels: Schema.optional<Schema.Number>;
41
+ readonly encoding: Schema.optional<Schema.String>;
42
+ readonly format: Schema.optional<Schema.String>;
43
+ }>;
44
+ export type Info = Schema.Schema.Type<typeof Info>;
45
+ /** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */
46
+ export declare const Notice: Schema.Struct<{
47
+ readonly type: Schema.Literals<readonly ["moderated", "filtered", "other"]>;
48
+ readonly message: Schema.String;
49
+ readonly providerMetadata: Schema.optional<Schema.$Record<Schema.String, Schema.$Record<Schema.String, Schema.Unknown>>>;
50
+ }>;
51
+ export type Notice = Schema.Schema.Type<typeof Notice>;
52
+ /** Synchronous view of an inline payload; `undefined` for `url` and `ref` sources, which carry no local bytes. */
53
+ export interface Inline {
54
+ readonly mime: string;
55
+ readonly base64: string;
56
+ readonly dataUrl: string;
57
+ }
58
+ export declare class Asset {
59
+ #private;
60
+ readonly source: Source;
61
+ /** Derived from the source: declared type, sniffed magic bytes, then `application/octet-stream`. */
62
+ readonly mediaType: string;
63
+ readonly kind: Kind;
64
+ readonly info?: Info;
65
+ /** Epoch milliseconds after which a `url` source stops resolving. */
66
+ readonly expiresAt?: number;
67
+ readonly providerMetadata?: ProviderMetadata;
68
+ constructor(input: Asset.Input);
69
+ /** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
70
+ inline(): Inline | undefined;
71
+ /** Decoded payload; downloads `url` sources through the request executor and caches the result. */
72
+ bytes(): Effect.Effect<Uint8Array, AIError, Service>;
73
+ base64(): Effect.Effect<string, AIError, Service>;
74
+ dataUrl(): Effect.Effect<string, AIError, Service>;
75
+ /**
76
+ * The `AssetEncoded` JSON form with `bytes` sources as base64, matching `Schema.toCodecJson(AssetSchema)`, so a
77
+ * plain `JSON.stringify` of messages or events stays lossless and decodes back through the JSON codec.
78
+ */
79
+ toJSON(): {
80
+ source: {
81
+ readonly type: "base64";
82
+ readonly data: string;
83
+ readonly mediaType: string;
84
+ } | {
85
+ readonly type: "url";
86
+ readonly url: string;
87
+ readonly headers?: {
88
+ readonly [x: string]: string;
89
+ } | undefined;
90
+ readonly mediaType?: string | undefined;
91
+ readonly expiresAt?: number | undefined;
92
+ } | {
93
+ readonly id: string;
94
+ readonly type: "ref";
95
+ readonly provider: string & import("effect/Brand").Brand<"AI.ProviderID">;
96
+ readonly mediaType?: string | undefined;
97
+ } | {
98
+ data: string;
99
+ type: "bytes";
100
+ mediaType: string;
101
+ };
102
+ info: {
103
+ readonly format?: string | undefined;
104
+ readonly width?: number | undefined;
105
+ readonly height?: number | undefined;
106
+ readonly durationSeconds?: number | undefined;
107
+ readonly sampleRate?: number | undefined;
108
+ readonly channels?: number | undefined;
109
+ readonly encoding?: string | undefined;
110
+ } | undefined;
111
+ providerMetadata: {
112
+ readonly [x: string]: {
113
+ readonly [x: string]: unknown;
114
+ };
115
+ } | undefined;
116
+ };
117
+ /** Pull `url` sources into owned bytes before the URL expires. Inline sources return themselves. */
118
+ materialize(): Effect.Effect<Asset, AIError, Service>;
119
+ }
120
+ export declare namespace Asset {
121
+ interface Input {
122
+ readonly source: Source;
123
+ readonly info?: Info;
124
+ readonly providerMetadata?: ProviderMetadata;
125
+ }
126
+ }
127
+ /** JSON form of an asset: the serializable `Source` plus caller-supplied metadata. `bytes` sources encode as base64. */
128
+ export declare const AssetEncoded: Schema.Struct<{
129
+ readonly source: Schema.Union<readonly [Schema.Struct<{
130
+ readonly type: Schema.Literal<"bytes">;
131
+ readonly data: Schema.Uint8Array;
132
+ readonly mediaType: Schema.String;
133
+ }>, Schema.Struct<{
134
+ readonly type: Schema.Literal<"base64">;
135
+ readonly data: Schema.String;
136
+ readonly mediaType: Schema.String;
137
+ }>, Schema.Struct<{
138
+ readonly type: Schema.Literal<"url">;
139
+ readonly url: Schema.String;
140
+ readonly mediaType: Schema.optional<Schema.String>;
141
+ /** Epoch milliseconds after which the provider no longer serves the URL. */
142
+ readonly expiresAt: Schema.optional<Schema.Number>;
143
+ /** Headers required to fetch the URL, such as provider auth for Veo downloads. */
144
+ readonly headers: Schema.optional<Schema.$Record<Schema.String, Schema.String>>;
145
+ }>, Schema.Struct<{
146
+ readonly type: Schema.Literal<"ref">;
147
+ readonly provider: Schema.brand<Schema.String, "AI.ProviderID">;
148
+ readonly id: Schema.String;
149
+ readonly mediaType: Schema.optional<Schema.String>;
150
+ }>]>;
151
+ readonly info: Schema.optional<Schema.Struct<{
152
+ readonly width: Schema.optional<Schema.Number>;
153
+ readonly height: Schema.optional<Schema.Number>;
154
+ readonly durationSeconds: Schema.optional<Schema.Number>;
155
+ readonly sampleRate: Schema.optional<Schema.Number>;
156
+ readonly channels: Schema.optional<Schema.Number>;
157
+ readonly encoding: Schema.optional<Schema.String>;
158
+ readonly format: Schema.optional<Schema.String>;
159
+ }>>;
160
+ readonly providerMetadata: Schema.optional<Schema.$Record<Schema.String, Schema.$Record<Schema.String, Schema.Unknown>>>;
161
+ }>;
162
+ /** `Asset` in the type domain and `AssetEncoded` on the wire, so messages and events holding assets serialize. */
163
+ export declare const AssetSchema: Schema.decodeTo<Schema.declare<Asset, Asset>, Schema.Struct<{
164
+ readonly source: Schema.Union<readonly [Schema.Struct<{
165
+ readonly type: Schema.Literal<"bytes">;
166
+ readonly data: Schema.Uint8Array;
167
+ readonly mediaType: Schema.String;
168
+ }>, Schema.Struct<{
169
+ readonly type: Schema.Literal<"base64">;
170
+ readonly data: Schema.String;
171
+ readonly mediaType: Schema.String;
172
+ }>, Schema.Struct<{
173
+ readonly type: Schema.Literal<"url">;
174
+ readonly url: Schema.String;
175
+ readonly mediaType: Schema.optional<Schema.String>;
176
+ /** Epoch milliseconds after which the provider no longer serves the URL. */
177
+ readonly expiresAt: Schema.optional<Schema.Number>;
178
+ /** Headers required to fetch the URL, such as provider auth for Veo downloads. */
179
+ readonly headers: Schema.optional<Schema.$Record<Schema.String, Schema.String>>;
180
+ }>, Schema.Struct<{
181
+ readonly type: Schema.Literal<"ref">;
182
+ readonly provider: Schema.brand<Schema.String, "AI.ProviderID">;
183
+ readonly id: Schema.String;
184
+ readonly mediaType: Schema.optional<Schema.String>;
185
+ }>]>;
186
+ readonly info: Schema.optional<Schema.Struct<{
187
+ readonly width: Schema.optional<Schema.Number>;
188
+ readonly height: Schema.optional<Schema.Number>;
189
+ readonly durationSeconds: Schema.optional<Schema.Number>;
190
+ readonly sampleRate: Schema.optional<Schema.Number>;
191
+ readonly channels: Schema.optional<Schema.Number>;
192
+ readonly encoding: Schema.optional<Schema.String>;
193
+ readonly format: Schema.optional<Schema.String>;
194
+ }>>;
195
+ readonly providerMetadata: Schema.optional<Schema.$Record<Schema.String, Schema.$Record<Schema.String, Schema.Unknown>>>;
196
+ }>, never, never>;
197
+ export type AssetOptions = Omit<Asset.Input, "source">;
198
+ export declare const from: (source: Source, options?: AssetOptions) => Asset;
199
+ export declare const bytes: (data: Uint8Array, mediaType?: string, options?: AssetOptions) => Asset;
200
+ export declare const base64: (data: string, mediaType: string, options?: AssetOptions) => Asset;
201
+ export declare const url: (value: string, options?: AssetOptions & Omit<Extract<Source, {
202
+ readonly type: "url";
203
+ }>, "type" | "url">) => Asset;
204
+ export declare const ref: (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) => Asset;
205
+ /** Parse a `data:<mime>;base64,<data>` URL, or `undefined` when the value is not a base64 data URL. */
206
+ export declare const parseDataUrl: (value: string, options?: AssetOptions) => Asset | undefined;
207
+ /** Parse a `data:<mime>;base64,<data>` URL. Malformed input throws a typed `AIError` because constructors are sync. */
208
+ export declare const fromDataUrl: (dataUrl: string, options?: AssetOptions) => Asset;
209
+ /** Read a file through `FileSystem` and sniff its media type from magic bytes, then the extension. */
210
+ export declare const file: (path: string, options?: AssetOptions) => Effect.Effect<Asset, AIError, FileSystem.FileSystem>;
211
+ /** Materialize an asset and write its bytes through `FileSystem`. */
212
+ export declare const write: (asset: Asset, path: string) => Effect.Effect<void, AIError, FileSystem.FileSystem | Service>;
package/dist/media.js ADDED
@@ -0,0 +1,228 @@
1
+ export * as Media from "./media.js";
2
+ import { Effect, Encoding, FileSystem, Schema, SchemaGetter } from "effect";
3
+ import { HttpClientRequest } from "effect/unstable/http";
4
+ import { ProviderID } from "./schema/ids.js";
5
+ import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js";
6
+ import { ProviderMetadata } from "./schema/options.js";
7
+ import { Service } from "./route/executor-service.js";
8
+ import { detectMediaType, extensionMediaType } from "./utils/media-type.js";
9
+ export { detectMediaType } from "./utils/media-type.js";
10
+ const OCTET_STREAM = "application/octet-stream";
11
+ // ---------------------------------------------------------------------------
12
+ // Source — the serializable wire/persistence form of a media asset
13
+ // ---------------------------------------------------------------------------
14
+ const BytesSource = Schema.Struct({
15
+ type: Schema.Literal("bytes"),
16
+ data: Schema.Uint8Array,
17
+ mediaType: Schema.String,
18
+ });
19
+ const Base64Source = Schema.Struct({
20
+ type: Schema.Literal("base64"),
21
+ data: Schema.String,
22
+ mediaType: Schema.String,
23
+ });
24
+ const UrlSource = Schema.Struct({
25
+ type: Schema.Literal("url"),
26
+ url: Schema.String,
27
+ mediaType: Schema.optional(Schema.String),
28
+ /** Epoch milliseconds after which the provider no longer serves the URL. */
29
+ expiresAt: Schema.optional(Schema.Number),
30
+ /** Headers required to fetch the URL, such as provider auth for Veo downloads. */
31
+ headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
32
+ });
33
+ /** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
34
+ const RefSource = Schema.Struct({
35
+ type: Schema.Literal("ref"),
36
+ provider: ProviderID,
37
+ id: Schema.String,
38
+ mediaType: Schema.optional(Schema.String),
39
+ });
40
+ export const Source = Schema.Union([BytesSource, Base64Source, UrlSource, RefSource])
41
+ .pipe(Schema.toTaggedUnion("type"))
42
+ .annotate({ identifier: "Media.Source" });
43
+ // ---------------------------------------------------------------------------
44
+ // Kind, Info, Notice
45
+ // ---------------------------------------------------------------------------
46
+ export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"]);
47
+ export const kindOf = (mediaType) => {
48
+ const lower = mediaType.toLowerCase();
49
+ if (lower.startsWith("image/"))
50
+ return "image";
51
+ if (lower.startsWith("video/"))
52
+ return "video";
53
+ if (lower.startsWith("audio/"))
54
+ return "audio";
55
+ if (lower === "application/pdf" || lower.startsWith("text/"))
56
+ return "document";
57
+ return "other";
58
+ };
59
+ /** Container-independent facts about the payload; raw PCM audio relies on these because it has no header. */
60
+ export const Info = Schema.Struct({
61
+ width: Schema.optional(Schema.Number),
62
+ height: Schema.optional(Schema.Number),
63
+ durationSeconds: Schema.optional(Schema.Number),
64
+ sampleRate: Schema.optional(Schema.Number),
65
+ channels: Schema.optional(Schema.Number),
66
+ encoding: Schema.optional(Schema.String),
67
+ format: Schema.optional(Schema.String),
68
+ }).annotate({ identifier: "Media.Info" });
69
+ /** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */
70
+ export const Notice = Schema.Struct({
71
+ type: Schema.Literals(["moderated", "filtered", "other"]),
72
+ message: Schema.String,
73
+ providerMetadata: Schema.optional(ProviderMetadata),
74
+ }).annotate({ identifier: "Media.Notice" });
75
+ // ---------------------------------------------------------------------------
76
+ // Asset
77
+ // ---------------------------------------------------------------------------
78
+ const invalid = (message, cause) => new AIError({ reason: new InvalidRequestError({ message, cause }) });
79
+ export class Asset {
80
+ source;
81
+ /** Derived from the source: declared type, sniffed magic bytes, then `application/octet-stream`. */
82
+ mediaType;
83
+ kind;
84
+ info;
85
+ /** Epoch milliseconds after which a `url` source stops resolving. */
86
+ expiresAt;
87
+ providerMetadata;
88
+ // Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
89
+ // cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
90
+ // through `source`, so round-tripping through `Media.from(asset.source)` stays lossless.
91
+ #bytes;
92
+ #base64;
93
+ constructor(input) {
94
+ this.source = input.source;
95
+ this.mediaType =
96
+ input.source.mediaType ??
97
+ (input.source.type === "bytes" ? detectMediaType(input.source.data) : undefined) ??
98
+ OCTET_STREAM;
99
+ this.kind = kindOf(this.mediaType);
100
+ this.info = input.info;
101
+ this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined;
102
+ this.providerMetadata = input.providerMetadata;
103
+ }
104
+ /** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
105
+ inline() {
106
+ const source = this.source;
107
+ if (source.type !== "bytes" && source.type !== "base64")
108
+ return undefined;
109
+ const base64 = source.type === "base64" ? source.data : (this.#base64 ??= Encoding.encodeBase64(source.data));
110
+ const mime = this.mediaType.toLowerCase();
111
+ return { mime, base64, dataUrl: `data:${mime};base64,${base64}` };
112
+ }
113
+ /** Decoded payload; downloads `url` sources through the request executor and caches the result. */
114
+ bytes() {
115
+ return Effect.suspend(() => {
116
+ const source = this.source;
117
+ if (source.type === "bytes")
118
+ return Effect.succeed(source.data);
119
+ if (this.#bytes !== undefined)
120
+ return Effect.succeed(this.#bytes);
121
+ if (source.type === "ref")
122
+ return Effect.fail(invalid(`Cannot materialize provider ref ${source.provider}:${source.id}`));
123
+ const decoded = source.type === "base64"
124
+ ? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)))
125
+ : download(source);
126
+ return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))));
127
+ });
128
+ }
129
+ base64() {
130
+ return Effect.suspend(() => {
131
+ const source = this.source;
132
+ if (source.type === "base64")
133
+ return Effect.succeed(source.data);
134
+ if (this.#base64 !== undefined)
135
+ return Effect.succeed(this.#base64);
136
+ return this.bytes().pipe(Effect.map((data) => (this.#base64 = Encoding.encodeBase64(data))));
137
+ });
138
+ }
139
+ dataUrl() {
140
+ return this.base64().pipe(Effect.map((data) => `data:${this.mediaType};base64,${data}`));
141
+ }
142
+ /**
143
+ * The `AssetEncoded` JSON form with `bytes` sources as base64, matching `Schema.toCodecJson(AssetSchema)`, so a
144
+ * plain `JSON.stringify` of messages or events stays lossless and decodes back through the JSON codec.
145
+ */
146
+ toJSON() {
147
+ const source = this.source;
148
+ return {
149
+ source: source.type === "bytes" ? { ...source, data: Encoding.encodeBase64(source.data) } : source,
150
+ info: this.info,
151
+ providerMetadata: this.providerMetadata,
152
+ };
153
+ }
154
+ /** Pull `url` sources into owned bytes before the URL expires. Inline sources return themselves. */
155
+ materialize() {
156
+ if (this.source.type === "bytes" || this.source.type === "base64")
157
+ return Effect.succeed(this);
158
+ return this.bytes().pipe(Effect.map((data) => bytes(data, this.source.mediaType, { info: this.info, providerMetadata: this.providerMetadata })));
159
+ }
160
+ }
161
+ /** JSON form of an asset: the serializable `Source` plus caller-supplied metadata. `bytes` sources encode as base64. */
162
+ export const AssetEncoded = Schema.Struct({
163
+ source: Source,
164
+ info: Schema.optional(Info),
165
+ providerMetadata: Schema.optional(ProviderMetadata),
166
+ }).annotate({ identifier: "Media.AssetEncoded" });
167
+ const encodeAsset = (asset) => ({
168
+ source: asset.source,
169
+ info: asset.info,
170
+ providerMetadata: asset.providerMetadata,
171
+ });
172
+ const AssetInstance = Schema.declare((value) => value instanceof Asset, {
173
+ expected: "Media.Asset",
174
+ });
175
+ /** `Asset` in the type domain and `AssetEncoded` on the wire, so messages and events holding assets serialize. */
176
+ export const AssetSchema = AssetEncoded.pipe(Schema.decodeTo(AssetInstance, {
177
+ decode: SchemaGetter.transform((encoded) => new Asset(encoded)),
178
+ encode: SchemaGetter.transform(encodeAsset),
179
+ }));
180
+ const download = Effect.fn("Media.download")(function* (source) {
181
+ const executor = yield* Service;
182
+ const response = yield* executor.execute(HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})));
183
+ const buffer = yield* response.arrayBuffer.pipe(Effect.mapError((cause) => new AIError({
184
+ reason: new InvalidProviderOutputError({
185
+ message: `Failed to read media from ${source.url}`,
186
+ http: new HttpContext({ url: response.request.url, status: response.status, headers: response.headers }),
187
+ cause,
188
+ }),
189
+ })));
190
+ return new Uint8Array(buffer);
191
+ });
192
+ export const from = (source, options) => new Asset({ ...options, source });
193
+ export const bytes = (data, mediaType, options) => from({ type: "bytes", data, mediaType: mediaType ?? detectMediaType(data) ?? OCTET_STREAM }, options);
194
+ export const base64 = (data, mediaType, options) => from({ type: "base64", data, mediaType }, options);
195
+ export const url = (value, options) => {
196
+ const { mediaType, expiresAt, headers, ...rest } = options ?? {};
197
+ return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest);
198
+ };
199
+ export const ref = (provider, id, mediaType, options) => from({ type: "ref", provider: ProviderID.make(provider), id, mediaType }, options);
200
+ const DATA_URL = /^data:([^;,]+)(?:;[^,]*)*;base64,(.*)$/s;
201
+ /** Parse a `data:<mime>;base64,<data>` URL, or `undefined` when the value is not a base64 data URL. */
202
+ export const parseDataUrl = (value, options) => {
203
+ const match = DATA_URL.exec(value);
204
+ return match === null ? undefined : base64(match[2], match[1], options);
205
+ };
206
+ /** Parse a `data:<mime>;base64,<data>` URL. Malformed input throws a typed `AIError` because constructors are sync. */
207
+ export const fromDataUrl = (dataUrl, options) => {
208
+ const asset = parseDataUrl(dataUrl, options);
209
+ if (asset === undefined)
210
+ throw invalid("Media data URLs must contain a MIME type and base64 data");
211
+ return asset;
212
+ };
213
+ /** Read a file through `FileSystem` and sniff its media type from magic bytes, then the extension. */
214
+ export const file = (path, options) => Effect.gen(function* () {
215
+ const fs = yield* FileSystem.FileSystem;
216
+ const data = yield* fs
217
+ .readFile(path)
218
+ .pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)));
219
+ return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options);
220
+ });
221
+ /** Materialize an asset and write its bytes through `FileSystem`. */
222
+ export const write = (asset, path) => Effect.gen(function* () {
223
+ const fs = yield* FileSystem.FileSystem;
224
+ const data = yield* asset.bytes();
225
+ yield* fs
226
+ .writeFile(path, data)
227
+ .pipe(Effect.mapError((cause) => invalid(`Failed to write media file ${path}`, cause)));
228
+ });