@opencode/ai 2.0.15 → 2.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +286 -2
  2. package/dist/generation.d.ts +36 -22
  3. package/dist/generation.js +53 -24
  4. package/dist/image-client.d.ts +14 -7
  5. package/dist/image-client.js +24 -8
  6. package/dist/image.d.ts +398 -47
  7. package/dist/image.js +47 -45
  8. package/dist/index.d.ts +13 -1
  9. package/dist/index.js +9 -0
  10. package/dist/media-model.d.ts +44 -0
  11. package/dist/media-model.js +49 -0
  12. package/dist/media.d.ts +10 -9
  13. package/dist/media.js +9 -10
  14. package/dist/promise.d.ts +428 -8
  15. package/dist/promise.js +40 -3
  16. package/dist/protocols/alibaba-chat.d.ts +12 -0
  17. package/dist/protocols/alibaba-responses.d.ts +2 -2
  18. package/dist/protocols/anthropic-messages.js +1 -2
  19. package/dist/protocols/assemblyai-transcription.d.ts +40 -0
  20. package/dist/protocols/assemblyai-transcription.js +138 -0
  21. package/dist/protocols/bedrock-converse.js +5 -11
  22. package/dist/protocols/bfl-images.d.ts +32 -0
  23. package/dist/protocols/bfl-images.js +153 -0
  24. package/dist/protocols/cartesia-speech.d.ts +127 -0
  25. package/dist/protocols/cartesia-speech.js +126 -0
  26. package/dist/protocols/deepgram-speech.d.ts +119 -0
  27. package/dist/protocols/deepgram-speech.js +92 -0
  28. package/dist/protocols/deepgram-transcription.d.ts +25 -0
  29. package/dist/protocols/deepgram-transcription.js +129 -0
  30. package/dist/protocols/elevenlabs-speech.d.ts +122 -0
  31. package/dist/protocols/elevenlabs-speech.js +115 -0
  32. package/dist/protocols/fal-images.d.ts +24 -0
  33. package/dist/protocols/fal-images.js +114 -0
  34. package/dist/protocols/fal-video.d.ts +29 -0
  35. package/dist/protocols/fal-video.js +88 -0
  36. package/dist/protocols/gemini.d.ts +9 -9
  37. package/dist/protocols/gemini.js +8 -34
  38. package/dist/protocols/google-images.js +2 -14
  39. package/dist/protocols/google-speech.d.ts +130 -0
  40. package/dist/protocols/google-speech.js +84 -0
  41. package/dist/protocols/google-transcription.d.ts +173 -0
  42. package/dist/protocols/google-transcription.js +138 -0
  43. package/dist/protocols/google-video.d.ts +26 -0
  44. package/dist/protocols/google-video.js +158 -0
  45. package/dist/protocols/meta-images.js +2 -9
  46. package/dist/protocols/meta-responses.d.ts +4 -4
  47. package/dist/protocols/meta-responses.js +1 -1
  48. package/dist/protocols/open-responses.d.ts +6 -6
  49. package/dist/protocols/open-responses.js +1 -2
  50. package/dist/protocols/openai-chat.d.ts +84 -0
  51. package/dist/protocols/openai-chat.js +26 -14
  52. package/dist/protocols/openai-compatible-chat.d.ts +12 -0
  53. package/dist/protocols/openai-compatible-responses.d.ts +2 -2
  54. package/dist/protocols/openai-images.d.ts +124 -3
  55. package/dist/protocols/openai-images.js +107 -54
  56. package/dist/protocols/openai-responses.d.ts +15 -15
  57. package/dist/protocols/openai-responses.js +5 -6
  58. package/dist/protocols/openai-speech.d.ts +116 -0
  59. package/dist/protocols/openai-speech.js +98 -0
  60. package/dist/protocols/openai-transcription.d.ts +207 -0
  61. package/dist/protocols/openai-transcription.js +190 -0
  62. package/dist/protocols/replicate-images.d.ts +28 -0
  63. package/dist/protocols/replicate-images.js +133 -0
  64. package/dist/protocols/runway-video.d.ts +38 -0
  65. package/dist/protocols/runway-video.js +146 -0
  66. package/dist/protocols/shared.d.ts +13 -3
  67. package/dist/protocols/shared.js +23 -3
  68. package/dist/protocols/stability-images.d.ts +38 -0
  69. package/dist/protocols/stability-images.js +148 -0
  70. package/dist/protocols/utils/fal-queue.d.ts +28 -0
  71. package/dist/protocols/utils/fal-queue.js +69 -0
  72. package/dist/protocols/utils/gemini-generate-content.d.ts +65 -0
  73. package/dist/protocols/utils/gemini-generate-content.js +65 -0
  74. package/dist/protocols/utils/gemini-json-schema.d.ts +3 -0
  75. package/dist/protocols/utils/gemini-json-schema.js +76 -0
  76. package/dist/protocols/utils/media-input.d.ts +8 -0
  77. package/dist/protocols/utils/media-input.js +18 -0
  78. package/dist/protocols/utils/speech-stream.d.ts +49 -0
  79. package/dist/protocols/utils/speech-stream.js +67 -0
  80. package/dist/protocols/utils/tool-schema.d.ts +2 -2
  81. package/dist/protocols/utils/tool-schema.js +40 -17
  82. package/dist/protocols/xai-images.js +1 -12
  83. package/dist/protocols/xai-responses.d.ts +2 -2
  84. package/dist/protocols/xai-video.d.ts +34 -0
  85. package/dist/protocols/xai-video.js +147 -0
  86. package/dist/protocols/zai-chat.d.ts +13 -1
  87. package/dist/provider-error.js +3 -0
  88. package/dist/providers/alibaba.d.ts +14 -2
  89. package/dist/providers/amazon-bedrock-mantle.d.ts +14 -2
  90. package/dist/providers/assemblyai.d.ts +25 -0
  91. package/dist/providers/assemblyai.js +29 -0
  92. package/dist/providers/azure.d.ts +18 -6
  93. package/dist/providers/baseten.d.ts +24 -0
  94. package/dist/providers/black-forest-labs.d.ts +25 -0
  95. package/dist/providers/black-forest-labs.js +28 -0
  96. package/dist/providers/cartesia.d.ts +24 -0
  97. package/dist/providers/cartesia.js +22 -0
  98. package/dist/providers/cerebras.d.ts +24 -0
  99. package/dist/providers/cloudflare-ai-gateway.d.ts +30 -6
  100. package/dist/providers/cloudflare-workers-ai.d.ts +24 -0
  101. package/dist/providers/deepgram.d.ts +29 -0
  102. package/dist/providers/deepgram.js +31 -0
  103. package/dist/providers/deepinfra.d.ts +24 -0
  104. package/dist/providers/deepseek.d.ts +24 -0
  105. package/dist/providers/elevenlabs.d.ts +24 -0
  106. package/dist/providers/elevenlabs.js +28 -0
  107. package/dist/providers/fal.d.ts +29 -0
  108. package/dist/providers/fal.js +33 -0
  109. package/dist/providers/fireworks.d.ts +24 -0
  110. package/dist/providers/google-vertex-chat.d.ts +12 -0
  111. package/dist/providers/google-vertex-responses.d.ts +2 -2
  112. package/dist/providers/google-vertex.d.ts +3 -3
  113. package/dist/providers/google.d.ts +18 -3
  114. package/dist/providers/google.js +11 -2
  115. package/dist/providers/groq.d.ts +24 -0
  116. package/dist/providers/index.d.ts +9 -0
  117. package/dist/providers/index.js +9 -0
  118. package/dist/providers/meta.d.ts +14 -2
  119. package/dist/providers/minimax.d.ts +14 -2
  120. package/dist/providers/moonshot.d.ts +14 -2
  121. package/dist/providers/moonshot.js +3 -3
  122. package/dist/providers/openai-compatible-responses.d.ts +2 -2
  123. package/dist/providers/openai-compatible.d.ts +12 -0
  124. package/dist/providers/openai.d.ts +25 -3
  125. package/dist/providers/openai.js +10 -1
  126. package/dist/providers/openrouter.d.ts +48 -0
  127. package/dist/providers/replicate.d.ts +25 -0
  128. package/dist/providers/replicate.js +22 -0
  129. package/dist/providers/runway.d.ts +24 -0
  130. package/dist/providers/runway.js +22 -0
  131. package/dist/providers/stability.d.ts +28 -0
  132. package/dist/providers/stability.js +23 -0
  133. package/dist/providers/togetherai.d.ts +24 -0
  134. package/dist/providers/xai.d.ts +17 -0
  135. package/dist/providers/xai.js +5 -2
  136. package/dist/providers/zai-coding-plan.d.ts +15 -3
  137. package/dist/providers/zai.d.ts +13 -1
  138. package/dist/route/auth.d.ts +4 -1
  139. package/dist/route/auth.js +6 -0
  140. package/dist/route/framing.d.ts +5 -1
  141. package/dist/route/framing.js +9 -0
  142. package/dist/route/media-protocol.d.ts +116 -3
  143. package/dist/route/media-protocol.js +60 -4
  144. package/dist/route/media.d.ts +58 -7
  145. package/dist/route/media.js +201 -29
  146. package/dist/schema/events.d.ts +0 -6
  147. package/dist/schema/messages.d.ts +0 -3
  148. package/dist/schema/options.d.ts +4 -3
  149. package/dist/schema/options.js +3 -2
  150. package/dist/speech-client.d.ts +21 -0
  151. package/dist/speech-client.js +25 -0
  152. package/dist/speech.d.ts +1150 -0
  153. package/dist/speech.js +119 -0
  154. package/dist/transcription-client.d.ts +28 -0
  155. package/dist/transcription-client.js +44 -0
  156. package/dist/transcription.d.ts +1504 -0
  157. package/dist/transcription.js +133 -0
  158. package/dist/utils/bytes.d.ts +1 -0
  159. package/dist/utils/bytes.js +10 -0
  160. package/dist/utils/media-type.d.ts +1 -0
  161. package/dist/utils/media-type.js +22 -1
  162. package/dist/video-client.d.ts +28 -0
  163. package/dist/video-client.js +40 -0
  164. package/dist/video.d.ts +1359 -0
  165. package/dist/video.js +119 -0
  166. package/package.json +3 -3
  167. package/dist/protocols/utils/gemini-tool-schema.d.ts +0 -2
  168. package/dist/protocols/utils/gemini-tool-schema.js +0 -103
@@ -0,0 +1,207 @@
1
+ import { Schema } from "effect";
2
+ import { MediaProtocol } from "../route/media-protocol.js";
3
+ import { MediaRoute } from "../route/media.js";
4
+ import { TranscriptionModel, type TranscriptionRequestFor, type TranscriptionSegment } from "../transcription.js";
5
+ export declare const DEFAULT_BASE_URL = "https://api.openai.com/v1";
6
+ export declare const PATH = "/audio/transcriptions";
7
+ export type OpenAITranscriptionOptions = {
8
+ readonly chunking_strategy?: "auto" | {
9
+ readonly type: "server_vad";
10
+ readonly prefix_padding_ms?: number;
11
+ readonly silence_duration_ms?: number;
12
+ readonly threshold?: number;
13
+ };
14
+ readonly include?: ReadonlyArray<"logprobs">;
15
+ readonly keywords?: ReadonlyArray<string>;
16
+ readonly known_speaker_names?: ReadonlyArray<string>;
17
+ readonly known_speaker_references?: ReadonlyArray<string>;
18
+ readonly temperature?: number;
19
+ } & Record<string, unknown>;
20
+ export type Request = TranscriptionRequestFor<OpenAITranscriptionOptions>;
21
+ declare const Transcript: Schema.Struct<{
22
+ text: Schema.String;
23
+ language: Schema.optional<Schema.String>;
24
+ languages: Schema.optional<Schema.$Array<Schema.Struct<{
25
+ readonly code: Schema.String;
26
+ }>>>;
27
+ duration: Schema.optional<Schema.Number>;
28
+ segments: Schema.optional<Schema.$Array<Schema.Struct<{
29
+ readonly text: Schema.String;
30
+ readonly start: Schema.Number;
31
+ readonly end: Schema.Number;
32
+ readonly speaker: Schema.optional<Schema.String>;
33
+ }>>>;
34
+ words: Schema.optional<Schema.$Array<Schema.Struct<{
35
+ readonly word: Schema.String;
36
+ readonly start: Schema.Number;
37
+ readonly end: Schema.Number;
38
+ }>>>;
39
+ usage: Schema.optional<Schema.Union<readonly [Schema.Struct<{
40
+ readonly type: Schema.Literal<"tokens">;
41
+ readonly input_tokens: Schema.optional<Schema.Number>;
42
+ readonly output_tokens: Schema.optional<Schema.Number>;
43
+ readonly total_tokens: Schema.optional<Schema.Number>;
44
+ }>, Schema.Struct<{
45
+ readonly type: Schema.Literal<"duration">;
46
+ readonly seconds: Schema.Number;
47
+ }>]>>;
48
+ }>;
49
+ type Transcript = Schema.Schema.Type<typeof Transcript>;
50
+ type Frame = string | {
51
+ readonly document: string;
52
+ };
53
+ interface State {
54
+ readonly segments: Array<TranscriptionSegment>;
55
+ readonly transcript?: Transcript;
56
+ }
57
+ export declare const protocol: MediaProtocol.Streamed<Request, {
58
+ readonly id: string;
59
+ readonly type: "generation-queued";
60
+ readonly position?: number | undefined;
61
+ } | {
62
+ readonly id: string;
63
+ readonly type: "generation-progress";
64
+ readonly progress?: number | undefined;
65
+ } | {
66
+ readonly type: "text-delta";
67
+ readonly delta: string;
68
+ } | {
69
+ readonly type: "segment";
70
+ readonly segment: {
71
+ readonly text: string;
72
+ readonly startSeconds: number;
73
+ readonly endSeconds: number;
74
+ readonly speaker?: string | undefined;
75
+ };
76
+ } | {
77
+ readonly type: "finish";
78
+ readonly text: string;
79
+ readonly durationSeconds?: number | undefined;
80
+ readonly providerMetadata?: {
81
+ readonly [x: string]: {
82
+ readonly [x: string]: unknown;
83
+ };
84
+ } | undefined;
85
+ readonly usage?: {
86
+ readonly type: "tokens";
87
+ readonly input?: number | undefined;
88
+ readonly output?: number | undefined;
89
+ readonly total?: number | undefined;
90
+ readonly details?: {
91
+ readonly [x: string]: unknown;
92
+ } | undefined;
93
+ } | {
94
+ readonly type: "seconds";
95
+ readonly seconds: number;
96
+ } | {
97
+ readonly type: "characters";
98
+ readonly characters: number;
99
+ } | {
100
+ readonly type: "credits";
101
+ readonly credits: number;
102
+ } | {
103
+ readonly type: "compute";
104
+ readonly seconds: number;
105
+ } | undefined;
106
+ readonly notices?: readonly {
107
+ readonly type: "other" | "moderated" | "filtered";
108
+ readonly message: string;
109
+ readonly providerMetadata?: {
110
+ readonly [x: string]: {
111
+ readonly [x: string]: unknown;
112
+ };
113
+ } | undefined;
114
+ }[] | undefined;
115
+ readonly language?: string | undefined;
116
+ readonly segments?: readonly {
117
+ readonly text: string;
118
+ readonly startSeconds: number;
119
+ readonly endSeconds: number;
120
+ readonly speaker?: string | undefined;
121
+ }[] | undefined;
122
+ readonly words?: readonly {
123
+ readonly text: string;
124
+ readonly startSeconds: number;
125
+ readonly endSeconds: number;
126
+ readonly speaker?: string | undefined;
127
+ readonly confidence?: number | undefined;
128
+ }[] | undefined;
129
+ }, Frame, State>;
130
+ export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<OpenAITranscriptionOptions>;
131
+ export declare const OpenAITranscription: {
132
+ readonly protocol: MediaProtocol.Streamed<Request, {
133
+ readonly id: string;
134
+ readonly type: "generation-queued";
135
+ readonly position?: number | undefined;
136
+ } | {
137
+ readonly id: string;
138
+ readonly type: "generation-progress";
139
+ readonly progress?: number | undefined;
140
+ } | {
141
+ readonly type: "text-delta";
142
+ readonly delta: string;
143
+ } | {
144
+ readonly type: "segment";
145
+ readonly segment: {
146
+ readonly text: string;
147
+ readonly startSeconds: number;
148
+ readonly endSeconds: number;
149
+ readonly speaker?: string | undefined;
150
+ };
151
+ } | {
152
+ readonly type: "finish";
153
+ readonly text: string;
154
+ readonly durationSeconds?: number | undefined;
155
+ readonly providerMetadata?: {
156
+ readonly [x: string]: {
157
+ readonly [x: string]: unknown;
158
+ };
159
+ } | undefined;
160
+ readonly usage?: {
161
+ readonly type: "tokens";
162
+ readonly input?: number | undefined;
163
+ readonly output?: number | undefined;
164
+ readonly total?: number | undefined;
165
+ readonly details?: {
166
+ readonly [x: string]: unknown;
167
+ } | undefined;
168
+ } | {
169
+ readonly type: "seconds";
170
+ readonly seconds: number;
171
+ } | {
172
+ readonly type: "characters";
173
+ readonly characters: number;
174
+ } | {
175
+ readonly type: "credits";
176
+ readonly credits: number;
177
+ } | {
178
+ readonly type: "compute";
179
+ readonly seconds: number;
180
+ } | undefined;
181
+ readonly notices?: readonly {
182
+ readonly type: "other" | "moderated" | "filtered";
183
+ readonly message: string;
184
+ readonly providerMetadata?: {
185
+ readonly [x: string]: {
186
+ readonly [x: string]: unknown;
187
+ };
188
+ } | undefined;
189
+ }[] | undefined;
190
+ readonly language?: string | undefined;
191
+ readonly segments?: readonly {
192
+ readonly text: string;
193
+ readonly startSeconds: number;
194
+ readonly endSeconds: number;
195
+ readonly speaker?: string | undefined;
196
+ }[] | undefined;
197
+ readonly words?: readonly {
198
+ readonly text: string;
199
+ readonly startSeconds: number;
200
+ readonly endSeconds: number;
201
+ readonly speaker?: string | undefined;
202
+ readonly confidence?: number | undefined;
203
+ }[] | undefined;
204
+ }, Frame, State>;
205
+ readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<OpenAITranscriptionOptions>;
206
+ };
207
+ export {};
@@ -0,0 +1,190 @@
1
+ import { Effect, Schema, Stream } from "effect";
2
+ import { Framing } from "../route/framing.js";
3
+ import { MediaProtocol } from "../route/media-protocol.js";
4
+ import { MediaRoute } from "../route/media.js";
5
+ import { ProviderID, mergeJsonRecords } from "../schema/index.js";
6
+ import { TranscriptionFinishEvent, TranscriptionModel, TranscriptionSegmentEvent, TranscriptionTextDeltaEvent, } from "../transcription.js";
7
+ import { mediaTypeExtension } from "../utils/media-type.js";
8
+ import { ProviderShared } from "./shared.js";
9
+ import { MediaInput } from "./utils/media-input.js";
10
+ const ADAPTER = "openai-transcription";
11
+ const NAME = "OpenAI Transcription";
12
+ const PROVIDER = ProviderID.make("openai");
13
+ export const DEFAULT_BASE_URL = "https://api.openai.com/v1";
14
+ export const PATH = "/audio/transcriptions";
15
+ // ---------------------------------------------------------------------------
16
+ // 3. Streaming event schema
17
+ // ---------------------------------------------------------------------------
18
+ const Segment = Schema.Struct({
19
+ text: Schema.String,
20
+ start: Schema.Number,
21
+ end: Schema.Number,
22
+ speaker: Schema.optional(Schema.String),
23
+ });
24
+ const Usage = Schema.Union([
25
+ Schema.Struct({
26
+ type: Schema.Literal("tokens"),
27
+ input_tokens: Schema.optional(Schema.Number),
28
+ output_tokens: Schema.optional(Schema.Number),
29
+ total_tokens: Schema.optional(Schema.Number),
30
+ }),
31
+ Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
32
+ ]);
33
+ const transcriptFields = {
34
+ text: Schema.String,
35
+ language: Schema.optional(Schema.String),
36
+ languages: Schema.optional(Schema.Array(Schema.Struct({ code: Schema.String }))),
37
+ duration: Schema.optional(Schema.Number),
38
+ segments: Schema.optional(Schema.Array(Segment)),
39
+ words: Schema.optional(Schema.Array(Schema.Struct({ word: Schema.String, start: Schema.Number, end: Schema.Number }))),
40
+ usage: Schema.optional(Usage),
41
+ };
42
+ const Event = Schema.Union([
43
+ Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
44
+ Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
45
+ Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
46
+ ]);
47
+ const Transcript = Schema.Struct(transcriptFields);
48
+ const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, Event);
49
+ const decodeTranscript = MediaProtocol.decodeFrame(ADAPTER, NAME, Transcript);
50
+ const TRANSCRIBE = { stream: true, timestamps: [], diarize: false, languageField: "language" };
51
+ const capabilities = (model) => {
52
+ if (model.startsWith("whisper"))
53
+ return { ...TRANSCRIBE, stream: false, timestamps: ["segment", "word"] };
54
+ if (model.includes("diarize"))
55
+ return { ...TRANSCRIBE, timestamps: ["segment"], diarize: true };
56
+ // `gpt-transcribe` replaces `language` with `languages[]` and rejects both together.
57
+ if (model.startsWith("gpt-transcribe"))
58
+ return { ...TRANSCRIBE, languageField: "languages" };
59
+ return TRANSCRIBE;
60
+ };
61
+ const unsupported = (operation, message) => Effect.fail(ProviderShared.unsupportedOperation({ operation, provider: PROVIDER, route: ADAPTER, message }));
62
+ const validate = (request, model) => {
63
+ const id = request.model.id;
64
+ if (request.mode === "stream" && !model.stream)
65
+ return unsupported("media.stream", `${id} does not stream; use Transcription.generate`);
66
+ if (request.diarize === true && !model.diarize)
67
+ return unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`);
68
+ if (request.prompt !== undefined && model.diarize)
69
+ return unsupported("media.prompt", `${id} does not accept a prompt`);
70
+ if (request.timestamps === undefined ||
71
+ request.timestamps === "none" ||
72
+ model.timestamps.includes(request.timestamps))
73
+ return Effect.void;
74
+ return unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`);
75
+ };
76
+ const RESERVED_FORM_FIELDS = new Set([
77
+ "file",
78
+ "model",
79
+ "prompt",
80
+ "language",
81
+ "response_format",
82
+ "timestamp_granularities",
83
+ "stream",
84
+ ]);
85
+ const appendField = (form, key, value) => {
86
+ if (Array.isArray(value))
87
+ return value.forEach((item) => form.append(`${key}[]`, String(item)));
88
+ form.append(key, typeof value === "object" && value !== null ? ProviderShared.encodeJson(value) : String(value));
89
+ };
90
+ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request) {
91
+ const model = capabilities(request.model.id);
92
+ yield* validate(request, model);
93
+ // The API detects the audio format from the upload's filename extension.
94
+ const extension = mediaTypeExtension(request.audio.mediaType);
95
+ if (extension === undefined)
96
+ return yield* ProviderShared.invalidRequest(`${NAME} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`);
97
+ const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio);
98
+ const responseFormat = model.diarize
99
+ ? "diarized_json"
100
+ : request.timestamps === undefined || request.timestamps === "none"
101
+ ? undefined
102
+ : "verbose_json";
103
+ const native = Object.entries(mergeJsonRecords(request.providerOptions, request.http?.body) ?? {}).filter(([key]) => !RESERVED_FORM_FIELDS.has(key));
104
+ const fields = mergeJsonRecords({
105
+ model: request.model.id,
106
+ language: model.languageField === "language" ? request.language : undefined,
107
+ languages: model.languageField === "languages" && request.language !== undefined ? [request.language] : undefined,
108
+ prompt: request.prompt,
109
+ response_format: responseFormat,
110
+ timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
111
+ // Diarizing audio longer than 30 seconds requires a chunking strategy.
112
+ chunking_strategy: model.diarize ? "auto" : undefined,
113
+ stream: request.mode === "stream" ? true : undefined,
114
+ }, Object.fromEntries(native));
115
+ const form = new FormData();
116
+ form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`);
117
+ Object.entries(fields ?? {}).forEach(([key, value]) => appendField(form, key, value));
118
+ return MediaProtocol.multipart(form);
119
+ });
120
+ // ---------------------------------------------------------------------------
121
+ // 6. Stream parsing
122
+ // ---------------------------------------------------------------------------
123
+ const segment = (value) => ({
124
+ text: value.text.trim(),
125
+ startSeconds: value.start,
126
+ endSeconds: value.end,
127
+ speaker: value.speaker,
128
+ });
129
+ const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state, frame) {
130
+ const event = yield* decodeEvent(frame);
131
+ if (event.type === "transcript.text.done")
132
+ return [{ ...state, transcript: event }, []];
133
+ if (event.type === "transcript.text.delta")
134
+ return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]];
135
+ const next = segment(event);
136
+ state.segments.push(next);
137
+ return [state, [TranscriptionSegmentEvent.make({ segment: next })]];
138
+ });
139
+ const step = (state, frame) => typeof frame === "string"
140
+ ? onEvent(state, frame)
141
+ : decodeTranscript(frame.document).pipe(Effect.map((transcript) => [{ ...state, transcript }, []]));
142
+ const usage = (value) => {
143
+ if (value === undefined)
144
+ return undefined;
145
+ if (value.type === "duration")
146
+ return { type: "seconds", seconds: value.seconds };
147
+ return {
148
+ type: "tokens",
149
+ input: value.input_tokens,
150
+ output: value.output_tokens,
151
+ total: value.total_tokens,
152
+ details: { openai: value },
153
+ };
154
+ };
155
+ const finish = (state) => {
156
+ const transcript = state.transcript;
157
+ if (transcript === undefined)
158
+ return Effect.fail(MediaProtocol.incomplete(ADAPTER));
159
+ const segments = transcript.segments?.map(segment) ?? state.segments;
160
+ return Effect.succeed([
161
+ TranscriptionFinishEvent.make({
162
+ text: transcript.text,
163
+ segments: segments.length === 0 ? undefined : segments,
164
+ words: transcript.words?.map((word) => ({ text: word.word, startSeconds: word.start, endSeconds: word.end })),
165
+ language: (transcript.language ?? transcript.languages?.[0]?.code)?.toLowerCase(),
166
+ durationSeconds: transcript.duration,
167
+ usage: usage(transcript.usage),
168
+ }),
169
+ ]);
170
+ };
171
+ // ---------------------------------------------------------------------------
172
+ // 7. Protocol and route
173
+ // ---------------------------------------------------------------------------
174
+ export const protocol = MediaProtocol.stream({
175
+ id: ADAPTER,
176
+ name: NAME,
177
+ unsupported: ["speakers"],
178
+ body: { from: fromRequest },
179
+ frames: (bytes, context) => context.request.mode === "stream"
180
+ ? Framing.sse.frame(bytes)
181
+ : Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
182
+ initial: () => ({ segments: [] }),
183
+ step,
184
+ finish,
185
+ });
186
+ export const model = (input) => TranscriptionModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
187
+ export const OpenAITranscription = {
188
+ protocol,
189
+ model,
190
+ };
@@ -0,0 +1,28 @@
1
+ import { Schema } from "effect";
2
+ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js";
3
+ import { MediaProtocol } from "../route/media-protocol.js";
4
+ import { MediaRoute } from "../route/media.js";
5
+ export declare const DEFAULT_BASE_URL = "https://api.replicate.com";
6
+ /** Model-defined `input` fields under the model's own names; `Media.Asset` values lower to URLs or data URLs. */
7
+ export type ReplicateImageOptions = Record<string, unknown>;
8
+ export type Request = ImageRequestFor<ReplicateImageOptions>;
9
+ export declare const Token: Schema.Struct<{
10
+ readonly id: Schema.String;
11
+ readonly getURL: Schema.String;
12
+ readonly cancelURL: Schema.String;
13
+ }>;
14
+ export type Token = Schema.Schema.Type<typeof Token>;
15
+ export declare const protocol: MediaProtocol.Queued<Request, ImageResponse, {
16
+ readonly id: string;
17
+ readonly getURL: string;
18
+ readonly cancelURL: string;
19
+ }>;
20
+ export declare const model: (input: MediaRoute.ModelInput) => ImageModel<ReplicateImageOptions>;
21
+ export declare const ReplicateImages: {
22
+ readonly protocol: MediaProtocol.Queued<Request, ImageResponse, {
23
+ readonly id: string;
24
+ readonly getURL: string;
25
+ readonly cancelURL: string;
26
+ }>;
27
+ readonly model: (input: MediaRoute.ModelInput) => ImageModel<ReplicateImageOptions>;
28
+ };
@@ -0,0 +1,133 @@
1
+ import { Duration, Effect, Schema } from "effect";
2
+ import { ImageModel, ImageResponse } from "../image.js";
3
+ import { Media } from "../media.js";
4
+ import { MediaProtocol } from "../route/media-protocol.js";
5
+ import { MediaRoute } from "../route/media.js";
6
+ import { ProviderID, mergeJsonRecords } from "../schema/index.js";
7
+ import { ProviderShared, optionalNull } from "./shared.js";
8
+ const ADAPTER = "replicate-images";
9
+ const NAME = "Replicate";
10
+ const PROVIDER = ProviderID.make("replicate");
11
+ export const DEFAULT_BASE_URL = "https://api.replicate.com";
12
+ const OUTPUT_RETENTION = Duration.hours(1);
13
+ const MAX_DATA_URL_BYTES = 256 * 1024;
14
+ // ---------------------------------------------------------------------------
15
+ // 2. Token and response schemas
16
+ // ---------------------------------------------------------------------------
17
+ export const Token = Schema.Struct({ id: Schema.String, getURL: Schema.String, cancelURL: Schema.String });
18
+ const Prediction = Schema.Struct({
19
+ id: Schema.String,
20
+ status: Schema.String,
21
+ output: optionalNull(Schema.Unknown),
22
+ error: optionalNull(Schema.Unknown),
23
+ data_removed: optionalNull(Schema.Boolean),
24
+ completed_at: optionalNull(Schema.String),
25
+ metrics: optionalNull(Schema.Struct({ predict_time: optionalNull(Schema.Number) })),
26
+ urls: Schema.Struct({ get: Schema.String, cancel: Schema.String }),
27
+ });
28
+ const Output = Schema.Union([Schema.String, Schema.Array(Schema.String)]);
29
+ const isOutput = Schema.is(Output);
30
+ const STATUS = {
31
+ starting: "queued",
32
+ processing: "running",
33
+ succeeded: "completed",
34
+ failed: "failed",
35
+ canceled: "cancelled",
36
+ // The prediction hit its `Cancel-After` deadline before it started running.
37
+ aborted: "expired",
38
+ };
39
+ // ---------------------------------------------------------------------------
40
+ // 5. Request body construction
41
+ // ---------------------------------------------------------------------------
42
+ // Official models run by `owner/name`; anything else (`owner/name:version` or a bare version id) is a pinned version.
43
+ const isOfficial = (model) => /^[^/:]+\/[^/:]+$/.test(model);
44
+ const inlineSize = (source) => {
45
+ if (source.type === "bytes")
46
+ return source.data.byteLength;
47
+ if (source.type === "base64")
48
+ return source.data.length * 0.75;
49
+ return 0;
50
+ };
51
+ const fileInput = (asset) => {
52
+ if (inlineSize(asset.source) > MAX_DATA_URL_BYTES)
53
+ return Effect.fail(ProviderShared.invalidRequest(`${NAME} data URL inputs are limited to 256 KB; pass a larger file by https URL`));
54
+ return ProviderShared.mediaReference(asset, undefined, NAME).pipe(Effect.map((reference) => reference.value));
55
+ };
56
+ const inputValue = (value) => {
57
+ if (value instanceof Media.Asset)
58
+ return fileInput(value);
59
+ if (Array.isArray(value) && value.some((item) => item instanceof Media.Asset))
60
+ return Effect.forEach(value, (item) => (item instanceof Media.Asset ? fileInput(item) : Effect.succeed(item)));
61
+ return Effect.succeed(value);
62
+ };
63
+ const fromRequest = Effect.fn("ReplicateImages.fromRequest")(function* (request) {
64
+ const native = yield* Effect.forEach(Object.entries(request.providerOptions ?? {}), ([key, value]) => inputValue(value).pipe(Effect.map((lowered) => [key, lowered])));
65
+ const input = mergeJsonRecords({ prompt: request.prompt }, Object.fromEntries(native));
66
+ const model = request.model.id;
67
+ return MediaProtocol.json(mergeJsonRecords(isOfficial(model) ? { input } : { version: model, input }, request.http?.body) ?? {});
68
+ });
69
+ // ---------------------------------------------------------------------------
70
+ // 6. Response decoding
71
+ // ---------------------------------------------------------------------------
72
+ const decodePrediction = MediaProtocol.decodeJson(ADAPTER, NAME, Prediction);
73
+ const decodeStart = Effect.fn("ReplicateImages.decodeStart")(function* (response) {
74
+ const output = yield* decodePrediction(response);
75
+ const prediction = output.value;
76
+ return {
77
+ token: { id: prediction.id, getURL: prediction.urls.get, cancelURL: prediction.urls.cancel },
78
+ // `Prefer: wait` can return an already-finished prediction, so `await` skips straight to the result.
79
+ snapshot: { id: prediction.id, status: yield* MediaProtocol.status(STATUS, prediction.status, output) },
80
+ };
81
+ });
82
+ const decodeStatus = Effect.fn("ReplicateImages.decodeStatus")(function* (response, context) {
83
+ const output = yield* decodePrediction(response);
84
+ return { id: context.token.id, status: yield* MediaProtocol.status(STATUS, output.value.status, output) };
85
+ });
86
+ const decodeResult = Effect.fn("ReplicateImages.decodeResult")(function* (response, context) {
87
+ const output = yield* decodePrediction(response);
88
+ const prediction = output.value;
89
+ const status = yield* MediaProtocol.status(STATUS, prediction.status, output);
90
+ if (status === "failed" || status === "cancelled" || status === "expired")
91
+ return yield* output.ended(status, `${NAME} prediction ${context.token.id} ${prediction.status}${typeof prediction.error === "string" ? `: ${prediction.error}` : ""}`);
92
+ if (status !== "completed")
93
+ return yield* output.invalid(`${NAME} prediction ${context.token.id} has not finished`);
94
+ if (prediction.data_removed === true)
95
+ return yield* output.ended("expired", `${NAME} removed the output of prediction ${context.token.id}`);
96
+ if (!isOutput(prediction.output))
97
+ return yield* output.invalid(`${NAME} prediction ${context.token.id} returned output that is not image URLs`);
98
+ const urls = typeof prediction.output === "string" ? [prediction.output] : prediction.output;
99
+ if (urls.length === 0)
100
+ return yield* output.invalid(`${NAME} prediction ${context.token.id} returned no images`);
101
+ const predictTime = prediction.metrics?.predict_time ?? undefined;
102
+ const completedAt = prediction.completed_at ?? undefined;
103
+ const expiresAt = completedAt === undefined ? undefined : Date.parse(completedAt) + Duration.toMillis(OUTPUT_RETENTION);
104
+ return new ImageResponse({
105
+ images: urls.map((url) => Media.url(url, { expiresAt })),
106
+ usage: predictTime === undefined ? undefined : { type: "compute", seconds: predictTime },
107
+ providerMetadata: { replicate: { predictionId: prediction.id } },
108
+ });
109
+ });
110
+ // ---------------------------------------------------------------------------
111
+ // 7. Protocol and route
112
+ // ---------------------------------------------------------------------------
113
+ export const protocol = MediaProtocol.queued({
114
+ id: ADAPTER,
115
+ name: NAME,
116
+ token: Token,
117
+ unsupported: ["images", "mask", "n", "size", "aspectRatio", "seed", "format"],
118
+ start: { body: { from: fromRequest }, decode: decodeStart },
119
+ status: { path: (token) => token.getURL, decode: decodeStatus },
120
+ result: { path: (token) => token.getURL, decode: decodeResult },
121
+ cancel: { method: "POST", path: (token) => token.cancelURL },
122
+ });
123
+ export const model = (input) => ImageModel.fromRoute({
124
+ id: ADAPTER,
125
+ provider: PROVIDER,
126
+ protocol,
127
+ baseURL: DEFAULT_BASE_URL,
128
+ path: ({ request }) => isOfficial(request.model.id) ? `/v1/models/${request.model.id}/predictions` : "/v1/predictions",
129
+ }, input);
130
+ export const ReplicateImages = {
131
+ protocol,
132
+ model,
133
+ };
@@ -0,0 +1,38 @@
1
+ import { Schema } from "effect";
2
+ import { MediaProtocol } from "../route/media-protocol.js";
3
+ import { MediaRoute } from "../route/media.js";
4
+ import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js";
5
+ export declare const DEFAULT_BASE_URL = "https://api.dev.runwayml.com/v1";
6
+ /** Every Runway request must pin the API version. */
7
+ export declare const API_VERSION = "2024-11-06";
8
+ export declare const TEXT_TO_VIDEO_PATH = "/text_to_video";
9
+ export declare const IMAGE_TO_VIDEO_PATH = "/image_to_video";
10
+ export declare const VIDEO_TO_VIDEO_PATH = "/video_to_video";
11
+ export declare const TASKS_PATH = "/tasks";
12
+ export type RunwayVideoString<Known extends string> = Known | (string & {});
13
+ /**
14
+ * Provider-native options. Common fields lower to Runway's names: `aspectRatio` → `ratio` (Runway expects pixel
15
+ * ratios such as `1280:720` for most models), `durationSeconds` → `duration`, `audio`, `negativePrompt`,
16
+ * `resolution`, `references`, and `frames` → `promptImage`.
17
+ */
18
+ export type RunwayVideoOptions = {
19
+ readonly contentModeration?: {
20
+ readonly publicFigureThreshold?: RunwayVideoString<"auto" | "low">;
21
+ };
22
+ readonly outputFormat?: RunwayVideoString<"mp4" | "prores" | "png_sequence">;
23
+ } & Record<string, unknown>;
24
+ export type Request = VideoRequestFor<RunwayVideoOptions>;
25
+ export declare const Token: Schema.Struct<{
26
+ readonly taskID: Schema.String;
27
+ }>;
28
+ export type Token = Schema.Schema.Type<typeof Token>;
29
+ export declare const protocol: MediaProtocol.Queued<Request, VideoResponse, {
30
+ readonly taskID: string;
31
+ }>;
32
+ export declare const model: (input: MediaRoute.ModelInput) => VideoModel<RunwayVideoOptions>;
33
+ export declare const RunwayVideo: {
34
+ readonly protocol: MediaProtocol.Queued<Request, VideoResponse, {
35
+ readonly taskID: string;
36
+ }>;
37
+ readonly model: (input: MediaRoute.ModelInput) => VideoModel<RunwayVideoOptions>;
38
+ };