@opencode/ai 0.0.0-dev-20048 → 0.0.0-dev-20052

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +65 -1
  2. package/dist/generation.d.ts +13 -9
  3. package/dist/generation.js +11 -0
  4. package/dist/index.d.ts +4 -0
  5. package/dist/index.js +3 -0
  6. package/dist/promise.d.ts +164 -3
  7. package/dist/promise.js +11 -1
  8. package/dist/protocols/assemblyai-transcription.d.ts +40 -0
  9. package/dist/protocols/assemblyai-transcription.js +138 -0
  10. package/dist/protocols/deepgram-speech.js +17 -15
  11. package/dist/protocols/deepgram-transcription.d.ts +25 -0
  12. package/dist/protocols/deepgram-transcription.js +129 -0
  13. package/dist/protocols/gemini.d.ts +2 -2
  14. package/dist/protocols/gemini.js +2 -6
  15. package/dist/protocols/google-images.js +2 -14
  16. package/dist/protocols/google-speech.d.ts +2 -32
  17. package/dist/protocols/google-speech.js +22 -60
  18. package/dist/protocols/google-transcription.d.ts +173 -0
  19. package/dist/protocols/google-transcription.js +138 -0
  20. package/dist/protocols/openai-images.js +4 -7
  21. package/dist/protocols/openai-transcription.d.ts +207 -0
  22. package/dist/protocols/openai-transcription.js +190 -0
  23. package/dist/protocols/utils/gemini-generate-content.d.ts +65 -0
  24. package/dist/protocols/utils/gemini-generate-content.js +65 -0
  25. package/dist/protocols/utils/media-input.d.ts +4 -0
  26. package/dist/protocols/utils/media-input.js +14 -0
  27. package/dist/providers/assemblyai.d.ts +25 -0
  28. package/dist/providers/assemblyai.js +29 -0
  29. package/dist/providers/deepgram.d.ts +5 -0
  30. package/dist/providers/deepgram.js +5 -2
  31. package/dist/providers/google.d.ts +5 -0
  32. package/dist/providers/google.js +3 -0
  33. package/dist/providers/index.d.ts +1 -0
  34. package/dist/providers/index.js +1 -0
  35. package/dist/providers/openai.d.ts +5 -0
  36. package/dist/providers/openai.js +4 -0
  37. package/dist/route/media-protocol.d.ts +15 -3
  38. package/dist/route/media-protocol.js +6 -0
  39. package/dist/route/media.d.ts +5 -2
  40. package/dist/route/media.js +62 -40
  41. package/dist/transcription-client.d.ts +28 -0
  42. package/dist/transcription-client.js +69 -0
  43. package/dist/transcription.d.ts +1504 -0
  44. package/dist/transcription.js +140 -0
  45. package/dist/utils/media-type.d.ts +1 -0
  46. package/dist/utils/media-type.js +14 -0
  47. package/dist/video-client.d.ts +1 -1
  48. package/dist/video-client.js +3 -6
  49. package/dist/video.d.ts +1 -15
  50. package/dist/video.js +2 -14
  51. package/package.json +3 -3
@@ -0,0 +1,25 @@
1
+ import { MediaProtocol } from "../route/media-protocol.js";
2
+ import { MediaRoute } from "../route/media.js";
3
+ import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js";
4
+ export declare const DEFAULT_BASE_URL = "https://api.deepgram.com";
5
+ export declare const PATH = "/v1/listen";
6
+ export type DeepgramTranscriptionOptions = {
7
+ readonly smart_format?: boolean;
8
+ readonly punctuate?: boolean;
9
+ readonly paragraphs?: boolean;
10
+ readonly utterances?: boolean;
11
+ readonly detect_language?: boolean | ReadonlyArray<string>;
12
+ readonly keyterm?: ReadonlyArray<string>;
13
+ readonly diarize_model?: "latest" | "v1" | "v2" | (string & {});
14
+ readonly filler_words?: boolean;
15
+ readonly numerals?: boolean;
16
+ readonly mip_opt_out?: boolean;
17
+ readonly tag?: string | ReadonlyArray<string>;
18
+ } & Record<string, unknown>;
19
+ export type Request = TranscriptionRequestFor<DeepgramTranscriptionOptions>;
20
+ export declare const protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
21
+ export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<DeepgramTranscriptionOptions>;
22
+ export declare const DeepgramTranscription: {
23
+ readonly protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
24
+ readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<DeepgramTranscriptionOptions>;
25
+ };
@@ -0,0 +1,129 @@
1
+ import { Effect, Schema } from "effect";
2
+ import { MediaProtocol } from "../route/media-protocol.js";
3
+ import { MediaRoute } from "../route/media.js";
4
+ import { ProviderID, mergeJsonRecords } from "../schema/index.js";
5
+ import { TranscriptionModel, TranscriptionResponse } from "../transcription.js";
6
+ import { ProviderShared } from "./shared.js";
7
+ import { MediaInput } from "./utils/media-input.js";
8
+ const ADAPTER = "deepgram-transcription";
9
+ const NAME = "Deepgram";
10
+ const PROVIDER = ProviderID.make("deepgram");
11
+ export const DEFAULT_BASE_URL = "https://api.deepgram.com";
12
+ export const PATH = "/v1/listen";
13
+ // ---------------------------------------------------------------------------
14
+ // 2. Response schema
15
+ // ---------------------------------------------------------------------------
16
+ const Word = Schema.Struct({
17
+ word: Schema.String,
18
+ start: Schema.Number,
19
+ end: Schema.Number,
20
+ confidence: Schema.optional(Schema.Number),
21
+ speaker: Schema.optional(Schema.Number),
22
+ punctuated_word: Schema.optional(Schema.String),
23
+ });
24
+ const ListenResponse = Schema.Struct({
25
+ metadata: Schema.optional(Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) })),
26
+ results: Schema.Struct({
27
+ channels: Schema.Array(Schema.Struct({
28
+ alternatives: Schema.optional(Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) }))),
29
+ detected_language: Schema.optional(Schema.String),
30
+ })),
31
+ utterances: Schema.optional(Schema.Array(Schema.Struct({
32
+ start: Schema.Number,
33
+ end: Schema.Number,
34
+ transcript: Schema.String,
35
+ speaker: Schema.optional(Schema.Number),
36
+ words: Schema.optional(Schema.Array(Word)),
37
+ }))),
38
+ }),
39
+ });
40
+ // ---------------------------------------------------------------------------
41
+ // 5. Request body construction
42
+ // ---------------------------------------------------------------------------
43
+ const query = (request) => MediaInput.query(ADAPTER, mergeJsonRecords({
44
+ model: request.model.id,
45
+ smart_format: true,
46
+ language: request.language,
47
+ // Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection.
48
+ detect_language: request.language === undefined ? true : undefined,
49
+ // `diarize=true` is deprecated in favor of choosing a diarization model.
50
+ diarize_model: request.diarize === true ? "latest" : undefined,
51
+ utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined,
52
+ }, request.providerOptions) ?? {});
53
+ const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request) {
54
+ const url = ProviderShared.mediaUrl(request.audio);
55
+ if (url !== undefined)
56
+ return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request));
57
+ if (request.http?.body !== undefined)
58
+ return yield* ProviderShared.invalidRequest(`${NAME} sends inline audio as the raw body, so http.body cannot apply`);
59
+ const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio);
60
+ return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request));
61
+ });
62
+ // ---------------------------------------------------------------------------
63
+ // 6. Response decoding
64
+ // ---------------------------------------------------------------------------
65
+ const decodeListen = MediaProtocol.decodeJson(ADAPTER, NAME, ListenResponse);
66
+ const speaker = (value) => (value === undefined ? undefined : String(value));
67
+ const wordText = (word) => word.punctuated_word ?? word.word;
68
+ // Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
69
+ // words change speaker, so segments split each utterance at speaker changes.
70
+ const speakerTurns = (words) => words.reduce((turns, word) => {
71
+ const last = turns.at(-1);
72
+ if (last === undefined || last[0].speaker !== word.speaker)
73
+ return [...turns, [word]];
74
+ last.push(word);
75
+ return turns;
76
+ }, []);
77
+ const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (response) {
78
+ const output = yield* decodeListen(response);
79
+ const channel = output.value.results.channels[0];
80
+ const alternative = channel?.alternatives?.[0];
81
+ if (alternative === undefined)
82
+ return yield* output.invalid(`${NAME} returned no transcript`);
83
+ const duration = output.value.metadata?.duration;
84
+ const requestID = output.value.metadata?.request_id;
85
+ return new TranscriptionResponse({
86
+ text: alternative.transcript,
87
+ segments: output.value.results.utterances?.flatMap((utterance) => utterance.words === undefined || utterance.words.length === 0
88
+ ? [
89
+ {
90
+ text: utterance.transcript,
91
+ startSeconds: utterance.start,
92
+ endSeconds: utterance.end,
93
+ speaker: speaker(utterance.speaker),
94
+ },
95
+ ]
96
+ : speakerTurns(utterance.words).map((turn) => ({
97
+ text: turn.map(wordText).join(" "),
98
+ startSeconds: turn[0].start,
99
+ endSeconds: turn[turn.length - 1].end,
100
+ speaker: speaker(turn[0].speaker),
101
+ }))),
102
+ words: alternative.words?.map((word) => ({
103
+ text: wordText(word),
104
+ startSeconds: word.start,
105
+ endSeconds: word.end,
106
+ speaker: speaker(word.speaker),
107
+ confidence: word.confidence,
108
+ })),
109
+ language: channel?.detected_language?.toLowerCase(),
110
+ durationSeconds: duration,
111
+ usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
112
+ providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } },
113
+ });
114
+ });
115
+ // ---------------------------------------------------------------------------
116
+ // 7. Protocol and route
117
+ // ---------------------------------------------------------------------------
118
+ export const protocol = MediaProtocol.inline({
119
+ id: ADAPTER,
120
+ name: NAME,
121
+ unsupported: ["prompt", "speakers"],
122
+ body: { from: fromRequest },
123
+ response: { decode: decodeResponse },
124
+ });
125
+ export const model = (input) => TranscriptionModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
126
+ export const DeepgramTranscription = {
127
+ protocol,
128
+ model,
129
+ };
@@ -206,11 +206,11 @@ export declare const protocol: Protocol<{
206
206
  readonly safetyRatings?: unknown;
207
207
  } | null | undefined;
208
208
  readonly usageMetadata?: {
209
- readonly cachedContentTokenCount?: number | null | undefined;
210
- readonly thoughtsTokenCount?: number | null | undefined;
211
209
  readonly promptTokenCount?: number | null | undefined;
212
210
  readonly candidatesTokenCount?: number | null | undefined;
213
211
  readonly totalTokenCount?: number | null | undefined;
212
+ readonly cachedContentTokenCount?: number | null | undefined;
213
+ readonly thoughtsTokenCount?: number | null | undefined;
214
214
  } | null | undefined;
215
215
  }, {
216
216
  route: string;
@@ -9,6 +9,7 @@ import { AIError, LLMEvent, Usage, } from "../schema/index.js";
9
9
  import { classifyProviderFailure } from "../provider-error.js";
10
10
  import { Media } from "../media.js";
11
11
  import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js";
12
+ import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
12
13
  import { GeminiToolSchema } from "./utils/gemini-tool-schema.js";
13
14
  import { Lifecycle } from "./utils/lifecycle.js";
14
15
  import { ToolSchemaProjection } from "./utils/tool-schema.js";
@@ -221,15 +222,10 @@ const lowerToolConfig = (toolChoice) => ProviderShared.matchToolChoice("Gemini",
221
222
  required: () => ({ functionCallingConfig: { mode: "ANY" } }),
222
223
  tool: (name) => ({ functionCallingConfig: { mode: "ANY", allowedFunctionNames: [name] } }),
223
224
  });
224
- // Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
225
225
  const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part) {
226
226
  if (part.type === "text")
227
227
  return { text: part.text };
228
- const source = part.media.source;
229
- if (source.type === "ref" && source.provider === "google")
230
- return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } };
231
- const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media);
232
- return { inlineData: { mimeType: media.mime, data: media.base64 } };
228
+ return yield* GeminiGenerateContent.mediaPart("Gemini", part.media);
233
229
  });
234
230
  const providerMetadata = (key, metadata) => ({ [key]: metadata });
235
231
  const thoughtSignature = (metadata, key) => {
@@ -1,10 +1,10 @@
1
1
  import { Effect, Schema } from "effect";
2
2
  import { ImageModel, ImageResponse } from "../image.js";
3
- import { Media } from "../media.js";
4
3
  import { MediaProtocol } from "../route/media-protocol.js";
5
4
  import { MediaRoute } from "../route/media.js";
6
5
  import { ProviderID, mergeJsonRecords } from "../schema/index.js";
7
6
  import { ProviderShared } from "./shared.js";
7
+ import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
8
8
  import { MediaInput } from "./utils/media-input.js";
9
9
  const ADAPTER = "google-images";
10
10
  const NAME = "Google Images";
@@ -61,18 +61,6 @@ const generationConfig = (request) => {
61
61
  thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
62
62
  }, native) ?? { responseModalities: ["IMAGE"] });
63
63
  };
64
- // Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
65
- const imagePart = (asset) => {
66
- const inline = asset.inline();
67
- if (inline)
68
- return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } });
69
- const id = MediaInput.refID(asset, PROVIDER);
70
- if (id)
71
- return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } });
72
- if (asset.source.type === "ref")
73
- return Effect.fail(ProviderShared.invalidRequest("Google generateContent requires Gemini file references rather than other providers' file IDs"));
74
- return Effect.fail(ProviderShared.invalidRequest("Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference"));
75
- };
76
64
  const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request) {
77
65
  if (request.n !== undefined && request.n > 1)
78
66
  return yield* ProviderShared.unsupportedOperation({
@@ -81,7 +69,7 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request) {
81
69
  route: ADAPTER,
82
70
  message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
83
71
  });
84
- const parts = yield* Effect.forEach(request.images ?? [], imagePart);
72
+ const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image));
85
73
  return MediaProtocol.json(mergeJsonRecords({
86
74
  contents: [{ role: "user", parts: [{ text: request.prompt }, ...parts] }],
87
75
  generationConfig: generationConfig(request),
@@ -1,7 +1,7 @@
1
- import { Schema } from "effect";
2
1
  import { MediaProtocol } from "../route/media-protocol.js";
3
2
  import { MediaRoute } from "../route/media.js";
4
3
  import { SpeechModel, type SpeechRequestFor } from "../speech.js";
4
+ import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
5
5
  import { SpeechStream } from "./utils/speech-stream.js";
6
6
  export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
7
7
  /** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
@@ -22,38 +22,8 @@ export type GoogleSpeechOptions = {
22
22
  };
23
23
  } & Record<string, unknown>;
24
24
  export type Request = SpeechRequestFor<GoogleSpeechOptions>;
25
- declare const GenerateContentChunk: Schema.Struct<{
26
- readonly candidates: Schema.optional<Schema.$Array<Schema.Struct<{
27
- readonly content: Schema.optional<Schema.Struct<{
28
- readonly parts: Schema.optional<Schema.$Array<Schema.Struct<{
29
- readonly text: Schema.optional<Schema.String>;
30
- readonly inlineData: Schema.optional<Schema.Struct<{
31
- readonly mimeType: Schema.String;
32
- readonly data: Schema.Uint8ArrayFromBase64;
33
- }>>;
34
- }>>>;
35
- }>>;
36
- readonly finishReason: Schema.optional<Schema.String>;
37
- }>>>;
38
- readonly promptFeedback: Schema.optional<Schema.Struct<{
39
- readonly blockReason: Schema.optional<Schema.String>;
40
- readonly blockReasonMessage: Schema.optional<Schema.String>;
41
- }>>;
42
- readonly usageMetadata: Schema.optional<Schema.Struct<{
43
- readonly promptTokenCount: Schema.optional<Schema.Number>;
44
- readonly candidatesTokenCount: Schema.optional<Schema.Number>;
45
- readonly totalTokenCount: Schema.optional<Schema.Number>;
46
- }>>;
47
- readonly modelVersion: Schema.optional<Schema.String>;
48
- readonly responseId: Schema.optional<Schema.String>;
49
- }>;
50
- type GenerateContentChunk = Schema.Schema.Type<typeof GenerateContentChunk>;
51
- interface State extends SpeechStream.Audio {
25
+ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
52
26
  readonly mimeType?: string;
53
- readonly usage?: GenerateContentChunk["usageMetadata"];
54
- readonly finishReason?: string;
55
- readonly modelVersion?: string;
56
- readonly responseId?: string;
57
27
  }
58
28
  export declare const protocol: MediaProtocol.Streamed<Request, {
59
29
  readonly type: "audio-delta";
@@ -1,10 +1,9 @@
1
1
  import { Effect, Schema } from "effect";
2
- import { Framing } from "../route/framing.js";
3
2
  import { MediaProtocol } from "../route/media-protocol.js";
4
3
  import { MediaRoute } from "../route/media.js";
5
- import { AIError, ContentPolicyError, ProviderID, mergeJsonRecords } from "../schema/index.js";
4
+ import { ProviderID, mergeJsonRecords } from "../schema/index.js";
6
5
  import { SpeechModel } from "../speech.js";
7
- import { ProviderShared } from "./shared.js";
6
+ import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
8
7
  import { SpeechStream } from "./utils/speech-stream.js";
9
8
  const ADAPTER = "google-speech";
10
9
  const NAME = "Google Speech";
@@ -14,25 +13,10 @@ const DEFAULT_SAMPLE_RATE = 24000;
14
13
  // ---------------------------------------------------------------------------
15
14
  // 3. Streaming event schema
16
15
  // ---------------------------------------------------------------------------
17
- const GenerateContentChunk = Schema.Struct({
18
- candidates: Schema.optional(Schema.Array(Schema.Struct({
19
- content: Schema.optional(Schema.Struct({
20
- parts: Schema.optional(Schema.Array(Schema.Struct({
21
- text: Schema.optional(Schema.String),
22
- inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
23
- }))),
24
- })),
25
- finishReason: Schema.optional(Schema.String),
26
- }))),
27
- promptFeedback: Schema.optional(Schema.Struct({ blockReason: Schema.optional(Schema.String), blockReasonMessage: Schema.optional(Schema.String) })),
28
- usageMetadata: Schema.optional(Schema.Struct({
29
- promptTokenCount: Schema.optional(Schema.Number),
30
- candidatesTokenCount: Schema.optional(Schema.Number),
31
- totalTokenCount: Schema.optional(Schema.Number),
32
- })),
33
- modelVersion: Schema.optional(Schema.String),
34
- responseId: Schema.optional(Schema.String),
35
- });
16
+ const GenerateContentChunk = GeminiGenerateContent.chunk(Schema.Struct({
17
+ text: Schema.optional(Schema.String),
18
+ inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
19
+ }));
36
20
  const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk);
37
21
  // ---------------------------------------------------------------------------
38
22
  // 5. Request body construction
@@ -52,52 +36,24 @@ const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request) {
52
36
  }, request.providerOptions),
53
37
  }, request.http?.body) ?? {});
54
38
  });
55
- // Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
56
- const path = (request) => request.mode === "stream"
57
- ? `/models/${request.model.id}:streamGenerateContent?alt=sse`
58
- : `/models/${request.model.id}:generateContent`;
59
39
  // ---------------------------------------------------------------------------
60
40
  // 6. Stream parsing
61
41
  // ---------------------------------------------------------------------------
62
42
  const step = Effect.fn("GoogleSpeech.step")(function* (state, frame) {
63
43
  const chunk = yield* decodeChunk(frame);
64
- const blockReason = chunk.promptFeedback?.blockReason;
65
- if (blockReason !== undefined)
66
- return yield* new AIError({
67
- reason: new ContentPolicyError({
68
- message: `${NAME} blocked the prompt (${blockReason})${chunk.promptFeedback?.blockReasonMessage === undefined ? "" : `: ${chunk.promptFeedback.blockReasonMessage}`}`,
69
- body: frame,
70
- }),
71
- });
72
- const candidate = chunk.candidates?.[0];
73
- const audio = (candidate?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
74
- const next = {
75
- ...state,
76
- mimeType: state.mimeType ?? audio[0]?.mimeType,
77
- usage: chunk.usageMetadata ?? state.usage,
78
- finishReason: candidate?.finishReason ?? state.finishReason,
79
- modelVersion: chunk.modelVersion ?? state.modelVersion,
80
- responseId: chunk.responseId ?? state.responseId,
81
- };
44
+ const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame);
45
+ if (blocked !== undefined)
46
+ return yield* blocked;
47
+ const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
48
+ const next = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType };
82
49
  return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])];
83
50
  });
84
51
  const finish = (state) => {
85
- const usage = state.usage;
86
52
  const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE;
87
53
  return SpeechStream.finish(ADAPTER, state, {
88
54
  ...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
89
- usage: usage === undefined
90
- ? undefined
91
- : {
92
- type: "tokens",
93
- input: usage.promptTokenCount,
94
- output: usage.candidatesTokenCount,
95
- total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
96
- details: { google: usage },
97
- },
98
- providerMetadata: {
99
- google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
100
- },
55
+ usage: GeminiGenerateContent.usage(state.usage),
56
+ providerMetadata: GeminiGenerateContent.providerMetadata(state),
101
57
  detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
102
58
  });
103
59
  };
@@ -109,13 +65,19 @@ export const protocol = MediaProtocol.stream({
109
65
  name: NAME,
110
66
  unsupported: ["instructions", "speed", "timestamps"],
111
67
  body: { from: fromRequest },
112
- // `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame.
113
- frames: (bytes, context) => context.request.mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes),
68
+ frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
114
69
  initial: () => ({ chunks: [] }),
115
70
  step,
116
71
  finish,
117
72
  });
118
- export const model = (input) => SpeechModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) }, input);
73
+ export const model = (input) => SpeechModel.fromRoute({
74
+ id: ADAPTER,
75
+ provider: PROVIDER,
76
+ protocol,
77
+ baseURL: DEFAULT_BASE_URL,
78
+ // Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
79
+ path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
80
+ }, input);
119
81
  export const GoogleSpeech = {
120
82
  protocol,
121
83
  model,
@@ -0,0 +1,173 @@
1
+ import { MediaProtocol } from "../route/media-protocol.js";
2
+ import { MediaRoute } from "../route/media.js";
3
+ import { TranscriptionModel, type TranscriptionRequestFor, type TranscriptionSegment, type TranscriptionWord } from "../transcription.js";
4
+ import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
5
+ export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
6
+ /**
7
+ * Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word
8
+ * timestamps.
9
+ */
10
+ export type GoogleTranscriptionOptions = {
11
+ readonly audioTranscriptionConfig?: {
12
+ readonly mode?: "VERBATIM" | "SMART" | (string & {});
13
+ readonly customVocabulary?: ReadonlyArray<string>;
14
+ readonly languageCodes?: ReadonlyArray<string>;
15
+ };
16
+ } & Record<string, unknown>;
17
+ export type Request = TranscriptionRequestFor<GoogleTranscriptionOptions>;
18
+ interface State extends GeminiGenerateContent.Metadata {
19
+ readonly text: string;
20
+ readonly segments: Array<TranscriptionSegment>;
21
+ readonly words: Array<TranscriptionWord>;
22
+ }
23
+ export declare const protocol: MediaProtocol.Streamed<Request, {
24
+ readonly id: string;
25
+ readonly type: "generation-queued";
26
+ readonly position?: number | undefined;
27
+ } | {
28
+ readonly id: string;
29
+ readonly type: "generation-progress";
30
+ readonly progress?: number | undefined;
31
+ } | {
32
+ readonly type: "text-delta";
33
+ readonly delta: string;
34
+ } | {
35
+ readonly type: "segment";
36
+ readonly segment: {
37
+ readonly text: string;
38
+ readonly startSeconds: number;
39
+ readonly endSeconds: number;
40
+ readonly speaker?: string | undefined;
41
+ };
42
+ } | {
43
+ readonly type: "finish";
44
+ readonly text: string;
45
+ readonly durationSeconds?: number | undefined;
46
+ readonly providerMetadata?: {
47
+ readonly [x: string]: {
48
+ readonly [x: string]: unknown;
49
+ };
50
+ } | undefined;
51
+ readonly usage?: {
52
+ readonly type: "tokens";
53
+ readonly input?: number | undefined;
54
+ readonly output?: number | undefined;
55
+ readonly total?: number | undefined;
56
+ readonly details?: {
57
+ readonly [x: string]: unknown;
58
+ } | undefined;
59
+ } | {
60
+ readonly type: "seconds";
61
+ readonly seconds: number;
62
+ } | {
63
+ readonly type: "characters";
64
+ readonly characters: number;
65
+ } | {
66
+ readonly type: "credits";
67
+ readonly credits: number;
68
+ } | {
69
+ readonly type: "compute";
70
+ readonly seconds: number;
71
+ } | undefined;
72
+ readonly notices?: readonly {
73
+ readonly type: "other" | "moderated" | "filtered";
74
+ readonly message: string;
75
+ readonly providerMetadata?: {
76
+ readonly [x: string]: {
77
+ readonly [x: string]: unknown;
78
+ };
79
+ } | undefined;
80
+ }[] | undefined;
81
+ readonly language?: string | undefined;
82
+ readonly segments?: readonly {
83
+ readonly text: string;
84
+ readonly startSeconds: number;
85
+ readonly endSeconds: number;
86
+ readonly speaker?: string | undefined;
87
+ }[] | undefined;
88
+ readonly words?: readonly {
89
+ readonly text: string;
90
+ readonly startSeconds: number;
91
+ readonly endSeconds: number;
92
+ readonly speaker?: string | undefined;
93
+ readonly confidence?: number | undefined;
94
+ }[] | undefined;
95
+ }, string, State>;
96
+ export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<GoogleTranscriptionOptions>;
97
+ export declare const GoogleTranscription: {
98
+ readonly protocol: MediaProtocol.Streamed<Request, {
99
+ readonly id: string;
100
+ readonly type: "generation-queued";
101
+ readonly position?: number | undefined;
102
+ } | {
103
+ readonly id: string;
104
+ readonly type: "generation-progress";
105
+ readonly progress?: number | undefined;
106
+ } | {
107
+ readonly type: "text-delta";
108
+ readonly delta: string;
109
+ } | {
110
+ readonly type: "segment";
111
+ readonly segment: {
112
+ readonly text: string;
113
+ readonly startSeconds: number;
114
+ readonly endSeconds: number;
115
+ readonly speaker?: string | undefined;
116
+ };
117
+ } | {
118
+ readonly type: "finish";
119
+ readonly text: string;
120
+ readonly durationSeconds?: number | undefined;
121
+ readonly providerMetadata?: {
122
+ readonly [x: string]: {
123
+ readonly [x: string]: unknown;
124
+ };
125
+ } | undefined;
126
+ readonly usage?: {
127
+ readonly type: "tokens";
128
+ readonly input?: number | undefined;
129
+ readonly output?: number | undefined;
130
+ readonly total?: number | undefined;
131
+ readonly details?: {
132
+ readonly [x: string]: unknown;
133
+ } | undefined;
134
+ } | {
135
+ readonly type: "seconds";
136
+ readonly seconds: number;
137
+ } | {
138
+ readonly type: "characters";
139
+ readonly characters: number;
140
+ } | {
141
+ readonly type: "credits";
142
+ readonly credits: number;
143
+ } | {
144
+ readonly type: "compute";
145
+ readonly seconds: number;
146
+ } | undefined;
147
+ readonly notices?: readonly {
148
+ readonly type: "other" | "moderated" | "filtered";
149
+ readonly message: string;
150
+ readonly providerMetadata?: {
151
+ readonly [x: string]: {
152
+ readonly [x: string]: unknown;
153
+ };
154
+ } | undefined;
155
+ }[] | undefined;
156
+ readonly language?: string | undefined;
157
+ readonly segments?: readonly {
158
+ readonly text: string;
159
+ readonly startSeconds: number;
160
+ readonly endSeconds: number;
161
+ readonly speaker?: string | undefined;
162
+ }[] | undefined;
163
+ readonly words?: readonly {
164
+ readonly text: string;
165
+ readonly startSeconds: number;
166
+ readonly endSeconds: number;
167
+ readonly speaker?: string | undefined;
168
+ readonly confidence?: number | undefined;
169
+ }[] | undefined;
170
+ }, string, State>;
171
+ readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<GoogleTranscriptionOptions>;
172
+ };
173
+ export {};