@opencode/ai 0.0.0-dev-20048 → 0.0.0-dev-20052
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -1
- package/dist/generation.d.ts +13 -9
- package/dist/generation.js +11 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/dist/promise.d.ts +164 -3
- package/dist/promise.js +11 -1
- package/dist/protocols/assemblyai-transcription.d.ts +40 -0
- package/dist/protocols/assemblyai-transcription.js +138 -0
- package/dist/protocols/deepgram-speech.js +17 -15
- package/dist/protocols/deepgram-transcription.d.ts +25 -0
- package/dist/protocols/deepgram-transcription.js +129 -0
- package/dist/protocols/gemini.d.ts +2 -2
- package/dist/protocols/gemini.js +2 -6
- package/dist/protocols/google-images.js +2 -14
- package/dist/protocols/google-speech.d.ts +2 -32
- package/dist/protocols/google-speech.js +22 -60
- package/dist/protocols/google-transcription.d.ts +173 -0
- package/dist/protocols/google-transcription.js +138 -0
- package/dist/protocols/openai-images.js +4 -7
- package/dist/protocols/openai-transcription.d.ts +207 -0
- package/dist/protocols/openai-transcription.js +190 -0
- package/dist/protocols/utils/gemini-generate-content.d.ts +65 -0
- package/dist/protocols/utils/gemini-generate-content.js +65 -0
- package/dist/protocols/utils/media-input.d.ts +4 -0
- package/dist/protocols/utils/media-input.js +14 -0
- package/dist/providers/assemblyai.d.ts +25 -0
- package/dist/providers/assemblyai.js +29 -0
- package/dist/providers/deepgram.d.ts +5 -0
- package/dist/providers/deepgram.js +5 -2
- package/dist/providers/google.d.ts +5 -0
- package/dist/providers/google.js +3 -0
- package/dist/providers/index.d.ts +1 -0
- package/dist/providers/index.js +1 -0
- package/dist/providers/openai.d.ts +5 -0
- package/dist/providers/openai.js +4 -0
- package/dist/route/media-protocol.d.ts +15 -3
- package/dist/route/media-protocol.js +6 -0
- package/dist/route/media.d.ts +5 -2
- package/dist/route/media.js +62 -40
- package/dist/transcription-client.d.ts +28 -0
- package/dist/transcription-client.js +69 -0
- package/dist/transcription.d.ts +1504 -0
- package/dist/transcription.js +140 -0
- package/dist/utils/media-type.d.ts +1 -0
- package/dist/utils/media-type.js +14 -0
- package/dist/video-client.d.ts +1 -1
- package/dist/video-client.js +3 -6
- package/dist/video.d.ts +1 -15
- package/dist/video.js +2 -14
- package/package.json +3 -3
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js";
|
|
4
|
+
export declare const DEFAULT_BASE_URL = "https://api.deepgram.com";
|
|
5
|
+
export declare const PATH = "/v1/listen";
|
|
6
|
+
export type DeepgramTranscriptionOptions = {
|
|
7
|
+
readonly smart_format?: boolean;
|
|
8
|
+
readonly punctuate?: boolean;
|
|
9
|
+
readonly paragraphs?: boolean;
|
|
10
|
+
readonly utterances?: boolean;
|
|
11
|
+
readonly detect_language?: boolean | ReadonlyArray<string>;
|
|
12
|
+
readonly keyterm?: ReadonlyArray<string>;
|
|
13
|
+
readonly diarize_model?: "latest" | "v1" | "v2" | (string & {});
|
|
14
|
+
readonly filler_words?: boolean;
|
|
15
|
+
readonly numerals?: boolean;
|
|
16
|
+
readonly mip_opt_out?: boolean;
|
|
17
|
+
readonly tag?: string | ReadonlyArray<string>;
|
|
18
|
+
} & Record<string, unknown>;
|
|
19
|
+
export type Request = TranscriptionRequestFor<DeepgramTranscriptionOptions>;
|
|
20
|
+
export declare const protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
|
|
21
|
+
export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<DeepgramTranscriptionOptions>;
|
|
22
|
+
export declare const DeepgramTranscription: {
|
|
23
|
+
readonly protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
|
|
24
|
+
readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<DeepgramTranscriptionOptions>;
|
|
25
|
+
};
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
3
|
+
import { MediaRoute } from "../route/media.js";
|
|
4
|
+
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
5
|
+
import { TranscriptionModel, TranscriptionResponse } from "../transcription.js";
|
|
6
|
+
import { ProviderShared } from "./shared.js";
|
|
7
|
+
import { MediaInput } from "./utils/media-input.js";
|
|
8
|
+
const ADAPTER = "deepgram-transcription";
|
|
9
|
+
const NAME = "Deepgram";
|
|
10
|
+
const PROVIDER = ProviderID.make("deepgram");
|
|
11
|
+
export const DEFAULT_BASE_URL = "https://api.deepgram.com";
|
|
12
|
+
export const PATH = "/v1/listen";
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
// 2. Response schema
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
const Word = Schema.Struct({
|
|
17
|
+
word: Schema.String,
|
|
18
|
+
start: Schema.Number,
|
|
19
|
+
end: Schema.Number,
|
|
20
|
+
confidence: Schema.optional(Schema.Number),
|
|
21
|
+
speaker: Schema.optional(Schema.Number),
|
|
22
|
+
punctuated_word: Schema.optional(Schema.String),
|
|
23
|
+
});
|
|
24
|
+
const ListenResponse = Schema.Struct({
|
|
25
|
+
metadata: Schema.optional(Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) })),
|
|
26
|
+
results: Schema.Struct({
|
|
27
|
+
channels: Schema.Array(Schema.Struct({
|
|
28
|
+
alternatives: Schema.optional(Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) }))),
|
|
29
|
+
detected_language: Schema.optional(Schema.String),
|
|
30
|
+
})),
|
|
31
|
+
utterances: Schema.optional(Schema.Array(Schema.Struct({
|
|
32
|
+
start: Schema.Number,
|
|
33
|
+
end: Schema.Number,
|
|
34
|
+
transcript: Schema.String,
|
|
35
|
+
speaker: Schema.optional(Schema.Number),
|
|
36
|
+
words: Schema.optional(Schema.Array(Word)),
|
|
37
|
+
}))),
|
|
38
|
+
}),
|
|
39
|
+
});
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
// 5. Request body construction
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
const query = (request) => MediaInput.query(ADAPTER, mergeJsonRecords({
|
|
44
|
+
model: request.model.id,
|
|
45
|
+
smart_format: true,
|
|
46
|
+
language: request.language,
|
|
47
|
+
// Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection.
|
|
48
|
+
detect_language: request.language === undefined ? true : undefined,
|
|
49
|
+
// `diarize=true` is deprecated in favor of choosing a diarization model.
|
|
50
|
+
diarize_model: request.diarize === true ? "latest" : undefined,
|
|
51
|
+
utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined,
|
|
52
|
+
}, request.providerOptions) ?? {});
|
|
53
|
+
const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request) {
|
|
54
|
+
const url = ProviderShared.mediaUrl(request.audio);
|
|
55
|
+
if (url !== undefined)
|
|
56
|
+
return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request));
|
|
57
|
+
if (request.http?.body !== undefined)
|
|
58
|
+
return yield* ProviderShared.invalidRequest(`${NAME} sends inline audio as the raw body, so http.body cannot apply`);
|
|
59
|
+
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio);
|
|
60
|
+
return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request));
|
|
61
|
+
});
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
// 6. Response decoding
|
|
64
|
+
// ---------------------------------------------------------------------------
|
|
65
|
+
const decodeListen = MediaProtocol.decodeJson(ADAPTER, NAME, ListenResponse);
|
|
66
|
+
const speaker = (value) => (value === undefined ? undefined : String(value));
|
|
67
|
+
const wordText = (word) => word.punctuated_word ?? word.word;
|
|
68
|
+
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
|
|
69
|
+
// words change speaker, so segments split each utterance at speaker changes.
|
|
70
|
+
const speakerTurns = (words) => words.reduce((turns, word) => {
|
|
71
|
+
const last = turns.at(-1);
|
|
72
|
+
if (last === undefined || last[0].speaker !== word.speaker)
|
|
73
|
+
return [...turns, [word]];
|
|
74
|
+
last.push(word);
|
|
75
|
+
return turns;
|
|
76
|
+
}, []);
|
|
77
|
+
const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (response) {
|
|
78
|
+
const output = yield* decodeListen(response);
|
|
79
|
+
const channel = output.value.results.channels[0];
|
|
80
|
+
const alternative = channel?.alternatives?.[0];
|
|
81
|
+
if (alternative === undefined)
|
|
82
|
+
return yield* output.invalid(`${NAME} returned no transcript`);
|
|
83
|
+
const duration = output.value.metadata?.duration;
|
|
84
|
+
const requestID = output.value.metadata?.request_id;
|
|
85
|
+
return new TranscriptionResponse({
|
|
86
|
+
text: alternative.transcript,
|
|
87
|
+
segments: output.value.results.utterances?.flatMap((utterance) => utterance.words === undefined || utterance.words.length === 0
|
|
88
|
+
? [
|
|
89
|
+
{
|
|
90
|
+
text: utterance.transcript,
|
|
91
|
+
startSeconds: utterance.start,
|
|
92
|
+
endSeconds: utterance.end,
|
|
93
|
+
speaker: speaker(utterance.speaker),
|
|
94
|
+
},
|
|
95
|
+
]
|
|
96
|
+
: speakerTurns(utterance.words).map((turn) => ({
|
|
97
|
+
text: turn.map(wordText).join(" "),
|
|
98
|
+
startSeconds: turn[0].start,
|
|
99
|
+
endSeconds: turn[turn.length - 1].end,
|
|
100
|
+
speaker: speaker(turn[0].speaker),
|
|
101
|
+
}))),
|
|
102
|
+
words: alternative.words?.map((word) => ({
|
|
103
|
+
text: wordText(word),
|
|
104
|
+
startSeconds: word.start,
|
|
105
|
+
endSeconds: word.end,
|
|
106
|
+
speaker: speaker(word.speaker),
|
|
107
|
+
confidence: word.confidence,
|
|
108
|
+
})),
|
|
109
|
+
language: channel?.detected_language?.toLowerCase(),
|
|
110
|
+
durationSeconds: duration,
|
|
111
|
+
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
|
|
112
|
+
providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } },
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
// ---------------------------------------------------------------------------
|
|
116
|
+
// 7. Protocol and route
|
|
117
|
+
// ---------------------------------------------------------------------------
|
|
118
|
+
export const protocol = MediaProtocol.inline({
|
|
119
|
+
id: ADAPTER,
|
|
120
|
+
name: NAME,
|
|
121
|
+
unsupported: ["prompt", "speakers"],
|
|
122
|
+
body: { from: fromRequest },
|
|
123
|
+
response: { decode: decodeResponse },
|
|
124
|
+
});
|
|
125
|
+
export const model = (input) => TranscriptionModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
|
|
126
|
+
export const DeepgramTranscription = {
|
|
127
|
+
protocol,
|
|
128
|
+
model,
|
|
129
|
+
};
|
|
@@ -206,11 +206,11 @@ export declare const protocol: Protocol<{
|
|
|
206
206
|
readonly safetyRatings?: unknown;
|
|
207
207
|
} | null | undefined;
|
|
208
208
|
readonly usageMetadata?: {
|
|
209
|
-
readonly cachedContentTokenCount?: number | null | undefined;
|
|
210
|
-
readonly thoughtsTokenCount?: number | null | undefined;
|
|
211
209
|
readonly promptTokenCount?: number | null | undefined;
|
|
212
210
|
readonly candidatesTokenCount?: number | null | undefined;
|
|
213
211
|
readonly totalTokenCount?: number | null | undefined;
|
|
212
|
+
readonly cachedContentTokenCount?: number | null | undefined;
|
|
213
|
+
readonly thoughtsTokenCount?: number | null | undefined;
|
|
214
214
|
} | null | undefined;
|
|
215
215
|
}, {
|
|
216
216
|
route: string;
|
package/dist/protocols/gemini.js
CHANGED
|
@@ -9,6 +9,7 @@ import { AIError, LLMEvent, Usage, } from "../schema/index.js";
|
|
|
9
9
|
import { classifyProviderFailure } from "../provider-error.js";
|
|
10
10
|
import { Media } from "../media.js";
|
|
11
11
|
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js";
|
|
12
|
+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
|
|
12
13
|
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js";
|
|
13
14
|
import { Lifecycle } from "./utils/lifecycle.js";
|
|
14
15
|
import { ToolSchemaProjection } from "./utils/tool-schema.js";
|
|
@@ -221,15 +222,10 @@ const lowerToolConfig = (toolChoice) => ProviderShared.matchToolChoice("Gemini",
|
|
|
221
222
|
required: () => ({ functionCallingConfig: { mode: "ANY" } }),
|
|
222
223
|
tool: (name) => ({ functionCallingConfig: { mode: "ANY", allowedFunctionNames: [name] } }),
|
|
223
224
|
});
|
|
224
|
-
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
|
|
225
225
|
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part) {
|
|
226
226
|
if (part.type === "text")
|
|
227
227
|
return { text: part.text };
|
|
228
|
-
|
|
229
|
-
if (source.type === "ref" && source.provider === "google")
|
|
230
|
-
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } };
|
|
231
|
-
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media);
|
|
232
|
-
return { inlineData: { mimeType: media.mime, data: media.base64 } };
|
|
228
|
+
return yield* GeminiGenerateContent.mediaPart("Gemini", part.media);
|
|
233
229
|
});
|
|
234
230
|
const providerMetadata = (key, metadata) => ({ [key]: metadata });
|
|
235
231
|
const thoughtSignature = (metadata, key) => {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Effect, Schema } from "effect";
|
|
2
2
|
import { ImageModel, ImageResponse } from "../image.js";
|
|
3
|
-
import { Media } from "../media.js";
|
|
4
3
|
import { MediaProtocol } from "../route/media-protocol.js";
|
|
5
4
|
import { MediaRoute } from "../route/media.js";
|
|
6
5
|
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
7
6
|
import { ProviderShared } from "./shared.js";
|
|
7
|
+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
|
|
8
8
|
import { MediaInput } from "./utils/media-input.js";
|
|
9
9
|
const ADAPTER = "google-images";
|
|
10
10
|
const NAME = "Google Images";
|
|
@@ -61,18 +61,6 @@ const generationConfig = (request) => {
|
|
|
61
61
|
thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
|
|
62
62
|
}, native) ?? { responseModalities: ["IMAGE"] });
|
|
63
63
|
};
|
|
64
|
-
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
|
|
65
|
-
const imagePart = (asset) => {
|
|
66
|
-
const inline = asset.inline();
|
|
67
|
-
if (inline)
|
|
68
|
-
return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } });
|
|
69
|
-
const id = MediaInput.refID(asset, PROVIDER);
|
|
70
|
-
if (id)
|
|
71
|
-
return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } });
|
|
72
|
-
if (asset.source.type === "ref")
|
|
73
|
-
return Effect.fail(ProviderShared.invalidRequest("Google generateContent requires Gemini file references rather than other providers' file IDs"));
|
|
74
|
-
return Effect.fail(ProviderShared.invalidRequest("Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference"));
|
|
75
|
-
};
|
|
76
64
|
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request) {
|
|
77
65
|
if (request.n !== undefined && request.n > 1)
|
|
78
66
|
return yield* ProviderShared.unsupportedOperation({
|
|
@@ -81,7 +69,7 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request) {
|
|
|
81
69
|
route: ADAPTER,
|
|
82
70
|
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
|
|
83
71
|
});
|
|
84
|
-
const parts = yield* Effect.forEach(request.images ?? [],
|
|
72
|
+
const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image));
|
|
85
73
|
return MediaProtocol.json(mergeJsonRecords({
|
|
86
74
|
contents: [{ role: "user", parts: [{ text: request.prompt }, ...parts] }],
|
|
87
75
|
generationConfig: generationConfig(request),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Schema } from "effect";
|
|
2
1
|
import { MediaProtocol } from "../route/media-protocol.js";
|
|
3
2
|
import { MediaRoute } from "../route/media.js";
|
|
4
3
|
import { SpeechModel, type SpeechRequestFor } from "../speech.js";
|
|
4
|
+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
|
|
5
5
|
import { SpeechStream } from "./utils/speech-stream.js";
|
|
6
6
|
export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
7
7
|
/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
|
|
@@ -22,38 +22,8 @@ export type GoogleSpeechOptions = {
|
|
|
22
22
|
};
|
|
23
23
|
} & Record<string, unknown>;
|
|
24
24
|
export type Request = SpeechRequestFor<GoogleSpeechOptions>;
|
|
25
|
-
|
|
26
|
-
readonly candidates: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
27
|
-
readonly content: Schema.optional<Schema.Struct<{
|
|
28
|
-
readonly parts: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
29
|
-
readonly text: Schema.optional<Schema.String>;
|
|
30
|
-
readonly inlineData: Schema.optional<Schema.Struct<{
|
|
31
|
-
readonly mimeType: Schema.String;
|
|
32
|
-
readonly data: Schema.Uint8ArrayFromBase64;
|
|
33
|
-
}>>;
|
|
34
|
-
}>>>;
|
|
35
|
-
}>>;
|
|
36
|
-
readonly finishReason: Schema.optional<Schema.String>;
|
|
37
|
-
}>>>;
|
|
38
|
-
readonly promptFeedback: Schema.optional<Schema.Struct<{
|
|
39
|
-
readonly blockReason: Schema.optional<Schema.String>;
|
|
40
|
-
readonly blockReasonMessage: Schema.optional<Schema.String>;
|
|
41
|
-
}>>;
|
|
42
|
-
readonly usageMetadata: Schema.optional<Schema.Struct<{
|
|
43
|
-
readonly promptTokenCount: Schema.optional<Schema.Number>;
|
|
44
|
-
readonly candidatesTokenCount: Schema.optional<Schema.Number>;
|
|
45
|
-
readonly totalTokenCount: Schema.optional<Schema.Number>;
|
|
46
|
-
}>>;
|
|
47
|
-
readonly modelVersion: Schema.optional<Schema.String>;
|
|
48
|
-
readonly responseId: Schema.optional<Schema.String>;
|
|
49
|
-
}>;
|
|
50
|
-
type GenerateContentChunk = Schema.Schema.Type<typeof GenerateContentChunk>;
|
|
51
|
-
interface State extends SpeechStream.Audio {
|
|
25
|
+
interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
|
|
52
26
|
readonly mimeType?: string;
|
|
53
|
-
readonly usage?: GenerateContentChunk["usageMetadata"];
|
|
54
|
-
readonly finishReason?: string;
|
|
55
|
-
readonly modelVersion?: string;
|
|
56
|
-
readonly responseId?: string;
|
|
57
27
|
}
|
|
58
28
|
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
59
29
|
readonly type: "audio-delta";
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { Effect, Schema } from "effect";
|
|
2
|
-
import { Framing } from "../route/framing.js";
|
|
3
2
|
import { MediaProtocol } from "../route/media-protocol.js";
|
|
4
3
|
import { MediaRoute } from "../route/media.js";
|
|
5
|
-
import {
|
|
4
|
+
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
6
5
|
import { SpeechModel } from "../speech.js";
|
|
7
|
-
import {
|
|
6
|
+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
|
|
8
7
|
import { SpeechStream } from "./utils/speech-stream.js";
|
|
9
8
|
const ADAPTER = "google-speech";
|
|
10
9
|
const NAME = "Google Speech";
|
|
@@ -14,25 +13,10 @@ const DEFAULT_SAMPLE_RATE = 24000;
|
|
|
14
13
|
// ---------------------------------------------------------------------------
|
|
15
14
|
// 3. Streaming event schema
|
|
16
15
|
// ---------------------------------------------------------------------------
|
|
17
|
-
const GenerateContentChunk = Schema.Struct({
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
text: Schema.optional(Schema.String),
|
|
22
|
-
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
|
|
23
|
-
}))),
|
|
24
|
-
})),
|
|
25
|
-
finishReason: Schema.optional(Schema.String),
|
|
26
|
-
}))),
|
|
27
|
-
promptFeedback: Schema.optional(Schema.Struct({ blockReason: Schema.optional(Schema.String), blockReasonMessage: Schema.optional(Schema.String) })),
|
|
28
|
-
usageMetadata: Schema.optional(Schema.Struct({
|
|
29
|
-
promptTokenCount: Schema.optional(Schema.Number),
|
|
30
|
-
candidatesTokenCount: Schema.optional(Schema.Number),
|
|
31
|
-
totalTokenCount: Schema.optional(Schema.Number),
|
|
32
|
-
})),
|
|
33
|
-
modelVersion: Schema.optional(Schema.String),
|
|
34
|
-
responseId: Schema.optional(Schema.String),
|
|
35
|
-
});
|
|
16
|
+
const GenerateContentChunk = GeminiGenerateContent.chunk(Schema.Struct({
|
|
17
|
+
text: Schema.optional(Schema.String),
|
|
18
|
+
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
|
|
19
|
+
}));
|
|
36
20
|
const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk);
|
|
37
21
|
// ---------------------------------------------------------------------------
|
|
38
22
|
// 5. Request body construction
|
|
@@ -52,52 +36,24 @@ const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request) {
|
|
|
52
36
|
}, request.providerOptions),
|
|
53
37
|
}, request.http?.body) ?? {});
|
|
54
38
|
});
|
|
55
|
-
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
|
|
56
|
-
const path = (request) => request.mode === "stream"
|
|
57
|
-
? `/models/${request.model.id}:streamGenerateContent?alt=sse`
|
|
58
|
-
: `/models/${request.model.id}:generateContent`;
|
|
59
39
|
// ---------------------------------------------------------------------------
|
|
60
40
|
// 6. Stream parsing
|
|
61
41
|
// ---------------------------------------------------------------------------
|
|
62
42
|
const step = Effect.fn("GoogleSpeech.step")(function* (state, frame) {
|
|
63
43
|
const chunk = yield* decodeChunk(frame);
|
|
64
|
-
const
|
|
65
|
-
if (
|
|
66
|
-
return yield*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
body: frame,
|
|
70
|
-
}),
|
|
71
|
-
});
|
|
72
|
-
const candidate = chunk.candidates?.[0];
|
|
73
|
-
const audio = (candidate?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
|
|
74
|
-
const next = {
|
|
75
|
-
...state,
|
|
76
|
-
mimeType: state.mimeType ?? audio[0]?.mimeType,
|
|
77
|
-
usage: chunk.usageMetadata ?? state.usage,
|
|
78
|
-
finishReason: candidate?.finishReason ?? state.finishReason,
|
|
79
|
-
modelVersion: chunk.modelVersion ?? state.modelVersion,
|
|
80
|
-
responseId: chunk.responseId ?? state.responseId,
|
|
81
|
-
};
|
|
44
|
+
const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame);
|
|
45
|
+
if (blocked !== undefined)
|
|
46
|
+
return yield* blocked;
|
|
47
|
+
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
|
|
48
|
+
const next = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType };
|
|
82
49
|
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])];
|
|
83
50
|
});
|
|
84
51
|
const finish = (state) => {
|
|
85
|
-
const usage = state.usage;
|
|
86
52
|
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE;
|
|
87
53
|
return SpeechStream.finish(ADAPTER, state, {
|
|
88
54
|
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
|
|
89
|
-
usage: usage
|
|
90
|
-
|
|
91
|
-
: {
|
|
92
|
-
type: "tokens",
|
|
93
|
-
input: usage.promptTokenCount,
|
|
94
|
-
output: usage.candidatesTokenCount,
|
|
95
|
-
total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
|
|
96
|
-
details: { google: usage },
|
|
97
|
-
},
|
|
98
|
-
providerMetadata: {
|
|
99
|
-
google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
|
|
100
|
-
},
|
|
55
|
+
usage: GeminiGenerateContent.usage(state.usage),
|
|
56
|
+
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
|
101
57
|
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
|
|
102
58
|
});
|
|
103
59
|
};
|
|
@@ -109,13 +65,19 @@ export const protocol = MediaProtocol.stream({
|
|
|
109
65
|
name: NAME,
|
|
110
66
|
unsupported: ["instructions", "speed", "timestamps"],
|
|
111
67
|
body: { from: fromRequest },
|
|
112
|
-
|
|
113
|
-
frames: (bytes, context) => context.request.mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes),
|
|
68
|
+
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
|
|
114
69
|
initial: () => ({ chunks: [] }),
|
|
115
70
|
step,
|
|
116
71
|
finish,
|
|
117
72
|
});
|
|
118
|
-
export const model = (input) => SpeechModel.fromRoute({
|
|
73
|
+
export const model = (input) => SpeechModel.fromRoute({
|
|
74
|
+
id: ADAPTER,
|
|
75
|
+
provider: PROVIDER,
|
|
76
|
+
protocol,
|
|
77
|
+
baseURL: DEFAULT_BASE_URL,
|
|
78
|
+
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
|
|
79
|
+
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
|
|
80
|
+
}, input);
|
|
119
81
|
export const GoogleSpeech = {
|
|
120
82
|
protocol,
|
|
121
83
|
model,
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { TranscriptionModel, type TranscriptionRequestFor, type TranscriptionSegment, type TranscriptionWord } from "../transcription.js";
|
|
4
|
+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
|
|
5
|
+
export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
6
|
+
/**
|
|
7
|
+
* Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word
|
|
8
|
+
* timestamps.
|
|
9
|
+
*/
|
|
10
|
+
export type GoogleTranscriptionOptions = {
|
|
11
|
+
readonly audioTranscriptionConfig?: {
|
|
12
|
+
readonly mode?: "VERBATIM" | "SMART" | (string & {});
|
|
13
|
+
readonly customVocabulary?: ReadonlyArray<string>;
|
|
14
|
+
readonly languageCodes?: ReadonlyArray<string>;
|
|
15
|
+
};
|
|
16
|
+
} & Record<string, unknown>;
|
|
17
|
+
export type Request = TranscriptionRequestFor<GoogleTranscriptionOptions>;
|
|
18
|
+
interface State extends GeminiGenerateContent.Metadata {
|
|
19
|
+
readonly text: string;
|
|
20
|
+
readonly segments: Array<TranscriptionSegment>;
|
|
21
|
+
readonly words: Array<TranscriptionWord>;
|
|
22
|
+
}
|
|
23
|
+
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
24
|
+
readonly id: string;
|
|
25
|
+
readonly type: "generation-queued";
|
|
26
|
+
readonly position?: number | undefined;
|
|
27
|
+
} | {
|
|
28
|
+
readonly id: string;
|
|
29
|
+
readonly type: "generation-progress";
|
|
30
|
+
readonly progress?: number | undefined;
|
|
31
|
+
} | {
|
|
32
|
+
readonly type: "text-delta";
|
|
33
|
+
readonly delta: string;
|
|
34
|
+
} | {
|
|
35
|
+
readonly type: "segment";
|
|
36
|
+
readonly segment: {
|
|
37
|
+
readonly text: string;
|
|
38
|
+
readonly startSeconds: number;
|
|
39
|
+
readonly endSeconds: number;
|
|
40
|
+
readonly speaker?: string | undefined;
|
|
41
|
+
};
|
|
42
|
+
} | {
|
|
43
|
+
readonly type: "finish";
|
|
44
|
+
readonly text: string;
|
|
45
|
+
readonly durationSeconds?: number | undefined;
|
|
46
|
+
readonly providerMetadata?: {
|
|
47
|
+
readonly [x: string]: {
|
|
48
|
+
readonly [x: string]: unknown;
|
|
49
|
+
};
|
|
50
|
+
} | undefined;
|
|
51
|
+
readonly usage?: {
|
|
52
|
+
readonly type: "tokens";
|
|
53
|
+
readonly input?: number | undefined;
|
|
54
|
+
readonly output?: number | undefined;
|
|
55
|
+
readonly total?: number | undefined;
|
|
56
|
+
readonly details?: {
|
|
57
|
+
readonly [x: string]: unknown;
|
|
58
|
+
} | undefined;
|
|
59
|
+
} | {
|
|
60
|
+
readonly type: "seconds";
|
|
61
|
+
readonly seconds: number;
|
|
62
|
+
} | {
|
|
63
|
+
readonly type: "characters";
|
|
64
|
+
readonly characters: number;
|
|
65
|
+
} | {
|
|
66
|
+
readonly type: "credits";
|
|
67
|
+
readonly credits: number;
|
|
68
|
+
} | {
|
|
69
|
+
readonly type: "compute";
|
|
70
|
+
readonly seconds: number;
|
|
71
|
+
} | undefined;
|
|
72
|
+
readonly notices?: readonly {
|
|
73
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
74
|
+
readonly message: string;
|
|
75
|
+
readonly providerMetadata?: {
|
|
76
|
+
readonly [x: string]: {
|
|
77
|
+
readonly [x: string]: unknown;
|
|
78
|
+
};
|
|
79
|
+
} | undefined;
|
|
80
|
+
}[] | undefined;
|
|
81
|
+
readonly language?: string | undefined;
|
|
82
|
+
readonly segments?: readonly {
|
|
83
|
+
readonly text: string;
|
|
84
|
+
readonly startSeconds: number;
|
|
85
|
+
readonly endSeconds: number;
|
|
86
|
+
readonly speaker?: string | undefined;
|
|
87
|
+
}[] | undefined;
|
|
88
|
+
readonly words?: readonly {
|
|
89
|
+
readonly text: string;
|
|
90
|
+
readonly startSeconds: number;
|
|
91
|
+
readonly endSeconds: number;
|
|
92
|
+
readonly speaker?: string | undefined;
|
|
93
|
+
readonly confidence?: number | undefined;
|
|
94
|
+
}[] | undefined;
|
|
95
|
+
}, string, State>;
|
|
96
|
+
export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<GoogleTranscriptionOptions>;
|
|
97
|
+
export declare const GoogleTranscription: {
|
|
98
|
+
readonly protocol: MediaProtocol.Streamed<Request, {
|
|
99
|
+
readonly id: string;
|
|
100
|
+
readonly type: "generation-queued";
|
|
101
|
+
readonly position?: number | undefined;
|
|
102
|
+
} | {
|
|
103
|
+
readonly id: string;
|
|
104
|
+
readonly type: "generation-progress";
|
|
105
|
+
readonly progress?: number | undefined;
|
|
106
|
+
} | {
|
|
107
|
+
readonly type: "text-delta";
|
|
108
|
+
readonly delta: string;
|
|
109
|
+
} | {
|
|
110
|
+
readonly type: "segment";
|
|
111
|
+
readonly segment: {
|
|
112
|
+
readonly text: string;
|
|
113
|
+
readonly startSeconds: number;
|
|
114
|
+
readonly endSeconds: number;
|
|
115
|
+
readonly speaker?: string | undefined;
|
|
116
|
+
};
|
|
117
|
+
} | {
|
|
118
|
+
readonly type: "finish";
|
|
119
|
+
readonly text: string;
|
|
120
|
+
readonly durationSeconds?: number | undefined;
|
|
121
|
+
readonly providerMetadata?: {
|
|
122
|
+
readonly [x: string]: {
|
|
123
|
+
readonly [x: string]: unknown;
|
|
124
|
+
};
|
|
125
|
+
} | undefined;
|
|
126
|
+
readonly usage?: {
|
|
127
|
+
readonly type: "tokens";
|
|
128
|
+
readonly input?: number | undefined;
|
|
129
|
+
readonly output?: number | undefined;
|
|
130
|
+
readonly total?: number | undefined;
|
|
131
|
+
readonly details?: {
|
|
132
|
+
readonly [x: string]: unknown;
|
|
133
|
+
} | undefined;
|
|
134
|
+
} | {
|
|
135
|
+
readonly type: "seconds";
|
|
136
|
+
readonly seconds: number;
|
|
137
|
+
} | {
|
|
138
|
+
readonly type: "characters";
|
|
139
|
+
readonly characters: number;
|
|
140
|
+
} | {
|
|
141
|
+
readonly type: "credits";
|
|
142
|
+
readonly credits: number;
|
|
143
|
+
} | {
|
|
144
|
+
readonly type: "compute";
|
|
145
|
+
readonly seconds: number;
|
|
146
|
+
} | undefined;
|
|
147
|
+
readonly notices?: readonly {
|
|
148
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
149
|
+
readonly message: string;
|
|
150
|
+
readonly providerMetadata?: {
|
|
151
|
+
readonly [x: string]: {
|
|
152
|
+
readonly [x: string]: unknown;
|
|
153
|
+
};
|
|
154
|
+
} | undefined;
|
|
155
|
+
}[] | undefined;
|
|
156
|
+
readonly language?: string | undefined;
|
|
157
|
+
readonly segments?: readonly {
|
|
158
|
+
readonly text: string;
|
|
159
|
+
readonly startSeconds: number;
|
|
160
|
+
readonly endSeconds: number;
|
|
161
|
+
readonly speaker?: string | undefined;
|
|
162
|
+
}[] | undefined;
|
|
163
|
+
readonly words?: readonly {
|
|
164
|
+
readonly text: string;
|
|
165
|
+
readonly startSeconds: number;
|
|
166
|
+
readonly endSeconds: number;
|
|
167
|
+
readonly speaker?: string | undefined;
|
|
168
|
+
readonly confidence?: number | undefined;
|
|
169
|
+
}[] | undefined;
|
|
170
|
+
}, string, State>;
|
|
171
|
+
readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<GoogleTranscriptionOptions>;
|
|
172
|
+
};
|
|
173
|
+
export {};
|