@opencode/ai 2.0.18 → 2.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -12
- package/dist/cache-policy.js +7 -0
- package/dist/experimental/evaluation.d.ts +7 -0
- package/dist/experimental/evaluation.js +2 -0
- package/dist/experimental/system-one.js +10 -7
- package/dist/generation.d.ts +1 -1
- package/dist/generation.js +23 -18
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/llm.d.ts +1 -1
- package/dist/promise.d.ts +3 -3
- package/dist/promise.js +4 -3
- package/dist/protocols/alibaba-chat.d.ts +8 -2
- package/dist/protocols/alibaba-responses.d.ts +14 -14
- package/dist/protocols/anthropic-messages.js +2 -15
- package/dist/protocols/bedrock-converse.d.ts +1 -1
- package/dist/protocols/bedrock-converse.js +38 -12
- package/dist/protocols/deepgram-speech.js +9 -7
- package/dist/protocols/deepgram-transcription.js +4 -10
- package/dist/protocols/elevenlabs-transcription.d.ts +28 -0
- package/dist/protocols/elevenlabs-transcription.js +137 -0
- package/dist/protocols/gemini.js +1 -10
- package/dist/protocols/google-speech.js +9 -2
- package/dist/protocols/google-transcription.js +4 -0
- package/dist/protocols/google-video.js +11 -2
- package/dist/protocols/meta-messages.d.ts +5 -1
- package/dist/protocols/meta-messages.js +11 -4
- package/dist/protocols/meta-responses.d.ts +16 -16
- package/dist/protocols/mistral-chat.d.ts +6 -0
- package/dist/protocols/mistral-chat.js +8 -5
- package/dist/protocols/open-responses.d.ts +31 -31
- package/dist/protocols/openai-chat.d.ts +53 -10
- package/dist/protocols/openai-chat.js +21 -8
- package/dist/protocols/openai-compatible-chat.d.ts +7 -2
- package/dist/protocols/openai-compatible-responses.d.ts +2 -2
- package/dist/protocols/openai-responses.d.ts +22 -22
- package/dist/protocols/openai-speech.js +6 -1
- package/dist/protocols/openai-transcription.js +1 -1
- package/dist/protocols/runway-video.js +2 -1
- package/dist/protocols/utils/claude-model.d.ts +8 -0
- package/dist/protocols/utils/claude-model.js +15 -0
- package/dist/protocols/utils/gemini-generate-content.d.ts +6 -0
- package/dist/protocols/utils/gemini-generate-content.js +33 -0
- package/dist/protocols/utils/media-input.d.ts +4 -3
- package/dist/protocols/utils/media-input.js +5 -4
- package/dist/protocols/utils/speaker-turns.d.ts +3 -0
- package/dist/protocols/utils/speaker-turns.js +9 -0
- package/dist/protocols/utils/speech-stream.d.ts +1 -0
- package/dist/protocols/utils/speech-stream.js +1 -0
- package/dist/protocols/utils/tool-stream.d.ts +4 -3
- package/dist/protocols/utils/tool-stream.js +1 -1
- package/dist/protocols/xai-responses.d.ts +14 -14
- package/dist/protocols/xai-video.js +7 -1
- package/dist/protocols/zai-chat.d.ts +8 -2
- package/dist/provider-error.d.ts +6 -0
- package/dist/provider-error.js +65 -2
- package/dist/providers/alibaba.d.ts +9 -4
- package/dist/providers/amazon-bedrock-mantle.d.ts +9 -4
- package/dist/providers/anthropic-compatible.js +3 -2
- package/dist/providers/azure.d.ts +11 -6
- package/dist/providers/baseten.d.ts +14 -4
- package/dist/providers/cerebras.d.ts +14 -4
- package/dist/providers/cloudflare-ai-gateway.d.ts +18 -8
- package/dist/providers/cloudflare-workers-ai.d.ts +14 -4
- package/dist/providers/deepinfra.d.ts +14 -4
- package/dist/providers/deepseek.d.ts +14 -4
- package/dist/providers/elevenlabs.d.ts +5 -0
- package/dist/providers/elevenlabs.js +4 -0
- package/dist/providers/fireworks.d.ts +14 -4
- package/dist/providers/google-vertex-chat.d.ts +7 -2
- package/dist/providers/google-vertex-responses.d.ts +2 -2
- package/dist/providers/groq.d.ts +15 -4
- package/dist/providers/meta.d.ts +14 -5
- package/dist/providers/minimax.d.ts +9 -4
- package/dist/providers/moonshot.d.ts +9 -4
- package/dist/providers/openai-compatible-responses.d.ts +2 -2
- package/dist/providers/openai-compatible.d.ts +7 -2
- package/dist/providers/openai.d.ts +9 -4
- package/dist/providers/openrouter.d.ts +27 -6
- package/dist/providers/togetherai.d.ts +14 -4
- package/dist/providers/xai.d.ts +7 -2
- package/dist/providers/xai.js +3 -3
- package/dist/providers/zai-coding-plan.d.ts +9 -4
- package/dist/providers/zai.d.ts +7 -2
- package/dist/route/client.d.ts +1 -1
- package/dist/route/executor.js +12 -8
- package/dist/route/media-protocol.d.ts +13 -3
- package/dist/route/media-protocol.js +16 -6
- package/dist/route/media.js +18 -4
- package/dist/schema/events.d.ts +30 -30
- package/dist/testing.d.ts +8 -8
- package/dist/transcription.d.ts +1 -1
- package/dist/transcription.js +1 -1
- package/package.json +3 -3
|
@@ -50,23 +50,25 @@ const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request)
|
|
|
50
50
|
// ---------------------------------------------------------------------------
|
|
51
51
|
// 6. Stream parsing
|
|
52
52
|
// ---------------------------------------------------------------------------
|
|
53
|
+
/** Deepgram wraps raw encodings in WAV unless `container` is `none`, and defaults their sample rate per encoding. */
|
|
53
54
|
const HEADERLESS_ENCODINGS = {
|
|
54
|
-
linear16: "pcm_s16le",
|
|
55
|
-
mulaw: "pcm_mulaw",
|
|
56
|
-
alaw: "pcm_alaw",
|
|
55
|
+
linear16: { encoding: "pcm_s16le", sampleRate: 24000 },
|
|
56
|
+
mulaw: { encoding: "pcm_mulaw", sampleRate: 8000 },
|
|
57
|
+
alaw: { encoding: "pcm_alaw", sampleRate: 8000 },
|
|
57
58
|
};
|
|
58
59
|
const finish = (state, context) => {
|
|
59
60
|
const headers = context.http.headers;
|
|
60
61
|
const mediaType = headers["content-type"];
|
|
61
62
|
const format = audioFormat(context.request);
|
|
62
|
-
const
|
|
63
|
+
const headerless = HEADERLESS_ENCODINGS[format.encoding ?? ""];
|
|
64
|
+
const container = format.container ?? (headerless === undefined ? undefined : "wav");
|
|
63
65
|
const requestID = headers["dg-request-id"];
|
|
64
66
|
const modelName = headers["dg-model-name"];
|
|
65
67
|
return SpeechStream.finish(route, state, {
|
|
66
|
-
...(
|
|
67
|
-
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
|
|
68
|
+
...(container === "none" && headerless !== undefined
|
|
69
|
+
? SpeechStream.pcm(headerless.encoding, SpeechStream.sampleRate(mediaType) ?? context.request.providerOptions?.sampleRate ?? headerless.sampleRate, mediaType)
|
|
68
70
|
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
|
|
69
|
-
{ mediaType, info: { format:
|
|
71
|
+
{ mediaType, info: { format: container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
|
|
70
72
|
usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
|
|
71
73
|
providerMetadata: requestID === undefined && modelName === undefined
|
|
72
74
|
? undefined
|
|
@@ -5,6 +5,7 @@ import { mergeJsonRecords } from "../schema/index.js";
|
|
|
5
5
|
import { TranscriptionModel, TranscriptionResponse } from "../transcription.js";
|
|
6
6
|
import { ProviderShared } from "./shared.js";
|
|
7
7
|
import { MediaInput } from "./utils/media-input.js";
|
|
8
|
+
import { SpeakerTurns } from "./utils/speaker-turns.js";
|
|
8
9
|
const route = MediaProtocol.identity({ id: "deepgram-transcription", name: "Deepgram", provider: "deepgram" });
|
|
9
10
|
export const DEFAULT_BASE_URL = "https://api.deepgram.com";
|
|
10
11
|
export const PATH = "/v1/listen";
|
|
@@ -63,15 +64,6 @@ const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (re
|
|
|
63
64
|
const decodeListen = route.decodeJson(ListenResponse);
|
|
64
65
|
const speaker = (value) => (value === undefined ? undefined : String(value));
|
|
65
66
|
const wordText = (word) => word.punctuated_word ?? word.word;
|
|
66
|
-
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
|
|
67
|
-
// words change speaker, so segments split each utterance at speaker changes.
|
|
68
|
-
const speakerTurns = (words) => words.reduce((turns, word) => {
|
|
69
|
-
const last = turns.at(-1);
|
|
70
|
-
if (last === undefined || last[0].speaker !== word.speaker)
|
|
71
|
-
return [...turns, [word]];
|
|
72
|
-
last.push(word);
|
|
73
|
-
return turns;
|
|
74
|
-
}, []);
|
|
75
67
|
const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (response) {
|
|
76
68
|
const output = yield* decodeListen(response);
|
|
77
69
|
const channel = output.value.results.channels[0];
|
|
@@ -82,6 +74,8 @@ const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(functio
|
|
|
82
74
|
const requestID = output.value.metadata?.request_id;
|
|
83
75
|
return new TranscriptionResponse({
|
|
84
76
|
text: alternative.transcript,
|
|
77
|
+
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when
|
|
78
|
+
// its words change speaker, so segments split each utterance at speaker changes.
|
|
85
79
|
segments: output.value.results.utterances?.flatMap((utterance) => utterance.words === undefined || utterance.words.length === 0
|
|
86
80
|
? [
|
|
87
81
|
{
|
|
@@ -91,7 +85,7 @@ const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(functio
|
|
|
91
85
|
speaker: speaker(utterance.speaker),
|
|
92
86
|
},
|
|
93
87
|
]
|
|
94
|
-
:
|
|
88
|
+
: SpeakerTurns.group(utterance.words, (word) => word.speaker).map((turn) => ({
|
|
95
89
|
text: turn.map(wordText).join(" "),
|
|
96
90
|
startSeconds: turn[0].start,
|
|
97
91
|
endSeconds: turn[turn.length - 1].end,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { type OpenString } from "../schema/index.js";
|
|
4
|
+
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js";
|
|
5
|
+
export declare const DEFAULT_BASE_URL = "https://api.elevenlabs.io";
|
|
6
|
+
export declare const PATH = "/v1/speech-to-text";
|
|
7
|
+
export type ElevenLabsTranscriptionOptions = {
|
|
8
|
+
readonly tag_audio_events?: boolean;
|
|
9
|
+
readonly timestamps_granularity?: OpenString<"none" | "word" | "character">;
|
|
10
|
+
readonly diarization_threshold?: number;
|
|
11
|
+
readonly file_format?: OpenString<"pcm_s16le_16" | "other">;
|
|
12
|
+
readonly temperature?: number;
|
|
13
|
+
readonly seed?: number;
|
|
14
|
+
readonly keyterms?: ReadonlyArray<string>;
|
|
15
|
+
readonly no_verbatim?: boolean;
|
|
16
|
+
readonly detect_speaker_roles?: boolean;
|
|
17
|
+
readonly use_speaker_library?: boolean;
|
|
18
|
+
readonly entity_detection?: string | ReadonlyArray<string>;
|
|
19
|
+
readonly entity_redaction?: string | ReadonlyArray<string>;
|
|
20
|
+
readonly entity_redaction_mode?: OpenString<"redacted" | "entity_type" | "enumerated_entity_type">;
|
|
21
|
+
} & Record<string, unknown>;
|
|
22
|
+
export type Request = TranscriptionRequestFor<ElevenLabsTranscriptionOptions>;
|
|
23
|
+
export declare const protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
|
|
24
|
+
export declare const model: (input: MediaRoute.ModelInput) => TranscriptionModel<ElevenLabsTranscriptionOptions>;
|
|
25
|
+
export declare const ElevenLabsTranscription: {
|
|
26
|
+
readonly protocol: MediaProtocol.Inline<Request, TranscriptionResponse>;
|
|
27
|
+
readonly model: (input: MediaRoute.ModelInput) => TranscriptionModel<ElevenLabsTranscriptionOptions>;
|
|
28
|
+
};
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
3
|
+
import { MediaRoute } from "../route/media.js";
|
|
4
|
+
import { mergeJsonRecords } from "../schema/index.js";
|
|
5
|
+
import { TranscriptionModel, TranscriptionResponse } from "../transcription.js";
|
|
6
|
+
import { mediaTypeExtension } from "../utils/media-type.js";
|
|
7
|
+
import { ProviderShared, optionalNull } from "./shared.js";
|
|
8
|
+
import { MediaInput } from "./utils/media-input.js";
|
|
9
|
+
import { SpeakerTurns } from "./utils/speaker-turns.js";
|
|
10
|
+
const route = MediaProtocol.identity({
|
|
11
|
+
id: "elevenlabs-transcription",
|
|
12
|
+
name: "ElevenLabs Transcription",
|
|
13
|
+
provider: "elevenlabs",
|
|
14
|
+
});
|
|
15
|
+
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io";
|
|
16
|
+
export const PATH = "/v1/speech-to-text";
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// 2. Response schema
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
/** `type` is `word`, `spacing` (the whitespace between words), or `audio_event` (`(laughter)`). */
|
|
21
|
+
const Token = Schema.Struct({
|
|
22
|
+
text: Schema.String,
|
|
23
|
+
type: Schema.String,
|
|
24
|
+
start: optionalNull(Schema.Number),
|
|
25
|
+
end: optionalNull(Schema.Number),
|
|
26
|
+
speaker_id: optionalNull(Schema.String),
|
|
27
|
+
logprob: optionalNull(Schema.Number),
|
|
28
|
+
});
|
|
29
|
+
const Transcript = Schema.Struct({
|
|
30
|
+
language_code: optionalNull(Schema.String),
|
|
31
|
+
text: Schema.String,
|
|
32
|
+
words: optionalNull(Schema.Array(Token)),
|
|
33
|
+
transcription_id: optionalNull(Schema.String),
|
|
34
|
+
audio_duration_secs: optionalNull(Schema.Number),
|
|
35
|
+
});
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// 5. Request body construction
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
/** Speaker turns are the only segments ElevenLabs can produce, and `num_speakers` only applies to diarization. */
|
|
40
|
+
const diarizes = (request) => request.diarize === true || request.timestamps === "segment" || request.speakers !== undefined;
|
|
41
|
+
const RESERVED_FORM_FIELDS = new Set([
|
|
42
|
+
"file",
|
|
43
|
+
"cloud_storage_url",
|
|
44
|
+
"source_url",
|
|
45
|
+
"model_id",
|
|
46
|
+
"language_code",
|
|
47
|
+
"diarize",
|
|
48
|
+
"num_speakers",
|
|
49
|
+
]);
|
|
50
|
+
const validate = (request, overlay) => {
|
|
51
|
+
// Webhook requests return 202 with no transcript; the result arrives at a configured webhook instead.
|
|
52
|
+
if (overlay.webhook === true)
|
|
53
|
+
return Effect.fail(route.unsupported("transcription.webhook", `${route.name} does not deliver to webhooks`));
|
|
54
|
+
// Separate multichannel output replaces the transcript with one transcript per channel.
|
|
55
|
+
if (overlay.use_multi_channel === true && overlay.multichannel_output_style !== "combined")
|
|
56
|
+
return Effect.fail(route.unsupported("transcription.multichannel", `${route.name} returns a single transcript; set multichannel_output_style: "combined" to merge channels`));
|
|
57
|
+
if (overlay.timestamps_granularity === "none" && (request.timestamps === "word" || diarizes(request)))
|
|
58
|
+
return Effect.fail(route.unsupported("media.timestamps", `${route.name} cannot return word timestamps or speaker turns with timestamps_granularity: "none"`));
|
|
59
|
+
return Effect.void;
|
|
60
|
+
};
|
|
61
|
+
const fromRequest = Effect.fn("ElevenLabsTranscription.fromRequest")(function* (request) {
|
|
62
|
+
const overlay = mergeJsonRecords(request.providerOptions, request.http?.body) ?? {};
|
|
63
|
+
yield* validate(request, overlay);
|
|
64
|
+
const form = new FormData();
|
|
65
|
+
const url = ProviderShared.mediaUrl(request.audio);
|
|
66
|
+
if (url === undefined) {
|
|
67
|
+
const extension = mediaTypeExtension(request.audio.mediaType);
|
|
68
|
+
const audio = yield* MediaInput.inlineBytes(route.id, request.audio);
|
|
69
|
+
form.append("file", MediaInput.blob(audio, request.audio.mediaType), extension === undefined ? "audio" : `audio.${extension}`);
|
|
70
|
+
}
|
|
71
|
+
MediaInput.appendFields(form, {
|
|
72
|
+
model_id: request.model.id,
|
|
73
|
+
// `cloud_storage_url` is deprecated in favor of `source_url`, which accepts any hosted audio or video URL.
|
|
74
|
+
source_url: url,
|
|
75
|
+
language_code: request.language,
|
|
76
|
+
diarize: diarizes(request) ? true : undefined,
|
|
77
|
+
num_speakers: request.speakers,
|
|
78
|
+
}, { overlay, reserved: RESERVED_FORM_FIELDS, repeatArrays: "key" });
|
|
79
|
+
return MediaProtocol.multipart(form);
|
|
80
|
+
});
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
// 6. Response decoding
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
const decodeTranscript = route.decodeJson(Transcript);
|
|
85
|
+
const isTimedWord = (token) => token.type === "word" && typeof token.start === "number" && typeof token.end === "number";
|
|
86
|
+
/** Turn text keeps the provider's own spacing tokens, so languages written without spaces are not re-spaced. */
|
|
87
|
+
const speakerTurns = (tokens) => SpeakerTurns.group(tokens.filter((token) => token.type === "word" || token.type === "spacing"), (token) => token.speaker_id).flatMap((turn) => {
|
|
88
|
+
const words = turn.filter(isTimedWord);
|
|
89
|
+
if (words.length === 0)
|
|
90
|
+
return [];
|
|
91
|
+
return [
|
|
92
|
+
{
|
|
93
|
+
text: turn
|
|
94
|
+
.map((token) => token.text)
|
|
95
|
+
.join("")
|
|
96
|
+
.trim(),
|
|
97
|
+
startSeconds: words[0].start,
|
|
98
|
+
endSeconds: words[words.length - 1].end,
|
|
99
|
+
speaker: turn[0].speaker_id ?? undefined,
|
|
100
|
+
},
|
|
101
|
+
];
|
|
102
|
+
});
|
|
103
|
+
const decodeResponse = Effect.fn("ElevenLabsTranscription.decodeResponse")(function* (response, context) {
|
|
104
|
+
const output = yield* decodeTranscript(response);
|
|
105
|
+
const transcript = output.value;
|
|
106
|
+
const tokens = transcript.words ?? [];
|
|
107
|
+
const duration = transcript.audio_duration_secs ?? undefined;
|
|
108
|
+
const transcriptionID = transcript.transcription_id ?? undefined;
|
|
109
|
+
return new TranscriptionResponse({
|
|
110
|
+
text: transcript.text,
|
|
111
|
+
segments: diarizes(context.request) ? speakerTurns(tokens) : undefined,
|
|
112
|
+
words: tokens.filter(isTimedWord).map((word) => ({
|
|
113
|
+
text: word.text,
|
|
114
|
+
startSeconds: word.start,
|
|
115
|
+
endSeconds: word.end,
|
|
116
|
+
speaker: word.speaker_id ?? undefined,
|
|
117
|
+
confidence: typeof word.logprob === "number" ? Math.exp(word.logprob) : undefined,
|
|
118
|
+
})),
|
|
119
|
+
language: transcript.language_code?.toLowerCase(),
|
|
120
|
+
durationSeconds: duration,
|
|
121
|
+
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
|
|
122
|
+
providerMetadata: transcriptionID === undefined ? undefined : { elevenlabs: { transcriptionId: transcriptionID } },
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
// ---------------------------------------------------------------------------
|
|
126
|
+
// 7. Protocol and route
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
export const protocol = MediaProtocol.inline(route, {
|
|
129
|
+
unsupported: ["prompt"],
|
|
130
|
+
body: { from: fromRequest },
|
|
131
|
+
response: { decode: decodeResponse },
|
|
132
|
+
});
|
|
133
|
+
export const model = (input) => TranscriptionModel.fromRoute({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
|
|
134
|
+
export const ElevenLabsTranscription = {
|
|
135
|
+
protocol,
|
|
136
|
+
model,
|
|
137
|
+
};
|
package/dist/protocols/gemini.js
CHANGED
|
@@ -435,16 +435,7 @@ const mapFinishReason = (finishReason, hasToolCalls) => {
|
|
|
435
435
|
return hasToolCalls ? "tool-calls" : "stop";
|
|
436
436
|
if (finishReason === "MAX_TOKENS")
|
|
437
437
|
return "length";
|
|
438
|
-
if (finishReason
|
|
439
|
-
finishReason === "RECITATION" ||
|
|
440
|
-
finishReason === "SAFETY" ||
|
|
441
|
-
finishReason === "BLOCKLIST" ||
|
|
442
|
-
finishReason === "PROHIBITED_CONTENT" ||
|
|
443
|
-
finishReason === "SPII" ||
|
|
444
|
-
finishReason === "MODEL_ARMOR" ||
|
|
445
|
-
finishReason === "IMAGE_PROHIBITED_CONTENT" ||
|
|
446
|
-
finishReason === "IMAGE_RECITATION" ||
|
|
447
|
-
finishReason === "LANGUAGE")
|
|
438
|
+
if (GeminiGenerateContent.contentFiltered(finishReason))
|
|
448
439
|
return "content-filter";
|
|
449
440
|
if (finishReason === "MALFORMED_FUNCTION_CALL" ||
|
|
450
441
|
finishReason === "UNEXPECTED_TOOL_CALL" ||
|
|
@@ -49,9 +49,15 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state, frame) {
|
|
|
49
49
|
return yield* blocked;
|
|
50
50
|
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
|
|
51
51
|
const next = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType };
|
|
52
|
-
|
|
52
|
+
const events = audio.flatMap((part) => SpeechStream.delta(next, part.data)[1]);
|
|
53
|
+
const withheld = next.chunks.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined;
|
|
54
|
+
if (withheld !== undefined)
|
|
55
|
+
return yield* withheld;
|
|
56
|
+
return [next, events];
|
|
53
57
|
});
|
|
54
58
|
const finish = (state, context) => {
|
|
59
|
+
if (state.finishReason === undefined)
|
|
60
|
+
return Effect.fail(route.incomplete());
|
|
55
61
|
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE;
|
|
56
62
|
const output = state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
|
|
57
63
|
? SpeechStream.container("wav", sampleRate)
|
|
@@ -61,8 +67,9 @@ const finish = (state, context) => {
|
|
|
61
67
|
return SpeechStream.finish(route, state, {
|
|
62
68
|
...output,
|
|
63
69
|
usage: GeminiGenerateContent.usage(state.usage),
|
|
70
|
+
notices: GeminiGenerateContent.notices(route.name, state),
|
|
64
71
|
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
|
65
|
-
detail:
|
|
72
|
+
detail: `finish reason: ${state.finishReason}`,
|
|
66
73
|
});
|
|
67
74
|
};
|
|
68
75
|
// ---------------------------------------------------------------------------
|
|
@@ -85,6 +85,9 @@ const step = Effect.fn("GoogleTranscription.step")(function* (state, frame) {
|
|
|
85
85
|
.filter((item) => item.length > 0)
|
|
86
86
|
.join(" ");
|
|
87
87
|
const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}`;
|
|
88
|
+
const withheld = state.text.length + delta.length === 0 ? GeminiGenerateContent.withheld(route.name, chunk, frame) : undefined;
|
|
89
|
+
if (withheld !== undefined)
|
|
90
|
+
return yield* withheld;
|
|
88
91
|
const events = [
|
|
89
92
|
...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]),
|
|
90
93
|
...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })),
|
|
@@ -100,6 +103,7 @@ const finish = (state) => {
|
|
|
100
103
|
segments: state.segments.length === 0 ? undefined : state.segments,
|
|
101
104
|
words: state.words.length === 0 ? undefined : state.words,
|
|
102
105
|
usage: GeminiGenerateContent.usage(state.usage),
|
|
106
|
+
notices: GeminiGenerateContent.notices(route.name, state),
|
|
103
107
|
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
|
104
108
|
}),
|
|
105
109
|
]);
|
|
@@ -17,7 +17,7 @@ export const Token = Schema.Struct({ operation: Schema.String });
|
|
|
17
17
|
const StartResponse = Schema.Struct({ name: Schema.String });
|
|
18
18
|
const Operation = Schema.Struct({
|
|
19
19
|
done: Schema.optional(Schema.Boolean),
|
|
20
|
-
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
|
|
20
|
+
error: Schema.optional(Schema.Struct({ code: Schema.optional(Schema.Number), message: Schema.optional(Schema.String) })),
|
|
21
21
|
response: Schema.optional(Schema.Struct({
|
|
22
22
|
generateVideoResponse: Schema.optional(Schema.Struct({
|
|
23
23
|
generatedSamples: optionalArray(Schema.Struct({
|
|
@@ -32,6 +32,15 @@ const Operation = Schema.Struct({
|
|
|
32
32
|
})),
|
|
33
33
|
metadata: Schema.optional(Schema.Unknown),
|
|
34
34
|
});
|
|
35
|
+
// Operation errors are `google.rpc.Status`; unlisted codes (INTERNAL, UNAVAILABLE, ...) are provider-side.
|
|
36
|
+
const FAILURE = {
|
|
37
|
+
3: "InvalidRequest", // INVALID_ARGUMENT
|
|
38
|
+
7: "Authentication", // PERMISSION_DENIED
|
|
39
|
+
8: "RateLimit", // RESOURCE_EXHAUSTED
|
|
40
|
+
9: "InvalidRequest", // FAILED_PRECONDITION
|
|
41
|
+
11: "InvalidRequest", // OUT_OF_RANGE
|
|
42
|
+
16: "Authentication", // UNAUTHENTICATED
|
|
43
|
+
};
|
|
35
44
|
// ---------------------------------------------------------------------------
|
|
36
45
|
// 5. Request body construction
|
|
37
46
|
// ---------------------------------------------------------------------------
|
|
@@ -92,7 +101,7 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (response,
|
|
|
92
101
|
if (status === "running")
|
|
93
102
|
return yield* output.pending(context.token.operation);
|
|
94
103
|
if (status === "failed")
|
|
95
|
-
return yield* output.ended("failed", `${route.name} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}
|
|
104
|
+
return yield* output.ended("failed", `${route.name} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`, MediaProtocol.failure(FAILURE, operation.error?.code));
|
|
96
105
|
const generated = operation.response?.generateVideoResponse;
|
|
97
106
|
// Downloads require the same API key as the poll; the asset carries it transiently and follows the redirect.
|
|
98
107
|
const videos = yield* Effect.forEach((generated?.generatedSamples ?? []).flatMap((sample) => sample.video?.uri === undefined ? [] : [{ uri: sample.video.uri, mimeType: sample.video.mimeType }]), (video) => MediaProtocol.expiringUrl(video.uri, FILE_RETENTION, {
|
|
@@ -203,11 +203,15 @@ export declare const protocol: Protocol<{
|
|
|
203
203
|
readonly timezone?: string | undefined;
|
|
204
204
|
} | undefined;
|
|
205
205
|
} | {
|
|
206
|
-
readonly name: string;
|
|
207
206
|
readonly description: string;
|
|
207
|
+
readonly name: string;
|
|
208
208
|
readonly input_schema: {
|
|
209
209
|
readonly [x: string]: unknown;
|
|
210
210
|
};
|
|
211
|
+
readonly cache_control?: {
|
|
212
|
+
readonly type: "ephemeral";
|
|
213
|
+
readonly ttl?: "1h" | "5m" | undefined;
|
|
214
|
+
} | undefined;
|
|
211
215
|
})[] | undefined;
|
|
212
216
|
readonly system?: readonly {
|
|
213
217
|
readonly type: "text";
|
|
@@ -8,12 +8,19 @@ const WebSearch = Schema.Struct({
|
|
|
8
8
|
name: Schema.Literal("web_search"),
|
|
9
9
|
user_location: MetaResponses.WebSearch.fields.user_location,
|
|
10
10
|
});
|
|
11
|
+
const MetaCacheControl = Schema.Struct({
|
|
12
|
+
type: Schema.tag("ephemeral"),
|
|
13
|
+
ttl: Schema.optional(Schema.Literals(["5m", "1h"])),
|
|
14
|
+
});
|
|
15
|
+
const FunctionTool = Schema.Struct({
|
|
16
|
+
name: Schema.String,
|
|
17
|
+
description: Schema.String,
|
|
18
|
+
input_schema: JsonObject,
|
|
19
|
+
cache_control: Schema.optional(MetaCacheControl),
|
|
20
|
+
});
|
|
11
21
|
const Body = Schema.Struct({
|
|
12
22
|
...AnthropicMessages.AnthropicMessagesBody.fields,
|
|
13
|
-
tools: optionalArray(Schema.Union([
|
|
14
|
-
Schema.Struct({ name: Schema.String, description: Schema.String, input_schema: JsonObject }),
|
|
15
|
-
WebSearch,
|
|
16
|
-
])),
|
|
23
|
+
tools: optionalArray(Schema.Union([FunctionTool, WebSearch])),
|
|
17
24
|
});
|
|
18
25
|
const fromRequest = Effect.fn("MetaMessages.fromRequest")(function* (request) {
|
|
19
26
|
const projected = ProviderShared.flattenToolRequest(request);
|
|
@@ -92,8 +92,8 @@ export declare const protocol: Protocol<{
|
|
|
92
92
|
} | {
|
|
93
93
|
readonly type: "input_file";
|
|
94
94
|
readonly filename: string;
|
|
95
|
-
readonly file_data?: string | undefined;
|
|
96
95
|
readonly detail?: string | undefined;
|
|
96
|
+
readonly file_data?: string | undefined;
|
|
97
97
|
readonly file_url?: string | undefined;
|
|
98
98
|
} | {
|
|
99
99
|
readonly type: "input_text";
|
|
@@ -129,8 +129,8 @@ export declare const protocol: Protocol<{
|
|
|
129
129
|
} | {
|
|
130
130
|
readonly type: "input_file";
|
|
131
131
|
readonly filename: string;
|
|
132
|
-
readonly file_data?: string | undefined;
|
|
133
132
|
readonly detail?: string | undefined;
|
|
133
|
+
readonly file_data?: string | undefined;
|
|
134
134
|
readonly file_url?: string | undefined;
|
|
135
135
|
} | {
|
|
136
136
|
readonly type: "input_text";
|
|
@@ -219,21 +219,10 @@ export declare const protocol: Protocol<{
|
|
|
219
219
|
readonly [x: string]: unknown;
|
|
220
220
|
readonly type: string;
|
|
221
221
|
readonly status?: unknown;
|
|
222
|
-
readonly code?: unknown;
|
|
223
222
|
readonly message?: unknown;
|
|
223
|
+
readonly code?: unknown;
|
|
224
224
|
readonly text?: string | undefined;
|
|
225
225
|
readonly error?: unknown;
|
|
226
|
-
readonly item?: {
|
|
227
|
-
readonly [x: string]: unknown;
|
|
228
|
-
readonly type: string;
|
|
229
|
-
readonly id?: string | undefined;
|
|
230
|
-
readonly name?: string | undefined;
|
|
231
|
-
readonly namespace?: string | undefined;
|
|
232
|
-
readonly arguments?: string | undefined;
|
|
233
|
-
readonly encrypted_content?: string | null | undefined;
|
|
234
|
-
readonly call_id?: string | undefined;
|
|
235
|
-
} | null | undefined;
|
|
236
|
-
readonly delta?: string | undefined;
|
|
237
226
|
readonly response?: {
|
|
238
227
|
readonly [x: string]: unknown;
|
|
239
228
|
readonly id?: string | undefined;
|
|
@@ -265,6 +254,17 @@ export declare const protocol: Protocol<{
|
|
|
265
254
|
readonly reason?: string | undefined;
|
|
266
255
|
} | null | undefined;
|
|
267
256
|
} | undefined;
|
|
257
|
+
readonly item?: {
|
|
258
|
+
readonly [x: string]: unknown;
|
|
259
|
+
readonly type: string;
|
|
260
|
+
readonly id?: string | undefined;
|
|
261
|
+
readonly name?: string | undefined;
|
|
262
|
+
readonly namespace?: string | undefined;
|
|
263
|
+
readonly arguments?: string | undefined;
|
|
264
|
+
readonly encrypted_content?: string | null | undefined;
|
|
265
|
+
readonly call_id?: string | undefined;
|
|
266
|
+
} | null | undefined;
|
|
267
|
+
readonly delta?: string | undefined;
|
|
268
268
|
readonly arguments?: string | undefined;
|
|
269
269
|
readonly item_id?: string | undefined;
|
|
270
270
|
readonly output_index?: number | undefined;
|
|
@@ -338,8 +338,8 @@ export declare const httpTransport: HttpTransport.HttpJsonTransport<{
|
|
|
338
338
|
} | {
|
|
339
339
|
readonly type: "input_file";
|
|
340
340
|
readonly filename: string;
|
|
341
|
-
readonly file_data?: string | undefined;
|
|
342
341
|
readonly detail?: string | undefined;
|
|
342
|
+
readonly file_data?: string | undefined;
|
|
343
343
|
readonly file_url?: string | undefined;
|
|
344
344
|
} | {
|
|
345
345
|
readonly type: "input_text";
|
|
@@ -375,8 +375,8 @@ export declare const httpTransport: HttpTransport.HttpJsonTransport<{
|
|
|
375
375
|
} | {
|
|
376
376
|
readonly type: "input_file";
|
|
377
377
|
readonly filename: string;
|
|
378
|
-
readonly file_data?: string | undefined;
|
|
379
378
|
readonly detail?: string | undefined;
|
|
379
|
+
readonly file_data?: string | undefined;
|
|
380
380
|
readonly file_url?: string | undefined;
|
|
381
381
|
} | {
|
|
382
382
|
readonly type: "input_text";
|
|
@@ -8,6 +8,11 @@ import { Lifecycle } from "./utils/lifecycle.js";
|
|
|
8
8
|
import { ToolStream } from "./utils/tool-stream.js";
|
|
9
9
|
export declare const DEFAULT_BASE_URL = "https://api.mistral.ai/v1";
|
|
10
10
|
export declare const PATH = "/chat/completions";
|
|
11
|
+
declare const MistralThinkingUnit: Schema.StructWithRest<Schema.Struct<{
|
|
12
|
+
readonly type: Schema.optional<Schema.String>;
|
|
13
|
+
readonly text: Schema.optional<Schema.String>;
|
|
14
|
+
}>, readonly [Schema.$Record<Schema.String, Schema.Unknown>]>;
|
|
15
|
+
type MistralThinkingUnit = Schema.Schema.Type<typeof MistralThinkingUnit>;
|
|
11
16
|
declare const MistralThinkingContent: Schema.StructWithRest<Schema.Struct<{
|
|
12
17
|
readonly type: Schema.Literal<"thinking">;
|
|
13
18
|
readonly thinking: Schema.$Array<Schema.StructWithRest<Schema.Struct<{
|
|
@@ -212,6 +217,7 @@ interface ActiveContent {
|
|
|
212
217
|
readonly type: "text" | "reasoning";
|
|
213
218
|
readonly id: string;
|
|
214
219
|
readonly thinking?: MistralThinkingContent;
|
|
220
|
+
readonly thinkingUnits?: MistralThinkingUnit[];
|
|
215
221
|
}
|
|
216
222
|
export interface ParserState {
|
|
217
223
|
readonly tools: ToolStream.State<ToolKey>;
|
|
@@ -365,7 +365,7 @@ const closeActive = (state, events) => {
|
|
|
365
365
|
return state;
|
|
366
366
|
const lifecycle = state.active.type === "text"
|
|
367
367
|
? Lifecycle.textEnd(state.lifecycle, events, state.active.id)
|
|
368
|
-
: Lifecycle.reasoningEnd(state.lifecycle, events, state.active.id, thinkingMetadata(state.active.thinking
|
|
368
|
+
: Lifecycle.reasoningEnd(state.lifecycle, events, state.active.id, thinkingMetadata({ ...state.active.thinking, type: "thinking", thinking: state.active.thinkingUnits ?? [] }), thinkingText(state.active.thinkingUnits ?? []));
|
|
369
369
|
return { ...state, lifecycle, active: undefined };
|
|
370
370
|
};
|
|
371
371
|
const appendText = (state, events, text) => {
|
|
@@ -384,19 +384,22 @@ const appendThinking = (state, events, part) => {
|
|
|
384
384
|
const current = state.active?.type === "reasoning" ? state : closeActive(state, events);
|
|
385
385
|
const units = thinkingUnits(part.thinking);
|
|
386
386
|
const active = current.active ?? { type: "reasoning", id: `reasoning-${current.nextContent}` };
|
|
387
|
+
// Keep native units out of streamed events until the block is complete.
|
|
388
|
+
const accumulated = active.thinkingUnits ?? [];
|
|
389
|
+
accumulated.push(...units);
|
|
387
390
|
const thinking = {
|
|
388
391
|
...active.thinking,
|
|
389
392
|
...part,
|
|
390
393
|
type: "thinking",
|
|
391
|
-
thinking: [
|
|
394
|
+
thinking: [],
|
|
392
395
|
};
|
|
393
396
|
const text = thinkingText(units);
|
|
394
397
|
return {
|
|
395
398
|
...current,
|
|
396
399
|
lifecycle: text.length > 0
|
|
397
|
-
? Lifecycle.reasoningDelta(current.lifecycle, events, active.id, text
|
|
398
|
-
: Lifecycle.reasoningStart(current.lifecycle, events, active.id
|
|
399
|
-
active: { ...active, thinking },
|
|
400
|
+
? Lifecycle.reasoningDelta(current.lifecycle, events, active.id, text)
|
|
401
|
+
: Lifecycle.reasoningStart(current.lifecycle, events, active.id),
|
|
402
|
+
active: { ...active, thinking, thinkingUnits: accumulated },
|
|
400
403
|
nextContent: current.active ? current.nextContent : current.nextContent + 1,
|
|
401
404
|
};
|
|
402
405
|
};
|