@effect-uai/google 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +11 -2
  2. package/dist/Gemini.d.mts +1 -1
  3. package/dist/GeminiEmbedding.d.mts +1 -1
  4. package/dist/GeminiImageGenerator.d.mts +1 -1
  5. package/dist/GeminiLiveSession.d.mts +17 -0
  6. package/dist/GeminiLiveSession.d.mts.map +1 -0
  7. package/dist/GeminiLiveSession.mjs +26 -0
  8. package/dist/GeminiLiveSession.mjs.map +1 -0
  9. package/dist/GeminiSynthesizer.d.mts +1 -1
  10. package/dist/GeminiSynthesizer.test.mjs +1 -1
  11. package/dist/GoogleDeepResearch.d.mts +1 -1
  12. package/dist/GoogleDeepResearch.test.mjs +1 -1
  13. package/dist/LyriaGenerator.d.mts +1 -1
  14. package/dist/LyriaGenerator.test.mjs +1 -1
  15. package/dist/codec.d.mts +5 -2
  16. package/dist/codec.d.mts.map +1 -1
  17. package/dist/codec.mjs +4 -1
  18. package/dist/codec.mjs.map +1 -1
  19. package/dist/geminiSpeechCodec.test.mjs +1 -1
  20. package/dist/geminiTools.test.mjs +1 -1
  21. package/dist/index.d.mts +3 -2
  22. package/dist/index.mjs +2 -1
  23. package/dist/models-BVSKJwCx.d.mts +87 -0
  24. package/dist/models-BVSKJwCx.d.mts.map +1 -0
  25. package/dist/models.d.mts +2 -75
  26. package/dist/realtimeSession-7HSIjWh4.d.mts +62 -0
  27. package/dist/realtimeSession-7HSIjWh4.d.mts.map +1 -0
  28. package/dist/realtimeSession.d.mts +2 -0
  29. package/dist/realtimeSession.mjs +406 -0
  30. package/dist/realtimeSession.mjs.map +1 -0
  31. package/dist/realtimeSession.test.d.mts +1 -0
  32. package/dist/realtimeSession.test.mjs +505 -0
  33. package/dist/realtimeSession.test.mjs.map +1 -0
  34. package/dist/{test.DNmyFkvJ-N3QqMRwZ.mjs → test.DNmyFkvJ-BPz3G1HS.mjs} +531 -22
  35. package/dist/test.DNmyFkvJ-BPz3G1HS.mjs.map +1 -0
  36. package/package.json +6 -2
  37. package/src/GeminiLiveSession.ts +45 -0
  38. package/src/codec.ts +3 -2
  39. package/src/index.ts +1 -0
  40. package/src/models.ts +19 -0
  41. package/src/realtimeSession.test.ts +415 -0
  42. package/src/realtimeSession.ts +807 -0
  43. package/dist/models.d.mts.map +0 -1
  44. package/dist/test.DNmyFkvJ-N3QqMRwZ.mjs.map +0 -1
package/dist/models.d.mts CHANGED
@@ -1,75 +1,2 @@
1
- //#region src/models.d.ts
2
- /**
3
- * Known Gemini model identifiers (as of April 2026). The `(string & {})`
4
- * tail keeps autocomplete on the literals while still accepting any string,
5
- * so newly-released models work without an SDK update.
6
- *
7
- * Reference: https://ai.google.dev/gemini-api/docs/models
8
- */
9
- type GoogleModel = "gemini-3.1-pro-preview" | "gemini-3-flash-preview" | "gemini-3.1-flash-lite-preview" | "gemini-3.1-flash-live-preview" | "gemini-3.1-flash-tts-preview" | "gemini-2.5-pro" | "gemini-2.5-flash" | "gemini-2.5-flash-lite" | "gemini-3.1-flash-image" | "gemini-3.1-flash-lite-image" | "gemini-3-pro-image" | (string & {});
10
- /**
11
- * Known Gemini image-generation model identifiers. The `-preview` ids
12
- * were shut down 2026-06-25 and are not listed; neither is the
13
- * previous-generation `gemini-2.5-flash-image`, which the `(string & {})`
14
- * tail still accepts.
15
- *
16
- * - `gemini-3.1-flash-image` — 512 / 1K / 2K / 4K, web and image search.
17
- * - `gemini-3.1-flash-lite-image` — 1K only, cheapest, no search.
18
- * - `gemini-3-pro-image` — 1K / 2K / 4K, the most reference images.
19
- *
20
- * Reference: https://ai.google.dev/gemini-api/docs/models
21
- */
22
- type GeminiImageModel = "gemini-3.1-flash-image" | "gemini-3.1-flash-lite-image" | "gemini-3-pro-image" | (string & {});
23
- /**
24
- * Deep-research agent identifiers for the Interactions API (`POST
25
- * /v1beta/interactions`). `-preview` is the fast tier, `-max-` the
26
- * comprehensive one. Preview surface: verify against live docs before relying on
27
- * exact ids. The `(string & {})` tail accepts any string so a renamed agent
28
- * works without an SDK update.
29
- *
30
- * Reference: https://ai.google.dev/gemini-api/docs/interactions/deep-research
31
- */
32
- type GoogleDeepResearchModel = "deep-research-preview-04-2026" | "deep-research-max-preview-04-2026" | (string & {});
33
- /**
34
- * Known Gemini embedding model identifiers.
35
- *
36
- * - `gemini-embedding-2` — natively multimodal (text, image, audio, video,
37
- * PDF). Does *not* honour the `taskType` field; instead prepend a task
38
- * instruction to the text yourself.
39
- * - `gemini-embedding-001` — text-only, supports the full `taskType` enum
40
- * (`RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` / classification / etc.).
41
- *
42
- * The `(string & {})` tail accepts any string so newly-released models
43
- * work without an SDK update.
44
- *
45
- * Reference: https://ai.google.dev/gemini-api/docs/embeddings
46
- */
47
- /**
48
- * Known Gemini TTS model identifiers. `(string & {})` tail keeps
49
- * autocomplete on the literals while still accepting any string.
50
- *
51
- * Reference: https://ai.google.dev/gemini-api/docs/speech-generation
52
- */
53
- type GeminiTtsModel = "gemini-2.5-flash-preview-tts" | "gemini-2.5-pro-preview-tts" | "gemini-3.1-flash-tts-preview" | (string & {});
54
- /**
55
- * 30 prebuilt voice names for Gemini TTS. No custom-voice / cloning path
56
- * is exposed on this surface, so the tail is stock-only (no
57
- * `(string & {})` escape).
58
- *
59
- * Reference: https://ai.google.dev/gemini-api/docs/speech-generation
60
- */
61
- type GeminiVoiceName = "Zephyr" | "Puck" | "Charon" | "Kore" | "Fenrir" | "Leda" | "Orus" | "Aoede" | "Callirrhoe" | "Autonoe" | "Enceladus" | "Iapetus" | "Umbriel" | "Algieba" | "Despina" | "Erinome" | "Algenib" | "Rasalgethi" | "Laomedeia" | "Achernar" | "Alnilam" | "Schedar" | "Gacrux" | "Pulcherrima" | "Achird" | "Zubenelgenubi" | "Vindemiatrix" | "Sadachbia" | "Sadaltager" | "Sulafat";
62
- /**
63
- * Known Lyria (music generation) model identifiers. `(string & {})` tail
64
- * keeps autocomplete on the literals while still accepting any string.
65
- *
66
- * - `lyria-3-clip-preview` — fixed 30s output, MP3 only.
67
- * - `lyria-3-pro-preview` — up to a couple of minutes, MP3 or WAV.
68
- *
69
- * Reference: https://ai.google.dev/gemini-api/docs/music-generation
70
- */
71
- type LyriaModel = "lyria-3-clip-preview" | "lyria-3-pro-preview" | (string & {});
72
- type GoogleEmbeddingModel = "gemini-embedding-2" | "gemini-embedding-001" | (string & {});
73
- //#endregion
74
- export { GeminiImageModel, GeminiTtsModel, GeminiVoiceName, GoogleDeepResearchModel, GoogleEmbeddingModel, GoogleModel, LyriaModel };
75
- //# sourceMappingURL=models.d.mts.map
1
+ import { a as GoogleDeepResearchModel, c as LyriaModel, i as GeminiVoiceName, n as GeminiLiveModel, o as GoogleEmbeddingModel, r as GeminiTtsModel, s as GoogleModel, t as GeminiImageModel } from "./models-BVSKJwCx.mjs";
2
+ export { GeminiImageModel, GeminiLiveModel, GeminiTtsModel, GeminiVoiceName, GoogleDeepResearchModel, GoogleEmbeddingModel, GoogleModel, LyriaModel };
@@ -0,0 +1,62 @@
1
+ import { n as GeminiLiveModel } from "./models-BVSKJwCx.mjs";
2
+ import { Effect, Redacted, Scope } from "effect";
3
+ import * as AiError from "@effect-uai/core/AiError";
4
+ import { RealtimeSessionHandle } from "@effect-uai/core/RealtimeSession";
5
+ import * as Socket from "effect/unstable/socket/Socket";
6
+ import { CommonSessionRequest } from "@effect-uai/core/Realtime";
7
+ //#region src/realtimeSession.d.ts
8
+ type Config = {
9
+ readonly apiKey: Redacted.Redacted;
10
+ readonly baseUrl?: string;
11
+ /** Build the socket yourself, for a proxy or an in-memory transport. */
12
+ readonly webSocket?: Socket.WebSocketConstructor["Service"];
13
+ };
14
+ /**
15
+ * Gemini-typed session request. Everything past `CommonSessionRequest` is
16
+ * provider-shaped. The configuration is immutable for the connection: changing
17
+ * any of it means opening a new session.
18
+ */
19
+ type GeminiLiveRequest = Omit<CommonSessionRequest, "model"> & {
20
+ readonly model: GeminiLiveModel;
21
+ /** Replaces the plain `turnDetection` switch when you need the knobs. */
22
+ readonly vad?: {
23
+ readonly startSensitivity?: "high" | "low";
24
+ readonly endSensitivity?: "high" | "low";
25
+ readonly prefixPaddingMs?: number;
26
+ readonly silenceDurationMs?: number;
27
+ };
28
+ /** `"none"` turns barge-in off. Defaults to interrupting. */
29
+ readonly activityHandling?: "interrupt" | "none";
30
+ /**
31
+ * When a tool result reaches the model, for the models that keep generating
32
+ * while a tool runs. Defaults to `"when-idle"`: the server's own default is
33
+ * to interrupt, which cuts off whatever the model was saying while it
34
+ * waited, including the "let me look that up" it just spoke. `"interrupt"`
35
+ * is right when the result cannot wait, `"silent"` when it is context the
36
+ * model should have rather than something to announce.
37
+ */
38
+ readonly toolScheduling?: "when-idle" | "interrupt" | "silent";
39
+ readonly turnCoverage?: "activity" | "all" | "audioActivityAndAllVideo";
40
+ /** 3.1 only. Higher levels delay the first audio. */
41
+ readonly thinkingLevel?: "minimal" | "low" | "medium" | "high";
42
+ readonly mediaResolution?: "low" | "medium" | "high";
43
+ readonly languageCode?: string;
44
+ readonly temperature?: number;
45
+ readonly maxOutputTokens?: number;
46
+ /** What turns an otherwise time-boxed session into an unbounded one. */
47
+ readonly contextCompression?: {
48
+ readonly triggerTokens?: number;
49
+ readonly targetTokens?: number;
50
+ };
51
+ /** Grounding. Cannot be combined with `tools`; the pair is a 400. */
52
+ readonly googleSearch?: boolean;
53
+ /** 2.5 only. */
54
+ readonly proactiveAudio?: boolean;
55
+ /** 2.5 only. */
56
+ readonly affectiveDialog?: boolean;
57
+ };
58
+ declare const buildWsUrl: (cfg: Config) => string;
59
+ declare const openSession: (cfg: Config) => (request: GeminiLiveRequest) => Effect.Effect<RealtimeSessionHandle, AiError.AiError, Scope.Scope>;
60
+ //#endregion
61
+ export { openSession as i, GeminiLiveRequest as n, buildWsUrl as r, Config as t };
62
+ //# sourceMappingURL=realtimeSession-7HSIjWh4.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"realtimeSession-7HSIjWh4.d.mts","names":[],"sources":["../src/realtimeSession.ts"],"mappings":";;;;;;;KA8CY;WACD,QAAQ,SAAS;WACjB;;WAEA,YAAY,OAAO;;;;;;;KAQlB,oBAAoB,KAAK;WAC1B,OAAO;;WAEP;aACE;aACA;aACA;aACA;;;WAGF;;;;;;;;;WASA;WACA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;aACE;aACA;;;WAGF;;WAEA;;WAEA;;cAilBE,aAAU,KAAS;cAqCnB,cAAW,KAChB,YAAM,SAED,sBACR,OAAO,OAAO,uBAAuB,QAAQ,SAAS,MAAM"}
@@ -0,0 +1,2 @@
1
+ import { i as openSession, n as GeminiLiveRequest, r as buildWsUrl, t as Config } from "./realtimeSession-7HSIjWh4.mjs";
2
+ export { Config, GeminiLiveRequest, buildWsUrl, openSession };
@@ -0,0 +1,406 @@
1
+ import { parsedResponse, toolDescriptorsToTools } from "./codec.mjs";
2
+ import { Cause, Duration, Effect, Encoding, Match, Pull, Queue, Redacted, Ref, Result, Schema, Stream } from "effect";
3
+ import * as AiError from "@effect-uai/core/AiError";
4
+ import * as Capabilities from "@effect-uai/core/Capabilities";
5
+ import * as Socket from "effect/unstable/socket/Socket";
6
+ import { RealtimeEvent } from "@effect-uai/core/Realtime";
7
+ import * as WebSocketSession from "@effect-uai/core/WebSocketSession";
8
+ //#region src/realtimeSession.ts
9
+ /**
10
+ * Gemini Live speech-to-speech over `BidiGenerateContent`.
11
+ *
12
+ * Auth is a query parameter, so this needs no `ws` package and no headers:
13
+ * the platform `WebSocket` reaches it from Node, Bun, Deno and the browser.
14
+ *
15
+ * Two shapes differ from OpenAI Realtime and drive most of this file. There
16
+ * are no response ids on the wire, so a turn id is minted here and every event
17
+ * of that turn carries it; and there is no truncate op, so a
18
+ * `PlaybackPosition` has nothing to send.
19
+ */
20
+ const PROVIDER = "gemini-live";
21
+ const DEFAULT_BASE_URL = "wss://generativelanguage.googleapis.com";
22
+ const RPC = "ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
23
+ /** Output is 24 kHz whatever the input rate, and cannot be configured. */
24
+ const OUTPUT_RATE = 24e3;
25
+ const unsupportedFormat = (format, field, expected) => new AiError.Unsupported({
26
+ provider: PROVIDER,
27
+ capability: field,
28
+ reason: `Gemini Live accepts ${expected}. Got ${JSON.stringify(format)}.`
29
+ });
30
+ /** Any rate is accepted on input; 16 kHz is what the model works in. */
31
+ const inputRate = (format) => Match.value(format).pipe(Match.when({
32
+ container: "raw",
33
+ encoding: "pcm_s16le",
34
+ channels: 1
35
+ }, (f) => Effect.succeed(f.sampleRate)), Match.orElse((f) => Effect.fail(unsupportedFormat(f, "inputFormat", "mono pcm_s16le at any rate"))));
36
+ const checkOutputFormat = (format) => Match.value(format).pipe(Match.when({
37
+ container: "raw",
38
+ encoding: "pcm_s16le",
39
+ sampleRate: OUTPUT_RATE,
40
+ channels: 1
41
+ }, () => Effect.void), Match.orElse((f) => Effect.fail(unsupportedFormat(f, "outputFormat", `mono pcm_s16le at ${OUTPUT_RATE}`))));
42
+ const SENSITIVITY = {
43
+ high: "HIGH",
44
+ low: "LOW"
45
+ };
46
+ const TURN_COVERAGE = {
47
+ activity: "TURN_INCLUDES_ONLY_ACTIVITY",
48
+ all: "TURN_INCLUDES_ALL_INPUT",
49
+ audioActivityAndAllVideo: "TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO"
50
+ };
51
+ const realtimeInputConfig = (request) => ({
52
+ automaticActivityDetection: {
53
+ ...request.turnDetection === "manual" && { disabled: true },
54
+ ...request.vad?.startSensitivity !== void 0 && { startOfSpeechSensitivity: `START_SENSITIVITY_${SENSITIVITY[request.vad.startSensitivity]}` },
55
+ ...request.vad?.endSensitivity !== void 0 && { endOfSpeechSensitivity: `END_SENSITIVITY_${SENSITIVITY[request.vad.endSensitivity]}` },
56
+ ...request.vad?.prefixPaddingMs !== void 0 && { prefixPaddingMs: request.vad.prefixPaddingMs },
57
+ ...request.vad?.silenceDurationMs !== void 0 && { silenceDurationMs: request.vad.silenceDurationMs }
58
+ },
59
+ ...request.activityHandling !== void 0 && { activityHandling: request.activityHandling === "none" ? "NO_INTERRUPTION" : "START_OF_ACTIVITY_INTERRUPTS" },
60
+ ...request.turnCoverage !== void 0 && { turnCoverage: TURN_COVERAGE[request.turnCoverage] }
61
+ });
62
+ const generationConfig = (request) => ({
63
+ responseModalities: ["AUDIO"],
64
+ ...request.voiceId !== void 0 && { speechConfig: {
65
+ voiceConfig: { prebuiltVoiceConfig: { voiceName: request.voiceId } },
66
+ ...request.languageCode !== void 0 && { languageCode: request.languageCode }
67
+ } },
68
+ ...request.temperature !== void 0 && { temperature: request.temperature },
69
+ ...request.maxOutputTokens !== void 0 && { maxOutputTokens: request.maxOutputTokens },
70
+ ...request.mediaResolution !== void 0 && { mediaResolution: `MEDIA_RESOLUTION_${request.mediaResolution.toUpperCase()}` },
71
+ ...request.thinkingLevel !== void 0 && { thinkingConfig: { thinkingLevel: request.thinkingLevel.toUpperCase() } },
72
+ ...request.affectiveDialog === true && { enableAffectiveDialog: true }
73
+ });
74
+ const toolsOf = (request) => request.googleSearch === true ? [{ googleSearch: {} }] : toolDescriptorsToTools(request.tools ?? []);
75
+ const setupFrame = (request) => {
76
+ const tools = toolsOf(request);
77
+ return JSON.stringify({ setup: {
78
+ model: `models/${request.model}`,
79
+ generationConfig: generationConfig(request),
80
+ ...request.instructions !== void 0 && { systemInstruction: { parts: [{ text: request.instructions }] } },
81
+ ...tools.length > 0 && { tools },
82
+ realtimeInputConfig: realtimeInputConfig(request),
83
+ ...request.transcribeInput !== false && { inputAudioTranscription: {} },
84
+ outputAudioTranscription: {},
85
+ sessionResumption: request.resume === void 0 ? {} : { handle: request.resume },
86
+ ...request.contextCompression !== void 0 && { contextWindowCompression: {
87
+ slidingWindow: { ...request.contextCompression.targetTokens !== void 0 && { targetTokens: request.contextCompression.targetTokens } },
88
+ ...request.contextCompression.triggerTokens !== void 0 && { triggerTokens: request.contextCompression.triggerTokens }
89
+ } },
90
+ ...request.proactiveAudio === true && { proactivity: { proactiveAudio: true } },
91
+ ...(request.history?.length ?? 0) > 0 && { historyConfig: { initialHistoryInClientContent: true } }
92
+ } });
93
+ };
94
+ const messageText = (item) => item.type !== "message" ? "" : item.content.map((block) => block.type === "input_text" || block.type === "output_text" ? block.text : "").join("");
95
+ /**
96
+ * Text-only seed, which is all `history` promises across providers. Anything
97
+ * with no text of its own, a tool call or its output, has no place in the seed
98
+ * and is reported once per kind rather than vanishing.
99
+ */
100
+ const historyFrame = (history) => Effect.gen(function* () {
101
+ const kinds = new Set(history.filter((i) => messageText(i).length === 0).map((i) => i.type));
102
+ yield* Effect.forEach(kinds, (kind) => Capabilities.warnDropped({
103
+ provider: PROVIDER,
104
+ capability: "history",
105
+ field: "history",
106
+ value: kind,
107
+ reason: `Gemini Live seeds a session with text turns only, so \`${kind}\` items are left out.`
108
+ }), { discard: true });
109
+ return JSON.stringify({ clientContent: {
110
+ turns: history.filter((item) => messageText(item).length > 0).map((item) => ({
111
+ role: item.type === "message" && item.role === "assistant" ? "model" : "user",
112
+ parts: [{ text: messageText(item) }]
113
+ })),
114
+ turnComplete: true
115
+ } });
116
+ });
117
+ const Part = Schema.Struct({
118
+ text: Schema.optional(Schema.String),
119
+ thought: Schema.optional(Schema.Boolean),
120
+ inlineData: Schema.optional(Schema.Struct({
121
+ mimeType: Schema.optional(Schema.String),
122
+ data: Schema.optional(Schema.String)
123
+ }))
124
+ });
125
+ const Transcription = Schema.Struct({ text: Schema.optional(Schema.String) });
126
+ const ServerContent = Schema.Struct({
127
+ modelTurn: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(Part)) })),
128
+ turnComplete: Schema.optional(Schema.Boolean),
129
+ generationComplete: Schema.optional(Schema.Boolean),
130
+ interrupted: Schema.optional(Schema.Boolean),
131
+ inputTranscription: Schema.optional(Transcription),
132
+ interimInputTranscription: Schema.optional(Transcription),
133
+ outputTranscription: Schema.optional(Transcription)
134
+ });
135
+ const UsageMetadata = Schema.Struct({
136
+ promptTokenCount: Schema.optional(Schema.Number),
137
+ responseTokenCount: Schema.optional(Schema.Number),
138
+ totalTokenCount: Schema.optional(Schema.Number)
139
+ });
140
+ const FunctionCall = Schema.Struct({
141
+ id: Schema.optional(Schema.String),
142
+ name: Schema.String,
143
+ args: Schema.optional(Schema.Unknown)
144
+ });
145
+ const ServerMessage = Schema.Struct({
146
+ setupComplete: Schema.optional(Schema.Unknown),
147
+ serverContent: Schema.optional(ServerContent),
148
+ toolCall: Schema.optional(Schema.Struct({ functionCalls: Schema.optional(Schema.Array(FunctionCall)) })),
149
+ toolCallCancellation: Schema.optional(Schema.Struct({ ids: Schema.optional(Schema.Array(Schema.String)) })),
150
+ goAway: Schema.optional(Schema.Struct({ timeLeft: Schema.optional(Schema.String) })),
151
+ sessionResumptionUpdate: Schema.optional(Schema.Struct({
152
+ newHandle: Schema.optional(Schema.String),
153
+ resumable: Schema.optional(Schema.Boolean)
154
+ })),
155
+ usageMetadata: Schema.optional(UsageMetadata)
156
+ });
157
+ const initialState = {
158
+ responseId: void 0,
159
+ turns: 0,
160
+ callNames: {},
161
+ usage: void 0,
162
+ goingAway: false
163
+ };
164
+ /**
165
+ * The id every event of the current turn carries, minting one and announcing
166
+ * the turn if this is its first event.
167
+ */
168
+ const startTurn = (state) => Ref.modify(state, (s) => {
169
+ const responseId = s.responseId ?? `turn_${s.turns + 1}`;
170
+ return s.responseId !== void 0 ? [[responseId, []], s] : [[responseId, [RealtimeEvent.ResponseStarted({ responseId })]], {
171
+ ...s,
172
+ responseId,
173
+ turns: s.turns + 1
174
+ }];
175
+ });
176
+ const usageOf = (metadata) => ({
177
+ ...metadata.promptTokenCount !== void 0 && { input_tokens: metadata.promptTokenCount },
178
+ ...metadata.responseTokenCount !== void 0 && { output_tokens: metadata.responseTokenCount },
179
+ ...metadata.totalTokenCount !== void 0 && { total_tokens: metadata.totalTokenCount }
180
+ });
181
+ const decodeAudio = (b64) => Result.match(Encoding.decodeBase64(b64), {
182
+ onSuccess: Effect.succeed,
183
+ onFailure: (cause) => Effect.fail(new AiError.GenerationFailed({
184
+ provider: PROVIDER,
185
+ raw: {
186
+ message: "audio part base64 decode failed",
187
+ cause
188
+ }
189
+ }))
190
+ });
191
+ /** `timeLeft` is a proto duration, so `"60s"` rather than a number. */
192
+ const parseTimeLeft = (timeLeft) => {
193
+ if (timeLeft === void 0) return void 0;
194
+ const seconds = Number.parseFloat(timeLeft.replace(/s$/, ""));
195
+ return Number.isFinite(seconds) ? Duration.seconds(seconds) : void 0;
196
+ };
197
+ /** Thought parts are the model reasoning, not something to speak or show. */
198
+ const partToEvents = (responseId, part) => Match.value(part).pipe(Match.when({ thought: true }, () => Effect.succeed([])), Match.when({ inlineData: { data: Match.string } }, (p) => Effect.map(decodeAudio(p.inlineData.data), (bytes) => [RealtimeEvent.AudioDelta({
199
+ responseId,
200
+ bytes
201
+ })])), Match.when({ text: Match.string }, (p) => Effect.succeed([RealtimeEvent.OutputTranscriptDelta({
202
+ responseId,
203
+ text: p.text
204
+ })])), Match.orElse(() => Effect.succeed([])));
205
+ const inputTranscripts = (content) => [...content.interimInputTranscription?.text === void 0 ? [] : [RealtimeEvent.InputTranscript({
206
+ text: content.interimInputTranscription.text,
207
+ final: false
208
+ })], ...content.inputTranscription?.text === void 0 ? [] : [RealtimeEvent.InputTranscript({
209
+ text: content.inputTranscription.text,
210
+ final: true
211
+ })]];
212
+ /**
213
+ * One `serverContent` to its events. Several fields can arrive together, and
214
+ * 3.1 packs several parts into one message, so this answers with a batch.
215
+ */
216
+ const contentToEvents = (state, content) => Effect.gen(function* () {
217
+ const input = inputTranscripts(content);
218
+ const parts = content.modelTurn?.parts ?? [];
219
+ const spokenText = content.outputTranscription?.text;
220
+ const generating = parts.length > 0 || spokenText !== void 0;
221
+ const ending = content.interrupted === true || content.turnComplete === true;
222
+ if (!generating && !ending) return input;
223
+ const [responseId, started] = generating ? yield* startTurn(state) : [(yield* Ref.get(state)).responseId, []];
224
+ if (responseId === void 0) return input;
225
+ const spoken = yield* Effect.forEach(parts, (part) => partToEvents(responseId, part));
226
+ const transcript = spokenText === void 0 ? [] : [RealtimeEvent.OutputTranscriptDelta({
227
+ responseId,
228
+ text: spokenText
229
+ })];
230
+ const body = [
231
+ ...input,
232
+ ...started,
233
+ ...spoken.flat(),
234
+ ...transcript
235
+ ];
236
+ if (!ending) return body;
237
+ const usage = (yield* Ref.get(state)).usage;
238
+ yield* Ref.update(state, (s) => ({
239
+ ...s,
240
+ responseId: void 0,
241
+ usage: void 0
242
+ }));
243
+ return [
244
+ ...body,
245
+ ...content.interrupted === true ? [RealtimeEvent.Interrupted({ responseId })] : [],
246
+ RealtimeEvent.ResponseDone({
247
+ responseId,
248
+ reason: content.interrupted === true ? "interrupted" : "complete",
249
+ ...usage !== void 0 && { usage }
250
+ })
251
+ ];
252
+ });
253
+ const toolCallEvents = (state, calls) => Effect.gen(function* () {
254
+ const [responseId, started] = yield* startTurn(state);
255
+ yield* Ref.update(state, (s) => ({
256
+ ...s,
257
+ callNames: calls.reduce((names, call) => ({
258
+ ...names,
259
+ [call.id ?? call.name]: call.name
260
+ }), s.callNames)
261
+ }));
262
+ return [...started, ...calls.map((call) => RealtimeEvent.ToolCall({
263
+ responseId,
264
+ call: {
265
+ type: "function_call",
266
+ call_id: call.id ?? call.name,
267
+ name: call.name,
268
+ arguments: JSON.stringify(call.args ?? {}),
269
+ providerData: void 0
270
+ }
271
+ }))];
272
+ });
273
+ const toEvents = (state, frame) => Effect.gen(function* () {
274
+ if (frame.usageMetadata !== void 0) {
275
+ const usage = usageOf(frame.usageMetadata);
276
+ yield* Ref.update(state, (s) => ({
277
+ ...s,
278
+ usage
279
+ }));
280
+ }
281
+ if (frame.serverContent !== void 0) return yield* contentToEvents(state, frame.serverContent);
282
+ if (frame.toolCall !== void 0) return yield* toolCallEvents(state, frame.toolCall.functionCalls ?? []);
283
+ const callIds = frame.toolCallCancellation?.ids ?? [];
284
+ if (callIds.length > 0) return [RealtimeEvent.ToolCallCancelled({ callIds })];
285
+ if (frame.goAway !== void 0) {
286
+ const timeLeft = parseTimeLeft(frame.goAway.timeLeft);
287
+ yield* Ref.update(state, (s) => ({
288
+ ...s,
289
+ goingAway: true
290
+ }));
291
+ return [RealtimeEvent.SessionEnding({ ...timeLeft !== void 0 && { timeLeft } })];
292
+ }
293
+ const handle = frame.sessionResumptionUpdate?.newHandle;
294
+ return frame.sessionResumptionUpdate?.resumable === true && handle !== void 0 && handle !== "" ? [RealtimeEvent.ResumptionHandle({ handle })] : [];
295
+ });
296
+ const droppedActivity = (field) => Capabilities.warnDropped({
297
+ provider: PROVIDER,
298
+ capability: "manualTurns",
299
+ field,
300
+ reason: "The session uses automatic activity detection; boundaries are the server's."
301
+ });
302
+ const videoPart = (frame) => Match.value(frame).pipe(Match.tag("base64", (s) => Effect.succeed({
303
+ data: s.base64,
304
+ mimeType: s.mimeType
305
+ })), Match.tag("bytes", (s) => Effect.succeed({
306
+ data: Encoding.encodeBase64(s.bytes),
307
+ mimeType: s.mimeType
308
+ })), Match.tag("url", () => Effect.fail(new AiError.Unsupported({
309
+ provider: PROVIDER,
310
+ capability: "videoInput",
311
+ reason: "A live video frame must carry its bytes; a URL would need the Files API."
312
+ }))), Match.exhaustive);
313
+ const schedulingOf = (request) => Match.value(request.toolScheduling ?? "when-idle").pipe(Match.when("when-idle", () => "WHEN_IDLE"), Match.when("interrupt", () => "INTERRUPT"), Match.when("silent", () => "SILENT"), Match.exhaustive);
314
+ const handleInput = (send, state, options, input) => Match.value(input).pipe(Match.tag("Audio", (i) => send(JSON.stringify({ realtimeInput: { audio: {
315
+ data: Encoding.encodeBase64(i.bytes),
316
+ mimeType: `audio/pcm;rate=${options.rate}`
317
+ } } }))), Match.tag("Text", (i) => i.role === "system" ? Effect.fail(new AiError.Unsupported({
318
+ provider: PROVIDER,
319
+ capability: "systemTurn",
320
+ reason: "Gemini Live fixes the system instruction at setup for the connection."
321
+ })) : send(JSON.stringify({ realtimeInput: { text: i.text } }))), Match.tag("ToolResult", (i) => Effect.gen(function* () {
322
+ const name = (yield* Ref.get(state)).callNames[i.output.call_id];
323
+ if (name === void 0) return yield* Effect.fail(new AiError.InvalidRequest({
324
+ provider: PROVIDER,
325
+ raw: `no tool call ${i.output.call_id} in this session`
326
+ }));
327
+ yield* send(JSON.stringify({ toolResponse: { functionResponses: [{
328
+ id: i.output.call_id,
329
+ name,
330
+ response: parsedResponse(i.output.output),
331
+ scheduling: options.scheduling
332
+ }] } }));
333
+ })), Match.tag("Interrupt", () => Effect.fail(new AiError.Unsupported({
334
+ provider: PROVIDER,
335
+ capability: "interrupt",
336
+ reason: "Gemini Live has no cancel op: the server interrupts when it hears speech. Pass activityHandling: \"none\" to turn that off."
337
+ }))), Match.tag("VideoFrame", (i) => Effect.flatMap(videoPart(i.frame), (video) => send(JSON.stringify({ realtimeInput: { video } })))), Match.tag("ActivityStart", () => options.manual ? send(JSON.stringify({ realtimeInput: { activityStart: {} } })) : droppedActivity("ActivityStart")), Match.tag("ActivityEnd", () => options.manual ? send(JSON.stringify({ realtimeInput: { activityEnd: {} } })) : droppedActivity("ActivityEnd")), Match.tag("PlaybackPosition", () => Capabilities.warnDropped({
338
+ provider: PROVIDER,
339
+ capability: "truncate",
340
+ field: "PlaybackPosition",
341
+ reason: "Gemini Live has no truncate op, so the unheard tail stays in context."
342
+ })), Match.exhaustive);
343
+ const buildWsUrl = (cfg) => {
344
+ const url = new URL(cfg.baseUrl ?? DEFAULT_BASE_URL);
345
+ url.protocol = url.protocol === "http:" ? "ws:" : "wss:";
346
+ url.pathname = `${url.pathname.replace(/\/$/, "")}/${RPC}`;
347
+ url.searchParams.set("key", Redacted.value(cfg.apiKey));
348
+ return url.toString();
349
+ };
350
+ const withSocket = (cfg) => (effect) => cfg.webSocket === void 0 ? Effect.provide(effect, Socket.layerWebSocketConstructorGlobal) : Effect.provideService(effect, Socket.WebSocketConstructor, cfg.webSocket);
351
+ /** Drop frames until `setupComplete`; a close before it is a failed handshake. */
352
+ const awaitSetupComplete = (session) => Queue.take(session.frames).pipe(Effect.catchCause((cause) => Pull.isDoneCause(cause) ? Effect.fail(new AiError.Unavailable({
353
+ provider: PROVIDER,
354
+ raw: "the socket closed during the handshake"
355
+ })) : Effect.failCause(cause)), Effect.flatMap((frame) => frame.setupComplete === void 0 ? awaitSetupComplete(session) : Effect.void));
356
+ const openSession = (cfg) => (request) => Effect.gen(function* () {
357
+ if (request.googleSearch === true && (request.tools?.length ?? 0) > 0) return yield* Effect.fail(new AiError.InvalidRequest({
358
+ provider: PROVIDER,
359
+ raw: "Gemini rejects googleSearch alongside function declarations; pick one per session."
360
+ }));
361
+ const rate = yield* inputRate(request.inputFormat);
362
+ const scheduling = schedulingOf(request);
363
+ yield* checkOutputFormat(request.outputFormat);
364
+ const session = yield* WebSocketSession.open({
365
+ url: buildWsUrl(cfg),
366
+ provider: PROVIDER,
367
+ schema: ServerMessage,
368
+ openTimeout: Duration.seconds(30)
369
+ }).pipe(withSocket(cfg));
370
+ yield* session.send(setupFrame(request));
371
+ yield* awaitSetupComplete(session);
372
+ const history = request.history ?? [];
373
+ if (history.length > 0) {
374
+ const seed = yield* historyFrame(history);
375
+ yield* session.send(seed);
376
+ }
377
+ const state = yield* Ref.make(initialState);
378
+ const out = yield* Queue.bounded(128);
379
+ const pump = Stream.fromQueue(session.frames).pipe(Stream.mapEffect((frame) => toEvents(state, frame)), Stream.runForEach((events) => Queue.offerAll(out, events)));
380
+ const finish = (cause) => Effect.gen(function* () {
381
+ const { goingAway, responseId } = yield* Ref.get(state);
382
+ if (responseId !== void 0) return yield* Queue.fail(out, new AiError.IncompleteTurn({ raw: cause ?? "the socket closed mid turn" }));
383
+ if (goingAway) return yield* Queue.fail(out, new AiError.SessionExpired({
384
+ provider: PROVIDER,
385
+ raw: cause ?? "the socket closed after the server announced it was going away"
386
+ }));
387
+ return yield* cause !== void 0 && Cause.hasFails(cause) ? Queue.failCause(out, cause) : Queue.end(out);
388
+ });
389
+ yield* pump.pipe(Effect.matchCauseEffect({
390
+ onFailure: finish,
391
+ onSuccess: () => finish(void 0)
392
+ }), Effect.forkScoped);
393
+ const manual = request.turnDetection === "manual";
394
+ return {
395
+ send: (input) => handleInput(session.send, state, {
396
+ manual,
397
+ rate,
398
+ scheduling
399
+ }, input),
400
+ events: Stream.fromQueue(out)
401
+ };
402
+ });
403
+ //#endregion
404
+ export { buildWsUrl, openSession };
405
+
406
+ //# sourceMappingURL=realtimeSession.mjs.map