@effect-uai/google 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/dist/Gemini.d.mts +1 -1
- package/dist/GeminiEmbedding.d.mts +1 -1
- package/dist/GeminiImageGenerator.d.mts +1 -1
- package/dist/GeminiLiveSession.d.mts +17 -0
- package/dist/GeminiLiveSession.d.mts.map +1 -0
- package/dist/GeminiLiveSession.mjs +26 -0
- package/dist/GeminiLiveSession.mjs.map +1 -0
- package/dist/GeminiSynthesizer.d.mts +1 -1
- package/dist/GeminiSynthesizer.test.mjs +1 -1
- package/dist/GoogleDeepResearch.d.mts +1 -1
- package/dist/GoogleDeepResearch.test.mjs +1 -1
- package/dist/LyriaGenerator.d.mts +1 -1
- package/dist/LyriaGenerator.test.mjs +1 -1
- package/dist/codec.d.mts +5 -2
- package/dist/codec.d.mts.map +1 -1
- package/dist/codec.mjs +4 -1
- package/dist/codec.mjs.map +1 -1
- package/dist/geminiSpeechCodec.test.mjs +1 -1
- package/dist/geminiTools.test.mjs +1 -1
- package/dist/index.d.mts +3 -2
- package/dist/index.mjs +2 -1
- package/dist/models-BVSKJwCx.d.mts +87 -0
- package/dist/models-BVSKJwCx.d.mts.map +1 -0
- package/dist/models.d.mts +2 -75
- package/dist/realtimeSession-7HSIjWh4.d.mts +62 -0
- package/dist/realtimeSession-7HSIjWh4.d.mts.map +1 -0
- package/dist/realtimeSession.d.mts +2 -0
- package/dist/realtimeSession.mjs +406 -0
- package/dist/realtimeSession.mjs.map +1 -0
- package/dist/realtimeSession.test.d.mts +1 -0
- package/dist/realtimeSession.test.mjs +505 -0
- package/dist/realtimeSession.test.mjs.map +1 -0
- package/dist/{test.DNmyFkvJ-N3QqMRwZ.mjs → test.DNmyFkvJ-BPz3G1HS.mjs} +531 -22
- package/dist/test.DNmyFkvJ-BPz3G1HS.mjs.map +1 -0
- package/package.json +6 -2
- package/src/GeminiLiveSession.ts +45 -0
- package/src/codec.ts +3 -2
- package/src/index.ts +1 -0
- package/src/models.ts +19 -0
- package/src/realtimeSession.test.ts +415 -0
- package/src/realtimeSession.ts +807 -0
- package/dist/models.d.mts.map +0 -1
- package/dist/test.DNmyFkvJ-N3QqMRwZ.mjs.map +0 -1
package/dist/models.d.mts
CHANGED
|
@@ -1,75 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
* Known Gemini model identifiers (as of April 2026). The `(string & {})`
|
|
4
|
-
* tail keeps autocomplete on the literals while still accepting any string,
|
|
5
|
-
* so newly-released models work without an SDK update.
|
|
6
|
-
*
|
|
7
|
-
* Reference: https://ai.google.dev/gemini-api/docs/models
|
|
8
|
-
*/
|
|
9
|
-
type GoogleModel = "gemini-3.1-pro-preview" | "gemini-3-flash-preview" | "gemini-3.1-flash-lite-preview" | "gemini-3.1-flash-live-preview" | "gemini-3.1-flash-tts-preview" | "gemini-2.5-pro" | "gemini-2.5-flash" | "gemini-2.5-flash-lite" | "gemini-3.1-flash-image" | "gemini-3.1-flash-lite-image" | "gemini-3-pro-image" | (string & {});
|
|
10
|
-
/**
|
|
11
|
-
* Known Gemini image-generation model identifiers. The `-preview` ids
|
|
12
|
-
* were shut down 2026-06-25 and are not listed; neither is the
|
|
13
|
-
* previous-generation `gemini-2.5-flash-image`, which the `(string & {})`
|
|
14
|
-
* tail still accepts.
|
|
15
|
-
*
|
|
16
|
-
* - `gemini-3.1-flash-image` — 512 / 1K / 2K / 4K, web and image search.
|
|
17
|
-
* - `gemini-3.1-flash-lite-image` — 1K only, cheapest, no search.
|
|
18
|
-
* - `gemini-3-pro-image` — 1K / 2K / 4K, the most reference images.
|
|
19
|
-
*
|
|
20
|
-
* Reference: https://ai.google.dev/gemini-api/docs/models
|
|
21
|
-
*/
|
|
22
|
-
type GeminiImageModel = "gemini-3.1-flash-image" | "gemini-3.1-flash-lite-image" | "gemini-3-pro-image" | (string & {});
|
|
23
|
-
/**
|
|
24
|
-
* Deep-research agent identifiers for the Interactions API (`POST
|
|
25
|
-
* /v1beta/interactions`). `-preview` is the fast tier, `-max-` the
|
|
26
|
-
* comprehensive one. Preview surface: verify against live docs before relying on
|
|
27
|
-
* exact ids. The `(string & {})` tail accepts any string so a renamed agent
|
|
28
|
-
* works without an SDK update.
|
|
29
|
-
*
|
|
30
|
-
* Reference: https://ai.google.dev/gemini-api/docs/interactions/deep-research
|
|
31
|
-
*/
|
|
32
|
-
type GoogleDeepResearchModel = "deep-research-preview-04-2026" | "deep-research-max-preview-04-2026" | (string & {});
|
|
33
|
-
/**
|
|
34
|
-
* Known Gemini embedding model identifiers.
|
|
35
|
-
*
|
|
36
|
-
* - `gemini-embedding-2` — natively multimodal (text, image, audio, video,
|
|
37
|
-
* PDF). Does *not* honour the `taskType` field; instead prepend a task
|
|
38
|
-
* instruction to the text yourself.
|
|
39
|
-
* - `gemini-embedding-001` — text-only, supports the full `taskType` enum
|
|
40
|
-
* (`RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` / classification / etc.).
|
|
41
|
-
*
|
|
42
|
-
* The `(string & {})` tail accepts any string so newly-released models
|
|
43
|
-
* work without an SDK update.
|
|
44
|
-
*
|
|
45
|
-
* Reference: https://ai.google.dev/gemini-api/docs/embeddings
|
|
46
|
-
*/
|
|
47
|
-
/**
|
|
48
|
-
* Known Gemini TTS model identifiers. `(string & {})` tail keeps
|
|
49
|
-
* autocomplete on the literals while still accepting any string.
|
|
50
|
-
*
|
|
51
|
-
* Reference: https://ai.google.dev/gemini-api/docs/speech-generation
|
|
52
|
-
*/
|
|
53
|
-
type GeminiTtsModel = "gemini-2.5-flash-preview-tts" | "gemini-2.5-pro-preview-tts" | "gemini-3.1-flash-tts-preview" | (string & {});
|
|
54
|
-
/**
|
|
55
|
-
* 30 prebuilt voice names for Gemini TTS. No custom-voice / cloning path
|
|
56
|
-
* is exposed on this surface, so the tail is stock-only (no
|
|
57
|
-
* `(string & {})` escape).
|
|
58
|
-
*
|
|
59
|
-
* Reference: https://ai.google.dev/gemini-api/docs/speech-generation
|
|
60
|
-
*/
|
|
61
|
-
type GeminiVoiceName = "Zephyr" | "Puck" | "Charon" | "Kore" | "Fenrir" | "Leda" | "Orus" | "Aoede" | "Callirrhoe" | "Autonoe" | "Enceladus" | "Iapetus" | "Umbriel" | "Algieba" | "Despina" | "Erinome" | "Algenib" | "Rasalgethi" | "Laomedeia" | "Achernar" | "Alnilam" | "Schedar" | "Gacrux" | "Pulcherrima" | "Achird" | "Zubenelgenubi" | "Vindemiatrix" | "Sadachbia" | "Sadaltager" | "Sulafat";
|
|
62
|
-
/**
|
|
63
|
-
* Known Lyria (music generation) model identifiers. `(string & {})` tail
|
|
64
|
-
* keeps autocomplete on the literals while still accepting any string.
|
|
65
|
-
*
|
|
66
|
-
* - `lyria-3-clip-preview` — fixed 30s output, MP3 only.
|
|
67
|
-
* - `lyria-3-pro-preview` — up to a couple of minutes, MP3 or WAV.
|
|
68
|
-
*
|
|
69
|
-
* Reference: https://ai.google.dev/gemini-api/docs/music-generation
|
|
70
|
-
*/
|
|
71
|
-
type LyriaModel = "lyria-3-clip-preview" | "lyria-3-pro-preview" | (string & {});
|
|
72
|
-
type GoogleEmbeddingModel = "gemini-embedding-2" | "gemini-embedding-001" | (string & {});
|
|
73
|
-
//#endregion
|
|
74
|
-
export { GeminiImageModel, GeminiTtsModel, GeminiVoiceName, GoogleDeepResearchModel, GoogleEmbeddingModel, GoogleModel, LyriaModel };
|
|
75
|
-
//# sourceMappingURL=models.d.mts.map
|
|
1
|
+
import { a as GoogleDeepResearchModel, c as LyriaModel, i as GeminiVoiceName, n as GeminiLiveModel, o as GoogleEmbeddingModel, r as GeminiTtsModel, s as GoogleModel, t as GeminiImageModel } from "./models-BVSKJwCx.mjs";
|
|
2
|
+
export { GeminiImageModel, GeminiLiveModel, GeminiTtsModel, GeminiVoiceName, GoogleDeepResearchModel, GoogleEmbeddingModel, GoogleModel, LyriaModel };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { n as GeminiLiveModel } from "./models-BVSKJwCx.mjs";
|
|
2
|
+
import { Effect, Redacted, Scope } from "effect";
|
|
3
|
+
import * as AiError from "@effect-uai/core/AiError";
|
|
4
|
+
import { RealtimeSessionHandle } from "@effect-uai/core/RealtimeSession";
|
|
5
|
+
import * as Socket from "effect/unstable/socket/Socket";
|
|
6
|
+
import { CommonSessionRequest } from "@effect-uai/core/Realtime";
|
|
7
|
+
//#region src/realtimeSession.d.ts
|
|
8
|
+
type Config = {
|
|
9
|
+
readonly apiKey: Redacted.Redacted;
|
|
10
|
+
readonly baseUrl?: string;
|
|
11
|
+
/** Build the socket yourself, for a proxy or an in-memory transport. */
|
|
12
|
+
readonly webSocket?: Socket.WebSocketConstructor["Service"];
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Gemini-typed session request. Everything past `CommonSessionRequest` is
|
|
16
|
+
* provider-shaped. The configuration is immutable for the connection: changing
|
|
17
|
+
* any of it means opening a new session.
|
|
18
|
+
*/
|
|
19
|
+
type GeminiLiveRequest = Omit<CommonSessionRequest, "model"> & {
|
|
20
|
+
readonly model: GeminiLiveModel;
|
|
21
|
+
/** Replaces the plain `turnDetection` switch when you need the knobs. */
|
|
22
|
+
readonly vad?: {
|
|
23
|
+
readonly startSensitivity?: "high" | "low";
|
|
24
|
+
readonly endSensitivity?: "high" | "low";
|
|
25
|
+
readonly prefixPaddingMs?: number;
|
|
26
|
+
readonly silenceDurationMs?: number;
|
|
27
|
+
};
|
|
28
|
+
/** `"none"` turns barge-in off. Defaults to interrupting. */
|
|
29
|
+
readonly activityHandling?: "interrupt" | "none";
|
|
30
|
+
/**
|
|
31
|
+
* When a tool result reaches the model, for the models that keep generating
|
|
32
|
+
* while a tool runs. Defaults to `"when-idle"`: the server's own default is
|
|
33
|
+
* to interrupt, which cuts off whatever the model was saying while it
|
|
34
|
+
* waited, including the "let me look that up" it just spoke. `"interrupt"`
|
|
35
|
+
* is right when the result cannot wait, `"silent"` when it is context the
|
|
36
|
+
* model should have rather than something to announce.
|
|
37
|
+
*/
|
|
38
|
+
readonly toolScheduling?: "when-idle" | "interrupt" | "silent";
|
|
39
|
+
readonly turnCoverage?: "activity" | "all" | "audioActivityAndAllVideo";
|
|
40
|
+
/** 3.1 only. Higher levels delay the first audio. */
|
|
41
|
+
readonly thinkingLevel?: "minimal" | "low" | "medium" | "high";
|
|
42
|
+
readonly mediaResolution?: "low" | "medium" | "high";
|
|
43
|
+
readonly languageCode?: string;
|
|
44
|
+
readonly temperature?: number;
|
|
45
|
+
readonly maxOutputTokens?: number;
|
|
46
|
+
/** What turns an otherwise time-boxed session into an unbounded one. */
|
|
47
|
+
readonly contextCompression?: {
|
|
48
|
+
readonly triggerTokens?: number;
|
|
49
|
+
readonly targetTokens?: number;
|
|
50
|
+
};
|
|
51
|
+
/** Grounding. Cannot be combined with `tools`; the pair is a 400. */
|
|
52
|
+
readonly googleSearch?: boolean;
|
|
53
|
+
/** 2.5 only. */
|
|
54
|
+
readonly proactiveAudio?: boolean;
|
|
55
|
+
/** 2.5 only. */
|
|
56
|
+
readonly affectiveDialog?: boolean;
|
|
57
|
+
};
|
|
58
|
+
declare const buildWsUrl: (cfg: Config) => string;
|
|
59
|
+
declare const openSession: (cfg: Config) => (request: GeminiLiveRequest) => Effect.Effect<RealtimeSessionHandle, AiError.AiError, Scope.Scope>;
|
|
60
|
+
//#endregion
|
|
61
|
+
export { openSession as i, GeminiLiveRequest as n, buildWsUrl as r, Config as t };
|
|
62
|
+
//# sourceMappingURL=realtimeSession-7HSIjWh4.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtimeSession-7HSIjWh4.d.mts","names":[],"sources":["../src/realtimeSession.ts"],"mappings":";;;;;;;KA8CY;WACD,QAAQ,SAAS;WACjB;;WAEA,YAAY,OAAO;;;;;;;KAQlB,oBAAoB,KAAK;WAC1B,OAAO;;WAEP;aACE;aACA;aACA;aACA;;;WAGF;;;;;;;;;WASA;WACA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;aACE;aACA;;;WAGF;;WAEA;;WAEA;;cAilBE,aAAU,KAAS;cAqCnB,cAAW,KAChB,YAAM,SAED,sBACR,OAAO,OAAO,uBAAuB,QAAQ,SAAS,MAAM"}
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { parsedResponse, toolDescriptorsToTools } from "./codec.mjs";
|
|
2
|
+
import { Cause, Duration, Effect, Encoding, Match, Pull, Queue, Redacted, Ref, Result, Schema, Stream } from "effect";
|
|
3
|
+
import * as AiError from "@effect-uai/core/AiError";
|
|
4
|
+
import * as Capabilities from "@effect-uai/core/Capabilities";
|
|
5
|
+
import * as Socket from "effect/unstable/socket/Socket";
|
|
6
|
+
import { RealtimeEvent } from "@effect-uai/core/Realtime";
|
|
7
|
+
import * as WebSocketSession from "@effect-uai/core/WebSocketSession";
|
|
8
|
+
//#region src/realtimeSession.ts
|
|
9
|
+
/**
|
|
10
|
+
* Gemini Live speech-to-speech over `BidiGenerateContent`.
|
|
11
|
+
*
|
|
12
|
+
* Auth is a query parameter, so this needs no `ws` package and no headers:
|
|
13
|
+
* the platform `WebSocket` reaches it from Node, Bun, Deno and the browser.
|
|
14
|
+
*
|
|
15
|
+
* Two shapes differ from OpenAI Realtime and drive most of this file. There
|
|
16
|
+
* are no response ids on the wire, so a turn id is minted here and every event
|
|
17
|
+
* of that turn carries it; and there is no truncate op, so a
|
|
18
|
+
* `PlaybackPosition` has nothing to send.
|
|
19
|
+
*/
|
|
20
|
+
const PROVIDER = "gemini-live";
|
|
21
|
+
const DEFAULT_BASE_URL = "wss://generativelanguage.googleapis.com";
|
|
22
|
+
const RPC = "ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
23
|
+
/** Output is 24 kHz whatever the input rate, and cannot be configured. */
|
|
24
|
+
const OUTPUT_RATE = 24e3;
|
|
25
|
+
const unsupportedFormat = (format, field, expected) => new AiError.Unsupported({
|
|
26
|
+
provider: PROVIDER,
|
|
27
|
+
capability: field,
|
|
28
|
+
reason: `Gemini Live accepts ${expected}. Got ${JSON.stringify(format)}.`
|
|
29
|
+
});
|
|
30
|
+
/** Any rate is accepted on input; 16 kHz is what the model works in. */
|
|
31
|
+
const inputRate = (format) => Match.value(format).pipe(Match.when({
|
|
32
|
+
container: "raw",
|
|
33
|
+
encoding: "pcm_s16le",
|
|
34
|
+
channels: 1
|
|
35
|
+
}, (f) => Effect.succeed(f.sampleRate)), Match.orElse((f) => Effect.fail(unsupportedFormat(f, "inputFormat", "mono pcm_s16le at any rate"))));
|
|
36
|
+
const checkOutputFormat = (format) => Match.value(format).pipe(Match.when({
|
|
37
|
+
container: "raw",
|
|
38
|
+
encoding: "pcm_s16le",
|
|
39
|
+
sampleRate: OUTPUT_RATE,
|
|
40
|
+
channels: 1
|
|
41
|
+
}, () => Effect.void), Match.orElse((f) => Effect.fail(unsupportedFormat(f, "outputFormat", `mono pcm_s16le at ${OUTPUT_RATE}`))));
|
|
42
|
+
const SENSITIVITY = {
|
|
43
|
+
high: "HIGH",
|
|
44
|
+
low: "LOW"
|
|
45
|
+
};
|
|
46
|
+
const TURN_COVERAGE = {
|
|
47
|
+
activity: "TURN_INCLUDES_ONLY_ACTIVITY",
|
|
48
|
+
all: "TURN_INCLUDES_ALL_INPUT",
|
|
49
|
+
audioActivityAndAllVideo: "TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO"
|
|
50
|
+
};
|
|
51
|
+
const realtimeInputConfig = (request) => ({
|
|
52
|
+
automaticActivityDetection: {
|
|
53
|
+
...request.turnDetection === "manual" && { disabled: true },
|
|
54
|
+
...request.vad?.startSensitivity !== void 0 && { startOfSpeechSensitivity: `START_SENSITIVITY_${SENSITIVITY[request.vad.startSensitivity]}` },
|
|
55
|
+
...request.vad?.endSensitivity !== void 0 && { endOfSpeechSensitivity: `END_SENSITIVITY_${SENSITIVITY[request.vad.endSensitivity]}` },
|
|
56
|
+
...request.vad?.prefixPaddingMs !== void 0 && { prefixPaddingMs: request.vad.prefixPaddingMs },
|
|
57
|
+
...request.vad?.silenceDurationMs !== void 0 && { silenceDurationMs: request.vad.silenceDurationMs }
|
|
58
|
+
},
|
|
59
|
+
...request.activityHandling !== void 0 && { activityHandling: request.activityHandling === "none" ? "NO_INTERRUPTION" : "START_OF_ACTIVITY_INTERRUPTS" },
|
|
60
|
+
...request.turnCoverage !== void 0 && { turnCoverage: TURN_COVERAGE[request.turnCoverage] }
|
|
61
|
+
});
|
|
62
|
+
const generationConfig = (request) => ({
|
|
63
|
+
responseModalities: ["AUDIO"],
|
|
64
|
+
...request.voiceId !== void 0 && { speechConfig: {
|
|
65
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: request.voiceId } },
|
|
66
|
+
...request.languageCode !== void 0 && { languageCode: request.languageCode }
|
|
67
|
+
} },
|
|
68
|
+
...request.temperature !== void 0 && { temperature: request.temperature },
|
|
69
|
+
...request.maxOutputTokens !== void 0 && { maxOutputTokens: request.maxOutputTokens },
|
|
70
|
+
...request.mediaResolution !== void 0 && { mediaResolution: `MEDIA_RESOLUTION_${request.mediaResolution.toUpperCase()}` },
|
|
71
|
+
...request.thinkingLevel !== void 0 && { thinkingConfig: { thinkingLevel: request.thinkingLevel.toUpperCase() } },
|
|
72
|
+
...request.affectiveDialog === true && { enableAffectiveDialog: true }
|
|
73
|
+
});
|
|
74
|
+
const toolsOf = (request) => request.googleSearch === true ? [{ googleSearch: {} }] : toolDescriptorsToTools(request.tools ?? []);
|
|
75
|
+
const setupFrame = (request) => {
|
|
76
|
+
const tools = toolsOf(request);
|
|
77
|
+
return JSON.stringify({ setup: {
|
|
78
|
+
model: `models/${request.model}`,
|
|
79
|
+
generationConfig: generationConfig(request),
|
|
80
|
+
...request.instructions !== void 0 && { systemInstruction: { parts: [{ text: request.instructions }] } },
|
|
81
|
+
...tools.length > 0 && { tools },
|
|
82
|
+
realtimeInputConfig: realtimeInputConfig(request),
|
|
83
|
+
...request.transcribeInput !== false && { inputAudioTranscription: {} },
|
|
84
|
+
outputAudioTranscription: {},
|
|
85
|
+
sessionResumption: request.resume === void 0 ? {} : { handle: request.resume },
|
|
86
|
+
...request.contextCompression !== void 0 && { contextWindowCompression: {
|
|
87
|
+
slidingWindow: { ...request.contextCompression.targetTokens !== void 0 && { targetTokens: request.contextCompression.targetTokens } },
|
|
88
|
+
...request.contextCompression.triggerTokens !== void 0 && { triggerTokens: request.contextCompression.triggerTokens }
|
|
89
|
+
} },
|
|
90
|
+
...request.proactiveAudio === true && { proactivity: { proactiveAudio: true } },
|
|
91
|
+
...(request.history?.length ?? 0) > 0 && { historyConfig: { initialHistoryInClientContent: true } }
|
|
92
|
+
} });
|
|
93
|
+
};
|
|
94
|
+
const messageText = (item) => item.type !== "message" ? "" : item.content.map((block) => block.type === "input_text" || block.type === "output_text" ? block.text : "").join("");
|
|
95
|
+
/**
|
|
96
|
+
* Text-only seed, which is all `history` promises across providers. Anything
|
|
97
|
+
* with no text of its own, a tool call or its output, has no place in the seed
|
|
98
|
+
* and is reported once per kind rather than vanishing.
|
|
99
|
+
*/
|
|
100
|
+
const historyFrame = (history) => Effect.gen(function* () {
|
|
101
|
+
const kinds = new Set(history.filter((i) => messageText(i).length === 0).map((i) => i.type));
|
|
102
|
+
yield* Effect.forEach(kinds, (kind) => Capabilities.warnDropped({
|
|
103
|
+
provider: PROVIDER,
|
|
104
|
+
capability: "history",
|
|
105
|
+
field: "history",
|
|
106
|
+
value: kind,
|
|
107
|
+
reason: `Gemini Live seeds a session with text turns only, so \`${kind}\` items are left out.`
|
|
108
|
+
}), { discard: true });
|
|
109
|
+
return JSON.stringify({ clientContent: {
|
|
110
|
+
turns: history.filter((item) => messageText(item).length > 0).map((item) => ({
|
|
111
|
+
role: item.type === "message" && item.role === "assistant" ? "model" : "user",
|
|
112
|
+
parts: [{ text: messageText(item) }]
|
|
113
|
+
})),
|
|
114
|
+
turnComplete: true
|
|
115
|
+
} });
|
|
116
|
+
});
|
|
117
|
+
const Part = Schema.Struct({
|
|
118
|
+
text: Schema.optional(Schema.String),
|
|
119
|
+
thought: Schema.optional(Schema.Boolean),
|
|
120
|
+
inlineData: Schema.optional(Schema.Struct({
|
|
121
|
+
mimeType: Schema.optional(Schema.String),
|
|
122
|
+
data: Schema.optional(Schema.String)
|
|
123
|
+
}))
|
|
124
|
+
});
|
|
125
|
+
const Transcription = Schema.Struct({ text: Schema.optional(Schema.String) });
|
|
126
|
+
const ServerContent = Schema.Struct({
|
|
127
|
+
modelTurn: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(Part)) })),
|
|
128
|
+
turnComplete: Schema.optional(Schema.Boolean),
|
|
129
|
+
generationComplete: Schema.optional(Schema.Boolean),
|
|
130
|
+
interrupted: Schema.optional(Schema.Boolean),
|
|
131
|
+
inputTranscription: Schema.optional(Transcription),
|
|
132
|
+
interimInputTranscription: Schema.optional(Transcription),
|
|
133
|
+
outputTranscription: Schema.optional(Transcription)
|
|
134
|
+
});
|
|
135
|
+
const UsageMetadata = Schema.Struct({
|
|
136
|
+
promptTokenCount: Schema.optional(Schema.Number),
|
|
137
|
+
responseTokenCount: Schema.optional(Schema.Number),
|
|
138
|
+
totalTokenCount: Schema.optional(Schema.Number)
|
|
139
|
+
});
|
|
140
|
+
const FunctionCall = Schema.Struct({
|
|
141
|
+
id: Schema.optional(Schema.String),
|
|
142
|
+
name: Schema.String,
|
|
143
|
+
args: Schema.optional(Schema.Unknown)
|
|
144
|
+
});
|
|
145
|
+
const ServerMessage = Schema.Struct({
|
|
146
|
+
setupComplete: Schema.optional(Schema.Unknown),
|
|
147
|
+
serverContent: Schema.optional(ServerContent),
|
|
148
|
+
toolCall: Schema.optional(Schema.Struct({ functionCalls: Schema.optional(Schema.Array(FunctionCall)) })),
|
|
149
|
+
toolCallCancellation: Schema.optional(Schema.Struct({ ids: Schema.optional(Schema.Array(Schema.String)) })),
|
|
150
|
+
goAway: Schema.optional(Schema.Struct({ timeLeft: Schema.optional(Schema.String) })),
|
|
151
|
+
sessionResumptionUpdate: Schema.optional(Schema.Struct({
|
|
152
|
+
newHandle: Schema.optional(Schema.String),
|
|
153
|
+
resumable: Schema.optional(Schema.Boolean)
|
|
154
|
+
})),
|
|
155
|
+
usageMetadata: Schema.optional(UsageMetadata)
|
|
156
|
+
});
|
|
157
|
+
const initialState = {
|
|
158
|
+
responseId: void 0,
|
|
159
|
+
turns: 0,
|
|
160
|
+
callNames: {},
|
|
161
|
+
usage: void 0,
|
|
162
|
+
goingAway: false
|
|
163
|
+
};
|
|
164
|
+
/**
|
|
165
|
+
* The id every event of the current turn carries, minting one and announcing
|
|
166
|
+
* the turn if this is its first event.
|
|
167
|
+
*/
|
|
168
|
+
const startTurn = (state) => Ref.modify(state, (s) => {
|
|
169
|
+
const responseId = s.responseId ?? `turn_${s.turns + 1}`;
|
|
170
|
+
return s.responseId !== void 0 ? [[responseId, []], s] : [[responseId, [RealtimeEvent.ResponseStarted({ responseId })]], {
|
|
171
|
+
...s,
|
|
172
|
+
responseId,
|
|
173
|
+
turns: s.turns + 1
|
|
174
|
+
}];
|
|
175
|
+
});
|
|
176
|
+
const usageOf = (metadata) => ({
|
|
177
|
+
...metadata.promptTokenCount !== void 0 && { input_tokens: metadata.promptTokenCount },
|
|
178
|
+
...metadata.responseTokenCount !== void 0 && { output_tokens: metadata.responseTokenCount },
|
|
179
|
+
...metadata.totalTokenCount !== void 0 && { total_tokens: metadata.totalTokenCount }
|
|
180
|
+
});
|
|
181
|
+
const decodeAudio = (b64) => Result.match(Encoding.decodeBase64(b64), {
|
|
182
|
+
onSuccess: Effect.succeed,
|
|
183
|
+
onFailure: (cause) => Effect.fail(new AiError.GenerationFailed({
|
|
184
|
+
provider: PROVIDER,
|
|
185
|
+
raw: {
|
|
186
|
+
message: "audio part base64 decode failed",
|
|
187
|
+
cause
|
|
188
|
+
}
|
|
189
|
+
}))
|
|
190
|
+
});
|
|
191
|
+
/** `timeLeft` is a proto duration, so `"60s"` rather than a number. */
|
|
192
|
+
const parseTimeLeft = (timeLeft) => {
|
|
193
|
+
if (timeLeft === void 0) return void 0;
|
|
194
|
+
const seconds = Number.parseFloat(timeLeft.replace(/s$/, ""));
|
|
195
|
+
return Number.isFinite(seconds) ? Duration.seconds(seconds) : void 0;
|
|
196
|
+
};
|
|
197
|
+
/** Thought parts are the model reasoning, not something to speak or show. */
|
|
198
|
+
const partToEvents = (responseId, part) => Match.value(part).pipe(Match.when({ thought: true }, () => Effect.succeed([])), Match.when({ inlineData: { data: Match.string } }, (p) => Effect.map(decodeAudio(p.inlineData.data), (bytes) => [RealtimeEvent.AudioDelta({
|
|
199
|
+
responseId,
|
|
200
|
+
bytes
|
|
201
|
+
})])), Match.when({ text: Match.string }, (p) => Effect.succeed([RealtimeEvent.OutputTranscriptDelta({
|
|
202
|
+
responseId,
|
|
203
|
+
text: p.text
|
|
204
|
+
})])), Match.orElse(() => Effect.succeed([])));
|
|
205
|
+
const inputTranscripts = (content) => [...content.interimInputTranscription?.text === void 0 ? [] : [RealtimeEvent.InputTranscript({
|
|
206
|
+
text: content.interimInputTranscription.text,
|
|
207
|
+
final: false
|
|
208
|
+
})], ...content.inputTranscription?.text === void 0 ? [] : [RealtimeEvent.InputTranscript({
|
|
209
|
+
text: content.inputTranscription.text,
|
|
210
|
+
final: true
|
|
211
|
+
})]];
|
|
212
|
+
/**
|
|
213
|
+
* One `serverContent` to its events. Several fields can arrive together, and
|
|
214
|
+
* 3.1 packs several parts into one message, so this answers with a batch.
|
|
215
|
+
*/
|
|
216
|
+
const contentToEvents = (state, content) => Effect.gen(function* () {
|
|
217
|
+
const input = inputTranscripts(content);
|
|
218
|
+
const parts = content.modelTurn?.parts ?? [];
|
|
219
|
+
const spokenText = content.outputTranscription?.text;
|
|
220
|
+
const generating = parts.length > 0 || spokenText !== void 0;
|
|
221
|
+
const ending = content.interrupted === true || content.turnComplete === true;
|
|
222
|
+
if (!generating && !ending) return input;
|
|
223
|
+
const [responseId, started] = generating ? yield* startTurn(state) : [(yield* Ref.get(state)).responseId, []];
|
|
224
|
+
if (responseId === void 0) return input;
|
|
225
|
+
const spoken = yield* Effect.forEach(parts, (part) => partToEvents(responseId, part));
|
|
226
|
+
const transcript = spokenText === void 0 ? [] : [RealtimeEvent.OutputTranscriptDelta({
|
|
227
|
+
responseId,
|
|
228
|
+
text: spokenText
|
|
229
|
+
})];
|
|
230
|
+
const body = [
|
|
231
|
+
...input,
|
|
232
|
+
...started,
|
|
233
|
+
...spoken.flat(),
|
|
234
|
+
...transcript
|
|
235
|
+
];
|
|
236
|
+
if (!ending) return body;
|
|
237
|
+
const usage = (yield* Ref.get(state)).usage;
|
|
238
|
+
yield* Ref.update(state, (s) => ({
|
|
239
|
+
...s,
|
|
240
|
+
responseId: void 0,
|
|
241
|
+
usage: void 0
|
|
242
|
+
}));
|
|
243
|
+
return [
|
|
244
|
+
...body,
|
|
245
|
+
...content.interrupted === true ? [RealtimeEvent.Interrupted({ responseId })] : [],
|
|
246
|
+
RealtimeEvent.ResponseDone({
|
|
247
|
+
responseId,
|
|
248
|
+
reason: content.interrupted === true ? "interrupted" : "complete",
|
|
249
|
+
...usage !== void 0 && { usage }
|
|
250
|
+
})
|
|
251
|
+
];
|
|
252
|
+
});
|
|
253
|
+
const toolCallEvents = (state, calls) => Effect.gen(function* () {
|
|
254
|
+
const [responseId, started] = yield* startTurn(state);
|
|
255
|
+
yield* Ref.update(state, (s) => ({
|
|
256
|
+
...s,
|
|
257
|
+
callNames: calls.reduce((names, call) => ({
|
|
258
|
+
...names,
|
|
259
|
+
[call.id ?? call.name]: call.name
|
|
260
|
+
}), s.callNames)
|
|
261
|
+
}));
|
|
262
|
+
return [...started, ...calls.map((call) => RealtimeEvent.ToolCall({
|
|
263
|
+
responseId,
|
|
264
|
+
call: {
|
|
265
|
+
type: "function_call",
|
|
266
|
+
call_id: call.id ?? call.name,
|
|
267
|
+
name: call.name,
|
|
268
|
+
arguments: JSON.stringify(call.args ?? {}),
|
|
269
|
+
providerData: void 0
|
|
270
|
+
}
|
|
271
|
+
}))];
|
|
272
|
+
});
|
|
273
|
+
const toEvents = (state, frame) => Effect.gen(function* () {
|
|
274
|
+
if (frame.usageMetadata !== void 0) {
|
|
275
|
+
const usage = usageOf(frame.usageMetadata);
|
|
276
|
+
yield* Ref.update(state, (s) => ({
|
|
277
|
+
...s,
|
|
278
|
+
usage
|
|
279
|
+
}));
|
|
280
|
+
}
|
|
281
|
+
if (frame.serverContent !== void 0) return yield* contentToEvents(state, frame.serverContent);
|
|
282
|
+
if (frame.toolCall !== void 0) return yield* toolCallEvents(state, frame.toolCall.functionCalls ?? []);
|
|
283
|
+
const callIds = frame.toolCallCancellation?.ids ?? [];
|
|
284
|
+
if (callIds.length > 0) return [RealtimeEvent.ToolCallCancelled({ callIds })];
|
|
285
|
+
if (frame.goAway !== void 0) {
|
|
286
|
+
const timeLeft = parseTimeLeft(frame.goAway.timeLeft);
|
|
287
|
+
yield* Ref.update(state, (s) => ({
|
|
288
|
+
...s,
|
|
289
|
+
goingAway: true
|
|
290
|
+
}));
|
|
291
|
+
return [RealtimeEvent.SessionEnding({ ...timeLeft !== void 0 && { timeLeft } })];
|
|
292
|
+
}
|
|
293
|
+
const handle = frame.sessionResumptionUpdate?.newHandle;
|
|
294
|
+
return frame.sessionResumptionUpdate?.resumable === true && handle !== void 0 && handle !== "" ? [RealtimeEvent.ResumptionHandle({ handle })] : [];
|
|
295
|
+
});
|
|
296
|
+
const droppedActivity = (field) => Capabilities.warnDropped({
|
|
297
|
+
provider: PROVIDER,
|
|
298
|
+
capability: "manualTurns",
|
|
299
|
+
field,
|
|
300
|
+
reason: "The session uses automatic activity detection; boundaries are the server's."
|
|
301
|
+
});
|
|
302
|
+
const videoPart = (frame) => Match.value(frame).pipe(Match.tag("base64", (s) => Effect.succeed({
|
|
303
|
+
data: s.base64,
|
|
304
|
+
mimeType: s.mimeType
|
|
305
|
+
})), Match.tag("bytes", (s) => Effect.succeed({
|
|
306
|
+
data: Encoding.encodeBase64(s.bytes),
|
|
307
|
+
mimeType: s.mimeType
|
|
308
|
+
})), Match.tag("url", () => Effect.fail(new AiError.Unsupported({
|
|
309
|
+
provider: PROVIDER,
|
|
310
|
+
capability: "videoInput",
|
|
311
|
+
reason: "A live video frame must carry its bytes; a URL would need the Files API."
|
|
312
|
+
}))), Match.exhaustive);
|
|
313
|
+
const schedulingOf = (request) => Match.value(request.toolScheduling ?? "when-idle").pipe(Match.when("when-idle", () => "WHEN_IDLE"), Match.when("interrupt", () => "INTERRUPT"), Match.when("silent", () => "SILENT"), Match.exhaustive);
|
|
314
|
+
const handleInput = (send, state, options, input) => Match.value(input).pipe(Match.tag("Audio", (i) => send(JSON.stringify({ realtimeInput: { audio: {
|
|
315
|
+
data: Encoding.encodeBase64(i.bytes),
|
|
316
|
+
mimeType: `audio/pcm;rate=${options.rate}`
|
|
317
|
+
} } }))), Match.tag("Text", (i) => i.role === "system" ? Effect.fail(new AiError.Unsupported({
|
|
318
|
+
provider: PROVIDER,
|
|
319
|
+
capability: "systemTurn",
|
|
320
|
+
reason: "Gemini Live fixes the system instruction at setup for the connection."
|
|
321
|
+
})) : send(JSON.stringify({ realtimeInput: { text: i.text } }))), Match.tag("ToolResult", (i) => Effect.gen(function* () {
|
|
322
|
+
const name = (yield* Ref.get(state)).callNames[i.output.call_id];
|
|
323
|
+
if (name === void 0) return yield* Effect.fail(new AiError.InvalidRequest({
|
|
324
|
+
provider: PROVIDER,
|
|
325
|
+
raw: `no tool call ${i.output.call_id} in this session`
|
|
326
|
+
}));
|
|
327
|
+
yield* send(JSON.stringify({ toolResponse: { functionResponses: [{
|
|
328
|
+
id: i.output.call_id,
|
|
329
|
+
name,
|
|
330
|
+
response: parsedResponse(i.output.output),
|
|
331
|
+
scheduling: options.scheduling
|
|
332
|
+
}] } }));
|
|
333
|
+
})), Match.tag("Interrupt", () => Effect.fail(new AiError.Unsupported({
|
|
334
|
+
provider: PROVIDER,
|
|
335
|
+
capability: "interrupt",
|
|
336
|
+
reason: "Gemini Live has no cancel op: the server interrupts when it hears speech. Pass activityHandling: \"none\" to turn that off."
|
|
337
|
+
}))), Match.tag("VideoFrame", (i) => Effect.flatMap(videoPart(i.frame), (video) => send(JSON.stringify({ realtimeInput: { video } })))), Match.tag("ActivityStart", () => options.manual ? send(JSON.stringify({ realtimeInput: { activityStart: {} } })) : droppedActivity("ActivityStart")), Match.tag("ActivityEnd", () => options.manual ? send(JSON.stringify({ realtimeInput: { activityEnd: {} } })) : droppedActivity("ActivityEnd")), Match.tag("PlaybackPosition", () => Capabilities.warnDropped({
|
|
338
|
+
provider: PROVIDER,
|
|
339
|
+
capability: "truncate",
|
|
340
|
+
field: "PlaybackPosition",
|
|
341
|
+
reason: "Gemini Live has no truncate op, so the unheard tail stays in context."
|
|
342
|
+
})), Match.exhaustive);
|
|
343
|
+
const buildWsUrl = (cfg) => {
|
|
344
|
+
const url = new URL(cfg.baseUrl ?? DEFAULT_BASE_URL);
|
|
345
|
+
url.protocol = url.protocol === "http:" ? "ws:" : "wss:";
|
|
346
|
+
url.pathname = `${url.pathname.replace(/\/$/, "")}/${RPC}`;
|
|
347
|
+
url.searchParams.set("key", Redacted.value(cfg.apiKey));
|
|
348
|
+
return url.toString();
|
|
349
|
+
};
|
|
350
|
+
const withSocket = (cfg) => (effect) => cfg.webSocket === void 0 ? Effect.provide(effect, Socket.layerWebSocketConstructorGlobal) : Effect.provideService(effect, Socket.WebSocketConstructor, cfg.webSocket);
|
|
351
|
+
/** Drop frames until `setupComplete`; a close before it is a failed handshake. */
|
|
352
|
+
const awaitSetupComplete = (session) => Queue.take(session.frames).pipe(Effect.catchCause((cause) => Pull.isDoneCause(cause) ? Effect.fail(new AiError.Unavailable({
|
|
353
|
+
provider: PROVIDER,
|
|
354
|
+
raw: "the socket closed during the handshake"
|
|
355
|
+
})) : Effect.failCause(cause)), Effect.flatMap((frame) => frame.setupComplete === void 0 ? awaitSetupComplete(session) : Effect.void));
|
|
356
|
+
const openSession = (cfg) => (request) => Effect.gen(function* () {
|
|
357
|
+
if (request.googleSearch === true && (request.tools?.length ?? 0) > 0) return yield* Effect.fail(new AiError.InvalidRequest({
|
|
358
|
+
provider: PROVIDER,
|
|
359
|
+
raw: "Gemini rejects googleSearch alongside function declarations; pick one per session."
|
|
360
|
+
}));
|
|
361
|
+
const rate = yield* inputRate(request.inputFormat);
|
|
362
|
+
const scheduling = schedulingOf(request);
|
|
363
|
+
yield* checkOutputFormat(request.outputFormat);
|
|
364
|
+
const session = yield* WebSocketSession.open({
|
|
365
|
+
url: buildWsUrl(cfg),
|
|
366
|
+
provider: PROVIDER,
|
|
367
|
+
schema: ServerMessage,
|
|
368
|
+
openTimeout: Duration.seconds(30)
|
|
369
|
+
}).pipe(withSocket(cfg));
|
|
370
|
+
yield* session.send(setupFrame(request));
|
|
371
|
+
yield* awaitSetupComplete(session);
|
|
372
|
+
const history = request.history ?? [];
|
|
373
|
+
if (history.length > 0) {
|
|
374
|
+
const seed = yield* historyFrame(history);
|
|
375
|
+
yield* session.send(seed);
|
|
376
|
+
}
|
|
377
|
+
const state = yield* Ref.make(initialState);
|
|
378
|
+
const out = yield* Queue.bounded(128);
|
|
379
|
+
const pump = Stream.fromQueue(session.frames).pipe(Stream.mapEffect((frame) => toEvents(state, frame)), Stream.runForEach((events) => Queue.offerAll(out, events)));
|
|
380
|
+
const finish = (cause) => Effect.gen(function* () {
|
|
381
|
+
const { goingAway, responseId } = yield* Ref.get(state);
|
|
382
|
+
if (responseId !== void 0) return yield* Queue.fail(out, new AiError.IncompleteTurn({ raw: cause ?? "the socket closed mid turn" }));
|
|
383
|
+
if (goingAway) return yield* Queue.fail(out, new AiError.SessionExpired({
|
|
384
|
+
provider: PROVIDER,
|
|
385
|
+
raw: cause ?? "the socket closed after the server announced it was going away"
|
|
386
|
+
}));
|
|
387
|
+
return yield* cause !== void 0 && Cause.hasFails(cause) ? Queue.failCause(out, cause) : Queue.end(out);
|
|
388
|
+
});
|
|
389
|
+
yield* pump.pipe(Effect.matchCauseEffect({
|
|
390
|
+
onFailure: finish,
|
|
391
|
+
onSuccess: () => finish(void 0)
|
|
392
|
+
}), Effect.forkScoped);
|
|
393
|
+
const manual = request.turnDetection === "manual";
|
|
394
|
+
return {
|
|
395
|
+
send: (input) => handleInput(session.send, state, {
|
|
396
|
+
manual,
|
|
397
|
+
rate,
|
|
398
|
+
scheduling
|
|
399
|
+
}, input),
|
|
400
|
+
events: Stream.fromQueue(out)
|
|
401
|
+
};
|
|
402
|
+
});
|
|
403
|
+
//#endregion
|
|
404
|
+
export { buildWsUrl, openSession };
|
|
405
|
+
|
|
406
|
+
//# sourceMappingURL=realtimeSession.mjs.map
|