@micdrop/gemini 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,96 @@
1
+ # @micdrop/gemini
2
+
3
+ [Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini)
4
+
5
+ Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
6
+
7
+ This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three.
8
+
9
+ ## Installation
10
+
11
+ ```bash
12
+ npm install @micdrop/gemini
13
+ ```
14
+
15
+ ## Gemini Agent, STT and TTS
16
+
17
+ ```typescript
18
+ import { GeminiAgent, GeminiSTT, GeminiTTS } from '@micdrop/gemini'
19
+ import { MicdropServer } from '@micdrop/server'
20
+
21
+ const apiKey = process.env.GEMINI_API_KEY || ''
22
+
23
+ new MicdropServer(socket, {
24
+ agent: new GeminiAgent({
25
+ apiKey,
26
+ model: 'gemini-3.8-flash', // Default model
27
+ thinkingLevel: 'low', // Optional, the lower the sooner the first word
28
+ systemPrompt: 'You are a helpful assistant',
29
+ }),
30
+ stt: new GeminiSTT({
31
+ apiKey,
32
+ model: 'gemini-3.5-transcribe-live', // Default model
33
+ language: 'en-US', // Optional, detected when left out
34
+ }),
35
+ tts: new GeminiTTS({
36
+ apiKey,
37
+ model: 'gemini-2.5-flash-preview-tts', // Default model
38
+ voice: 'Kore', // Default voice
39
+ }),
40
+ })
41
+ ```
42
+
43
+ Each one can be combined with the providers of other packages.
44
+
45
+ ## Gemini Live
46
+
47
+ ### Usage with MicdropServer
48
+
49
+ ```typescript
50
+ import { GeminiLive } from '@micdrop/gemini'
51
+ import { MicdropServer } from '@micdrop/server'
52
+
53
+ const realtime = new GeminiLive({
54
+ apiKey: process.env.GEMINI_API_KEY || '',
55
+ model: 'gemini-3.8-live', // Default model
56
+ systemPrompt: 'You are a helpful assistant',
57
+ voice: 'Kore', // Optional, a prebuilt voice
58
+
59
+ // Advanced features (optional)
60
+ autoEndCall: true, // Automatically end call when user requests
61
+ })
62
+
63
+ new MicdropServer(socket, {
64
+ realtime,
65
+ generateFirstMessage: true,
66
+ })
67
+ ```
68
+
69
+ The Micdrop client detects when the user speaks, so the automatic activity detection of Gemini is turned off. Tools are added with `addTool()` right after creating the model, since Gemini reads them when the session opens.
70
+
71
+ ### Events
72
+
73
+ | Event | Payload | Description |
74
+ | ----------------------- | ------------------------- | ---------------------------------------------------------------------- |
75
+ | `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits, 16 kHz, mono. |
76
+ | `PartialMessage` | `string` | The transcript of the answer so far. |
77
+ | `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
78
+ | `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
79
+ | `CancelLastUserMessage` | none | The last user message was dropped because it carried no intent. |
80
+ | `SkipAnswer` | none | The model gave no spoken answer. |
81
+ | `EndCall` | none | The model decided that the call is over. |
82
+ | `Failed` | none | The connection could not be restored after its retries. |
83
+
84
+ `gemini-3.8-live-extended-thinking` reasons before answering and needs a `thinkingLevel`, such as `'low'`.
85
+
86
+ ## Documentation
87
+
88
+ Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini) on the [website](https://micdrop.dev).
89
+
90
+ ## License
91
+
92
+ MIT
93
+
94
+ ## Author
95
+
96
+ Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
@@ -0,0 +1,167 @@
1
+ import { GoogleGenAI } from '@google/genai';
2
+ import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
3
+ import { PassThrough, Readable } from 'stream';
4
+
5
+ /**
6
+ * Gemini agent, through the Interactions API
7
+ *
8
+ * @see https://ai.google.dev/gemini-api/docs/text-generation
9
+ *
10
+ * Nothing is stored on Google's side: the whole conversation is sent on every
11
+ * turn, as the other agents do.
12
+ */
13
+ type GeminiAgentOptions = AgentOptions & GeminiOptions & {
14
+ model?: string;
15
+ thinkingLevel?: string;
16
+ settings?: Record<string, unknown>;
17
+ retryDelay?: number;
18
+ maxRetry?: number;
19
+ maxSteps?: number;
20
+ };
21
+ type GeminiOptions = {
22
+ apiKey: string;
23
+ } | {
24
+ genai: GoogleGenAI;
25
+ };
26
+ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
27
+ private genai;
28
+ private abortController?;
29
+ private signatures;
30
+ constructor(options: GeminiAgentOptions);
31
+ protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
32
+ cancel(): void;
33
+ private buildSystemInstruction;
34
+ private buildInput;
35
+ private buildTools;
36
+ }
37
+
38
+ /**
39
+ * Gemini Live API, a model hearing the user and answering with its voice
40
+ *
41
+ * @see https://ai.google.dev/gemini-api/docs/live-api
42
+ *
43
+ * The Micdrop client already tells when the user speaks, so the automatic
44
+ * activity detection of Gemini is off: a turn is framed by activityStart and
45
+ * activityEnd, and the model answers once it ends.
46
+ */
47
+ interface GeminiLiveOptions extends RealtimeOptions {
48
+ apiKey: string;
49
+ model?: string;
50
+ voice?: string;
51
+ thinkingLevel?: string;
52
+ connectionTimeout?: number;
53
+ retryDelay?: number;
54
+ maxRetry?: number;
55
+ }
56
+ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
57
+ private socket;
58
+ private everConnected;
59
+ private resumptionHandle?;
60
+ private resuming;
61
+ private goingAway;
62
+ private outputResampler;
63
+ private hasTranscribedTurn;
64
+ private ignoreUserTranscript;
65
+ private cancelledToolCalls;
66
+ private runningTools;
67
+ private toolFollowUp?;
68
+ private answerEndDeferred;
69
+ constructor(options: GeminiLiveOptions);
70
+ destroy(): void;
71
+ protected openTurn(): void;
72
+ protected appendAudio(chunk: Buffer): void;
73
+ protected closeTurn(): void;
74
+ protected clearTurn(): void;
75
+ protected generate(): void;
76
+ protected speakText(text: string): void;
77
+ protected cancelAnswer(): void;
78
+ protected get isIdle(): boolean;
79
+ protected updateTools(): void;
80
+ protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
81
+ private sendUserText;
82
+ private contentMessage;
83
+ private send;
84
+ private buildSetup;
85
+ private onReady;
86
+ private onReconnecting;
87
+ /** Moves to a new connection before Gemini closes this one */
88
+ private renewIfGoingAway;
89
+ private onMessage;
90
+ private onServerContent;
91
+ private onModelOutput;
92
+ /** Ends the answer the model stopped giving to wait for a tool */
93
+ private endDeferredAnswer;
94
+ private clearToolFollowUp;
95
+ /**
96
+ * The transcript of the turn comes before the answer, so the first piece of
97
+ * the answer closes it.
98
+ */
99
+ private commitTurnTranscript;
100
+ private onToolCall;
101
+ }
102
+
103
+ /**
104
+ * Gemini Transcribe, streamed over the Live API
105
+ *
106
+ * @see https://ai.google.dev/gemini-api/docs/live-api/live-transcribe
107
+ *
108
+ * The Micdrop client already tells when the user speaks, so the automatic
109
+ * activity detection of Gemini is off: each utterance is framed by
110
+ * activityStart and activityEnd, and its final transcript follows the end.
111
+ */
112
+ interface GeminiSTTOptions {
113
+ apiKey: string;
114
+ model?: string;
115
+ language?: string;
116
+ vocabulary?: string[];
117
+ mode?: 'SMART' | 'VERBATIM';
118
+ connectionTimeout?: number;
119
+ transcriptionTimeout?: number;
120
+ retryDelay?: number;
121
+ maxRetry?: number;
122
+ }
123
+ declare class GeminiSTT extends STT {
124
+ private readonly options;
125
+ private socket;
126
+ private goingAway;
127
+ private chunks;
128
+ private utteranceOpen;
129
+ private utteranceEnded;
130
+ private transcript;
131
+ private transcriptionTimeout?;
132
+ constructor(options: GeminiSTTOptions);
133
+ transcribe(audioStream: Readable): void;
134
+ destroy(): void;
135
+ private buildSetup;
136
+ private openUtterance;
137
+ private sendAudio;
138
+ /** Emits what was transcribed of the utterance, possibly nothing */
139
+ private finishUtterance;
140
+ private onMessage;
141
+ private onReconnecting;
142
+ private clearTranscriptionTimeout;
143
+ }
144
+
145
+ /**
146
+ * Gemini text to speech, through the Interactions API
147
+ *
148
+ * @see https://ai.google.dev/gemini-api/docs/speech-generation
149
+ *
150
+ * The model reads a whole text at once, so the answer is cut into sentences,
151
+ * and the audio of each one is emitted as the model streams it.
152
+ */
153
+ type GeminiTTSOptions = GeminiOptions & {
154
+ model?: string;
155
+ voice?: string;
156
+ instructions?: string;
157
+ maxRetry?: number;
158
+ };
159
+ declare class GeminiTTS extends SentenceTTS {
160
+ private readonly options;
161
+ private genai;
162
+ constructor(options: GeminiTTSOptions);
163
+ protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
164
+ private synthesizeOnce;
165
+ }
166
+
167
+ export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
@@ -0,0 +1,167 @@
1
+ import { GoogleGenAI } from '@google/genai';
2
+ import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
3
+ import { PassThrough, Readable } from 'stream';
4
+
5
+ /**
6
+ * Gemini agent, through the Interactions API
7
+ *
8
+ * @see https://ai.google.dev/gemini-api/docs/text-generation
9
+ *
10
+ * Nothing is stored on Google's side: the whole conversation is sent on every
11
+ * turn, as the other agents do.
12
+ */
13
+ type GeminiAgentOptions = AgentOptions & GeminiOptions & {
14
+ model?: string;
15
+ thinkingLevel?: string;
16
+ settings?: Record<string, unknown>;
17
+ retryDelay?: number;
18
+ maxRetry?: number;
19
+ maxSteps?: number;
20
+ };
21
+ type GeminiOptions = {
22
+ apiKey: string;
23
+ } | {
24
+ genai: GoogleGenAI;
25
+ };
26
+ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
27
+ private genai;
28
+ private abortController?;
29
+ private signatures;
30
+ constructor(options: GeminiAgentOptions);
31
+ protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
32
+ cancel(): void;
33
+ private buildSystemInstruction;
34
+ private buildInput;
35
+ private buildTools;
36
+ }
37
+
38
+ /**
39
+ * Gemini Live API, a model hearing the user and answering with its voice
40
+ *
41
+ * @see https://ai.google.dev/gemini-api/docs/live-api
42
+ *
43
+ * The Micdrop client already tells when the user speaks, so the automatic
44
+ * activity detection of Gemini is off: a turn is framed by activityStart and
45
+ * activityEnd, and the model answers once it ends.
46
+ */
47
+ interface GeminiLiveOptions extends RealtimeOptions {
48
+ apiKey: string;
49
+ model?: string;
50
+ voice?: string;
51
+ thinkingLevel?: string;
52
+ connectionTimeout?: number;
53
+ retryDelay?: number;
54
+ maxRetry?: number;
55
+ }
56
+ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
57
+ private socket;
58
+ private everConnected;
59
+ private resumptionHandle?;
60
+ private resuming;
61
+ private goingAway;
62
+ private outputResampler;
63
+ private hasTranscribedTurn;
64
+ private ignoreUserTranscript;
65
+ private cancelledToolCalls;
66
+ private runningTools;
67
+ private toolFollowUp?;
68
+ private answerEndDeferred;
69
+ constructor(options: GeminiLiveOptions);
70
+ destroy(): void;
71
+ protected openTurn(): void;
72
+ protected appendAudio(chunk: Buffer): void;
73
+ protected closeTurn(): void;
74
+ protected clearTurn(): void;
75
+ protected generate(): void;
76
+ protected speakText(text: string): void;
77
+ protected cancelAnswer(): void;
78
+ protected get isIdle(): boolean;
79
+ protected updateTools(): void;
80
+ protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
81
+ private sendUserText;
82
+ private contentMessage;
83
+ private send;
84
+ private buildSetup;
85
+ private onReady;
86
+ private onReconnecting;
87
+ /** Moves to a new connection before Gemini closes this one */
88
+ private renewIfGoingAway;
89
+ private onMessage;
90
+ private onServerContent;
91
+ private onModelOutput;
92
+ /** Ends the answer the model stopped giving to wait for a tool */
93
+ private endDeferredAnswer;
94
+ private clearToolFollowUp;
95
+ /**
96
+ * The transcript of the turn comes before the answer, so the first piece of
97
+ * the answer closes it.
98
+ */
99
+ private commitTurnTranscript;
100
+ private onToolCall;
101
+ }
102
+
103
+ /**
104
+ * Gemini Transcribe, streamed over the Live API
105
+ *
106
+ * @see https://ai.google.dev/gemini-api/docs/live-api/live-transcribe
107
+ *
108
+ * The Micdrop client already tells when the user speaks, so the automatic
109
+ * activity detection of Gemini is off: each utterance is framed by
110
+ * activityStart and activityEnd, and its final transcript follows the end.
111
+ */
112
+ interface GeminiSTTOptions {
113
+ apiKey: string;
114
+ model?: string;
115
+ language?: string;
116
+ vocabulary?: string[];
117
+ mode?: 'SMART' | 'VERBATIM';
118
+ connectionTimeout?: number;
119
+ transcriptionTimeout?: number;
120
+ retryDelay?: number;
121
+ maxRetry?: number;
122
+ }
123
+ declare class GeminiSTT extends STT {
124
+ private readonly options;
125
+ private socket;
126
+ private goingAway;
127
+ private chunks;
128
+ private utteranceOpen;
129
+ private utteranceEnded;
130
+ private transcript;
131
+ private transcriptionTimeout?;
132
+ constructor(options: GeminiSTTOptions);
133
+ transcribe(audioStream: Readable): void;
134
+ destroy(): void;
135
+ private buildSetup;
136
+ private openUtterance;
137
+ private sendAudio;
138
+ /** Emits what was transcribed of the utterance, possibly nothing */
139
+ private finishUtterance;
140
+ private onMessage;
141
+ private onReconnecting;
142
+ private clearTranscriptionTimeout;
143
+ }
144
+
145
+ /**
146
+ * Gemini text to speech, through the Interactions API
147
+ *
148
+ * @see https://ai.google.dev/gemini-api/docs/speech-generation
149
+ *
150
+ * The model reads a whole text at once, so the answer is cut into sentences,
151
+ * and the audio of each one is emitted as the model streams it.
152
+ */
153
+ type GeminiTTSOptions = GeminiOptions & {
154
+ model?: string;
155
+ voice?: string;
156
+ instructions?: string;
157
+ maxRetry?: number;
158
+ };
159
+ declare class GeminiTTS extends SentenceTTS {
160
+ private readonly options;
161
+ private genai;
162
+ constructor(options: GeminiTTSOptions);
163
+ protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
164
+ private synthesizeOnce;
165
+ }
166
+
167
+ export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };