@micdrop/gemini 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -0
- package/dist/index.d.mts +167 -0
- package/dist/index.d.ts +167 -0
- package/dist/index.js +959 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +922 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +56 -0
package/README.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# @micdrop/gemini
|
|
2
|
+
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini)
|
|
4
|
+
|
|
5
|
+
Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
|
+
|
|
7
|
+
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install @micdrop/gemini
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Gemini Agent, STT and TTS
|
|
16
|
+
|
|
17
|
+
```typescript
|
|
18
|
+
import { GeminiAgent, GeminiSTT, GeminiTTS } from '@micdrop/gemini'
|
|
19
|
+
import { MicdropServer } from '@micdrop/server'
|
|
20
|
+
|
|
21
|
+
const apiKey = process.env.GEMINI_API_KEY || ''
|
|
22
|
+
|
|
23
|
+
new MicdropServer(socket, {
|
|
24
|
+
agent: new GeminiAgent({
|
|
25
|
+
apiKey,
|
|
26
|
+
model: 'gemini-3.8-flash', // Default model
|
|
27
|
+
thinkingLevel: 'low', // Optional, the lower the sooner the first word
|
|
28
|
+
systemPrompt: 'You are a helpful assistant',
|
|
29
|
+
}),
|
|
30
|
+
stt: new GeminiSTT({
|
|
31
|
+
apiKey,
|
|
32
|
+
model: 'gemini-3.5-transcribe-live', // Default model
|
|
33
|
+
language: 'en-US', // Optional, detected when left out
|
|
34
|
+
}),
|
|
35
|
+
tts: new GeminiTTS({
|
|
36
|
+
apiKey,
|
|
37
|
+
model: 'gemini-2.5-flash-preview-tts', // Default model
|
|
38
|
+
voice: 'Kore', // Default voice
|
|
39
|
+
}),
|
|
40
|
+
})
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Each one can be combined with the providers of other packages.
|
|
44
|
+
|
|
45
|
+
## Gemini Live
|
|
46
|
+
|
|
47
|
+
### Usage with MicdropServer
|
|
48
|
+
|
|
49
|
+
```typescript
|
|
50
|
+
import { GeminiLive } from '@micdrop/gemini'
|
|
51
|
+
import { MicdropServer } from '@micdrop/server'
|
|
52
|
+
|
|
53
|
+
const realtime = new GeminiLive({
|
|
54
|
+
apiKey: process.env.GEMINI_API_KEY || '',
|
|
55
|
+
model: 'gemini-3.8-live', // Default model
|
|
56
|
+
systemPrompt: 'You are a helpful assistant',
|
|
57
|
+
voice: 'Kore', // Optional, a prebuilt voice
|
|
58
|
+
|
|
59
|
+
// Advanced features (optional)
|
|
60
|
+
autoEndCall: true, // Automatically end call when user requests
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
new MicdropServer(socket, {
|
|
64
|
+
realtime,
|
|
65
|
+
generateFirstMessage: true,
|
|
66
|
+
})
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The Micdrop client detects when the user speaks, so the automatic activity detection of Gemini is turned off. Tools are added with `addTool()` right after creating the model, since Gemini reads them when the session opens.
|
|
70
|
+
|
|
71
|
+
### Events
|
|
72
|
+
|
|
73
|
+
| Event | Payload | Description |
|
|
74
|
+
| ----------------------- | ------------------------- | ---------------------------------------------------------------------- |
|
|
75
|
+
| `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits, 16 kHz, mono. |
|
|
76
|
+
| `PartialMessage` | `string` | The transcript of the answer so far. |
|
|
77
|
+
| `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
|
|
78
|
+
| `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
|
|
79
|
+
| `CancelLastUserMessage` | none | The last user message was dropped because it carried no intent. |
|
|
80
|
+
| `SkipAnswer` | none | The model gave no spoken answer. |
|
|
81
|
+
| `EndCall` | none | The model decided that the call is over. |
|
|
82
|
+
| `Failed` | none | The connection could not be restored after its retries. |
|
|
83
|
+
|
|
84
|
+
`gemini-3.8-live-extended-thinking` reasons before answering and needs a `thinkingLevel`, such as `'low'`.
|
|
85
|
+
|
|
86
|
+
## Documentation
|
|
87
|
+
|
|
88
|
+
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini) on the [website](https://micdrop.dev).
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
|
|
92
|
+
MIT
|
|
93
|
+
|
|
94
|
+
## Author
|
|
95
|
+
|
|
96
|
+
Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { GoogleGenAI } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
|
+
import { PassThrough, Readable } from 'stream';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Gemini agent, through the Interactions API
|
|
7
|
+
*
|
|
8
|
+
* @see https://ai.google.dev/gemini-api/docs/text-generation
|
|
9
|
+
*
|
|
10
|
+
* Nothing is stored on Google's side: the whole conversation is sent on every
|
|
11
|
+
* turn, as the other agents do.
|
|
12
|
+
*/
|
|
13
|
+
type GeminiAgentOptions = AgentOptions & GeminiOptions & {
|
|
14
|
+
model?: string;
|
|
15
|
+
thinkingLevel?: string;
|
|
16
|
+
settings?: Record<string, unknown>;
|
|
17
|
+
retryDelay?: number;
|
|
18
|
+
maxRetry?: number;
|
|
19
|
+
maxSteps?: number;
|
|
20
|
+
};
|
|
21
|
+
type GeminiOptions = {
|
|
22
|
+
apiKey: string;
|
|
23
|
+
} | {
|
|
24
|
+
genai: GoogleGenAI;
|
|
25
|
+
};
|
|
26
|
+
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
private genai;
|
|
28
|
+
private abortController?;
|
|
29
|
+
private signatures;
|
|
30
|
+
constructor(options: GeminiAgentOptions);
|
|
31
|
+
protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
|
|
32
|
+
cancel(): void;
|
|
33
|
+
private buildSystemInstruction;
|
|
34
|
+
private buildInput;
|
|
35
|
+
private buildTools;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Gemini Live API, a model hearing the user and answering with its voice
|
|
40
|
+
*
|
|
41
|
+
* @see https://ai.google.dev/gemini-api/docs/live-api
|
|
42
|
+
*
|
|
43
|
+
* The Micdrop client already tells when the user speaks, so the automatic
|
|
44
|
+
* activity detection of Gemini is off: a turn is framed by activityStart and
|
|
45
|
+
* activityEnd, and the model answers once it ends.
|
|
46
|
+
*/
|
|
47
|
+
interface GeminiLiveOptions extends RealtimeOptions {
|
|
48
|
+
apiKey: string;
|
|
49
|
+
model?: string;
|
|
50
|
+
voice?: string;
|
|
51
|
+
thinkingLevel?: string;
|
|
52
|
+
connectionTimeout?: number;
|
|
53
|
+
retryDelay?: number;
|
|
54
|
+
maxRetry?: number;
|
|
55
|
+
}
|
|
56
|
+
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
57
|
+
private socket;
|
|
58
|
+
private everConnected;
|
|
59
|
+
private resumptionHandle?;
|
|
60
|
+
private resuming;
|
|
61
|
+
private goingAway;
|
|
62
|
+
private outputResampler;
|
|
63
|
+
private hasTranscribedTurn;
|
|
64
|
+
private ignoreUserTranscript;
|
|
65
|
+
private cancelledToolCalls;
|
|
66
|
+
private runningTools;
|
|
67
|
+
private toolFollowUp?;
|
|
68
|
+
private answerEndDeferred;
|
|
69
|
+
constructor(options: GeminiLiveOptions);
|
|
70
|
+
destroy(): void;
|
|
71
|
+
protected openTurn(): void;
|
|
72
|
+
protected appendAudio(chunk: Buffer): void;
|
|
73
|
+
protected closeTurn(): void;
|
|
74
|
+
protected clearTurn(): void;
|
|
75
|
+
protected generate(): void;
|
|
76
|
+
protected speakText(text: string): void;
|
|
77
|
+
protected cancelAnswer(): void;
|
|
78
|
+
protected get isIdle(): boolean;
|
|
79
|
+
protected updateTools(): void;
|
|
80
|
+
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
81
|
+
private sendUserText;
|
|
82
|
+
private contentMessage;
|
|
83
|
+
private send;
|
|
84
|
+
private buildSetup;
|
|
85
|
+
private onReady;
|
|
86
|
+
private onReconnecting;
|
|
87
|
+
/** Moves to a new connection before Gemini closes this one */
|
|
88
|
+
private renewIfGoingAway;
|
|
89
|
+
private onMessage;
|
|
90
|
+
private onServerContent;
|
|
91
|
+
private onModelOutput;
|
|
92
|
+
/** Ends the answer the model stopped giving to wait for a tool */
|
|
93
|
+
private endDeferredAnswer;
|
|
94
|
+
private clearToolFollowUp;
|
|
95
|
+
/**
|
|
96
|
+
* The transcript of the turn comes before the answer, so the first piece of
|
|
97
|
+
* the answer closes it.
|
|
98
|
+
*/
|
|
99
|
+
private commitTurnTranscript;
|
|
100
|
+
private onToolCall;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Gemini Transcribe, streamed over the Live API
|
|
105
|
+
*
|
|
106
|
+
* @see https://ai.google.dev/gemini-api/docs/live-api/live-transcribe
|
|
107
|
+
*
|
|
108
|
+
* The Micdrop client already tells when the user speaks, so the automatic
|
|
109
|
+
* activity detection of Gemini is off: each utterance is framed by
|
|
110
|
+
* activityStart and activityEnd, and its final transcript follows the end.
|
|
111
|
+
*/
|
|
112
|
+
interface GeminiSTTOptions {
|
|
113
|
+
apiKey: string;
|
|
114
|
+
model?: string;
|
|
115
|
+
language?: string;
|
|
116
|
+
vocabulary?: string[];
|
|
117
|
+
mode?: 'SMART' | 'VERBATIM';
|
|
118
|
+
connectionTimeout?: number;
|
|
119
|
+
transcriptionTimeout?: number;
|
|
120
|
+
retryDelay?: number;
|
|
121
|
+
maxRetry?: number;
|
|
122
|
+
}
|
|
123
|
+
declare class GeminiSTT extends STT {
|
|
124
|
+
private readonly options;
|
|
125
|
+
private socket;
|
|
126
|
+
private goingAway;
|
|
127
|
+
private chunks;
|
|
128
|
+
private utteranceOpen;
|
|
129
|
+
private utteranceEnded;
|
|
130
|
+
private transcript;
|
|
131
|
+
private transcriptionTimeout?;
|
|
132
|
+
constructor(options: GeminiSTTOptions);
|
|
133
|
+
transcribe(audioStream: Readable): void;
|
|
134
|
+
destroy(): void;
|
|
135
|
+
private buildSetup;
|
|
136
|
+
private openUtterance;
|
|
137
|
+
private sendAudio;
|
|
138
|
+
/** Emits what was transcribed of the utterance, possibly nothing */
|
|
139
|
+
private finishUtterance;
|
|
140
|
+
private onMessage;
|
|
141
|
+
private onReconnecting;
|
|
142
|
+
private clearTranscriptionTimeout;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Gemini text to speech, through the Interactions API
|
|
147
|
+
*
|
|
148
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation
|
|
149
|
+
*
|
|
150
|
+
* The model reads a whole text at once, so the answer is cut into sentences,
|
|
151
|
+
* and the audio of each one is emitted as the model streams it.
|
|
152
|
+
*/
|
|
153
|
+
type GeminiTTSOptions = GeminiOptions & {
|
|
154
|
+
model?: string;
|
|
155
|
+
voice?: string;
|
|
156
|
+
instructions?: string;
|
|
157
|
+
maxRetry?: number;
|
|
158
|
+
};
|
|
159
|
+
declare class GeminiTTS extends SentenceTTS {
|
|
160
|
+
private readonly options;
|
|
161
|
+
private genai;
|
|
162
|
+
constructor(options: GeminiTTSOptions);
|
|
163
|
+
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
164
|
+
private synthesizeOnce;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { GoogleGenAI } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
|
+
import { PassThrough, Readable } from 'stream';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Gemini agent, through the Interactions API
|
|
7
|
+
*
|
|
8
|
+
* @see https://ai.google.dev/gemini-api/docs/text-generation
|
|
9
|
+
*
|
|
10
|
+
* Nothing is stored on Google's side: the whole conversation is sent on every
|
|
11
|
+
* turn, as the other agents do.
|
|
12
|
+
*/
|
|
13
|
+
type GeminiAgentOptions = AgentOptions & GeminiOptions & {
|
|
14
|
+
model?: string;
|
|
15
|
+
thinkingLevel?: string;
|
|
16
|
+
settings?: Record<string, unknown>;
|
|
17
|
+
retryDelay?: number;
|
|
18
|
+
maxRetry?: number;
|
|
19
|
+
maxSteps?: number;
|
|
20
|
+
};
|
|
21
|
+
type GeminiOptions = {
|
|
22
|
+
apiKey: string;
|
|
23
|
+
} | {
|
|
24
|
+
genai: GoogleGenAI;
|
|
25
|
+
};
|
|
26
|
+
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
private genai;
|
|
28
|
+
private abortController?;
|
|
29
|
+
private signatures;
|
|
30
|
+
constructor(options: GeminiAgentOptions);
|
|
31
|
+
protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
|
|
32
|
+
cancel(): void;
|
|
33
|
+
private buildSystemInstruction;
|
|
34
|
+
private buildInput;
|
|
35
|
+
private buildTools;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Gemini Live API, a model hearing the user and answering with its voice
|
|
40
|
+
*
|
|
41
|
+
* @see https://ai.google.dev/gemini-api/docs/live-api
|
|
42
|
+
*
|
|
43
|
+
* The Micdrop client already tells when the user speaks, so the automatic
|
|
44
|
+
* activity detection of Gemini is off: a turn is framed by activityStart and
|
|
45
|
+
* activityEnd, and the model answers once it ends.
|
|
46
|
+
*/
|
|
47
|
+
interface GeminiLiveOptions extends RealtimeOptions {
|
|
48
|
+
apiKey: string;
|
|
49
|
+
model?: string;
|
|
50
|
+
voice?: string;
|
|
51
|
+
thinkingLevel?: string;
|
|
52
|
+
connectionTimeout?: number;
|
|
53
|
+
retryDelay?: number;
|
|
54
|
+
maxRetry?: number;
|
|
55
|
+
}
|
|
56
|
+
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
57
|
+
private socket;
|
|
58
|
+
private everConnected;
|
|
59
|
+
private resumptionHandle?;
|
|
60
|
+
private resuming;
|
|
61
|
+
private goingAway;
|
|
62
|
+
private outputResampler;
|
|
63
|
+
private hasTranscribedTurn;
|
|
64
|
+
private ignoreUserTranscript;
|
|
65
|
+
private cancelledToolCalls;
|
|
66
|
+
private runningTools;
|
|
67
|
+
private toolFollowUp?;
|
|
68
|
+
private answerEndDeferred;
|
|
69
|
+
constructor(options: GeminiLiveOptions);
|
|
70
|
+
destroy(): void;
|
|
71
|
+
protected openTurn(): void;
|
|
72
|
+
protected appendAudio(chunk: Buffer): void;
|
|
73
|
+
protected closeTurn(): void;
|
|
74
|
+
protected clearTurn(): void;
|
|
75
|
+
protected generate(): void;
|
|
76
|
+
protected speakText(text: string): void;
|
|
77
|
+
protected cancelAnswer(): void;
|
|
78
|
+
protected get isIdle(): boolean;
|
|
79
|
+
protected updateTools(): void;
|
|
80
|
+
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
81
|
+
private sendUserText;
|
|
82
|
+
private contentMessage;
|
|
83
|
+
private send;
|
|
84
|
+
private buildSetup;
|
|
85
|
+
private onReady;
|
|
86
|
+
private onReconnecting;
|
|
87
|
+
/** Moves to a new connection before Gemini closes this one */
|
|
88
|
+
private renewIfGoingAway;
|
|
89
|
+
private onMessage;
|
|
90
|
+
private onServerContent;
|
|
91
|
+
private onModelOutput;
|
|
92
|
+
/** Ends the answer the model stopped giving to wait for a tool */
|
|
93
|
+
private endDeferredAnswer;
|
|
94
|
+
private clearToolFollowUp;
|
|
95
|
+
/**
|
|
96
|
+
* The transcript of the turn comes before the answer, so the first piece of
|
|
97
|
+
* the answer closes it.
|
|
98
|
+
*/
|
|
99
|
+
private commitTurnTranscript;
|
|
100
|
+
private onToolCall;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Gemini Transcribe, streamed over the Live API
|
|
105
|
+
*
|
|
106
|
+
* @see https://ai.google.dev/gemini-api/docs/live-api/live-transcribe
|
|
107
|
+
*
|
|
108
|
+
* The Micdrop client already tells when the user speaks, so the automatic
|
|
109
|
+
* activity detection of Gemini is off: each utterance is framed by
|
|
110
|
+
* activityStart and activityEnd, and its final transcript follows the end.
|
|
111
|
+
*/
|
|
112
|
+
interface GeminiSTTOptions {
|
|
113
|
+
apiKey: string;
|
|
114
|
+
model?: string;
|
|
115
|
+
language?: string;
|
|
116
|
+
vocabulary?: string[];
|
|
117
|
+
mode?: 'SMART' | 'VERBATIM';
|
|
118
|
+
connectionTimeout?: number;
|
|
119
|
+
transcriptionTimeout?: number;
|
|
120
|
+
retryDelay?: number;
|
|
121
|
+
maxRetry?: number;
|
|
122
|
+
}
|
|
123
|
+
declare class GeminiSTT extends STT {
|
|
124
|
+
private readonly options;
|
|
125
|
+
private socket;
|
|
126
|
+
private goingAway;
|
|
127
|
+
private chunks;
|
|
128
|
+
private utteranceOpen;
|
|
129
|
+
private utteranceEnded;
|
|
130
|
+
private transcript;
|
|
131
|
+
private transcriptionTimeout?;
|
|
132
|
+
constructor(options: GeminiSTTOptions);
|
|
133
|
+
transcribe(audioStream: Readable): void;
|
|
134
|
+
destroy(): void;
|
|
135
|
+
private buildSetup;
|
|
136
|
+
private openUtterance;
|
|
137
|
+
private sendAudio;
|
|
138
|
+
/** Emits what was transcribed of the utterance, possibly nothing */
|
|
139
|
+
private finishUtterance;
|
|
140
|
+
private onMessage;
|
|
141
|
+
private onReconnecting;
|
|
142
|
+
private clearTranscriptionTimeout;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Gemini text to speech, through the Interactions API
|
|
147
|
+
*
|
|
148
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation
|
|
149
|
+
*
|
|
150
|
+
* The model reads a whole text at once, so the answer is cut into sentences,
|
|
151
|
+
* and the audio of each one is emitted as the model streams it.
|
|
152
|
+
*/
|
|
153
|
+
type GeminiTTSOptions = GeminiOptions & {
|
|
154
|
+
model?: string;
|
|
155
|
+
voice?: string;
|
|
156
|
+
instructions?: string;
|
|
157
|
+
maxRetry?: number;
|
|
158
|
+
};
|
|
159
|
+
declare class GeminiTTS extends SentenceTTS {
|
|
160
|
+
private readonly options;
|
|
161
|
+
private genai;
|
|
162
|
+
constructor(options: GeminiTTSOptions);
|
|
163
|
+
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
164
|
+
private synthesizeOnce;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|