@micdrop/gemini 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -5
- package/dist/index.d.mts +164 -4
- package/dist/index.d.ts +164 -4
- package/dist/index.js +255 -49
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +242 -41
- package/dist/index.mjs.map +1 -1
- package/package.json +7 -5
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# @micdrop/gemini
|
|
2
2
|
|
|
3
|
-
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/providers)
|
|
4
4
|
|
|
5
5
|
Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
7
|
-
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/
|
|
7
|
+
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three. `GeminiClassifier` answers typed questions about each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api).
|
|
8
8
|
|
|
9
9
|
## Installation
|
|
10
10
|
|
|
@@ -34,7 +34,7 @@ new MicdropServer(socket, {
|
|
|
34
34
|
}),
|
|
35
35
|
tts: new GeminiTTS({
|
|
36
36
|
apiKey,
|
|
37
|
-
model: 'gemini-
|
|
37
|
+
model: 'gemini-3.8-flash-tts', // Default model
|
|
38
38
|
voice: 'Kore', // Default voice
|
|
39
39
|
}),
|
|
40
40
|
})
|
|
@@ -72,7 +72,7 @@ The Micdrop client detects when the user speaks, so the automatic activity detec
|
|
|
72
72
|
|
|
73
73
|
| Event | Payload | Description |
|
|
74
74
|
| ----------------------- | ------------------------- | ---------------------------------------------------------------------- |
|
|
75
|
-
| `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits,
|
|
75
|
+
| `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits, 24 kHz, mono. |
|
|
76
76
|
| `PartialMessage` | `string` | The transcript of the answer so far. |
|
|
77
77
|
| `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
|
|
78
78
|
| `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
|
|
@@ -83,9 +83,38 @@ The Micdrop client detects when the user speaks, so the automatic activity detec
|
|
|
83
83
|
|
|
84
84
|
`gemini-3.8-live-extended-thinking` reasons before answering and needs a `thinkingLevel`, such as `'low'`.
|
|
85
85
|
|
|
86
|
+
## Gemini Classifier
|
|
87
|
+
|
|
88
|
+
Classifies each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api): yes or no questions, choices and scores answered with calibrated probabilities by `gemini-decision-1`, on text, images, audio, video or PDF.
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
import { choice, GeminiClassifier, noul } from '@micdrop/gemini'
|
|
92
|
+
import { MicdropServer } from '@micdrop/server'
|
|
93
|
+
|
|
94
|
+
const classifier = new GeminiClassifier({
|
|
95
|
+
apiKey: process.env.GEMINI_API_KEY || '',
|
|
96
|
+
questions: {
|
|
97
|
+
intent: choice('What does the user in `turn` want?', {
|
|
98
|
+
billing: 'A charge, an invoice, a refund',
|
|
99
|
+
outage: 'The service is down or slow',
|
|
100
|
+
other: 'Anything else',
|
|
101
|
+
}),
|
|
102
|
+
wantsHuman: noul('Does the user in `turn` ask for a human?'),
|
|
103
|
+
},
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
new MicdropServer(socket, {
|
|
107
|
+
stt,
|
|
108
|
+
agent,
|
|
109
|
+
tts,
|
|
110
|
+
classifier,
|
|
111
|
+
classifierOptions: { waitBeforeAnswer: true },
|
|
112
|
+
})
|
|
113
|
+
```
|
|
114
|
+
|
|
86
115
|
## Documentation
|
|
87
116
|
|
|
88
|
-
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/
|
|
117
|
+
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/providers) on the [website](https://micdrop.dev).
|
|
89
118
|
|
|
90
119
|
## License
|
|
91
120
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS, MicdropVoice } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -24,6 +24,16 @@ type GeminiOptions = {
|
|
|
24
24
|
genai: GoogleGenAI;
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
/**
|
|
28
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
29
|
+
*
|
|
30
|
+
* The list is fetched once and shared by every call of the process for a
|
|
31
|
+
* day. `refresh` fetches it again.
|
|
32
|
+
*/
|
|
33
|
+
static listModels(options: {
|
|
34
|
+
apiKey: string;
|
|
35
|
+
refresh?: boolean;
|
|
36
|
+
}): Promise<MicdropModel[]>;
|
|
27
37
|
private genai;
|
|
28
38
|
private signatures;
|
|
29
39
|
constructor(options: GeminiAgentOptions);
|
|
@@ -44,6 +54,116 @@ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
|
44
54
|
private buildTools;
|
|
45
55
|
}
|
|
46
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
59
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
60
|
+
* without writing a single token.
|
|
61
|
+
*
|
|
62
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
63
|
+
*/
|
|
64
|
+
/** What a question asks, a text or text blocks */
|
|
65
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
66
|
+
/** The probability that a condition is true */
|
|
67
|
+
interface GeminiNoulQuestion {
|
|
68
|
+
type: 'noul';
|
|
69
|
+
instructions: GeminiInstructions;
|
|
70
|
+
criteria?: Decisions.NoulCriteria;
|
|
71
|
+
}
|
|
72
|
+
/** One value out of several, without order */
|
|
73
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
74
|
+
type: 'choice';
|
|
75
|
+
instructions: GeminiInstructions;
|
|
76
|
+
criteria: Record<V, string>;
|
|
77
|
+
}
|
|
78
|
+
/** A position on ordered levels, from the lowest */
|
|
79
|
+
interface GeminiScoreQuestion {
|
|
80
|
+
type: 'score';
|
|
81
|
+
instructions: GeminiInstructions;
|
|
82
|
+
criteria: string[];
|
|
83
|
+
}
|
|
84
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
85
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
86
|
+
/**
|
|
87
|
+
* Asks for the probability that a condition is true
|
|
88
|
+
* @param instructions - The condition, written around what can be observed
|
|
89
|
+
* @param criteria - What makes it true, and what makes it false
|
|
90
|
+
*/
|
|
91
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
92
|
+
true?: string;
|
|
93
|
+
false?: string;
|
|
94
|
+
}): GeminiNoulQuestion;
|
|
95
|
+
/**
|
|
96
|
+
* Asks for one value out of several
|
|
97
|
+
* @param instructions - The question
|
|
98
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
99
|
+
*/
|
|
100
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
101
|
+
/**
|
|
102
|
+
* Asks for a score on ordered levels
|
|
103
|
+
* @param instructions - The question
|
|
104
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
105
|
+
*/
|
|
106
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
107
|
+
interface GeminiNoulAnswer {
|
|
108
|
+
type: 'noul';
|
|
109
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
110
|
+
noul: number;
|
|
111
|
+
/** Probability of the likelier outcome */
|
|
112
|
+
confidence: number;
|
|
113
|
+
}
|
|
114
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
115
|
+
type: 'choice';
|
|
116
|
+
choice: V;
|
|
117
|
+
confidence: number;
|
|
118
|
+
probabilities: Record<V, number>;
|
|
119
|
+
}
|
|
120
|
+
interface GeminiScoreAnswer {
|
|
121
|
+
type: 'score';
|
|
122
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
123
|
+
score: number;
|
|
124
|
+
confidence: number;
|
|
125
|
+
/** Probability of each level, by index */
|
|
126
|
+
probabilities: Record<number, number>;
|
|
127
|
+
/** Description of each level, by index */
|
|
128
|
+
legend: Record<number, string>;
|
|
129
|
+
}
|
|
130
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
131
|
+
/** What a Gemini classifier returns for its questions */
|
|
132
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
133
|
+
/** The answer to each question */
|
|
134
|
+
answers: {
|
|
135
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
136
|
+
};
|
|
137
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
138
|
+
}
|
|
139
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
140
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
141
|
+
model?: string;
|
|
142
|
+
/**
|
|
143
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
144
|
+
* `score()`. A function picks them per input.
|
|
145
|
+
*
|
|
146
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
147
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
148
|
+
* and `history`.
|
|
149
|
+
*/
|
|
150
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
151
|
+
/**
|
|
152
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
153
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
154
|
+
*/
|
|
155
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
156
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
157
|
+
timeout?: number;
|
|
158
|
+
};
|
|
159
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
160
|
+
private options;
|
|
161
|
+
private genai;
|
|
162
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
163
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
164
|
+
private buildInput;
|
|
165
|
+
}
|
|
166
|
+
|
|
47
167
|
/**
|
|
48
168
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
49
169
|
*
|
|
@@ -63,11 +183,20 @@ interface GeminiLiveOptions extends RealtimeOptions {
|
|
|
63
183
|
maxRetry?: number;
|
|
64
184
|
}
|
|
65
185
|
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
186
|
+
/**
|
|
187
|
+
* Lists the Live models, which hear and speak.
|
|
188
|
+
*
|
|
189
|
+
* The list is fetched once and shared by every call of the process for a
|
|
190
|
+
* day. `refresh` fetches it again.
|
|
191
|
+
*/
|
|
192
|
+
static listModels(options: {
|
|
193
|
+
apiKey: string;
|
|
194
|
+
refresh?: boolean;
|
|
195
|
+
}): Promise<MicdropModel[]>;
|
|
66
196
|
private socket;
|
|
67
197
|
private everConnected;
|
|
68
198
|
private resumptionHandle?;
|
|
69
199
|
private goingAway;
|
|
70
|
-
private outputResampler;
|
|
71
200
|
private hasTranscribedTurn;
|
|
72
201
|
private ignoreUserTranscript;
|
|
73
202
|
private cancelledToolCalls;
|
|
@@ -86,6 +215,7 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
86
215
|
protected clearTurn(): void;
|
|
87
216
|
protected generate(): void;
|
|
88
217
|
protected speakText(text: string): void;
|
|
218
|
+
get sampleRate(): number;
|
|
89
219
|
protected cancelAnswer(): void;
|
|
90
220
|
protected get isIdle(): boolean;
|
|
91
221
|
protected updateTools(): void;
|
|
@@ -141,6 +271,16 @@ interface GeminiSTTOptions {
|
|
|
141
271
|
}
|
|
142
272
|
declare class GeminiSTT extends STT {
|
|
143
273
|
private readonly options;
|
|
274
|
+
/**
|
|
275
|
+
* Lists the models that transcribe a Live session.
|
|
276
|
+
*
|
|
277
|
+
* The list is fetched once and shared by every call of the process for a
|
|
278
|
+
* day. `refresh` fetches it again.
|
|
279
|
+
*/
|
|
280
|
+
static listModels(options: {
|
|
281
|
+
apiKey: string;
|
|
282
|
+
refresh?: boolean;
|
|
283
|
+
}): Promise<MicdropModel[]>;
|
|
144
284
|
private socket;
|
|
145
285
|
private goingAway;
|
|
146
286
|
private utterances;
|
|
@@ -185,10 +325,30 @@ type GeminiTTSOptions = GeminiOptions & {
|
|
|
185
325
|
};
|
|
186
326
|
declare class GeminiTTS extends SentenceTTS {
|
|
187
327
|
private readonly options;
|
|
328
|
+
/**
|
|
329
|
+
* Lists the models that speak.
|
|
330
|
+
*
|
|
331
|
+
* The list is fetched once and shared by every call of the process for a
|
|
332
|
+
* day. `refresh` fetches it again.
|
|
333
|
+
*/
|
|
334
|
+
static listModels(options: {
|
|
335
|
+
apiKey: string;
|
|
336
|
+
refresh?: boolean;
|
|
337
|
+
}): Promise<MicdropModel[]>;
|
|
338
|
+
/**
|
|
339
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
340
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
341
|
+
* here.
|
|
342
|
+
*/
|
|
343
|
+
static listVoices(options?: {
|
|
344
|
+
language?: string;
|
|
345
|
+
}): Promise<MicdropVoice[]>;
|
|
188
346
|
private genai;
|
|
347
|
+
private rate;
|
|
348
|
+
get sampleRate(): number;
|
|
189
349
|
constructor(options: GeminiTTSOptions);
|
|
190
350
|
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
191
351
|
private synthesizeOnce;
|
|
192
352
|
}
|
|
193
353
|
|
|
194
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
354
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS, MicdropVoice } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -24,6 +24,16 @@ type GeminiOptions = {
|
|
|
24
24
|
genai: GoogleGenAI;
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
/**
|
|
28
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
29
|
+
*
|
|
30
|
+
* The list is fetched once and shared by every call of the process for a
|
|
31
|
+
* day. `refresh` fetches it again.
|
|
32
|
+
*/
|
|
33
|
+
static listModels(options: {
|
|
34
|
+
apiKey: string;
|
|
35
|
+
refresh?: boolean;
|
|
36
|
+
}): Promise<MicdropModel[]>;
|
|
27
37
|
private genai;
|
|
28
38
|
private signatures;
|
|
29
39
|
constructor(options: GeminiAgentOptions);
|
|
@@ -44,6 +54,116 @@ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
|
44
54
|
private buildTools;
|
|
45
55
|
}
|
|
46
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
59
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
60
|
+
* without writing a single token.
|
|
61
|
+
*
|
|
62
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
63
|
+
*/
|
|
64
|
+
/** What a question asks, a text or text blocks */
|
|
65
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
66
|
+
/** The probability that a condition is true */
|
|
67
|
+
interface GeminiNoulQuestion {
|
|
68
|
+
type: 'noul';
|
|
69
|
+
instructions: GeminiInstructions;
|
|
70
|
+
criteria?: Decisions.NoulCriteria;
|
|
71
|
+
}
|
|
72
|
+
/** One value out of several, without order */
|
|
73
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
74
|
+
type: 'choice';
|
|
75
|
+
instructions: GeminiInstructions;
|
|
76
|
+
criteria: Record<V, string>;
|
|
77
|
+
}
|
|
78
|
+
/** A position on ordered levels, from the lowest */
|
|
79
|
+
interface GeminiScoreQuestion {
|
|
80
|
+
type: 'score';
|
|
81
|
+
instructions: GeminiInstructions;
|
|
82
|
+
criteria: string[];
|
|
83
|
+
}
|
|
84
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
85
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
86
|
+
/**
|
|
87
|
+
* Asks for the probability that a condition is true
|
|
88
|
+
* @param instructions - The condition, written around what can be observed
|
|
89
|
+
* @param criteria - What makes it true, and what makes it false
|
|
90
|
+
*/
|
|
91
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
92
|
+
true?: string;
|
|
93
|
+
false?: string;
|
|
94
|
+
}): GeminiNoulQuestion;
|
|
95
|
+
/**
|
|
96
|
+
* Asks for one value out of several
|
|
97
|
+
* @param instructions - The question
|
|
98
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
99
|
+
*/
|
|
100
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
101
|
+
/**
|
|
102
|
+
* Asks for a score on ordered levels
|
|
103
|
+
* @param instructions - The question
|
|
104
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
105
|
+
*/
|
|
106
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
107
|
+
interface GeminiNoulAnswer {
|
|
108
|
+
type: 'noul';
|
|
109
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
110
|
+
noul: number;
|
|
111
|
+
/** Probability of the likelier outcome */
|
|
112
|
+
confidence: number;
|
|
113
|
+
}
|
|
114
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
115
|
+
type: 'choice';
|
|
116
|
+
choice: V;
|
|
117
|
+
confidence: number;
|
|
118
|
+
probabilities: Record<V, number>;
|
|
119
|
+
}
|
|
120
|
+
interface GeminiScoreAnswer {
|
|
121
|
+
type: 'score';
|
|
122
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
123
|
+
score: number;
|
|
124
|
+
confidence: number;
|
|
125
|
+
/** Probability of each level, by index */
|
|
126
|
+
probabilities: Record<number, number>;
|
|
127
|
+
/** Description of each level, by index */
|
|
128
|
+
legend: Record<number, string>;
|
|
129
|
+
}
|
|
130
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
131
|
+
/** What a Gemini classifier returns for its questions */
|
|
132
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
133
|
+
/** The answer to each question */
|
|
134
|
+
answers: {
|
|
135
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
136
|
+
};
|
|
137
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
138
|
+
}
|
|
139
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
140
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
141
|
+
model?: string;
|
|
142
|
+
/**
|
|
143
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
144
|
+
* `score()`. A function picks them per input.
|
|
145
|
+
*
|
|
146
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
147
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
148
|
+
* and `history`.
|
|
149
|
+
*/
|
|
150
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
151
|
+
/**
|
|
152
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
153
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
154
|
+
*/
|
|
155
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
156
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
157
|
+
timeout?: number;
|
|
158
|
+
};
|
|
159
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
160
|
+
private options;
|
|
161
|
+
private genai;
|
|
162
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
163
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
164
|
+
private buildInput;
|
|
165
|
+
}
|
|
166
|
+
|
|
47
167
|
/**
|
|
48
168
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
49
169
|
*
|
|
@@ -63,11 +183,20 @@ interface GeminiLiveOptions extends RealtimeOptions {
|
|
|
63
183
|
maxRetry?: number;
|
|
64
184
|
}
|
|
65
185
|
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
186
|
+
/**
|
|
187
|
+
* Lists the Live models, which hear and speak.
|
|
188
|
+
*
|
|
189
|
+
* The list is fetched once and shared by every call of the process for a
|
|
190
|
+
* day. `refresh` fetches it again.
|
|
191
|
+
*/
|
|
192
|
+
static listModels(options: {
|
|
193
|
+
apiKey: string;
|
|
194
|
+
refresh?: boolean;
|
|
195
|
+
}): Promise<MicdropModel[]>;
|
|
66
196
|
private socket;
|
|
67
197
|
private everConnected;
|
|
68
198
|
private resumptionHandle?;
|
|
69
199
|
private goingAway;
|
|
70
|
-
private outputResampler;
|
|
71
200
|
private hasTranscribedTurn;
|
|
72
201
|
private ignoreUserTranscript;
|
|
73
202
|
private cancelledToolCalls;
|
|
@@ -86,6 +215,7 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
86
215
|
protected clearTurn(): void;
|
|
87
216
|
protected generate(): void;
|
|
88
217
|
protected speakText(text: string): void;
|
|
218
|
+
get sampleRate(): number;
|
|
89
219
|
protected cancelAnswer(): void;
|
|
90
220
|
protected get isIdle(): boolean;
|
|
91
221
|
protected updateTools(): void;
|
|
@@ -141,6 +271,16 @@ interface GeminiSTTOptions {
|
|
|
141
271
|
}
|
|
142
272
|
declare class GeminiSTT extends STT {
|
|
143
273
|
private readonly options;
|
|
274
|
+
/**
|
|
275
|
+
* Lists the models that transcribe a Live session.
|
|
276
|
+
*
|
|
277
|
+
* The list is fetched once and shared by every call of the process for a
|
|
278
|
+
* day. `refresh` fetches it again.
|
|
279
|
+
*/
|
|
280
|
+
static listModels(options: {
|
|
281
|
+
apiKey: string;
|
|
282
|
+
refresh?: boolean;
|
|
283
|
+
}): Promise<MicdropModel[]>;
|
|
144
284
|
private socket;
|
|
145
285
|
private goingAway;
|
|
146
286
|
private utterances;
|
|
@@ -185,10 +325,30 @@ type GeminiTTSOptions = GeminiOptions & {
|
|
|
185
325
|
};
|
|
186
326
|
declare class GeminiTTS extends SentenceTTS {
|
|
187
327
|
private readonly options;
|
|
328
|
+
/**
|
|
329
|
+
* Lists the models that speak.
|
|
330
|
+
*
|
|
331
|
+
* The list is fetched once and shared by every call of the process for a
|
|
332
|
+
* day. `refresh` fetches it again.
|
|
333
|
+
*/
|
|
334
|
+
static listModels(options: {
|
|
335
|
+
apiKey: string;
|
|
336
|
+
refresh?: boolean;
|
|
337
|
+
}): Promise<MicdropModel[]>;
|
|
338
|
+
/**
|
|
339
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
340
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
341
|
+
* here.
|
|
342
|
+
*/
|
|
343
|
+
static listVoices(options?: {
|
|
344
|
+
language?: string;
|
|
345
|
+
}): Promise<MicdropVoice[]>;
|
|
188
346
|
private genai;
|
|
347
|
+
private rate;
|
|
348
|
+
get sampleRate(): number;
|
|
189
349
|
constructor(options: GeminiTTSOptions);
|
|
190
350
|
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
191
351
|
private synthesizeOnce;
|
|
192
352
|
}
|
|
193
353
|
|
|
194
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
354
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|