@micdrop/gemini 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -2
- package/dist/index.d.mts +156 -19
- package/dist/index.d.ts +156 -19
- package/dist/index.js +467 -323
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +456 -314
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
7
|
-
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three.
|
|
7
|
+
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three. `GeminiClassifier` answers typed questions about each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api).
|
|
8
8
|
|
|
9
9
|
## Installation
|
|
10
10
|
|
|
@@ -66,7 +66,7 @@ new MicdropServer(socket, {
|
|
|
66
66
|
})
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
-
The Micdrop client detects when the user speaks, so the automatic activity detection of Gemini is turned off. Tools are added with `addTool()` right after creating the model, since Gemini reads them when the session opens.
|
|
69
|
+
The Micdrop client detects when the user speaks, so the automatic activity detection of Gemini is turned off. Tools are added with `addTool()` right after creating the model, since Gemini reads them when the session opens. When the user interrupts the answer while a tool runs, its result goes in the conversation and reaches the model silently, so it keeps quiet about it.
|
|
70
70
|
|
|
71
71
|
### Events
|
|
72
72
|
|
|
@@ -83,6 +83,35 @@ The Micdrop client detects when the user speaks, so the automatic activity detec
|
|
|
83
83
|
|
|
84
84
|
`gemini-3.8-live-extended-thinking` reasons before answering and needs a `thinkingLevel`, such as `'low'`.
|
|
85
85
|
|
|
86
|
+
## Gemini Classifier
|
|
87
|
+
|
|
88
|
+
Classifies each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api): yes or no questions, choices and scores answered with calibrated probabilities by `gemini-decision-1`, on text, images, audio, video or PDF.
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
import { choice, GeminiClassifier, noul } from '@micdrop/gemini'
|
|
92
|
+
import { MicdropServer } from '@micdrop/server'
|
|
93
|
+
|
|
94
|
+
const classifier = new GeminiClassifier({
|
|
95
|
+
apiKey: process.env.GEMINI_API_KEY || '',
|
|
96
|
+
questions: {
|
|
97
|
+
intent: choice('What does the user in `turn` want?', {
|
|
98
|
+
billing: 'A charge, an invoice, a refund',
|
|
99
|
+
outage: 'The service is down or slow',
|
|
100
|
+
other: 'Anything else',
|
|
101
|
+
}),
|
|
102
|
+
wantsHuman: noul('Does the user in `turn` ask for a human?'),
|
|
103
|
+
},
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
new MicdropServer(socket, {
|
|
107
|
+
stt,
|
|
108
|
+
agent,
|
|
109
|
+
tts,
|
|
110
|
+
classifier,
|
|
111
|
+
classifierOptions: { waitBeforeAnswer: true },
|
|
112
|
+
})
|
|
113
|
+
```
|
|
114
|
+
|
|
86
115
|
## Documentation
|
|
87
116
|
|
|
88
117
|
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini) on the [website](https://micdrop.dev).
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -25,16 +25,135 @@ type GeminiOptions = {
|
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
27
|
private genai;
|
|
28
|
-
private abortController?;
|
|
29
28
|
private signatures;
|
|
30
29
|
constructor(options: GeminiAgentOptions);
|
|
31
|
-
protected generateAnswer(stream: PassThrough,
|
|
32
|
-
|
|
30
|
+
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Generates one step of the answer, retried on error.
|
|
33
|
+
*
|
|
34
|
+
* A retry starts the step over, so it only happens while nothing of the
|
|
35
|
+
* step reached the stream: what was said cannot be taken back, and saying
|
|
36
|
+
* it twice is worse than stopping there.
|
|
37
|
+
*/
|
|
38
|
+
private generateStep;
|
|
39
|
+
/** Streams one request, writing the text of the answer as it comes */
|
|
40
|
+
private streamStep;
|
|
41
|
+
private recordAnswer;
|
|
33
42
|
private buildSystemInstruction;
|
|
34
43
|
private buildInput;
|
|
35
44
|
private buildTools;
|
|
36
45
|
}
|
|
37
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
49
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
50
|
+
* without writing a single token.
|
|
51
|
+
*
|
|
52
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
53
|
+
*/
|
|
54
|
+
/** What a question asks, a text or text blocks */
|
|
55
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
56
|
+
/** The probability that a condition is true */
|
|
57
|
+
interface GeminiNoulQuestion {
|
|
58
|
+
type: 'noul';
|
|
59
|
+
instructions: GeminiInstructions;
|
|
60
|
+
criteria?: Decisions.NoulCriteria;
|
|
61
|
+
}
|
|
62
|
+
/** One value out of several, without order */
|
|
63
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
64
|
+
type: 'choice';
|
|
65
|
+
instructions: GeminiInstructions;
|
|
66
|
+
criteria: Record<V, string>;
|
|
67
|
+
}
|
|
68
|
+
/** A position on ordered levels, from the lowest */
|
|
69
|
+
interface GeminiScoreQuestion {
|
|
70
|
+
type: 'score';
|
|
71
|
+
instructions: GeminiInstructions;
|
|
72
|
+
criteria: string[];
|
|
73
|
+
}
|
|
74
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
75
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
76
|
+
/**
|
|
77
|
+
* Asks for the probability that a condition is true
|
|
78
|
+
* @param instructions - The condition, written around what can be observed
|
|
79
|
+
* @param criteria - What makes it true, and what makes it false
|
|
80
|
+
*/
|
|
81
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
82
|
+
true?: string;
|
|
83
|
+
false?: string;
|
|
84
|
+
}): GeminiNoulQuestion;
|
|
85
|
+
/**
|
|
86
|
+
* Asks for one value out of several
|
|
87
|
+
* @param instructions - The question
|
|
88
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
89
|
+
*/
|
|
90
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
91
|
+
/**
|
|
92
|
+
* Asks for a score on ordered levels
|
|
93
|
+
* @param instructions - The question
|
|
94
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
95
|
+
*/
|
|
96
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
97
|
+
interface GeminiNoulAnswer {
|
|
98
|
+
type: 'noul';
|
|
99
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
100
|
+
noul: number;
|
|
101
|
+
/** Probability of the likelier outcome */
|
|
102
|
+
confidence: number;
|
|
103
|
+
}
|
|
104
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
105
|
+
type: 'choice';
|
|
106
|
+
choice: V;
|
|
107
|
+
confidence: number;
|
|
108
|
+
probabilities: Record<V, number>;
|
|
109
|
+
}
|
|
110
|
+
interface GeminiScoreAnswer {
|
|
111
|
+
type: 'score';
|
|
112
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
113
|
+
score: number;
|
|
114
|
+
confidence: number;
|
|
115
|
+
/** Probability of each level, by index */
|
|
116
|
+
probabilities: Record<number, number>;
|
|
117
|
+
/** Description of each level, by index */
|
|
118
|
+
legend: Record<number, string>;
|
|
119
|
+
}
|
|
120
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
121
|
+
/** What a Gemini classifier returns for its questions */
|
|
122
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
123
|
+
/** The answer to each question */
|
|
124
|
+
answers: {
|
|
125
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
126
|
+
};
|
|
127
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
128
|
+
}
|
|
129
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
130
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
131
|
+
model?: string;
|
|
132
|
+
/**
|
|
133
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
134
|
+
* `score()`. A function picks them per input.
|
|
135
|
+
*
|
|
136
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
137
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
138
|
+
* and `history`.
|
|
139
|
+
*/
|
|
140
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
141
|
+
/**
|
|
142
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
143
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
144
|
+
*/
|
|
145
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
146
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
147
|
+
timeout?: number;
|
|
148
|
+
};
|
|
149
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
150
|
+
private options;
|
|
151
|
+
private genai;
|
|
152
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
153
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
154
|
+
private buildInput;
|
|
155
|
+
}
|
|
156
|
+
|
|
38
157
|
/**
|
|
39
158
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
40
159
|
*
|
|
@@ -57,16 +176,19 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
57
176
|
private socket;
|
|
58
177
|
private everConnected;
|
|
59
178
|
private resumptionHandle?;
|
|
60
|
-
private resuming;
|
|
61
179
|
private goingAway;
|
|
62
180
|
private outputResampler;
|
|
63
181
|
private hasTranscribedTurn;
|
|
64
182
|
private ignoreUserTranscript;
|
|
65
183
|
private cancelledToolCalls;
|
|
66
|
-
private
|
|
67
|
-
private
|
|
68
|
-
private answerEndDeferred;
|
|
184
|
+
private session;
|
|
185
|
+
private toolRound;
|
|
69
186
|
constructor(options: GeminiLiveOptions);
|
|
187
|
+
/**
|
|
188
|
+
* Stops the answer. The tools it is waiting for still send their results,
|
|
189
|
+
* silently, since a blocking call left unanswered would stall the model.
|
|
190
|
+
*/
|
|
191
|
+
cancel(): void;
|
|
70
192
|
destroy(): void;
|
|
71
193
|
protected openTurn(): void;
|
|
72
194
|
protected appendAudio(chunk: Buffer): void;
|
|
@@ -84,6 +206,14 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
84
206
|
private buildSetup;
|
|
85
207
|
private onReady;
|
|
86
208
|
private onReconnecting;
|
|
209
|
+
/** Forgets what belonged to the previous session */
|
|
210
|
+
private startFreshSession;
|
|
211
|
+
/**
|
|
212
|
+
* Starts a new round of tools. The tools still running belong to the old
|
|
213
|
+
* one: their results go out silently, and nothing waits for the model to
|
|
214
|
+
* talk about them.
|
|
215
|
+
*/
|
|
216
|
+
private resetToolRound;
|
|
87
217
|
/** Moves to a new connection before Gemini closes this one */
|
|
88
218
|
private renewIfGoingAway;
|
|
89
219
|
private onMessage;
|
|
@@ -91,7 +221,6 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
91
221
|
private onModelOutput;
|
|
92
222
|
/** Ends the answer the model stopped giving to wait for a tool */
|
|
93
223
|
private endDeferredAnswer;
|
|
94
|
-
private clearToolFollowUp;
|
|
95
224
|
/**
|
|
96
225
|
* The transcript of the turn comes before the answer, so the first piece of
|
|
97
226
|
* the answer closes it.
|
|
@@ -124,22 +253,30 @@ declare class GeminiSTT extends STT {
|
|
|
124
253
|
private readonly options;
|
|
125
254
|
private socket;
|
|
126
255
|
private goingAway;
|
|
127
|
-
private
|
|
128
|
-
private utteranceOpen;
|
|
129
|
-
private utteranceEnded;
|
|
130
|
-
private transcript;
|
|
131
|
-
private transcriptionTimeout?;
|
|
256
|
+
private utterances;
|
|
132
257
|
constructor(options: GeminiSTTOptions);
|
|
133
258
|
transcribe(audioStream: Readable): void;
|
|
134
259
|
destroy(): void;
|
|
135
260
|
private buildSetup;
|
|
136
261
|
private openUtterance;
|
|
262
|
+
private endUtterance;
|
|
263
|
+
/** Sends the first utterance to Gemini, with the audio it held meanwhile */
|
|
264
|
+
private sendFirst;
|
|
265
|
+
private sendEnd;
|
|
137
266
|
private sendAudio;
|
|
138
|
-
/**
|
|
139
|
-
|
|
267
|
+
/**
|
|
268
|
+
* Emits what was transcribed of an utterance, possibly nothing, and keeps
|
|
269
|
+
* it in line for what Gemini still sends about it
|
|
270
|
+
*/
|
|
271
|
+
private give;
|
|
272
|
+
private remove;
|
|
273
|
+
/** Moves to a new connection once no utterance waits on this one */
|
|
274
|
+
private renewIfGoingAway;
|
|
275
|
+
private onReady;
|
|
140
276
|
private onMessage;
|
|
141
277
|
private onReconnecting;
|
|
142
|
-
private
|
|
278
|
+
private onFailed;
|
|
279
|
+
private dropUtterances;
|
|
143
280
|
}
|
|
144
281
|
|
|
145
282
|
/**
|
|
@@ -164,4 +301,4 @@ declare class GeminiTTS extends SentenceTTS {
|
|
|
164
301
|
private synthesizeOnce;
|
|
165
302
|
}
|
|
166
303
|
|
|
167
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
304
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -25,16 +25,135 @@ type GeminiOptions = {
|
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
27
|
private genai;
|
|
28
|
-
private abortController?;
|
|
29
28
|
private signatures;
|
|
30
29
|
constructor(options: GeminiAgentOptions);
|
|
31
|
-
protected generateAnswer(stream: PassThrough,
|
|
32
|
-
|
|
30
|
+
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Generates one step of the answer, retried on error.
|
|
33
|
+
*
|
|
34
|
+
* A retry starts the step over, so it only happens while nothing of the
|
|
35
|
+
* step reached the stream: what was said cannot be taken back, and saying
|
|
36
|
+
* it twice is worse than stopping there.
|
|
37
|
+
*/
|
|
38
|
+
private generateStep;
|
|
39
|
+
/** Streams one request, writing the text of the answer as it comes */
|
|
40
|
+
private streamStep;
|
|
41
|
+
private recordAnswer;
|
|
33
42
|
private buildSystemInstruction;
|
|
34
43
|
private buildInput;
|
|
35
44
|
private buildTools;
|
|
36
45
|
}
|
|
37
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
49
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
50
|
+
* without writing a single token.
|
|
51
|
+
*
|
|
52
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
53
|
+
*/
|
|
54
|
+
/** What a question asks, a text or text blocks */
|
|
55
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
56
|
+
/** The probability that a condition is true */
|
|
57
|
+
interface GeminiNoulQuestion {
|
|
58
|
+
type: 'noul';
|
|
59
|
+
instructions: GeminiInstructions;
|
|
60
|
+
criteria?: Decisions.NoulCriteria;
|
|
61
|
+
}
|
|
62
|
+
/** One value out of several, without order */
|
|
63
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
64
|
+
type: 'choice';
|
|
65
|
+
instructions: GeminiInstructions;
|
|
66
|
+
criteria: Record<V, string>;
|
|
67
|
+
}
|
|
68
|
+
/** A position on ordered levels, from the lowest */
|
|
69
|
+
interface GeminiScoreQuestion {
|
|
70
|
+
type: 'score';
|
|
71
|
+
instructions: GeminiInstructions;
|
|
72
|
+
criteria: string[];
|
|
73
|
+
}
|
|
74
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
75
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
76
|
+
/**
|
|
77
|
+
* Asks for the probability that a condition is true
|
|
78
|
+
* @param instructions - The condition, written around what can be observed
|
|
79
|
+
* @param criteria - What makes it true, and what makes it false
|
|
80
|
+
*/
|
|
81
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
82
|
+
true?: string;
|
|
83
|
+
false?: string;
|
|
84
|
+
}): GeminiNoulQuestion;
|
|
85
|
+
/**
|
|
86
|
+
* Asks for one value out of several
|
|
87
|
+
* @param instructions - The question
|
|
88
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
89
|
+
*/
|
|
90
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
91
|
+
/**
|
|
92
|
+
* Asks for a score on ordered levels
|
|
93
|
+
* @param instructions - The question
|
|
94
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
95
|
+
*/
|
|
96
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
97
|
+
interface GeminiNoulAnswer {
|
|
98
|
+
type: 'noul';
|
|
99
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
100
|
+
noul: number;
|
|
101
|
+
/** Probability of the likelier outcome */
|
|
102
|
+
confidence: number;
|
|
103
|
+
}
|
|
104
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
105
|
+
type: 'choice';
|
|
106
|
+
choice: V;
|
|
107
|
+
confidence: number;
|
|
108
|
+
probabilities: Record<V, number>;
|
|
109
|
+
}
|
|
110
|
+
interface GeminiScoreAnswer {
|
|
111
|
+
type: 'score';
|
|
112
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
113
|
+
score: number;
|
|
114
|
+
confidence: number;
|
|
115
|
+
/** Probability of each level, by index */
|
|
116
|
+
probabilities: Record<number, number>;
|
|
117
|
+
/** Description of each level, by index */
|
|
118
|
+
legend: Record<number, string>;
|
|
119
|
+
}
|
|
120
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
121
|
+
/** What a Gemini classifier returns for its questions */
|
|
122
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
123
|
+
/** The answer to each question */
|
|
124
|
+
answers: {
|
|
125
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
126
|
+
};
|
|
127
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
128
|
+
}
|
|
129
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
130
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
131
|
+
model?: string;
|
|
132
|
+
/**
|
|
133
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
134
|
+
* `score()`. A function picks them per input.
|
|
135
|
+
*
|
|
136
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
137
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
138
|
+
* and `history`.
|
|
139
|
+
*/
|
|
140
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
141
|
+
/**
|
|
142
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
143
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
144
|
+
*/
|
|
145
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
146
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
147
|
+
timeout?: number;
|
|
148
|
+
};
|
|
149
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
150
|
+
private options;
|
|
151
|
+
private genai;
|
|
152
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
153
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
154
|
+
private buildInput;
|
|
155
|
+
}
|
|
156
|
+
|
|
38
157
|
/**
|
|
39
158
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
40
159
|
*
|
|
@@ -57,16 +176,19 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
57
176
|
private socket;
|
|
58
177
|
private everConnected;
|
|
59
178
|
private resumptionHandle?;
|
|
60
|
-
private resuming;
|
|
61
179
|
private goingAway;
|
|
62
180
|
private outputResampler;
|
|
63
181
|
private hasTranscribedTurn;
|
|
64
182
|
private ignoreUserTranscript;
|
|
65
183
|
private cancelledToolCalls;
|
|
66
|
-
private
|
|
67
|
-
private
|
|
68
|
-
private answerEndDeferred;
|
|
184
|
+
private session;
|
|
185
|
+
private toolRound;
|
|
69
186
|
constructor(options: GeminiLiveOptions);
|
|
187
|
+
/**
|
|
188
|
+
* Stops the answer. The tools it is waiting for still send their results,
|
|
189
|
+
* silently, since a blocking call left unanswered would stall the model.
|
|
190
|
+
*/
|
|
191
|
+
cancel(): void;
|
|
70
192
|
destroy(): void;
|
|
71
193
|
protected openTurn(): void;
|
|
72
194
|
protected appendAudio(chunk: Buffer): void;
|
|
@@ -84,6 +206,14 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
84
206
|
private buildSetup;
|
|
85
207
|
private onReady;
|
|
86
208
|
private onReconnecting;
|
|
209
|
+
/** Forgets what belonged to the previous session */
|
|
210
|
+
private startFreshSession;
|
|
211
|
+
/**
|
|
212
|
+
* Starts a new round of tools. The tools still running belong to the old
|
|
213
|
+
* one: their results go out silently, and nothing waits for the model to
|
|
214
|
+
* talk about them.
|
|
215
|
+
*/
|
|
216
|
+
private resetToolRound;
|
|
87
217
|
/** Moves to a new connection before Gemini closes this one */
|
|
88
218
|
private renewIfGoingAway;
|
|
89
219
|
private onMessage;
|
|
@@ -91,7 +221,6 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
91
221
|
private onModelOutput;
|
|
92
222
|
/** Ends the answer the model stopped giving to wait for a tool */
|
|
93
223
|
private endDeferredAnswer;
|
|
94
|
-
private clearToolFollowUp;
|
|
95
224
|
/**
|
|
96
225
|
* The transcript of the turn comes before the answer, so the first piece of
|
|
97
226
|
* the answer closes it.
|
|
@@ -124,22 +253,30 @@ declare class GeminiSTT extends STT {
|
|
|
124
253
|
private readonly options;
|
|
125
254
|
private socket;
|
|
126
255
|
private goingAway;
|
|
127
|
-
private
|
|
128
|
-
private utteranceOpen;
|
|
129
|
-
private utteranceEnded;
|
|
130
|
-
private transcript;
|
|
131
|
-
private transcriptionTimeout?;
|
|
256
|
+
private utterances;
|
|
132
257
|
constructor(options: GeminiSTTOptions);
|
|
133
258
|
transcribe(audioStream: Readable): void;
|
|
134
259
|
destroy(): void;
|
|
135
260
|
private buildSetup;
|
|
136
261
|
private openUtterance;
|
|
262
|
+
private endUtterance;
|
|
263
|
+
/** Sends the first utterance to Gemini, with the audio it held meanwhile */
|
|
264
|
+
private sendFirst;
|
|
265
|
+
private sendEnd;
|
|
137
266
|
private sendAudio;
|
|
138
|
-
/**
|
|
139
|
-
|
|
267
|
+
/**
|
|
268
|
+
* Emits what was transcribed of an utterance, possibly nothing, and keeps
|
|
269
|
+
* it in line for what Gemini still sends about it
|
|
270
|
+
*/
|
|
271
|
+
private give;
|
|
272
|
+
private remove;
|
|
273
|
+
/** Moves to a new connection once no utterance waits on this one */
|
|
274
|
+
private renewIfGoingAway;
|
|
275
|
+
private onReady;
|
|
140
276
|
private onMessage;
|
|
141
277
|
private onReconnecting;
|
|
142
|
-
private
|
|
278
|
+
private onFailed;
|
|
279
|
+
private dropUtterances;
|
|
143
280
|
}
|
|
144
281
|
|
|
145
282
|
/**
|
|
@@ -164,4 +301,4 @@ declare class GeminiTTS extends SentenceTTS {
|
|
|
164
301
|
private synthesizeOnce;
|
|
165
302
|
}
|
|
166
303
|
|
|
167
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
304
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|