@micdrop/gemini 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -1
- package/dist/index.d.mts +113 -3
- package/dist/index.d.ts +113 -3
- package/dist/index.js +94 -20
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +79 -9
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
7
|
-
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three.
|
|
7
|
+
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three. `GeminiClassifier` answers typed questions about each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api).
|
|
8
8
|
|
|
9
9
|
## Installation
|
|
10
10
|
|
|
@@ -83,6 +83,35 @@ The Micdrop client detects when the user speaks, so the automatic activity detec
|
|
|
83
83
|
|
|
84
84
|
`gemini-3.8-live-extended-thinking` reasons before answering and needs a `thinkingLevel`, such as `'low'`.
|
|
85
85
|
|
|
86
|
+
## Gemini Classifier
|
|
87
|
+
|
|
88
|
+
Classifies each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api): yes or no questions, choices and scores answered with calibrated probabilities by `gemini-decision-1`, on text, images, audio, video or PDF.
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
import { choice, GeminiClassifier, noul } from '@micdrop/gemini'
|
|
92
|
+
import { MicdropServer } from '@micdrop/server'
|
|
93
|
+
|
|
94
|
+
const classifier = new GeminiClassifier({
|
|
95
|
+
apiKey: process.env.GEMINI_API_KEY || '',
|
|
96
|
+
questions: {
|
|
97
|
+
intent: choice('What does the user in `turn` want?', {
|
|
98
|
+
billing: 'A charge, an invoice, a refund',
|
|
99
|
+
outage: 'The service is down or slow',
|
|
100
|
+
other: 'Anything else',
|
|
101
|
+
}),
|
|
102
|
+
wantsHuman: noul('Does the user in `turn` ask for a human?'),
|
|
103
|
+
},
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
new MicdropServer(socket, {
|
|
107
|
+
stt,
|
|
108
|
+
agent,
|
|
109
|
+
tts,
|
|
110
|
+
classifier,
|
|
111
|
+
classifierOptions: { waitBeforeAnswer: true },
|
|
112
|
+
})
|
|
113
|
+
```
|
|
114
|
+
|
|
86
115
|
## Documentation
|
|
87
116
|
|
|
88
117
|
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini) on the [website](https://micdrop.dev).
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -44,6 +44,116 @@ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
|
44
44
|
private buildTools;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
49
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
50
|
+
* without writing a single token.
|
|
51
|
+
*
|
|
52
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
53
|
+
*/
|
|
54
|
+
/** What a question asks, a text or text blocks */
|
|
55
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
56
|
+
/** The probability that a condition is true */
|
|
57
|
+
interface GeminiNoulQuestion {
|
|
58
|
+
type: 'noul';
|
|
59
|
+
instructions: GeminiInstructions;
|
|
60
|
+
criteria?: Decisions.NoulCriteria;
|
|
61
|
+
}
|
|
62
|
+
/** One value out of several, without order */
|
|
63
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
64
|
+
type: 'choice';
|
|
65
|
+
instructions: GeminiInstructions;
|
|
66
|
+
criteria: Record<V, string>;
|
|
67
|
+
}
|
|
68
|
+
/** A position on ordered levels, from the lowest */
|
|
69
|
+
interface GeminiScoreQuestion {
|
|
70
|
+
type: 'score';
|
|
71
|
+
instructions: GeminiInstructions;
|
|
72
|
+
criteria: string[];
|
|
73
|
+
}
|
|
74
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
75
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
76
|
+
/**
|
|
77
|
+
* Asks for the probability that a condition is true
|
|
78
|
+
* @param instructions - The condition, written around what can be observed
|
|
79
|
+
* @param criteria - What makes it true, and what makes it false
|
|
80
|
+
*/
|
|
81
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
82
|
+
true?: string;
|
|
83
|
+
false?: string;
|
|
84
|
+
}): GeminiNoulQuestion;
|
|
85
|
+
/**
|
|
86
|
+
* Asks for one value out of several
|
|
87
|
+
* @param instructions - The question
|
|
88
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
89
|
+
*/
|
|
90
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
91
|
+
/**
|
|
92
|
+
* Asks for a score on ordered levels
|
|
93
|
+
* @param instructions - The question
|
|
94
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
95
|
+
*/
|
|
96
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
97
|
+
interface GeminiNoulAnswer {
|
|
98
|
+
type: 'noul';
|
|
99
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
100
|
+
noul: number;
|
|
101
|
+
/** Probability of the likelier outcome */
|
|
102
|
+
confidence: number;
|
|
103
|
+
}
|
|
104
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
105
|
+
type: 'choice';
|
|
106
|
+
choice: V;
|
|
107
|
+
confidence: number;
|
|
108
|
+
probabilities: Record<V, number>;
|
|
109
|
+
}
|
|
110
|
+
interface GeminiScoreAnswer {
|
|
111
|
+
type: 'score';
|
|
112
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
113
|
+
score: number;
|
|
114
|
+
confidence: number;
|
|
115
|
+
/** Probability of each level, by index */
|
|
116
|
+
probabilities: Record<number, number>;
|
|
117
|
+
/** Description of each level, by index */
|
|
118
|
+
legend: Record<number, string>;
|
|
119
|
+
}
|
|
120
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
121
|
+
/** What a Gemini classifier returns for its questions */
|
|
122
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
123
|
+
/** The answer to each question */
|
|
124
|
+
answers: {
|
|
125
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
126
|
+
};
|
|
127
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
128
|
+
}
|
|
129
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
130
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
131
|
+
model?: string;
|
|
132
|
+
/**
|
|
133
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
134
|
+
* `score()`. A function picks them per input.
|
|
135
|
+
*
|
|
136
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
137
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
138
|
+
* and `history`.
|
|
139
|
+
*/
|
|
140
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
141
|
+
/**
|
|
142
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
143
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
144
|
+
*/
|
|
145
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
146
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
147
|
+
timeout?: number;
|
|
148
|
+
};
|
|
149
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
150
|
+
private options;
|
|
151
|
+
private genai;
|
|
152
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
153
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
154
|
+
private buildInput;
|
|
155
|
+
}
|
|
156
|
+
|
|
47
157
|
/**
|
|
48
158
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
49
159
|
*
|
|
@@ -191,4 +301,4 @@ declare class GeminiTTS extends SentenceTTS {
|
|
|
191
301
|
private synthesizeOnce;
|
|
192
302
|
}
|
|
193
303
|
|
|
194
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
304
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GoogleGenAI } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
1
|
+
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
+
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -44,6 +44,116 @@ declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
|
44
44
|
private buildTools;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Gemini classifier, running the Decisions API: typed questions (yes or no,
|
|
49
|
+
* choice, score) answered with calibrated probabilities by Gemini Decision 1,
|
|
50
|
+
* without writing a single token.
|
|
51
|
+
*
|
|
52
|
+
* @see https://aistudio.google.com/docs/decisions-api
|
|
53
|
+
*/
|
|
54
|
+
/** What a question asks, a text or text blocks */
|
|
55
|
+
type GeminiInstructions = string | Decisions.DecisionInstructionContent | Decisions.DecisionInstructionContent[];
|
|
56
|
+
/** The probability that a condition is true */
|
|
57
|
+
interface GeminiNoulQuestion {
|
|
58
|
+
type: 'noul';
|
|
59
|
+
instructions: GeminiInstructions;
|
|
60
|
+
criteria?: Decisions.NoulCriteria;
|
|
61
|
+
}
|
|
62
|
+
/** One value out of several, without order */
|
|
63
|
+
interface GeminiChoiceQuestion<V extends string = string> {
|
|
64
|
+
type: 'choice';
|
|
65
|
+
instructions: GeminiInstructions;
|
|
66
|
+
criteria: Record<V, string>;
|
|
67
|
+
}
|
|
68
|
+
/** A position on ordered levels, from the lowest */
|
|
69
|
+
interface GeminiScoreQuestion {
|
|
70
|
+
type: 'score';
|
|
71
|
+
instructions: GeminiInstructions;
|
|
72
|
+
criteria: string[];
|
|
73
|
+
}
|
|
74
|
+
type GeminiQuestion = GeminiNoulQuestion | GeminiChoiceQuestion<any> | GeminiScoreQuestion;
|
|
75
|
+
type GeminiQuestions = Record<string, GeminiQuestion>;
|
|
76
|
+
/**
|
|
77
|
+
* Asks for the probability that a condition is true
|
|
78
|
+
* @param instructions - The condition, written around what can be observed
|
|
79
|
+
* @param criteria - What makes it true, and what makes it false
|
|
80
|
+
*/
|
|
81
|
+
declare function noul(instructions: GeminiInstructions, criteria?: {
|
|
82
|
+
true?: string;
|
|
83
|
+
false?: string;
|
|
84
|
+
}): GeminiNoulQuestion;
|
|
85
|
+
/**
|
|
86
|
+
* Asks for one value out of several
|
|
87
|
+
* @param instructions - The question
|
|
88
|
+
* @param criteria - Each value with a description of when it applies, 2 to 255
|
|
89
|
+
*/
|
|
90
|
+
declare function choice<V extends string>(instructions: GeminiInstructions, criteria: Record<V, string>): GeminiChoiceQuestion<V>;
|
|
91
|
+
/**
|
|
92
|
+
* Asks for a score on ordered levels
|
|
93
|
+
* @param instructions - The question
|
|
94
|
+
* @param levels - Description of each level from the lowest up, 2 to 255
|
|
95
|
+
*/
|
|
96
|
+
declare function score(instructions: GeminiInstructions, levels: string[]): GeminiScoreQuestion;
|
|
97
|
+
interface GeminiNoulAnswer {
|
|
98
|
+
type: 'noul';
|
|
99
|
+
/** Probability that the condition is true, from 0.01 to 0.99 */
|
|
100
|
+
noul: number;
|
|
101
|
+
/** Probability of the likelier outcome */
|
|
102
|
+
confidence: number;
|
|
103
|
+
}
|
|
104
|
+
interface GeminiChoiceAnswer<V extends string = string> {
|
|
105
|
+
type: 'choice';
|
|
106
|
+
choice: V;
|
|
107
|
+
confidence: number;
|
|
108
|
+
probabilities: Record<V, number>;
|
|
109
|
+
}
|
|
110
|
+
interface GeminiScoreAnswer {
|
|
111
|
+
type: 'score';
|
|
112
|
+
/** Probability-weighted average of the level indices, from 0 */
|
|
113
|
+
score: number;
|
|
114
|
+
confidence: number;
|
|
115
|
+
/** Probability of each level, by index */
|
|
116
|
+
probabilities: Record<number, number>;
|
|
117
|
+
/** Description of each level, by index */
|
|
118
|
+
legend: Record<number, string>;
|
|
119
|
+
}
|
|
120
|
+
type GeminiAnswer<Q extends GeminiQuestion> = Q extends GeminiChoiceQuestion<infer V> ? GeminiChoiceAnswer<V> : Q extends GeminiScoreQuestion ? GeminiScoreAnswer : GeminiNoulAnswer;
|
|
121
|
+
/** What a Gemini classifier returns for its questions */
|
|
122
|
+
interface GeminiClassification<Q extends GeminiQuestions> {
|
|
123
|
+
/** The answer to each question */
|
|
124
|
+
answers: {
|
|
125
|
+
[K in keyof Q]: GeminiAnswer<Q[K]>;
|
|
126
|
+
};
|
|
127
|
+
usage: Decisions.CreateDecisionResponse['usage'];
|
|
128
|
+
}
|
|
129
|
+
type GeminiClassifierOptions<Q extends GeminiQuestions> = GeminiOptions & {
|
|
130
|
+
/** Model answering the questions, `gemini-decision-1` by default */
|
|
131
|
+
model?: string;
|
|
132
|
+
/**
|
|
133
|
+
* Questions asked about each input, built with `noul()`, `choice()` and
|
|
134
|
+
* `score()`. A function picks them per input.
|
|
135
|
+
*
|
|
136
|
+
* Every question is answered in a single request, up to 1,000. Point at
|
|
137
|
+
* the fields of the input by name, between backticks: in a call, `turn`
|
|
138
|
+
* and `history`.
|
|
139
|
+
*/
|
|
140
|
+
questions: Q | ((input: ClassifierInput) => Q);
|
|
141
|
+
/**
|
|
142
|
+
* Builds what the model reads from the input: a text, or blocks of text,
|
|
143
|
+
* image, audio, video or PDF. The input as JSON by default, a text as is.
|
|
144
|
+
*/
|
|
145
|
+
input?: (input: ClassifierInput) => Decisions.CreateDecisionRequest['input'];
|
|
146
|
+
/** Timeout of each request in ms, 3000 by default */
|
|
147
|
+
timeout?: number;
|
|
148
|
+
};
|
|
149
|
+
declare class GeminiClassifier<Q extends GeminiQuestions> extends Classifier<GeminiClassification<Q>> {
|
|
150
|
+
private options;
|
|
151
|
+
private genai;
|
|
152
|
+
constructor(options: GeminiClassifierOptions<Q>);
|
|
153
|
+
protected evaluate(input: ClassifierInput, signal: AbortSignal): Promise<GeminiClassification<Q>>;
|
|
154
|
+
private buildInput;
|
|
155
|
+
}
|
|
156
|
+
|
|
47
157
|
/**
|
|
48
158
|
* Gemini Live API, a model hearing the user and answering with its voice
|
|
49
159
|
*
|
|
@@ -191,4 +301,4 @@ declare class GeminiTTS extends SentenceTTS {
|
|
|
191
301
|
private synthesizeOnce;
|
|
192
302
|
}
|
|
193
303
|
|
|
194
|
-
export { GeminiAgent, type GeminiAgentOptions, GeminiLive, type GeminiLiveOptions, type GeminiOptions, GeminiSTT, type GeminiSTTOptions, GeminiTTS, type GeminiTTSOptions };
|
|
304
|
+
export { GeminiAgent, type GeminiAgentOptions, type GeminiAnswer, type GeminiChoiceAnswer, type GeminiChoiceQuestion, type GeminiClassification, GeminiClassifier, type GeminiClassifierOptions, type GeminiInstructions, GeminiLive, type GeminiLiveOptions, type GeminiNoulAnswer, type GeminiNoulQuestion, type GeminiOptions, type GeminiQuestion, type GeminiQuestions, GeminiSTT, type GeminiSTTOptions, type GeminiScoreAnswer, type GeminiScoreQuestion, GeminiTTS, type GeminiTTSOptions, choice, noul, score };
|
package/dist/index.js
CHANGED
|
@@ -31,9 +31,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
GeminiAgent: () => GeminiAgent,
|
|
34
|
+
GeminiClassifier: () => GeminiClassifier,
|
|
34
35
|
GeminiLive: () => GeminiLive,
|
|
35
36
|
GeminiSTT: () => GeminiSTT,
|
|
36
|
-
GeminiTTS: () => GeminiTTS
|
|
37
|
+
GeminiTTS: () => GeminiTTS,
|
|
38
|
+
choice: () => choice,
|
|
39
|
+
noul: () => noul,
|
|
40
|
+
score: () => score
|
|
37
41
|
});
|
|
38
42
|
module.exports = __toCommonJS(index_exports);
|
|
39
43
|
|
|
@@ -277,12 +281,78 @@ function parseArguments(parameters) {
|
|
|
277
281
|
}
|
|
278
282
|
}
|
|
279
283
|
|
|
284
|
+
// src/GeminiClassifier.ts
|
|
285
|
+
var import_genai2 = require("@google/genai");
|
|
286
|
+
var import_server2 = require("@micdrop/server");
|
|
287
|
+
function noul(instructions, criteria) {
|
|
288
|
+
return {
|
|
289
|
+
type: "noul",
|
|
290
|
+
instructions,
|
|
291
|
+
criteria: criteria && {
|
|
292
|
+
true_description: criteria.true,
|
|
293
|
+
false_description: criteria.false
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
function choice(instructions, criteria) {
|
|
298
|
+
return { type: "choice", instructions, criteria };
|
|
299
|
+
}
|
|
300
|
+
function score(instructions, levels) {
|
|
301
|
+
return { type: "score", instructions, criteria: levels };
|
|
302
|
+
}
|
|
303
|
+
var DEFAULT_MODEL2 = "gemini-decision-1";
|
|
304
|
+
var DEFAULT_TIMEOUT = 3e3;
|
|
305
|
+
var API_VERSION = "v1beta";
|
|
306
|
+
var GeminiClassifier = class extends import_server2.Classifier {
|
|
307
|
+
constructor(options) {
|
|
308
|
+
super();
|
|
309
|
+
this.options = options;
|
|
310
|
+
this.genai = "genai" in options ? options.genai : new import_genai2.GoogleGenAI({ apiKey: options.apiKey });
|
|
311
|
+
}
|
|
312
|
+
async evaluate(input, signal) {
|
|
313
|
+
const { questions, model, timeout } = this.options;
|
|
314
|
+
const asked = typeof questions === "function" ? questions(input) : questions;
|
|
315
|
+
const decision = await this.genai.decisions.create(
|
|
316
|
+
{
|
|
317
|
+
model: model || DEFAULT_MODEL2,
|
|
318
|
+
input: this.buildInput(input),
|
|
319
|
+
questions: asked,
|
|
320
|
+
// Only on v1beta, while the client defaults to v1
|
|
321
|
+
api_version: API_VERSION
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
// The SDK drops its timeout when a signal is given
|
|
325
|
+
signal: AbortSignal.any([
|
|
326
|
+
signal,
|
|
327
|
+
AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT)
|
|
328
|
+
]),
|
|
329
|
+
retries: { strategy: "none" }
|
|
330
|
+
}
|
|
331
|
+
);
|
|
332
|
+
const blocked = decision.prompt_feedback?.block_reason;
|
|
333
|
+
if (blocked) {
|
|
334
|
+
const message = decision.prompt_feedback?.block_reason_message;
|
|
335
|
+
throw new Error(
|
|
336
|
+
`Input blocked by Gemini (${blocked})${message ? `: ${message}` : ""}`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
answers: decision.answers ?? {},
|
|
341
|
+
usage: decision.usage
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
buildInput(input) {
|
|
345
|
+
if (this.options.input) return this.options.input(input);
|
|
346
|
+
return typeof input === "string" ? input : JSON.stringify(input);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
349
|
+
|
|
280
350
|
// src/GeminiLive.ts
|
|
281
|
-
var
|
|
351
|
+
var import_server4 = require("@micdrop/server");
|
|
282
352
|
var import_zod2 = require("zod");
|
|
283
353
|
|
|
284
354
|
// src/LiveSocket.ts
|
|
285
|
-
var
|
|
355
|
+
var import_server3 = require("@micdrop/server");
|
|
286
356
|
var import_eventemitter3 = require("eventemitter3");
|
|
287
357
|
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
288
358
|
var SAMPLE_RATE = 16e3;
|
|
@@ -329,7 +399,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
|
329
399
|
this.clearPending();
|
|
330
400
|
this.emit("Failed");
|
|
331
401
|
};
|
|
332
|
-
this.socket = new
|
|
402
|
+
this.socket = new import_server3.ReconnectingSocket({
|
|
333
403
|
url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
|
|
334
404
|
connectionTimeout: options.connectionTimeout,
|
|
335
405
|
retryDelay: options.retryDelay,
|
|
@@ -431,7 +501,7 @@ function parseMessage(text) {
|
|
|
431
501
|
}
|
|
432
502
|
|
|
433
503
|
// src/GeminiLive.ts
|
|
434
|
-
var
|
|
504
|
+
var DEFAULT_MODEL3 = "gemini-3.8-live";
|
|
435
505
|
var SAMPLE_RATE2 = 16e3;
|
|
436
506
|
var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
|
|
437
507
|
var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
|
|
@@ -439,12 +509,12 @@ function newToolRound() {
|
|
|
439
509
|
return { running: 0, answerEndDeferred: false };
|
|
440
510
|
}
|
|
441
511
|
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
442
|
-
var GeminiLive = class extends
|
|
512
|
+
var GeminiLive = class extends import_server4.Realtime {
|
|
443
513
|
constructor(options) {
|
|
444
514
|
super(options);
|
|
445
515
|
this.everConnected = false;
|
|
446
516
|
this.goingAway = false;
|
|
447
|
-
this.outputResampler = new
|
|
517
|
+
this.outputResampler = new import_server4.Pcm16Resampler(
|
|
448
518
|
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
449
519
|
SAMPLE_RATE2
|
|
450
520
|
);
|
|
@@ -606,7 +676,7 @@ var GeminiLive = class extends import_server3.Realtime {
|
|
|
606
676
|
}));
|
|
607
677
|
return {
|
|
608
678
|
setup: {
|
|
609
|
-
model: `models/${this.options.model ||
|
|
679
|
+
model: `models/${this.options.model || DEFAULT_MODEL3}`,
|
|
610
680
|
generationConfig: {
|
|
611
681
|
responseModalities: ["AUDIO"],
|
|
612
682
|
...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
|
|
@@ -768,11 +838,11 @@ var GeminiLive = class extends import_server3.Realtime {
|
|
|
768
838
|
};
|
|
769
839
|
|
|
770
840
|
// src/GeminiSTT.ts
|
|
771
|
-
var
|
|
772
|
-
var
|
|
841
|
+
var import_server5 = require("@micdrop/server");
|
|
842
|
+
var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
|
|
773
843
|
var SAMPLE_RATE3 = 16e3;
|
|
774
844
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
775
|
-
var GeminiSTT = class extends
|
|
845
|
+
var GeminiSTT = class extends import_server5.STT {
|
|
776
846
|
constructor(options) {
|
|
777
847
|
super();
|
|
778
848
|
this.options = options;
|
|
@@ -852,7 +922,7 @@ var GeminiSTT = class extends import_server4.STT {
|
|
|
852
922
|
buildSetup() {
|
|
853
923
|
return {
|
|
854
924
|
setup: {
|
|
855
|
-
model: `models/${this.options.model ||
|
|
925
|
+
model: `models/${this.options.model || DEFAULT_MODEL4}`,
|
|
856
926
|
generationConfig: { responseModalities: ["TEXT"] },
|
|
857
927
|
inputAudioTranscription: {
|
|
858
928
|
languageCodes: this.options.language ? [this.options.language] : [],
|
|
@@ -948,18 +1018,18 @@ var GeminiSTT = class extends import_server4.STT {
|
|
|
948
1018
|
};
|
|
949
1019
|
|
|
950
1020
|
// src/GeminiTTS.ts
|
|
951
|
-
var
|
|
952
|
-
var
|
|
953
|
-
var
|
|
1021
|
+
var import_genai3 = require("@google/genai");
|
|
1022
|
+
var import_server6 = require("@micdrop/server");
|
|
1023
|
+
var DEFAULT_MODEL5 = "gemini-2.5-flash-preview-tts";
|
|
954
1024
|
var DEFAULT_VOICE = "Kore";
|
|
955
1025
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
956
1026
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
957
1027
|
var DEFAULT_MAX_RETRY = 2;
|
|
958
|
-
var GeminiTTS = class extends
|
|
1028
|
+
var GeminiTTS = class extends import_server6.SentenceTTS {
|
|
959
1029
|
constructor(options) {
|
|
960
1030
|
super();
|
|
961
1031
|
this.options = options;
|
|
962
|
-
this.genai = "genai" in options ? options.genai : new
|
|
1032
|
+
this.genai = "genai" in options ? options.genai : new import_genai3.GoogleGenAI({ apiKey: options.apiKey });
|
|
963
1033
|
}
|
|
964
1034
|
async synthesize(text, signal) {
|
|
965
1035
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
@@ -981,7 +1051,7 @@ var GeminiTTS = class extends import_server5.SentenceTTS {
|
|
|
981
1051
|
async synthesizeOnce(input, signal, progress) {
|
|
982
1052
|
const stream = await this.genai.interactions.create(
|
|
983
1053
|
{
|
|
984
|
-
model: this.options.model ||
|
|
1054
|
+
model: this.options.model || DEFAULT_MODEL5,
|
|
985
1055
|
store: false,
|
|
986
1056
|
stream: true,
|
|
987
1057
|
input,
|
|
@@ -1006,7 +1076,7 @@ var GeminiTTS = class extends import_server5.SentenceTTS {
|
|
|
1006
1076
|
}
|
|
1007
1077
|
const { data, sample_rate } = event.delta;
|
|
1008
1078
|
if (!data) continue;
|
|
1009
|
-
resampler ?? (resampler = new
|
|
1079
|
+
resampler ?? (resampler = new import_server6.Pcm16Resampler(
|
|
1010
1080
|
sample_rate || GEMINI_SAMPLE_RATE,
|
|
1011
1081
|
OUTPUT_SAMPLE_RATE
|
|
1012
1082
|
));
|
|
@@ -1022,8 +1092,12 @@ function isBlocked(error) {
|
|
|
1022
1092
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1023
1093
|
0 && (module.exports = {
|
|
1024
1094
|
GeminiAgent,
|
|
1095
|
+
GeminiClassifier,
|
|
1025
1096
|
GeminiLive,
|
|
1026
1097
|
GeminiSTT,
|
|
1027
|
-
GeminiTTS
|
|
1098
|
+
GeminiTTS,
|
|
1099
|
+
choice,
|
|
1100
|
+
noul,
|
|
1101
|
+
score
|
|
1028
1102
|
});
|
|
1029
1103
|
//# sourceMappingURL=index.js.map
|