@aituber-onair/core 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +639 -7
- package/dist/constants/gemini.d.ts +1 -1
- package/dist/constants/gemini.js +1 -1
- package/dist/constants/gemini.js.map +1 -1
- package/dist/core/AITuberOnAirCore.d.ts +32 -1
- package/dist/core/AITuberOnAirCore.js +38 -2
- package/dist/core/AITuberOnAirCore.js.map +1 -1
- package/dist/core/ChatProcessor.d.ts +9 -1
- package/dist/core/ChatProcessor.js +116 -51
- package/dist/core/ChatProcessor.js.map +1 -1
- package/dist/core/MemoryManager.js +10 -9
- package/dist/core/MemoryManager.js.map +1 -1
- package/dist/core/ToolExecutor.d.ts +9 -0
- package/dist/core/ToolExecutor.js +39 -0
- package/dist/core/ToolExecutor.js.map +1 -0
- package/dist/services/chat/ChatService.d.ts +13 -0
- package/dist/services/chat/providers/claude/ClaudeChatService.d.ts +45 -4
- package/dist/services/chat/providers/claude/ClaudeChatService.js +227 -180
- package/dist/services/chat/providers/claude/ClaudeChatService.js.map +1 -1
- package/dist/services/chat/providers/claude/ClaudeChatServiceProvider.js +1 -1
- package/dist/services/chat/providers/claude/ClaudeChatServiceProvider.js.map +1 -1
- package/dist/services/chat/providers/gemini/GeminiChatService.d.ts +19 -16
- package/dist/services/chat/providers/gemini/GeminiChatService.js +376 -157
- package/dist/services/chat/providers/gemini/GeminiChatService.js.map +1 -1
- package/dist/services/chat/providers/gemini/GeminiChatServiceProvider.js +1 -1
- package/dist/services/chat/providers/gemini/GeminiChatServiceProvider.js.map +1 -1
- package/dist/services/chat/providers/openai/OpenAIChatService.d.ts +21 -3
- package/dist/services/chat/providers/openai/OpenAIChatService.js +205 -114
- package/dist/services/chat/providers/openai/OpenAIChatService.js.map +1 -1
- package/dist/services/chat/providers/openai/OpenAIChatServiceProvider.js +3 -1
- package/dist/services/chat/providers/openai/OpenAIChatServiceProvider.js.map +1 -1
- package/dist/services/voice/VoiceEngineAdapter.d.ts +1 -1
- package/dist/services/voice/VoiceEngineAdapter.js +1 -1
- package/dist/services/voice/VoiceService.d.ts +1 -1
- package/dist/types/chat.d.ts +2 -2
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -1
- package/dist/types/toolChat.d.ts +46 -0
- package/dist/types/toolChat.js +2 -0
- package/dist/types/toolChat.js.map +1 -0
- package/package.json +1 -1
- package/dist/constants/api.d.ts +0 -4
- package/dist/constants/api.js +0 -13
- package/dist/constants/api.js.map +0 -1
- package/dist/constants/openaiApi.d.ts +0 -15
- package/dist/constants/openaiApi.js +0 -15
- package/dist/constants/openaiApi.js.map +0 -1
- package/dist/services/chat/ClaudeChatService.d.ts +0 -64
- package/dist/services/chat/ClaudeChatService.js +0 -237
- package/dist/services/chat/ClaudeChatService.js.map +0 -1
- package/dist/services/chat/GeminiChatService.d.ts +0 -63
- package/dist/services/chat/GeminiChatService.js +0 -314
- package/dist/services/chat/GeminiChatService.js.map +0 -1
- package/dist/services/chat/OpenAIChatService.d.ts +0 -39
- package/dist/services/chat/OpenAIChatService.js +0 -171
- package/dist/services/chat/OpenAIChatService.js.map +0 -1
- package/dist/services/chat/OpenAISummarizer.d.ts +0 -25
- package/dist/services/chat/OpenAISummarizer.js +0 -70
- package/dist/services/chat/OpenAISummarizer.js.map +0 -1
- package/dist/services/chat/providers/ClaudeChatServiceProvider.d.ts +0 -39
- package/dist/services/chat/providers/ClaudeChatServiceProvider.js +0 -57
- package/dist/services/chat/providers/ClaudeChatServiceProvider.js.map +0 -1
- package/dist/services/chat/providers/GeminiChatServiceProvider.d.ts +0 -39
- package/dist/services/chat/providers/GeminiChatServiceProvider.js +0 -57
- package/dist/services/chat/providers/GeminiChatServiceProvider.js.map +0 -1
- package/dist/services/chat/providers/OpenAIChatServiceProvider.d.ts +0 -39
- package/dist/services/chat/providers/OpenAIChatServiceProvider.js +0 -57
- package/dist/services/chat/providers/OpenAIChatServiceProvider.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ClaudeChatServiceProvider.js","sourceRoot":"","sources":["../../../../../src/services/chat/providers/claude/ClaudeChatServiceProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAMxD;;GAEG;AACH,MAAM,OAAO,yBAAyB;IACpC;;;;OAIG;IACH,iBAAiB,CAAC,OAA2B;QAC3C,gFAAgF;QAChF,MAAM,WAAW,GACf,OAAO,CAAC,WAAW;YACnB,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnE,CAAC,CAAC,OAAO,CAAC,KAAK;gBACf,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9B,OAAO,IAAI,iBAAiB,CAC1B,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,EACvC,WAAW,
|
|
1
|
+
{"version":3,"file":"ClaudeChatServiceProvider.js","sourceRoot":"","sources":["../../../../../src/services/chat/providers/claude/ClaudeChatServiceProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAMxD;;GAEG;AACH,MAAM,OAAO,yBAAyB;IACpC;;;;OAIG;IACH,iBAAiB,CAAC,OAA2B;QAC3C,gFAAgF;QAChF,MAAM,WAAW,GACf,OAAO,CAAC,WAAW;YACnB,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnE,CAAC,CAAC,OAAO,CAAC,KAAK;gBACf,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9B,OAAO,IAAI,iBAAiB,CAC1B,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,EACvC,WAAW,EACX,OAAO,CAAC,KAAK,IAAI,EAAE,CACpB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,kBAAkB;QAChB,OAAO;YACL,oBAAoB;YACpB,sBAAsB;YACtB,uBAAuB;YACvB,uBAAuB;SACxB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,sBAAsB,CAAC,KAAa;QAClC,OAAO,8BAA8B,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC;CACF"}
|
|
@@ -1,21 +1,30 @@
|
|
|
1
1
|
import { ChatService } from '../../ChatService';
|
|
2
|
-
import { Message, MessageWithVision } from '../../../../types';
|
|
2
|
+
import { Message, MessageWithVision, ToolChatCompletion, ToolDefinition } from '../../../../types';
|
|
3
3
|
/**
|
|
4
4
|
* Gemini implementation of ChatService
|
|
5
5
|
*/
|
|
6
6
|
export declare class GeminiChatService implements ChatService {
|
|
7
|
+
/** Provider name */
|
|
8
|
+
readonly provider: string;
|
|
7
9
|
private apiKey;
|
|
8
10
|
private model;
|
|
9
11
|
private visionModel;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
private tools;
|
|
13
|
+
/** id(OpenAI) → name(Gemini) mapping */
|
|
14
|
+
private callIdMap;
|
|
15
|
+
private safeJsonParse;
|
|
16
|
+
private normalizeToolResult;
|
|
17
|
+
/**
|
|
18
|
+
* camelCase → snake_case conversion (v1beta)
|
|
19
|
+
*/
|
|
20
|
+
private adaptKeysForApi;
|
|
12
21
|
/**
|
|
13
22
|
* Constructor
|
|
14
23
|
* @param apiKey Google API key
|
|
15
24
|
* @param model Name of the model to use
|
|
16
25
|
* @param visionModel Name of the vision model
|
|
17
26
|
*/
|
|
18
|
-
constructor(apiKey: string, model?: string, visionModel?: string);
|
|
27
|
+
constructor(apiKey: string, model?: string, visionModel?: string, tools?: ToolDefinition[]);
|
|
19
28
|
/**
|
|
20
29
|
* Get the current model name
|
|
21
30
|
* @returns Model name
|
|
@@ -33,20 +42,9 @@ export declare class GeminiChatService implements ChatService {
|
|
|
33
42
|
* @param onCompleteResponse Callback to execute when response is complete
|
|
34
43
|
*/
|
|
35
44
|
processChat(messages: Message[], onPartialResponse: (text: string) => void, onCompleteResponse: (text: string) => Promise<void>): Promise<void>;
|
|
36
|
-
/**
|
|
37
|
-
* Process chat messages with images
|
|
38
|
-
* @param messages Array of messages to send (including images)
|
|
39
|
-
* @param onPartialResponse Callback to receive each part of streaming response
|
|
40
|
-
* @param onCompleteResponse Callback to execute when response is complete
|
|
41
|
-
* @throws Error if the selected model doesn't support vision
|
|
42
|
-
*/
|
|
43
45
|
processVisionChat(messages: MessageWithVision[], onPartialResponse: (text: string) => void, onCompleteResponse: (text: string) => Promise<void>): Promise<void>;
|
|
44
|
-
/**
|
|
45
|
-
* Convert AITuber OnAir messages to Gemini format
|
|
46
|
-
* @param messages Array of messages
|
|
47
|
-
* @returns Gemini formatted messages
|
|
48
|
-
*/
|
|
49
46
|
private convertMessagesToGeminiFormat;
|
|
47
|
+
private callGemini;
|
|
50
48
|
/**
|
|
51
49
|
* Convert AITuber OnAir vision messages to Gemini format
|
|
52
50
|
* @param messages Array of vision messages
|
|
@@ -65,4 +63,9 @@ export declare class GeminiChatService implements ChatService {
|
|
|
65
63
|
* @returns Gemini role
|
|
66
64
|
*/
|
|
67
65
|
private mapRoleToGemini;
|
|
66
|
+
private parseStream;
|
|
67
|
+
private parseOneShot;
|
|
68
|
+
chatOnce(messages: Message[], stream?: boolean, onPartialResponse?: (t: string) => void): Promise<ToolChatCompletion>;
|
|
69
|
+
visionChatOnce(messages: MessageWithVision[], stream?: boolean): Promise<ToolChatCompletion>;
|
|
70
|
+
private genUUID;
|
|
68
71
|
}
|
|
@@ -3,15 +3,56 @@ import { ENDPOINT_GEMINI_API, MODEL_GEMINI_2_0_FLASH_LITE, GEMINI_VISION_SUPPORT
|
|
|
3
3
|
* Gemini implementation of ChatService
|
|
4
4
|
*/
|
|
5
5
|
export class GeminiChatService {
|
|
6
|
+
/* ────────────────────────────────── */
|
|
7
|
+
/* Utilities */
|
|
8
|
+
/* ────────────────────────────────── */
|
|
9
|
+
safeJsonParse(str) {
|
|
10
|
+
try {
|
|
11
|
+
return JSON.parse(str);
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return str; // keep as string
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
normalizeToolResult(val) {
|
|
18
|
+
if (val === null)
|
|
19
|
+
return { content: null };
|
|
20
|
+
if (typeof val === 'object')
|
|
21
|
+
return val;
|
|
22
|
+
return { content: val }; // wrap primitive
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* camelCase → snake_case conversion (v1beta)
|
|
26
|
+
*/
|
|
27
|
+
adaptKeysForApi(obj) {
|
|
28
|
+
const map = {
|
|
29
|
+
toolConfig: 'tool_config',
|
|
30
|
+
functionCallingConfig: 'function_calling_config',
|
|
31
|
+
functionDeclarations: 'function_declarations',
|
|
32
|
+
functionCall: 'function_call',
|
|
33
|
+
functionResponse: 'function_response',
|
|
34
|
+
};
|
|
35
|
+
if (Array.isArray(obj))
|
|
36
|
+
return obj.map((v) => this.adaptKeysForApi(v));
|
|
37
|
+
if (obj && typeof obj === 'object') {
|
|
38
|
+
return Object.fromEntries(Object.entries(obj).map(([k, v]) => [
|
|
39
|
+
map[k] ?? k,
|
|
40
|
+
this.adaptKeysForApi(v),
|
|
41
|
+
]));
|
|
42
|
+
}
|
|
43
|
+
return obj;
|
|
44
|
+
}
|
|
6
45
|
/**
|
|
7
46
|
* Constructor
|
|
8
47
|
* @param apiKey Google API key
|
|
9
48
|
* @param model Name of the model to use
|
|
10
49
|
* @param visionModel Name of the vision model
|
|
11
50
|
*/
|
|
12
|
-
constructor(apiKey, model = MODEL_GEMINI_2_0_FLASH_LITE, visionModel = MODEL_GEMINI_2_0_FLASH_LITE) {
|
|
51
|
+
constructor(apiKey, model = MODEL_GEMINI_2_0_FLASH_LITE, visionModel = MODEL_GEMINI_2_0_FLASH_LITE, tools = []) {
|
|
13
52
|
/** Provider name */
|
|
14
53
|
this.provider = 'gemini';
|
|
54
|
+
/** id(OpenAI) → name(Gemini) mapping */
|
|
55
|
+
this.callIdMap = new Map();
|
|
15
56
|
this.apiKey = apiKey;
|
|
16
57
|
this.model = model;
|
|
17
58
|
// check if the vision model is supported
|
|
@@ -19,6 +60,7 @@ export class GeminiChatService {
|
|
|
19
60
|
throw new Error(`Model ${visionModel} does not support vision capabilities.`);
|
|
20
61
|
}
|
|
21
62
|
this.visionModel = visionModel;
|
|
63
|
+
this.tools = tools;
|
|
22
64
|
}
|
|
23
65
|
/**
|
|
24
66
|
* Get the current model name
|
|
@@ -42,188 +84,186 @@ export class GeminiChatService {
|
|
|
42
84
|
*/
|
|
43
85
|
async processChat(messages, onPartialResponse, onCompleteResponse) {
|
|
44
86
|
try {
|
|
45
|
-
//
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
body: JSON.stringify({
|
|
56
|
-
contents: geminiMessages,
|
|
57
|
-
generationConfig: {
|
|
58
|
-
temperature: 0.7,
|
|
59
|
-
topK: 40,
|
|
60
|
-
topP: 0.95,
|
|
61
|
-
maxOutputTokens: 1000,
|
|
62
|
-
},
|
|
63
|
-
}),
|
|
64
|
-
});
|
|
65
|
-
if (!response.ok) {
|
|
66
|
-
const errorData = await response.json();
|
|
67
|
-
throw new Error(`Gemini API error: ${errorData.error?.message || response.statusText}`);
|
|
87
|
+
// not use tools
|
|
88
|
+
if (this.tools.length === 0) {
|
|
89
|
+
const res = await this.callGemini(messages, this.model, true);
|
|
90
|
+
const { blocks } = await this.parseStream(res, onPartialResponse);
|
|
91
|
+
const full = blocks
|
|
92
|
+
.filter((b) => b.type === 'text')
|
|
93
|
+
.map((b) => b.text)
|
|
94
|
+
.join('');
|
|
95
|
+
await onCompleteResponse(full);
|
|
96
|
+
return;
|
|
68
97
|
}
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
98
|
+
/* with tools (1 turn) */
|
|
99
|
+
const { blocks, stop_reason } = await this.chatOnce(messages, true, onPartialResponse);
|
|
100
|
+
if (stop_reason === 'end') {
|
|
101
|
+
const full = blocks
|
|
102
|
+
.filter((b) => b.type === 'text')
|
|
103
|
+
.map((b) => b.text)
|
|
104
|
+
.join('');
|
|
105
|
+
await onCompleteResponse(full);
|
|
106
|
+
return;
|
|
75
107
|
}
|
|
76
|
-
|
|
77
|
-
let responseText = '';
|
|
78
|
-
while (true) {
|
|
79
|
-
const { done, value } = await reader.read();
|
|
80
|
-
if (done)
|
|
81
|
-
break;
|
|
82
|
-
const chunk = decoder.decode(value);
|
|
83
|
-
responseText += chunk;
|
|
84
|
-
}
|
|
85
|
-
// parse response
|
|
86
|
-
try {
|
|
87
|
-
const responseArray = JSON.parse(responseText);
|
|
88
|
-
// process each response
|
|
89
|
-
for (const item of responseArray) {
|
|
90
|
-
if (item.candidates && item.candidates.length > 0) {
|
|
91
|
-
const content = item.candidates[0].content;
|
|
92
|
-
if (content && content.parts && content.parts.length > 0) {
|
|
93
|
-
const text = content.parts[0].text || '';
|
|
94
|
-
if (text) {
|
|
95
|
-
fullText += text;
|
|
96
|
-
onPartialResponse(text);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
catch (err) {
|
|
103
|
-
console.error('Error parsing Gemini response:', err);
|
|
104
|
-
throw new Error(`Failed to parse Gemini response: ${err.message}`);
|
|
105
|
-
}
|
|
106
|
-
// Complete response callback
|
|
107
|
-
await onCompleteResponse(fullText);
|
|
108
|
+
throw new Error('Received functionCall. Use chatOnce() loop when tools are enabled.');
|
|
108
109
|
}
|
|
109
|
-
catch (
|
|
110
|
-
console.error('Error in processChat:',
|
|
111
|
-
throw
|
|
110
|
+
catch (err) {
|
|
111
|
+
console.error('Error in processChat:', err);
|
|
112
|
+
throw err;
|
|
112
113
|
}
|
|
113
114
|
}
|
|
114
|
-
/**
|
|
115
|
-
* Process chat messages with images
|
|
116
|
-
* @param messages Array of messages to send (including images)
|
|
117
|
-
* @param onPartialResponse Callback to receive each part of streaming response
|
|
118
|
-
* @param onCompleteResponse Callback to execute when response is complete
|
|
119
|
-
* @throws Error if the selected model doesn't support vision
|
|
120
|
-
*/
|
|
121
115
|
async processVisionChat(messages, onPartialResponse, onCompleteResponse) {
|
|
122
116
|
try {
|
|
123
|
-
// Check if the vision model supports vision capabilities
|
|
124
117
|
if (!GEMINI_VISION_SUPPORTED_MODELS.includes(this.visionModel)) {
|
|
125
118
|
throw new Error(`Model ${this.visionModel} does not support vision capabilities.`);
|
|
126
119
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
},
|
|
137
|
-
body: JSON.stringify({
|
|
138
|
-
contents: geminiMessages,
|
|
139
|
-
generationConfig: {
|
|
140
|
-
temperature: 0.7,
|
|
141
|
-
topK: 40,
|
|
142
|
-
topP: 0.95,
|
|
143
|
-
maxOutputTokens: 1000,
|
|
144
|
-
},
|
|
145
|
-
}),
|
|
146
|
-
});
|
|
147
|
-
if (!response.ok) {
|
|
148
|
-
const errorData = await response.json();
|
|
149
|
-
throw new Error(`Gemini API error: ${errorData.error?.message || response.statusText}`);
|
|
150
|
-
}
|
|
151
|
-
// Process streaming response
|
|
152
|
-
const reader = response.body?.getReader();
|
|
153
|
-
const decoder = new TextDecoder('utf-8');
|
|
154
|
-
let fullText = '';
|
|
155
|
-
if (!reader) {
|
|
156
|
-
throw new Error('Failed to get response reader');
|
|
157
|
-
}
|
|
158
|
-
// get full response
|
|
159
|
-
let responseText = '';
|
|
160
|
-
while (true) {
|
|
161
|
-
const { done, value } = await reader.read();
|
|
162
|
-
if (done)
|
|
163
|
-
break;
|
|
164
|
-
const chunk = decoder.decode(value);
|
|
165
|
-
responseText += chunk;
|
|
166
|
-
}
|
|
167
|
-
// parse response
|
|
168
|
-
try {
|
|
169
|
-
const responseArray = JSON.parse(responseText);
|
|
170
|
-
// process each response
|
|
171
|
-
for (const item of responseArray) {
|
|
172
|
-
if (item.candidates && item.candidates.length > 0) {
|
|
173
|
-
const content = item.candidates[0].content;
|
|
174
|
-
if (content && content.parts && content.parts.length > 0) {
|
|
175
|
-
const text = content.parts[0].text || '';
|
|
176
|
-
if (text) {
|
|
177
|
-
fullText += text;
|
|
178
|
-
onPartialResponse(text);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
120
|
+
if (this.tools.length === 0) {
|
|
121
|
+
const res = await this.callGemini(messages, this.visionModel, true);
|
|
122
|
+
const { blocks } = await this.parseStream(res, onPartialResponse);
|
|
123
|
+
const full = blocks
|
|
124
|
+
.filter((b) => b.type === 'text')
|
|
125
|
+
.map((b) => b.text)
|
|
126
|
+
.join('');
|
|
127
|
+
await onCompleteResponse(full);
|
|
128
|
+
return;
|
|
183
129
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
130
|
+
const { blocks, stop_reason } = await this.visionChatOnce(messages);
|
|
131
|
+
blocks
|
|
132
|
+
.filter((b) => b.type === 'text')
|
|
133
|
+
.forEach((b) => onPartialResponse(b.text));
|
|
134
|
+
if (stop_reason === 'end') {
|
|
135
|
+
const full = blocks
|
|
136
|
+
.filter((b) => b.type === 'text')
|
|
137
|
+
.map((b) => b.text)
|
|
138
|
+
.join('');
|
|
139
|
+
await onCompleteResponse(full);
|
|
140
|
+
return;
|
|
187
141
|
}
|
|
188
|
-
|
|
189
|
-
await onCompleteResponse(fullText);
|
|
142
|
+
throw new Error('Received functionCall. Use visionChatOnce() loop when tools are enabled.');
|
|
190
143
|
}
|
|
191
|
-
catch (
|
|
192
|
-
console.error('Error in processVisionChat:',
|
|
193
|
-
throw
|
|
144
|
+
catch (err) {
|
|
145
|
+
console.error('Error in processVisionChat:', err);
|
|
146
|
+
throw err;
|
|
194
147
|
}
|
|
195
148
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
* @returns Gemini formatted messages
|
|
200
|
-
*/
|
|
149
|
+
/* ────────────────────────────────── */
|
|
150
|
+
/* OpenAI → Gemini conversion */
|
|
151
|
+
/* ────────────────────────────────── */
|
|
201
152
|
convertMessagesToGeminiFormat(messages) {
|
|
202
|
-
const
|
|
153
|
+
const gemini = [];
|
|
203
154
|
let currentRole = null;
|
|
204
155
|
let currentParts = [];
|
|
156
|
+
const pushCurrent = () => {
|
|
157
|
+
if (currentRole && currentParts.length) {
|
|
158
|
+
gemini.push({ role: currentRole, parts: [...currentParts] });
|
|
159
|
+
currentParts = [];
|
|
160
|
+
}
|
|
161
|
+
};
|
|
205
162
|
for (const msg of messages) {
|
|
206
|
-
// Map AITuber OnAir roles to Gemini roles
|
|
207
163
|
const role = this.mapRoleToGemini(msg.role);
|
|
208
|
-
|
|
209
|
-
if (
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
164
|
+
/* assistant: tool_calls -> functionCall */
|
|
165
|
+
if (msg.tool_calls) {
|
|
166
|
+
pushCurrent();
|
|
167
|
+
for (const call of msg.tool_calls) {
|
|
168
|
+
this.callIdMap.set(call.id, call.function.name);
|
|
169
|
+
gemini.push({
|
|
170
|
+
role: 'model',
|
|
171
|
+
parts: [
|
|
172
|
+
{
|
|
173
|
+
functionCall: {
|
|
174
|
+
name: call.function.name,
|
|
175
|
+
args: JSON.parse(call.function.arguments || '{}'),
|
|
176
|
+
},
|
|
177
|
+
},
|
|
178
|
+
],
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
/* tool → functionResponse */
|
|
184
|
+
if (msg.role === 'tool') {
|
|
185
|
+
pushCurrent();
|
|
186
|
+
const funcName = msg.name ??
|
|
187
|
+
this.callIdMap.get(msg.tool_call_id) ??
|
|
188
|
+
'result';
|
|
189
|
+
gemini.push({
|
|
190
|
+
role: 'user',
|
|
191
|
+
parts: [
|
|
192
|
+
{
|
|
193
|
+
functionResponse: {
|
|
194
|
+
name: funcName,
|
|
195
|
+
response: this.normalizeToolResult(this.safeJsonParse(msg.content)),
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
],
|
|
213
199
|
});
|
|
214
|
-
|
|
200
|
+
continue;
|
|
215
201
|
}
|
|
202
|
+
/* normal text */
|
|
203
|
+
if (role !== currentRole)
|
|
204
|
+
pushCurrent();
|
|
216
205
|
currentRole = role;
|
|
217
206
|
currentParts.push({ text: msg.content });
|
|
218
207
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
208
|
+
pushCurrent();
|
|
209
|
+
return gemini;
|
|
210
|
+
}
|
|
211
|
+
/* ────────────────────────────────── */
|
|
212
|
+
/* HTTP call */
|
|
213
|
+
/* ────────────────────────────────── */
|
|
214
|
+
async callGemini(messages, model, stream = false) {
|
|
215
|
+
const hasVision = messages.some((m) => Array.isArray(m.content) &&
|
|
216
|
+
m.content.some((b) => b?.type === 'image_url' || b?.inlineData));
|
|
217
|
+
const contents = hasVision
|
|
218
|
+
? await this.convertVisionMessagesToGeminiFormat(messages)
|
|
219
|
+
: this.convertMessagesToGeminiFormat(messages);
|
|
220
|
+
const body = {
|
|
221
|
+
contents,
|
|
222
|
+
generationConfig: {
|
|
223
|
+
maxOutputTokens: 1000,
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
if (this.tools.length) {
|
|
227
|
+
body.tools = [
|
|
228
|
+
{
|
|
229
|
+
functionDeclarations: this.tools.map((t) => ({
|
|
230
|
+
name: t.name,
|
|
231
|
+
description: t.description,
|
|
232
|
+
parameters: t.parameters,
|
|
233
|
+
})),
|
|
234
|
+
},
|
|
235
|
+
];
|
|
236
|
+
body.toolConfig = { functionCallingConfig: { mode: 'AUTO' } };
|
|
225
237
|
}
|
|
226
|
-
|
|
238
|
+
const fetchOnce = async (ver, payload) => {
|
|
239
|
+
const fn = stream ? 'streamGenerateContent' : 'generateContent';
|
|
240
|
+
const alt = stream ? '?alt=sse' : '';
|
|
241
|
+
const url = `${ENDPOINT_GEMINI_API}/${ver}/models/${model}:${fn}${alt}${alt ? '&' : '?'}key=${this.apiKey}`;
|
|
242
|
+
return fetch(url, {
|
|
243
|
+
method: 'POST',
|
|
244
|
+
headers: { 'Content-Type': 'application/json' },
|
|
245
|
+
body: JSON.stringify(payload),
|
|
246
|
+
});
|
|
247
|
+
};
|
|
248
|
+
const isLite = /flash[-_]lite/.test(model);
|
|
249
|
+
const firstVer = isLite ? 'v1beta' : 'v1';
|
|
250
|
+
const tryApi = async () => {
|
|
251
|
+
try {
|
|
252
|
+
const payload = firstVer === 'v1' ? body : this.adaptKeysForApi(body); // snake_case conversion
|
|
253
|
+
return await fetchOnce(firstVer, payload);
|
|
254
|
+
}
|
|
255
|
+
catch (e) {
|
|
256
|
+
// Only retry v1beta if camel/snake case mismatch error occurs in non-Lite models
|
|
257
|
+
if (!isLite && /Unknown name|Cannot find field|404/.test(e.message)) {
|
|
258
|
+
return await fetchOnce('v1beta', this.adaptKeysForApi(body));
|
|
259
|
+
}
|
|
260
|
+
throw e; // otherwise, throw to upper layer
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
const res = await tryApi();
|
|
264
|
+
if (!res.ok)
|
|
265
|
+
throw new Error(`Gemini HTTP ${res.status}`);
|
|
266
|
+
return res;
|
|
227
267
|
}
|
|
228
268
|
/**
|
|
229
269
|
* Convert AITuber OnAir vision messages to Gemini format
|
|
@@ -237,6 +277,43 @@ export class GeminiChatService {
|
|
|
237
277
|
for (const msg of messages) {
|
|
238
278
|
// Map AITuber OnAir roles to Gemini roles
|
|
239
279
|
const role = this.mapRoleToGemini(msg.role);
|
|
280
|
+
/* ----------- OpenAI compatible tool metadata ----------- */
|
|
281
|
+
// assistant: { tool_calls:[{id,name,function:{arguments}}] }
|
|
282
|
+
if (msg.tool_calls) {
|
|
283
|
+
for (const call of msg.tool_calls) {
|
|
284
|
+
// Gemini does not need id. Insert functionCall into parts
|
|
285
|
+
geminiMessages.push({
|
|
286
|
+
role: 'model',
|
|
287
|
+
parts: [
|
|
288
|
+
{
|
|
289
|
+
functionCall: {
|
|
290
|
+
name: call.function.name,
|
|
291
|
+
args: JSON.parse(call.function.arguments || '{}'),
|
|
292
|
+
},
|
|
293
|
+
},
|
|
294
|
+
],
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
// tool role → user role + functionResponse
|
|
300
|
+
if (msg.role === 'tool') {
|
|
301
|
+
const funcName = msg.name ??
|
|
302
|
+
this.callIdMap.get(msg.tool_call_id) ??
|
|
303
|
+
'result';
|
|
304
|
+
geminiMessages.push({
|
|
305
|
+
role: 'user',
|
|
306
|
+
parts: [
|
|
307
|
+
{
|
|
308
|
+
functionResponse: {
|
|
309
|
+
name: funcName,
|
|
310
|
+
response: this.normalizeToolResult(this.safeJsonParse(msg.content)),
|
|
311
|
+
},
|
|
312
|
+
},
|
|
313
|
+
],
|
|
314
|
+
});
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
240
317
|
// If role changes, start a new message
|
|
241
318
|
if (role !== currentRole && currentParts.length > 0) {
|
|
242
319
|
geminiMessages.push({
|
|
@@ -321,5 +398,147 @@ export class GeminiChatService {
|
|
|
321
398
|
return 'user';
|
|
322
399
|
}
|
|
323
400
|
}
|
|
401
|
+
/* ────────────────────────────────────────────────────────── */
|
|
402
|
+
/* Convert NDJSON stream to common format */
|
|
403
|
+
/* ────────────────────────────────────────────────────────── */
|
|
404
|
+
async parseStream(res, onPartial) {
|
|
405
|
+
const reader = res.body.getReader();
|
|
406
|
+
const dec = new TextDecoder();
|
|
407
|
+
const textBlocks = [];
|
|
408
|
+
const toolBlocks = [];
|
|
409
|
+
let buf = '';
|
|
410
|
+
const flush = (payload) => {
|
|
411
|
+
if (!payload || payload === '[DONE]')
|
|
412
|
+
return;
|
|
413
|
+
let obj;
|
|
414
|
+
try {
|
|
415
|
+
obj = JSON.parse(payload);
|
|
416
|
+
}
|
|
417
|
+
catch {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
for (const cand of obj.candidates ?? []) {
|
|
421
|
+
for (const part of cand.content?.parts ?? []) {
|
|
422
|
+
if (part.text) {
|
|
423
|
+
onPartial(part.text);
|
|
424
|
+
textBlocks.push({ type: 'text', text: part.text });
|
|
425
|
+
}
|
|
426
|
+
if (part.functionCall) {
|
|
427
|
+
toolBlocks.push({
|
|
428
|
+
type: 'tool_use',
|
|
429
|
+
id: this.genUUID(),
|
|
430
|
+
name: part.functionCall.name,
|
|
431
|
+
input: part.functionCall.args ?? {},
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
if (part.functionResponse) {
|
|
435
|
+
toolBlocks.push({
|
|
436
|
+
type: 'tool_result',
|
|
437
|
+
tool_use_id: part.functionResponse.name,
|
|
438
|
+
content: JSON.stringify(part.functionResponse.response),
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
};
|
|
444
|
+
while (true) {
|
|
445
|
+
const { done, value } = await reader.read();
|
|
446
|
+
if (done)
|
|
447
|
+
break;
|
|
448
|
+
buf += dec.decode(value, { stream: true });
|
|
449
|
+
let nl;
|
|
450
|
+
while ((nl = buf.indexOf('\n')) !== -1) {
|
|
451
|
+
let line = buf.slice(0, nl);
|
|
452
|
+
buf = buf.slice(nl + 1);
|
|
453
|
+
if (line.endsWith('\r'))
|
|
454
|
+
line = line.slice(0, -1); // CRLF support
|
|
455
|
+
if (!line.trim()) {
|
|
456
|
+
flush('');
|
|
457
|
+
continue;
|
|
458
|
+
} // keep-alive empty line
|
|
459
|
+
if (line.startsWith('data:'))
|
|
460
|
+
line = line.slice(5).trim();
|
|
461
|
+
if (!line)
|
|
462
|
+
continue;
|
|
463
|
+
flush(line);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
if (buf)
|
|
467
|
+
flush(buf);
|
|
468
|
+
const blocks = [...textBlocks, ...toolBlocks];
|
|
469
|
+
return {
|
|
470
|
+
blocks,
|
|
471
|
+
stop_reason: toolBlocks.some((b) => b.type === 'tool_use')
|
|
472
|
+
? 'tool_use'
|
|
473
|
+
: 'end',
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
/* ────────────────────────────────────────────────────────── */
|
|
477
|
+
/* Convert JSON of non-stream (= generateContent) */
|
|
478
|
+
/* ────────────────────────────────────────────────────────── */
|
|
479
|
+
parseOneShot(data) {
|
|
480
|
+
const textBlocks = [];
|
|
481
|
+
const toolBlocks = [];
|
|
482
|
+
for (const cand of data.candidates ?? []) {
|
|
483
|
+
for (const part of cand.content?.parts ?? []) {
|
|
484
|
+
if (part.text) {
|
|
485
|
+
textBlocks.push({ type: 'text', text: part.text });
|
|
486
|
+
}
|
|
487
|
+
if (part.functionCall) {
|
|
488
|
+
toolBlocks.push({
|
|
489
|
+
type: 'tool_use',
|
|
490
|
+
id: this.genUUID(),
|
|
491
|
+
name: part.functionCall.name,
|
|
492
|
+
input: part.functionCall.args ?? {},
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
if (part.functionResponse) {
|
|
496
|
+
toolBlocks.push({
|
|
497
|
+
type: 'tool_result',
|
|
498
|
+
tool_use_id: part.functionResponse.name,
|
|
499
|
+
content: JSON.stringify(part.functionResponse.response),
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
const blocks = [...textBlocks, ...toolBlocks];
|
|
505
|
+
return {
|
|
506
|
+
blocks,
|
|
507
|
+
stop_reason: toolBlocks.some((b) => b.type === 'tool_use')
|
|
508
|
+
? 'tool_use'
|
|
509
|
+
: 'end',
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
/* ────────────────────────────────────────────────────────── */
|
|
513
|
+
/* chatOnce (text) */
|
|
514
|
+
/* ────────────────────────────────────────────────────────── */
|
|
515
|
+
async chatOnce(messages, stream = true, onPartialResponse = () => { }) {
|
|
516
|
+
const res = await this.callGemini(messages, this.model, stream);
|
|
517
|
+
return stream
|
|
518
|
+
? this.parseStream(res, onPartialResponse)
|
|
519
|
+
: this.parseOneShot(await res.json());
|
|
520
|
+
}
|
|
521
|
+
/* ────────────────────────────────────────────────────────── */
|
|
522
|
+
/* visionChatOnce (images) */
|
|
523
|
+
/* ────────────────────────────────────────────────────────── */
|
|
524
|
+
async visionChatOnce(messages, stream = false) {
|
|
525
|
+
const res = await this.callGemini(messages, this.visionModel, stream);
|
|
526
|
+
return stream
|
|
527
|
+
? this.parseStream(res,
|
|
528
|
+
/* vision is usually stream=false, but just in case */ () => { })
|
|
529
|
+
: this.parseOneShot(await res.json());
|
|
530
|
+
}
|
|
531
|
+
/* ────────────────────────────────────────────────────────── */
|
|
532
|
+
/* UUID helper */
|
|
533
|
+
/* ────────────────────────────────────────────────────────── */
|
|
534
|
+
genUUID() {
|
|
535
|
+
return typeof crypto !== 'undefined' && crypto.randomUUID
|
|
536
|
+
? crypto.randomUUID()
|
|
537
|
+
: 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
|
538
|
+
const r = (Math.random() * 16) | 0;
|
|
539
|
+
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
|
540
|
+
return v.toString(16);
|
|
541
|
+
});
|
|
542
|
+
}
|
|
324
543
|
}
|
|
325
544
|
//# sourceMappingURL=GeminiChatService.js.map
|