@celljs/ta-voice 0.0.0-stage → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +8 -0
- package/README.md +7 -2
- package/host/index.mjs +662 -0
- package/package.json +65 -4
package/README.en.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# @celljs/ta-voice
|
|
2
|
+
|
|
3
|
+
Real-time voice services for the calls of Ta: OpenAI Realtime, Qwen-Omni Realtime of Alibaba Cloud Model Studio, Hy Realtime of Tencent Hunyuan, or the real-time voice of Doubao on Volcengine.
|
|
4
|
+
Ta hears you as you speak, you can cut in while it talks, and it reads in a natural voice. The answers are still thought by the model of the conversation, with its tools and memory.
|
|
5
|
+
|
|
6
|
+
Add and choose one in Settings → Voice → Voice of calls. The keys stay on this computer, in `~/.ta/agent/voices.json` (0600): no page or phone gets them.
|
|
7
|
+
|
|
8
|
+
Another service is a package whose `pi.host` calls `host.voices.registerType(type, definition)`; see [docs/design/ta-voice.md](../../docs/design/ta-voice.md).
|
package/README.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @celljs/ta-voice
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Ta 的实时语音服务:给通话接上 OpenAI Realtime、阿里云百炼 Qwen-Omni Realtime、腾讯混元 Hy Realtime 或火山引擎豆包实时语音。
|
|
4
|
+
Ta 边听边断句,它说话时可以随时插话,声音也更自然。回答仍由对话原来的模型来想,工具和记忆照常可用。
|
|
5
|
+
|
|
6
|
+
在 设置 → 语音 →「通话用的声音」里添加服务、选用。Key 只存在这台电脑的 `~/.ta/agent/voices.json`(0600),页面和手机拿不到。
|
|
7
|
+
|
|
8
|
+
别的服务写一个包,在 `pi.host` 里调用 `host.voices.registerType(type, definition)` 即可,见 [docs/design/ta-voice.md](../../docs/design/ta-voice.md)。
|
package/host/index.mjs
ADDED
|
@@ -0,0 +1,662 @@
|
|
|
1
|
+
// src/doubao-realtime.ts
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
// src/doubao-frames.ts
|
|
5
|
+
var DoubaoFrames;
|
|
6
|
+
((DoubaoFrames) => {
|
|
7
|
+
DoubaoFrames.FULL_CLIENT = 1;
|
|
8
|
+
DoubaoFrames.AUDIO_CLIENT = 2;
|
|
9
|
+
DoubaoFrames.FULL_SERVER = 9;
|
|
10
|
+
DoubaoFrames.AUDIO_SERVER = 11;
|
|
11
|
+
DoubaoFrames.ERROR = 15;
|
|
12
|
+
const WITH_EVENT = 4;
|
|
13
|
+
const JSON_SERIAL = 1;
|
|
14
|
+
const RAW_SERIAL = 0;
|
|
15
|
+
DoubaoFrames.START_CONNECTION = 1;
|
|
16
|
+
DoubaoFrames.FINISH_CONNECTION = 2;
|
|
17
|
+
DoubaoFrames.CONNECTION_STARTED = 50;
|
|
18
|
+
DoubaoFrames.CONNECTION_FAILED = 51;
|
|
19
|
+
DoubaoFrames.START_SESSION = 100;
|
|
20
|
+
DoubaoFrames.FINISH_SESSION = 102;
|
|
21
|
+
DoubaoFrames.SESSION_STARTED = 150;
|
|
22
|
+
DoubaoFrames.SESSION_FINISHED = 152;
|
|
23
|
+
DoubaoFrames.SESSION_FAILED = 153;
|
|
24
|
+
DoubaoFrames.TASK_REQUEST = 200;
|
|
25
|
+
DoubaoFrames.TTS_SENTENCE_START = 350;
|
|
26
|
+
DoubaoFrames.TTS_RESPONSE = 352;
|
|
27
|
+
DoubaoFrames.TTS_ENDED = 359;
|
|
28
|
+
DoubaoFrames.ASR_INFO = 450;
|
|
29
|
+
DoubaoFrames.ASR_RESPONSE = 451;
|
|
30
|
+
DoubaoFrames.ASR_ENDED = 459;
|
|
31
|
+
DoubaoFrames.CHAT_TTS_TEXT = 500;
|
|
32
|
+
DoubaoFrames.DIALOG_ERROR = 599;
|
|
33
|
+
function encode(event, payload, session) {
|
|
34
|
+
const audio = payload instanceof Uint8Array;
|
|
35
|
+
const body = audio ? payload : new TextEncoder().encode(JSON.stringify(payload));
|
|
36
|
+
const id = session === undefined ? undefined : new TextEncoder().encode(session);
|
|
37
|
+
const out = new Uint8Array(4 + 4 + (id ? 4 + id.length : 0) + 4 + body.length);
|
|
38
|
+
const view = new DataView(out.buffer);
|
|
39
|
+
out[0] = 17;
|
|
40
|
+
out[1] = (audio ? DoubaoFrames.AUDIO_CLIENT : DoubaoFrames.FULL_CLIENT) << 4 | WITH_EVENT;
|
|
41
|
+
out[2] = (audio ? RAW_SERIAL : JSON_SERIAL) << 4 | 0;
|
|
42
|
+
out[3] = 0;
|
|
43
|
+
let at = 4;
|
|
44
|
+
view.setInt32(at, event);
|
|
45
|
+
at += 4;
|
|
46
|
+
if (id) {
|
|
47
|
+
view.setUint32(at, id.length);
|
|
48
|
+
out.set(id, at + 4);
|
|
49
|
+
at += 4 + id.length;
|
|
50
|
+
}
|
|
51
|
+
view.setUint32(at, body.length);
|
|
52
|
+
out.set(body, at + 4);
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
DoubaoFrames.encode = encode;
|
|
56
|
+
function decode(data) {
|
|
57
|
+
const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
|
|
58
|
+
if (bytes.length < 8) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
62
|
+
const headerSize = (bytes[0] & 15) * 4;
|
|
63
|
+
const type = bytes[1] >> 4;
|
|
64
|
+
const flags = bytes[1] & 15;
|
|
65
|
+
let at = headerSize;
|
|
66
|
+
const read = () => {
|
|
67
|
+
if (at + 4 > bytes.length) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const value = view.getInt32(at);
|
|
71
|
+
at += 4;
|
|
72
|
+
return value;
|
|
73
|
+
};
|
|
74
|
+
try {
|
|
75
|
+
if (type === DoubaoFrames.ERROR) {
|
|
76
|
+
const code = read();
|
|
77
|
+
const size2 = read() ?? 0;
|
|
78
|
+
return { type, code, payload: bytes.slice(at, at + size2) };
|
|
79
|
+
}
|
|
80
|
+
if (flags & 3) {
|
|
81
|
+
read();
|
|
82
|
+
}
|
|
83
|
+
const event = flags & WITH_EVENT ? read() : undefined;
|
|
84
|
+
let id;
|
|
85
|
+
if (event !== undefined) {
|
|
86
|
+
const size2 = read() ?? 0;
|
|
87
|
+
id = new TextDecoder().decode(bytes.slice(at, at + size2));
|
|
88
|
+
at += size2;
|
|
89
|
+
}
|
|
90
|
+
const size = read() ?? 0;
|
|
91
|
+
return { type, event, ...id ? { id } : {}, payload: bytes.slice(at, at + size) };
|
|
92
|
+
} catch {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
DoubaoFrames.decode = decode;
|
|
97
|
+
function json(frame) {
|
|
98
|
+
try {
|
|
99
|
+
const value = JSON.parse(new TextDecoder().decode(frame.payload));
|
|
100
|
+
return value && typeof value === "object" ? value : {};
|
|
101
|
+
} catch {
|
|
102
|
+
return {};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
DoubaoFrames.json = json;
|
|
106
|
+
})(DoubaoFrames ||= {});
|
|
107
|
+
|
|
108
|
+
// src/realtime-socket.ts
|
|
109
|
+
var RealtimeSocket;
|
|
110
|
+
((RealtimeSocket) => {
|
|
111
|
+
RealtimeSocket.OPEN_MS = 15000;
|
|
112
|
+
function open(url, headers) {
|
|
113
|
+
const Socket = WebSocket;
|
|
114
|
+
const socket = new Socket(url, { headers });
|
|
115
|
+
socket.binaryType = "arraybuffer";
|
|
116
|
+
return socket;
|
|
117
|
+
}
|
|
118
|
+
RealtimeSocket.open = open;
|
|
119
|
+
function until(socket, wire) {
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
let done = false;
|
|
122
|
+
const finish = (why) => {
|
|
123
|
+
if (done) {
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
done = true;
|
|
127
|
+
clearTimeout(timer);
|
|
128
|
+
if (why === undefined) {
|
|
129
|
+
resolve();
|
|
130
|
+
} else {
|
|
131
|
+
try {
|
|
132
|
+
socket.close();
|
|
133
|
+
} catch {}
|
|
134
|
+
reject(new Error(why));
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
const timer = setTimeout(() => finish("The voice service did not answer in time."), RealtimeSocket.OPEN_MS);
|
|
138
|
+
socket.addEventListener("close", (event) => finish(closeReason(event) || "The voice service closed the connection: check the address and the key."));
|
|
139
|
+
socket.addEventListener("error", () => finish("The voice service could not be reached: check the address and the key."));
|
|
140
|
+
wire(() => finish(), (why) => finish(why));
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
RealtimeSocket.until = until;
|
|
144
|
+
function closeReason(event) {
|
|
145
|
+
return event.reason ? `${event.reason} (${event.code})` : event.code && event.code !== 1000 && event.code !== 1005 ? `closed with ${event.code}` : undefined;
|
|
146
|
+
}
|
|
147
|
+
RealtimeSocket.closeReason = closeReason;
|
|
148
|
+
function text(value) {
|
|
149
|
+
return typeof value === "string" ? value.trim() : "";
|
|
150
|
+
}
|
|
151
|
+
RealtimeSocket.text = text;
|
|
152
|
+
function base64(bytes2) {
|
|
153
|
+
return Buffer.from(bytes2).toString("base64");
|
|
154
|
+
}
|
|
155
|
+
RealtimeSocket.base64 = base64;
|
|
156
|
+
function bytes(base642) {
|
|
157
|
+
return new Uint8Array(Buffer.from(base642, "base64"));
|
|
158
|
+
}
|
|
159
|
+
RealtimeSocket.bytes = bytes;
|
|
160
|
+
function reading(text2) {
|
|
161
|
+
return `Read the following text aloud exactly as written, word for word, in its own language. Add nothing, leave nothing out, answer nothing:
|
|
162
|
+
|
|
163
|
+
${text2}`;
|
|
164
|
+
}
|
|
165
|
+
RealtimeSocket.reading = reading;
|
|
166
|
+
function languageCode(language) {
|
|
167
|
+
return (language || "zh").split(/[-_]/)[0].toLowerCase();
|
|
168
|
+
}
|
|
169
|
+
RealtimeSocket.languageCode = languageCode;
|
|
170
|
+
})(RealtimeSocket ||= {});
|
|
171
|
+
|
|
172
|
+
// src/doubao-realtime.ts
|
|
173
|
+
var DOUBAO_URL = "wss://openspeech.bytedance.com/api/v3/realtime/dialogue";
|
|
174
|
+
var APP_KEY = "PlgvMymc7f3tQnJ6";
|
|
175
|
+
|
|
176
|
+
class DoubaoRealtimeEngine {
|
|
177
|
+
socket;
|
|
178
|
+
events;
|
|
179
|
+
inputRate = 16000;
|
|
180
|
+
outputRate = 24000;
|
|
181
|
+
session = randomUUID();
|
|
182
|
+
asked = [];
|
|
183
|
+
reading;
|
|
184
|
+
hushedUpTo = 0;
|
|
185
|
+
lastSay = 0;
|
|
186
|
+
heard = "";
|
|
187
|
+
closing = false;
|
|
188
|
+
constructor(socket, events) {
|
|
189
|
+
this.socket = socket;
|
|
190
|
+
this.events = events;
|
|
191
|
+
}
|
|
192
|
+
static async open(options, language, events) {
|
|
193
|
+
const socket = RealtimeSocket.open(RealtimeSocket.text(options.url) || DOUBAO_URL, {
|
|
194
|
+
"X-Api-App-ID": RealtimeSocket.text(options.appId),
|
|
195
|
+
"X-Api-Access-Key": RealtimeSocket.text(options.accessKey),
|
|
196
|
+
"X-Api-Resource-Id": "volc.speech.dialog",
|
|
197
|
+
"X-Api-App-Key": APP_KEY,
|
|
198
|
+
"X-Api-Connect-Id": randomUUID()
|
|
199
|
+
});
|
|
200
|
+
const engine = new DoubaoRealtimeEngine(socket, events);
|
|
201
|
+
await RealtimeSocket.until(socket, (ready, fail) => {
|
|
202
|
+
socket.addEventListener("open", () => engine.frame(DoubaoFrames.START_CONNECTION, {}));
|
|
203
|
+
socket.addEventListener("message", (event) => {
|
|
204
|
+
const frame = DoubaoFrames.decode(event.data);
|
|
205
|
+
if (!frame) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (frame.event === DoubaoFrames.CONNECTION_STARTED) {
|
|
209
|
+
engine.frame(DoubaoFrames.START_SESSION, DoubaoRealtimeEngine.start(options, language), engine.session);
|
|
210
|
+
} else if (frame.event === DoubaoFrames.SESSION_STARTED) {
|
|
211
|
+
ready();
|
|
212
|
+
} else if (frame.type === DoubaoFrames.ERROR || frame.event === DoubaoFrames.CONNECTION_FAILED || frame.event === DoubaoFrames.SESSION_FAILED) {
|
|
213
|
+
fail(DoubaoRealtimeEngine.why(frame));
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
engine.listen();
|
|
218
|
+
return engine;
|
|
219
|
+
}
|
|
220
|
+
static start(options, language) {
|
|
221
|
+
return {
|
|
222
|
+
asr: { audio_info: { format: "pcm", sample_rate: 16000, channel: 1 }, extra: { end_smooth_window_ms: 800 } },
|
|
223
|
+
tts: {
|
|
224
|
+
speaker: RealtimeSocket.text(options.speaker) || "zh_female_vv_jupiter_bigtts",
|
|
225
|
+
audio_config: { channel: 1, format: "pcm_s16le", sample_rate: 24000 }
|
|
226
|
+
},
|
|
227
|
+
dialog: {
|
|
228
|
+
bot_name: "Ta",
|
|
229
|
+
system_role: "You only read aloud the text you are given.",
|
|
230
|
+
extra: { model: RealtimeSocket.text(options.model) || "1.2.1.1", language: RealtimeSocket.languageCode(language) }
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
send(pcm) {
|
|
235
|
+
this.frame(DoubaoFrames.TASK_REQUEST, pcm, this.session);
|
|
236
|
+
}
|
|
237
|
+
say(text, say) {
|
|
238
|
+
this.lastSay = Math.max(this.lastSay, say);
|
|
239
|
+
this.asked.push(say);
|
|
240
|
+
this.frame(DoubaoFrames.CHAT_TTS_TEXT, { start: true, content: text, end: false }, this.session);
|
|
241
|
+
this.frame(DoubaoFrames.CHAT_TTS_TEXT, { start: false, content: "", end: true }, this.session);
|
|
242
|
+
}
|
|
243
|
+
hush() {
|
|
244
|
+
this.hushedUpTo = this.lastSay;
|
|
245
|
+
}
|
|
246
|
+
close() {
|
|
247
|
+
this.closing = true;
|
|
248
|
+
this.frame(DoubaoFrames.FINISH_SESSION, {}, this.session);
|
|
249
|
+
this.frame(DoubaoFrames.FINISH_CONNECTION, {});
|
|
250
|
+
try {
|
|
251
|
+
this.socket.close();
|
|
252
|
+
} catch {}
|
|
253
|
+
}
|
|
254
|
+
listen() {
|
|
255
|
+
this.socket.addEventListener("message", (event) => {
|
|
256
|
+
const frame = DoubaoFrames.decode(event.data);
|
|
257
|
+
if (frame) {
|
|
258
|
+
this.take(frame);
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
this.socket.addEventListener("close", (event) => {
|
|
262
|
+
if (!this.closing) {
|
|
263
|
+
this.events.closed(RealtimeSocket.closeReason(event) ?? "The voice service closed the connection.");
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
take(frame) {
|
|
268
|
+
switch (frame.event) {
|
|
269
|
+
case DoubaoFrames.ASR_INFO:
|
|
270
|
+
this.heard = "";
|
|
271
|
+
this.events.speech();
|
|
272
|
+
return;
|
|
273
|
+
case DoubaoFrames.ASR_RESPONSE: {
|
|
274
|
+
const results = DoubaoFrames.json(frame).results;
|
|
275
|
+
const result = results?.[0];
|
|
276
|
+
if (typeof result?.text === "string") {
|
|
277
|
+
this.heard = result.text;
|
|
278
|
+
this.events.text(this.heard, false);
|
|
279
|
+
}
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
case DoubaoFrames.ASR_ENDED:
|
|
283
|
+
if (this.heard) {
|
|
284
|
+
this.events.text(this.heard, true);
|
|
285
|
+
this.heard = "";
|
|
286
|
+
}
|
|
287
|
+
return;
|
|
288
|
+
case DoubaoFrames.TTS_SENTENCE_START:
|
|
289
|
+
this.reading = DoubaoFrames.json(frame).tts_type === "chat_tts_text" ? this.asked[0] : undefined;
|
|
290
|
+
return;
|
|
291
|
+
case DoubaoFrames.TTS_RESPONSE:
|
|
292
|
+
if (this.reading !== undefined && this.reading > this.hushedUpTo) {
|
|
293
|
+
this.events.audio(frame.payload, this.reading);
|
|
294
|
+
}
|
|
295
|
+
return;
|
|
296
|
+
case DoubaoFrames.TTS_ENDED: {
|
|
297
|
+
const done = this.asked.shift();
|
|
298
|
+
this.reading = undefined;
|
|
299
|
+
if (done !== undefined) {
|
|
300
|
+
this.events.said(done);
|
|
301
|
+
}
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
case DoubaoFrames.DIALOG_ERROR:
|
|
305
|
+
case DoubaoFrames.SESSION_FAILED:
|
|
306
|
+
this.events.closed(DoubaoRealtimeEngine.why(frame));
|
|
307
|
+
this.close();
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (frame.type === DoubaoFrames.ERROR) {
|
|
311
|
+
this.events.closed(DoubaoRealtimeEngine.why(frame));
|
|
312
|
+
this.close();
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
frame(event, payload, session) {
|
|
316
|
+
if (this.socket.readyState === 1) {
|
|
317
|
+
this.socket.send(DoubaoFrames.encode(event, payload, session));
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
static why(frame) {
|
|
321
|
+
const body = DoubaoFrames.json(frame);
|
|
322
|
+
const said = body.message ?? body.error ?? body.status_code;
|
|
323
|
+
return said ? `The voice service refused: ${said}` : `The voice service refused (${frame.code ?? frame.event}).`;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// src/openai-realtime.ts
|
|
328
|
+
var INSTRUCTIONS = "You are only a voice. Read aloud exactly the text you are given, and nothing else. Never answer on your own.";
|
|
329
|
+
var OPENAI_DIALECT = {
|
|
330
|
+
inputRate: 24000,
|
|
331
|
+
outputRate: 24000,
|
|
332
|
+
tagged: true,
|
|
333
|
+
session: (options, language) => ({
|
|
334
|
+
type: "realtime",
|
|
335
|
+
instructions: INSTRUCTIONS,
|
|
336
|
+
output_modalities: ["audio"],
|
|
337
|
+
audio: {
|
|
338
|
+
input: {
|
|
339
|
+
format: { type: "audio/pcm", rate: 24000 },
|
|
340
|
+
noise_reduction: { type: "near_field" },
|
|
341
|
+
transcription: { model: RealtimeSocket.text(options.transcribe) || "gpt-4o-mini-transcribe", language: RealtimeSocket.languageCode(language) },
|
|
342
|
+
turn_detection: { type: "semantic_vad", eagerness: "auto", create_response: false, interrupt_response: false }
|
|
343
|
+
},
|
|
344
|
+
output: { format: { type: "audio/pcm", rate: 24000 }, voice: RealtimeSocket.text(options.voice) || "marin" }
|
|
345
|
+
}
|
|
346
|
+
}),
|
|
347
|
+
read: (text, say) => [
|
|
348
|
+
{
|
|
349
|
+
type: "response.create",
|
|
350
|
+
response: {
|
|
351
|
+
conversation: "none",
|
|
352
|
+
output_modalities: ["audio"],
|
|
353
|
+
instructions: RealtimeSocket.reading(text),
|
|
354
|
+
input: [],
|
|
355
|
+
metadata: { ta_say: String(say) }
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
]
|
|
359
|
+
};
|
|
360
|
+
var QWEN_DIALECT = {
|
|
361
|
+
inputRate: 16000,
|
|
362
|
+
outputRate: 24000,
|
|
363
|
+
tagged: false,
|
|
364
|
+
session: (options) => ({
|
|
365
|
+
modalities: ["text", "audio"],
|
|
366
|
+
voice: RealtimeSocket.text(options.voice) || "Cherry",
|
|
367
|
+
instructions: INSTRUCTIONS,
|
|
368
|
+
input_audio_format: "pcm16",
|
|
369
|
+
output_audio_format: "pcm24",
|
|
370
|
+
input_audio_transcription: { model: RealtimeSocket.text(options.transcribe) || "gummy-realtime-v1" },
|
|
371
|
+
turn_detection: {
|
|
372
|
+
type: "server_vad",
|
|
373
|
+
threshold: 0.5,
|
|
374
|
+
prefix_padding_ms: 300,
|
|
375
|
+
silence_duration_ms: 800,
|
|
376
|
+
create_response: false,
|
|
377
|
+
interrupt_response: false
|
|
378
|
+
}
|
|
379
|
+
}),
|
|
380
|
+
read: (text) => [
|
|
381
|
+
{ type: "conversation.item.create", item: { type: "message", role: "user", content: [{ type: "input_text", text: RealtimeSocket.reading(text) }] } },
|
|
382
|
+
{ type: "response.create" }
|
|
383
|
+
]
|
|
384
|
+
};
|
|
385
|
+
var HY_DIALECT = {
|
|
386
|
+
inputRate: 16000,
|
|
387
|
+
outputRate: 24000,
|
|
388
|
+
tagged: true,
|
|
389
|
+
session: (options) => ({
|
|
390
|
+
instructions: INSTRUCTIONS,
|
|
391
|
+
output_modalities: ["audio"],
|
|
392
|
+
input_audio_channels: 1,
|
|
393
|
+
tools: [],
|
|
394
|
+
audio: {
|
|
395
|
+
input: {
|
|
396
|
+
format: { type: "audio/pcm", rate: 16000 },
|
|
397
|
+
transcription: { model: "server_asr" },
|
|
398
|
+
turn_detection: { type: "server_vad", create_response: false }
|
|
399
|
+
},
|
|
400
|
+
output: { format: { type: "audio/pcm", rate: 24000 }, voice: RealtimeSocket.text(options.voice) || "f1" }
|
|
401
|
+
}
|
|
402
|
+
}),
|
|
403
|
+
read: (text, say) => [
|
|
404
|
+
{
|
|
405
|
+
type: "response.create",
|
|
406
|
+
response: { input_scope: "independent", input_text: text, output_modalities: ["audio"], metadata: { ta_say: String(say) } }
|
|
407
|
+
}
|
|
408
|
+
]
|
|
409
|
+
};
|
|
410
|
+
|
|
411
|
+
class OpenAiRealtimeEngine {
|
|
412
|
+
socket;
|
|
413
|
+
dialect;
|
|
414
|
+
events;
|
|
415
|
+
inputRate;
|
|
416
|
+
outputRate;
|
|
417
|
+
asked = [];
|
|
418
|
+
responses = new Map;
|
|
419
|
+
hushedUpTo = 0;
|
|
420
|
+
lastSay = 0;
|
|
421
|
+
heard = "";
|
|
422
|
+
closing = false;
|
|
423
|
+
constructor(socket, dialect, events) {
|
|
424
|
+
this.socket = socket;
|
|
425
|
+
this.dialect = dialect;
|
|
426
|
+
this.events = events;
|
|
427
|
+
this.inputRate = dialect.inputRate;
|
|
428
|
+
this.outputRate = dialect.outputRate;
|
|
429
|
+
}
|
|
430
|
+
static async open(url, headers, dialect, options, language, events) {
|
|
431
|
+
const socket = RealtimeSocket.open(url, headers);
|
|
432
|
+
const engine = new OpenAiRealtimeEngine(socket, dialect, events);
|
|
433
|
+
await RealtimeSocket.until(socket, (ready, fail) => {
|
|
434
|
+
socket.addEventListener("message", (event) => {
|
|
435
|
+
const message = OpenAiRealtimeEngine.parse(event.data);
|
|
436
|
+
if (message?.type === "session.created") {
|
|
437
|
+
engine.message({ type: "session.update", session: dialect.session(options, language) });
|
|
438
|
+
} else if (message?.type === "session.updated") {
|
|
439
|
+
ready();
|
|
440
|
+
} else if (message?.type === "error") {
|
|
441
|
+
fail(message.error?.message || "The voice service refused the session.");
|
|
442
|
+
}
|
|
443
|
+
});
|
|
444
|
+
});
|
|
445
|
+
engine.listen();
|
|
446
|
+
return engine;
|
|
447
|
+
}
|
|
448
|
+
send(pcm) {
|
|
449
|
+
this.message({ type: "input_audio_buffer.append", audio: RealtimeSocket.base64(pcm) });
|
|
450
|
+
}
|
|
451
|
+
say(text, say) {
|
|
452
|
+
this.lastSay = Math.max(this.lastSay, say);
|
|
453
|
+
this.asked.push(say);
|
|
454
|
+
for (const message of this.dialect.read(text, say)) {
|
|
455
|
+
this.message(message);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
hush() {
|
|
459
|
+
this.hushedUpTo = this.lastSay;
|
|
460
|
+
for (const [id, say] of this.responses) {
|
|
461
|
+
if (say <= this.hushedUpTo) {
|
|
462
|
+
this.message({ type: "response.cancel", response_id: id });
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
close() {
|
|
467
|
+
this.closing = true;
|
|
468
|
+
try {
|
|
469
|
+
this.socket.close();
|
|
470
|
+
} catch {}
|
|
471
|
+
}
|
|
472
|
+
listen() {
|
|
473
|
+
this.socket.addEventListener("message", (event) => {
|
|
474
|
+
const message = OpenAiRealtimeEngine.parse(event.data);
|
|
475
|
+
if (message) {
|
|
476
|
+
this.take(message);
|
|
477
|
+
}
|
|
478
|
+
});
|
|
479
|
+
this.socket.addEventListener("close", (event) => {
|
|
480
|
+
if (!this.closing) {
|
|
481
|
+
this.events.closed(RealtimeSocket.closeReason(event) ?? "The voice service closed the connection.");
|
|
482
|
+
}
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
take(message) {
|
|
486
|
+
switch (message.type) {
|
|
487
|
+
case "input_audio_buffer.speech_started":
|
|
488
|
+
this.heard = "";
|
|
489
|
+
this.events.speech();
|
|
490
|
+
return;
|
|
491
|
+
case "conversation.item.input_audio_transcription.delta":
|
|
492
|
+
this.heard = typeof message.text === "string" ? message.text + (message.stash ?? "") : this.heard + (message.delta ?? "");
|
|
493
|
+
this.events.text(this.heard, false);
|
|
494
|
+
return;
|
|
495
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
496
|
+
this.heard = "";
|
|
497
|
+
this.events.text(message.transcript ?? "", true);
|
|
498
|
+
return;
|
|
499
|
+
case "response.created":
|
|
500
|
+
this.started(message.response ?? {});
|
|
501
|
+
return;
|
|
502
|
+
case "response.output_audio.delta":
|
|
503
|
+
case "response.audio.delta": {
|
|
504
|
+
const say = this.responses.get(message.response_id ?? "");
|
|
505
|
+
if (say !== undefined && say > this.hushedUpTo && message.delta) {
|
|
506
|
+
this.events.audio(RealtimeSocket.bytes(message.delta), say);
|
|
507
|
+
}
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
case "response.done": {
|
|
511
|
+
const id = message.response?.id ?? "";
|
|
512
|
+
const say = this.responses.get(id);
|
|
513
|
+
this.responses.delete(id);
|
|
514
|
+
if (say !== undefined) {
|
|
515
|
+
this.events.said(say);
|
|
516
|
+
}
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
started(response) {
|
|
522
|
+
const id = response.id ?? "";
|
|
523
|
+
const tagged = Number(response.metadata?.ta_say);
|
|
524
|
+
let say;
|
|
525
|
+
if (tagged > 0 && this.asked.includes(tagged)) {
|
|
526
|
+
say = tagged;
|
|
527
|
+
this.asked.splice(this.asked.indexOf(tagged), 1);
|
|
528
|
+
} else if (!this.dialect.tagged && this.asked.length) {
|
|
529
|
+
say = this.asked.shift();
|
|
530
|
+
}
|
|
531
|
+
if (say === undefined) {
|
|
532
|
+
this.message({ type: "response.cancel", response_id: id });
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
this.responses.set(id, say);
|
|
536
|
+
if (say <= this.hushedUpTo) {
|
|
537
|
+
this.message({ type: "response.cancel", response_id: id });
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
message(message) {
|
|
541
|
+
if (this.socket.readyState === 1) {
|
|
542
|
+
this.socket.send(JSON.stringify(message));
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
static parse(data) {
|
|
546
|
+
try {
|
|
547
|
+
const message = JSON.parse(typeof data === "string" ? data : new TextDecoder().decode(data));
|
|
548
|
+
return message && typeof message.type === "string" ? message : undefined;
|
|
549
|
+
} catch {
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// src/host.ts
|
|
556
|
+
var text = (zh, tw, en, ja) => ({ "zh-CN": zh, "zh-TW": tw, "en-US": en, "ja-JP": ja });
|
|
557
|
+
var KEY = { key: "apiKey", label: text("API Key", "API Key", "API key", "API キー"), secret: true };
|
|
558
|
+
var VOICE = (placeholder) => ({
|
|
559
|
+
key: "voice",
|
|
560
|
+
label: text("音色", "音色", "Voice", "音色"),
|
|
561
|
+
placeholder,
|
|
562
|
+
optional: true
|
|
563
|
+
});
|
|
564
|
+
function host(url) {
|
|
565
|
+
try {
|
|
566
|
+
return new URL(url).host;
|
|
567
|
+
} catch {
|
|
568
|
+
return url;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
var OPENAI_URL = "wss://api.openai.com/v1/realtime";
|
|
572
|
+
var QWEN_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime";
|
|
573
|
+
var OPENAI_REALTIME = {
|
|
574
|
+
title: text("OpenAI Realtime", "OpenAI Realtime", "OpenAI Realtime", "OpenAI Realtime"),
|
|
575
|
+
hint: text("OpenAI 的实时语音:断句准、插话自然。国内需要代理;同协议的服务可以填它的地址。", "OpenAI 的即時語音:斷句準、插話自然。同協定的服務可以填它的位址。", "OpenAI's real-time voice: it hears when you stop and lets you cut in. A service of the same protocol may give its own address.", "OpenAI のリアルタイム音声:区切りが正確で、自然に割り込めます。同じプロトコルのサービスはそのアドレスを入力します。"),
|
|
576
|
+
icon: "Sparkles",
|
|
577
|
+
speaks: true,
|
|
578
|
+
options: [
|
|
579
|
+
KEY,
|
|
580
|
+
{ key: "model", label: text("模型", "模型", "Model", "モデル"), placeholder: "gpt-realtime", optional: true },
|
|
581
|
+
VOICE("marin"),
|
|
582
|
+
{ key: "url", label: text("服务地址", "服務位址", "Address", "アドレス"), placeholder: OPENAI_URL, optional: true }
|
|
583
|
+
],
|
|
584
|
+
where: (options) => host(RealtimeSocket.text(options.url) || OPENAI_URL),
|
|
585
|
+
open: (options, call, events) => {
|
|
586
|
+
const base = RealtimeSocket.text(options.url) || OPENAI_URL;
|
|
587
|
+
const url = `${base}${base.includes("?") ? "&" : "?"}model=${encodeURIComponent(RealtimeSocket.text(options.model) || "gpt-realtime")}`;
|
|
588
|
+
return OpenAiRealtimeEngine.open(url, { Authorization: `Bearer ${RealtimeSocket.text(options.apiKey)}` }, OPENAI_DIALECT, options, call.language, events);
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
var QWEN_REALTIME = {
|
|
592
|
+
title: text("通义 Qwen-Omni Realtime", "通義 Qwen-Omni Realtime", "Qwen-Omni Realtime", "Qwen-Omni Realtime"),
|
|
593
|
+
hint: text("阿里云百炼的实时语音,国内直连。API Key 在百炼控制台创建。", "阿里雲百煉的即時語音。API Key 在百煉主控台建立。", "The real-time voice of Alibaba Cloud Model Studio. Create the API key in its console.", "Alibaba Cloud Model Studio のリアルタイム音声。API キーはコンソールで作成します。"),
|
|
594
|
+
icon: "Cloud",
|
|
595
|
+
speaks: true,
|
|
596
|
+
options: [
|
|
597
|
+
KEY,
|
|
598
|
+
{ key: "model", label: text("模型", "模型", "Model", "モデル"), placeholder: "qwen3-omni-flash-realtime", optional: true },
|
|
599
|
+
VOICE("Cherry"),
|
|
600
|
+
{ key: "url", label: text("服务地址", "服務位址", "Address", "アドレス"), placeholder: QWEN_URL, optional: true }
|
|
601
|
+
],
|
|
602
|
+
where: (options) => host(RealtimeSocket.text(options.url) || QWEN_URL),
|
|
603
|
+
open: (options, call, events) => {
|
|
604
|
+
const base = RealtimeSocket.text(options.url) || QWEN_URL;
|
|
605
|
+
const url = `${base}${base.includes("?") ? "&" : "?"}model=${encodeURIComponent(RealtimeSocket.text(options.model) || "qwen3-omni-flash-realtime")}`;
|
|
606
|
+
return OpenAiRealtimeEngine.open(url, { Authorization: `Bearer ${RealtimeSocket.text(options.apiKey)}` }, QWEN_DIALECT, options, call.language, events);
|
|
607
|
+
}
|
|
608
|
+
};
|
|
609
|
+
var HY_REALTIME = {
|
|
610
|
+
title: text("混元 Hy Realtime", "混元 Hy Realtime", "Hunyuan Hy Realtime", "Hunyuan Hy Realtime"),
|
|
611
|
+
hint: text("腾讯混元的实时语音。填服务方给的地址(以 /v1/openai/realtime 结尾);需要鉴权的部署再填 Key。", "騰訊混元的即時語音。填服務方給的位址(以 /v1/openai/realtime 結尾);需要驗證的部署再填 Key。", "The real-time voice of Tencent Hunyuan. Give the address its provider gave (ending in /v1/openai/realtime), and a key where the deployment asks for one.", "Tencent Hunyuan のリアルタイム音声。提供元のアドレス(/v1/openai/realtime で終わる)を入力し、認証が必要な環境ではキーも入力します。"),
|
|
612
|
+
icon: "Sparkles",
|
|
613
|
+
speaks: true,
|
|
614
|
+
options: [
|
|
615
|
+
{ key: "url", label: text("服务地址", "服務位址", "Address", "アドレス"), placeholder: "ws://…/v1/openai/realtime" },
|
|
616
|
+
{ ...KEY, optional: true },
|
|
617
|
+
VOICE("f1")
|
|
618
|
+
],
|
|
619
|
+
where: (options) => host(RealtimeSocket.text(options.url)),
|
|
620
|
+
open: (options, call, events) => OpenAiRealtimeEngine.open(RealtimeSocket.text(options.url), { Authorization: `Bearer ${RealtimeSocket.text(options.apiKey) || "ta"}` }, HY_DIALECT, options, call.language, events)
|
|
621
|
+
};
|
|
622
|
+
var DOUBAO_REALTIME = {
|
|
623
|
+
title: text("豆包实时语音", "豆包即時語音", "Doubao real-time voice", "Doubao リアルタイム音声"),
|
|
624
|
+
hint: text("火山引擎的端到端实时语音,国内直连,中文自然。在火山引擎控制台开通后,填应用的 APP ID 和 Access Token。", "火山引擎的端到端即時語音。開通後填應用的 APP ID 和 Access Token。", "The end-to-end real-time voice of Volcengine. Once it is on in the console, give the APP ID and Access Token of the app.", "Volcengine のエンドツーエンド・リアルタイム音声。コンソールで有効化し、アプリの APP ID と Access Token を入力します。"),
|
|
625
|
+
icon: "MessageCircle",
|
|
626
|
+
speaks: true,
|
|
627
|
+
options: [
|
|
628
|
+
{ key: "appId", label: text("APP ID", "APP ID", "APP ID", "APP ID"), placeholder: "1234567890" },
|
|
629
|
+
{ key: "accessKey", label: text("Access Token", "Access Token", "Access Token", "Access Token"), secret: true },
|
|
630
|
+
{ key: "speaker", label: text("音色", "音色", "Voice", "音色"), placeholder: "zh_female_vv_jupiter_bigtts", optional: true },
|
|
631
|
+
{ key: "url", label: text("服务地址", "服務位址", "Address", "アドレス"), placeholder: DOUBAO_URL, optional: true }
|
|
632
|
+
],
|
|
633
|
+
where: (options) => host(RealtimeSocket.text(options.url) || DOUBAO_URL),
|
|
634
|
+
open: (options, call, events) => DoubaoRealtimeEngine.open(options, call.language, events)
|
|
635
|
+
};
|
|
636
|
+
function activate(host2) {
|
|
637
|
+
if (typeof host2.voices?.registerType !== "function") {
|
|
638
|
+
return {};
|
|
639
|
+
}
|
|
640
|
+
const stops = [
|
|
641
|
+
host2.voices.registerType("openai-realtime", OPENAI_REALTIME),
|
|
642
|
+
host2.voices.registerType("qwen-realtime", QWEN_REALTIME),
|
|
643
|
+
host2.voices.registerType("hy-realtime", HY_REALTIME),
|
|
644
|
+
host2.voices.registerType("doubao-realtime", DOUBAO_REALTIME)
|
|
645
|
+
];
|
|
646
|
+
return {
|
|
647
|
+
dispose: () => {
|
|
648
|
+
for (const stop of stops) {
|
|
649
|
+
stop();
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
export {
|
|
655
|
+
activate as default,
|
|
656
|
+
QWEN_URL,
|
|
657
|
+
QWEN_REALTIME,
|
|
658
|
+
OPENAI_URL,
|
|
659
|
+
OPENAI_REALTIME,
|
|
660
|
+
HY_REALTIME,
|
|
661
|
+
DOUBAO_REALTIME
|
|
662
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,67 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celljs/ta-voice",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "4.18.0",
|
|
4
|
+
"description": "A pi package for Ta: real-time voice services for calls, OpenAI Realtime, Qwen-Omni Realtime of Alibaba Cloud Model Studio, Hy Realtime of Tencent Hunyuan and the real-time voice of Doubao on Volcengine, to listen with barge-in and read aloud",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pi-package"
|
|
7
|
+
],
|
|
8
|
+
"homepage": "https://github.com/cellbang/ta",
|
|
9
|
+
"bugs": {
|
|
10
|
+
"url": "https://github.com/cellbang/ta/issues"
|
|
11
|
+
},
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "https://github.com/cellbang/cell.git"
|
|
15
|
+
},
|
|
16
|
+
"license": "MIT",
|
|
17
|
+
"files": [
|
|
18
|
+
"esm",
|
|
19
|
+
"host",
|
|
20
|
+
"!**/*.map"
|
|
21
|
+
],
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "tsc -b && bun build src/host.ts --format esm --target node --outfile host/index.mjs",
|
|
24
|
+
"clean": "rm -rf lib host *.tsbuildinfo",
|
|
25
|
+
"test": "bun test ./lib",
|
|
26
|
+
"watch": "tsc -b -w --preserveWatchOutput"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@celljs/ta-protocol": "4.18.0"
|
|
30
|
+
},
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"access": "public"
|
|
33
|
+
},
|
|
34
|
+
"pi": {
|
|
35
|
+
"title": {
|
|
36
|
+
"en": "Real-time voice",
|
|
37
|
+
"zh-CN": "实时语音",
|
|
38
|
+
"zh-TW": "即時語音",
|
|
39
|
+
"ja-JP": "リアルタイム音声"
|
|
40
|
+
},
|
|
41
|
+
"summary": {
|
|
42
|
+
"en": "Calls with a real-time voice service: OpenAI Realtime, Qwen-Omni Realtime, Hunyuan Hy Realtime or Doubao. Ta hears you as you speak, you can cut in while it talks, and it reads in a natural voice. Add one in Settings → Voice.",
|
|
43
|
+
"zh-CN": "用实时语音服务打电话:OpenAI Realtime、通义 Qwen-Omni Realtime、混元 Hy Realtime 或豆包。Ta 边听边断句,它说话时你可以随时插话,声音也更自然。在 设置 → 语音 里添加。",
|
|
44
|
+
"zh-TW": "用即時語音服務打電話:OpenAI Realtime、通義 Qwen-Omni Realtime、混元 Hy Realtime 或豆包。Ta 邊聽邊斷句,它說話時你可以隨時插話,聲音也更自然。在 設定 → 語音 裡新增。",
|
|
45
|
+
"ja-JP": "リアルタイム音声サービスで通話:OpenAI Realtime、Qwen-Omni Realtime、Hunyuan Hy Realtime、Doubao。話しながら聞き取り、Ta が話している途中でも割り込めて、声も自然です。設定 → 音声 で追加します。"
|
|
46
|
+
},
|
|
47
|
+
"icon": "Phone",
|
|
48
|
+
"host": "./host/index.mjs",
|
|
49
|
+
"permissions": {
|
|
50
|
+
"host": {
|
|
51
|
+
"*": "settings.manage"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
"exports": {
|
|
56
|
+
"./package.json": "./package.json",
|
|
57
|
+
"./lib/*.js": {
|
|
58
|
+
"import": "./esm/*.js",
|
|
59
|
+
"default": "./lib/*.js"
|
|
60
|
+
},
|
|
61
|
+
"./lib/*": {
|
|
62
|
+
"import": "./esm/*.js",
|
|
63
|
+
"default": "./lib/*.js"
|
|
64
|
+
},
|
|
65
|
+
"./*": "./*"
|
|
66
|
+
}
|
|
67
|
+
}
|