@kmchai/cortico-world-voice 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +83 -70
- package/describe.json +2 -27
- package/dist/webui.css +1 -0
- package/dist/webui.js +1 -0
- package/package.json +33 -6
- package/src/ENV_PROMPT.md +6 -7
- package/src/audio/queue.ts +1 -1
- package/src/config.ts +60 -117
- package/src/definition.ts +3 -6
- package/src/models.ts +66 -0
- package/src/panel.ts +34 -0
- package/src/service.ts +371 -0
- package/src/speech/style.ts +2 -14
- package/src/speech/tags.ts +8 -6
- package/src/speech/text.ts +4 -1
- package/src/tools.ts +1 -16
- package/src/types.ts +46 -49
- package/src/world.ts +219 -267
- package/src/backends/audiocpp.ts +0 -98
- package/src/backends/custom.ts +0 -132
- package/src/backends/http.ts +0 -91
- package/src/backends/index.ts +0 -21
- package/src/backends/openai.ts +0 -51
- package/src/http.ts +0 -205
- package/src/options.ts +0 -15
- package/src/sing.ts +0 -73
package/src/world.ts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* 语音 World:把回复念出来。
|
|
3
3
|
*
|
|
4
4
|
* 触发层从主 session 输出流与事件库认出要念的话;文本层过滤与切句;
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* 合成层经 HTTP 调 Kmch AI · 服务;出口层本机播放、发事件或只返回附件。
|
|
6
|
+
* 这一层不解析任何 TTS 方言,也不监听端口。
|
|
7
7
|
*/
|
|
8
8
|
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
9
9
|
import { tmpdir } from 'node:os';
|
|
@@ -14,24 +14,26 @@ import type {
|
|
|
14
14
|
Logger, ToolDef, ToolOutcome, World, WorldHost, WorldLamp, WorldWebUiDecl,
|
|
15
15
|
} from 'cortico/core/types.ts';
|
|
16
16
|
import { nullLogger, nowIso } from 'cortico/core/util.ts';
|
|
17
|
-
import {
|
|
18
|
-
import { VOICE_DEFAULTS, VOICE_ID, VOICE_OPTIONS_KIND, voiceConfigGroups, voiceText } from './config.ts';
|
|
17
|
+
import { VOICE_DEFAULTS, VOICE_ID, voiceConfigGroups, voiceText } from './config.ts';
|
|
19
18
|
import type { AudioPlayer } from './audio/player.ts';
|
|
20
19
|
import { createPlayer } from './audio/player.ts';
|
|
21
20
|
import { SpeechQueue, type SpeechJob } from './audio/queue.ts';
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
21
|
+
import { modelPrompt } from './models.ts';
|
|
22
|
+
import { isPlainObject, type PanelModelState } from './panel.ts';
|
|
23
|
+
import { ServiceClient, type SpeechService } from './service.ts';
|
|
25
24
|
import { splitSpeech, truncateSpeech } from './speech/chunk.ts';
|
|
25
|
+
import { buildInstruct, resolveStyle } from './speech/style.ts';
|
|
26
26
|
import { filterSpeech } from './speech/text.ts';
|
|
27
27
|
import { EventPoller, eventTypes, hasKeyword, latestUserText, splitList, VoiceTap } from './triggers.ts';
|
|
28
28
|
import { VOICE_TOOL_DECLS } from './tools.ts';
|
|
29
|
-
import type {
|
|
29
|
+
import type {
|
|
30
|
+
ServiceHealth, TtsAudio, TtsRequest, VoiceConfigSection, VoiceModel,
|
|
31
|
+
} from './types.ts';
|
|
30
32
|
|
|
31
33
|
const ENV_PROMPT_FILE = fileURLToPath(new URL('./ENV_PROMPT.md', import.meta.url));
|
|
32
34
|
const DEDUPE_WINDOW_MS = 5000;
|
|
35
|
+
const HEALTH_INTERVAL_MS = 15_000;
|
|
33
36
|
const HEALTH_TTL_MS = 15_000;
|
|
34
|
-
const VOICES_TTL_MS = 10 * 60_000;
|
|
35
37
|
|
|
36
38
|
export interface VoiceWorldDeps {
|
|
37
39
|
fetch: typeof fetch;
|
|
@@ -39,8 +41,8 @@ export interface VoiceWorldDeps {
|
|
|
39
41
|
rng(): number;
|
|
40
42
|
now(): number;
|
|
41
43
|
sleep(ms: number): Promise<void>;
|
|
42
|
-
/**
|
|
43
|
-
|
|
44
|
+
/** 测试用:替换服务客户端。 */
|
|
45
|
+
service?: SpeechService;
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
export interface VoiceWorldOptions {
|
|
@@ -61,8 +63,12 @@ interface EnqueueOptions {
|
|
|
61
63
|
voice?: string;
|
|
62
64
|
speed?: number;
|
|
63
65
|
silent?: boolean;
|
|
64
|
-
|
|
65
|
-
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
interface ServiceState {
|
|
69
|
+
ok: boolean;
|
|
70
|
+
detail?: string;
|
|
71
|
+
at: number;
|
|
66
72
|
}
|
|
67
73
|
|
|
68
74
|
function defaultDeps(): VoiceWorldDeps {
|
|
@@ -84,6 +90,17 @@ function asNumber(value: unknown): number | undefined {
|
|
|
84
90
|
return Number.isFinite(n) ? n : undefined;
|
|
85
91
|
}
|
|
86
92
|
|
|
93
|
+
/** 徽标只显示主机与端口;地址不合法时退回原串。 */
|
|
94
|
+
function serviceHost(raw: string): string {
|
|
95
|
+
const value = raw.trim();
|
|
96
|
+
if (!value) return '—';
|
|
97
|
+
try {
|
|
98
|
+
return new URL(value).host || '—';
|
|
99
|
+
} catch {
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
87
104
|
/** 等一个 promise;超时返回 'timeout',不取消原任务。 */
|
|
88
105
|
async function within<T>(promise: Promise<T>, ms: number): Promise<T | 'timeout'> {
|
|
89
106
|
let timer: NodeJS.Timeout | undefined;
|
|
@@ -104,27 +121,29 @@ export class VoiceWorld implements World {
|
|
|
104
121
|
private readonly player: AudioPlayer | null;
|
|
105
122
|
private readonly queue: SpeechQueue;
|
|
106
123
|
private readonly tap: VoiceTap;
|
|
124
|
+
private readonly service: SpeechService;
|
|
107
125
|
private host: WorldHost | null = null;
|
|
108
|
-
private http: VoiceHttpHandle | null = null;
|
|
109
|
-
private httpError = '';
|
|
110
126
|
private poller: EventPoller | null = null;
|
|
111
127
|
private healthTimer: NodeJS.Timeout | null = null;
|
|
112
128
|
private started = false;
|
|
113
129
|
private tempDir = '';
|
|
114
130
|
private tempSeq = 0;
|
|
115
|
-
private
|
|
131
|
+
private serviceState: ServiceState | null = null;
|
|
116
132
|
private probing = false;
|
|
117
|
-
private
|
|
118
|
-
private
|
|
119
|
-
private
|
|
120
|
-
private voicesSignature = '';
|
|
121
|
-
private voicesLoading = false;
|
|
133
|
+
private catalog: VoiceModel[] | null = null;
|
|
134
|
+
private catalogError: string | undefined;
|
|
135
|
+
private catalogLoad: Promise<void> | null = null;
|
|
122
136
|
private readonly lastSpoken = new Map<string, number>();
|
|
123
137
|
private lastFailureAt = 0;
|
|
124
138
|
|
|
125
139
|
constructor(private readonly opts: VoiceWorldOptions) {
|
|
126
140
|
this.log = opts.log ?? nullLogger();
|
|
127
141
|
this.deps = { ...defaultDeps(), ...opts.deps };
|
|
142
|
+
this.service = this.deps.service ?? new ServiceClient({
|
|
143
|
+
fetch: this.deps.fetch,
|
|
144
|
+
secret: (name) => this.opts.secret(name),
|
|
145
|
+
cfg: () => this.cfg,
|
|
146
|
+
});
|
|
128
147
|
this.player = this.deps.createPlayer(this.log);
|
|
129
148
|
this.queue = new SpeechQueue({
|
|
130
149
|
player: this.player,
|
|
@@ -141,7 +160,6 @@ export class VoiceWorld implements World {
|
|
|
141
160
|
onText: (text, source) => this.consider(text, source),
|
|
142
161
|
onAbort: () => this.queue.hush(),
|
|
143
162
|
});
|
|
144
|
-
registerVoiceOptions(() => this.voiceOptions());
|
|
145
163
|
}
|
|
146
164
|
|
|
147
165
|
private get cfg(): VoiceConfigSection {
|
|
@@ -152,11 +170,10 @@ export class VoiceWorld implements World {
|
|
|
152
170
|
const cfg = this.cfg;
|
|
153
171
|
return {
|
|
154
172
|
'voice.auto': cfg.autoSpeak ? `${Math.round(cfg.autoProbability * 100)}%` : 'off',
|
|
155
|
-
'voice.backend': cfg.backend,
|
|
156
173
|
'voice.voice': cfg.voice || 'default',
|
|
157
174
|
'voice.emotion': cfg.defaultEmotion || 'neutral',
|
|
158
175
|
'voice.instruct': cfg.defaultInstruct || 'none',
|
|
159
|
-
'voice.
|
|
176
|
+
'voice.modelPrompt': modelPrompt(cfg.model),
|
|
160
177
|
};
|
|
161
178
|
}
|
|
162
179
|
|
|
@@ -168,7 +185,6 @@ export class VoiceWorld implements World {
|
|
|
168
185
|
});
|
|
169
186
|
return [
|
|
170
187
|
tool('voice_speak', (args) => this.handleSpeak(args)),
|
|
171
|
-
tool('voice_sing', (args) => this.handleSing(args)),
|
|
172
188
|
tool('voice_hush', async () => {
|
|
173
189
|
this.queue.hush();
|
|
174
190
|
return 'Voice stopped; queued lines were dropped.';
|
|
@@ -182,30 +198,42 @@ export class VoiceWorld implements World {
|
|
|
182
198
|
this.maybeRefresh();
|
|
183
199
|
const cfg = this.cfg;
|
|
184
200
|
const stats = this.queue.stats();
|
|
185
|
-
const
|
|
186
|
-
? { label: t.
|
|
187
|
-
: this.
|
|
188
|
-
? { label: t.
|
|
189
|
-
: { label: t.
|
|
190
|
-
const apiLamp: WorldLamp = !cfg.httpApi
|
|
191
|
-
? { label: t.lampApi, state: 'offline' }
|
|
192
|
-
: this.http
|
|
193
|
-
? { label: t.lampApi, state: 'online', hint: `127.0.0.1:${this.http.port}` }
|
|
194
|
-
: { label: t.lampApi, state: 'error', ...(this.httpError ? { hint: this.httpError } : {}) };
|
|
201
|
+
const serviceLamp: WorldLamp = this.serviceState === null
|
|
202
|
+
? { label: t.lampService, state: this.started ? 'loading' : 'offline' }
|
|
203
|
+
: this.serviceState.ok
|
|
204
|
+
? { label: t.lampService, state: 'online', ...(this.serviceState.detail ? { hint: this.serviceState.detail } : {}) }
|
|
205
|
+
: { label: t.lampService, state: 'error', ...(this.serviceState.detail ? { hint: this.serviceState.detail } : {}) };
|
|
195
206
|
return {
|
|
196
207
|
label: t.label,
|
|
197
208
|
lamps: [
|
|
198
|
-
|
|
199
|
-
{
|
|
200
|
-
|
|
201
|
-
|
|
209
|
+
serviceLamp,
|
|
210
|
+
{
|
|
211
|
+
label: t.lampQueue,
|
|
212
|
+
state: !this.started ? 'offline' : stats.speaking ? 'loading' : 'online',
|
|
213
|
+
hint: `${stats.queued}`,
|
|
214
|
+
},
|
|
202
215
|
],
|
|
203
216
|
badges: [
|
|
217
|
+
{ label: 'service', value: serviceHost(cfg.serviceUrl), tone: 'plain' },
|
|
218
|
+
{ label: 'model', value: cfg.model || '—', tone: 'plain' },
|
|
204
219
|
{ label: 'voice', value: cfg.voice || '—', tone: 'plain' },
|
|
205
|
-
{ label: 'emotion', value: cfg.defaultEmotion || '—', tone: 'plain' },
|
|
206
|
-
{ label: 'queue', value: stats.queued, tone: stats.queued ? 'on' : 'plain' },
|
|
207
220
|
],
|
|
221
|
+
panels: [{ id: 'model', title: t.modelPanel, getMethods: ['state'] }],
|
|
208
222
|
config: voiceConfigGroups(language),
|
|
223
|
+
links: [{ label: t.servicePage, href: '#/provider/world:service' }],
|
|
224
|
+
invoke: async (panel, method, args) => {
|
|
225
|
+
if (panel !== 'model') throw new Error(t.modelUnknownMethod);
|
|
226
|
+
switch (method) {
|
|
227
|
+
case 'state':
|
|
228
|
+
return this.modelPanelState();
|
|
229
|
+
case 'refresh':
|
|
230
|
+
return this.refreshModelPanel();
|
|
231
|
+
case 'save':
|
|
232
|
+
return this.saveModelPanel(language, args);
|
|
233
|
+
default:
|
|
234
|
+
throw new Error(t.modelUnknownMethod);
|
|
235
|
+
}
|
|
236
|
+
},
|
|
209
237
|
promptDocs: [{
|
|
210
238
|
key: 'worlds.voice.envPrompt',
|
|
211
239
|
title: t.envTitle,
|
|
@@ -214,16 +242,114 @@ export class VoiceWorld implements World {
|
|
|
214
242
|
role: 'envPrompt',
|
|
215
243
|
vars: [
|
|
216
244
|
{ name: 'voice.auto', description: 'Automatic reading probability, or off.' },
|
|
217
|
-
{ name: 'voice.backend', description: 'Active protocol id.' },
|
|
218
245
|
{ name: 'voice.voice', description: 'Current voice id.' },
|
|
219
246
|
{ name: 'voice.emotion', description: 'Standing emotion.' },
|
|
220
247
|
{ name: 'voice.instruct', description: 'Standing direction.' },
|
|
221
|
-
{ name: 'voice.
|
|
248
|
+
{ name: 'voice.modelPrompt', description: 'How to write text and emotion for the current model.' },
|
|
222
249
|
],
|
|
223
250
|
}],
|
|
224
251
|
};
|
|
225
252
|
}
|
|
226
253
|
|
|
254
|
+
// ———————— 模型面板 ————————
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* 第一次读面板时拉一次目录;之后直接回缓存。刷新失败保留上一次可用的目录,
|
|
258
|
+
* 只把错误一起交回面板。
|
|
259
|
+
*/
|
|
260
|
+
private async modelPanelState(): Promise<PanelModelState> {
|
|
261
|
+
if (!this.catalog) await this.loadCatalog(this.cfg.model);
|
|
262
|
+
return this.panelSnapshot();
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
private async refreshModelPanel(): Promise<PanelModelState> {
|
|
266
|
+
if (this.catalogLoad) await this.catalogLoad;
|
|
267
|
+
await this.loadCatalog(this.cfg.model);
|
|
268
|
+
return this.panelSnapshot();
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private panelSnapshot(): PanelModelState {
|
|
272
|
+
const cfg = this.cfg;
|
|
273
|
+
const saved = cfg.modelOptions?.[cfg.model];
|
|
274
|
+
return {
|
|
275
|
+
model: cfg.model,
|
|
276
|
+
voice: cfg.voice,
|
|
277
|
+
prompt: modelPrompt(cfg.model),
|
|
278
|
+
options: isPlainObject(saved) ? { ...saved } : {},
|
|
279
|
+
models: this.catalog ?? [],
|
|
280
|
+
...(this.catalogError ? { error: this.catalogError } : {}),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
private loadCatalog(selected: string): Promise<void> {
|
|
285
|
+
if (this.catalogLoad) return this.catalogLoad;
|
|
286
|
+
this.catalogLoad = this.fetchCatalog(selected).finally(() => {
|
|
287
|
+
this.catalogLoad = null;
|
|
288
|
+
});
|
|
289
|
+
return this.catalogLoad;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* 先取模型表,再尽力为当前模型补一次更细的音色表。模型表是权威来源,已经带
|
|
294
|
+
* 音色与参数;补音色失败只损失一次补全,不能让整份目录跟着作废。
|
|
295
|
+
*/
|
|
296
|
+
private async fetchCatalog(selected: string): Promise<void> {
|
|
297
|
+
let models: VoiceModel[];
|
|
298
|
+
try {
|
|
299
|
+
models = await this.service.models();
|
|
300
|
+
} catch (error) {
|
|
301
|
+
this.catalogError = this.catalogFailure(error);
|
|
302
|
+
this.markService({ ok: false, detail: this.catalogError });
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const id = selected.trim() || this.cfg.model.trim() || models[0]?.id || '';
|
|
307
|
+
let catalog = models;
|
|
308
|
+
if (id) {
|
|
309
|
+
try {
|
|
310
|
+
const voices = await this.service.voices(id);
|
|
311
|
+
const names = [...new Set(voices.map((voice) => voice.value.trim()).filter(Boolean))];
|
|
312
|
+
if (names.length > 0) {
|
|
313
|
+
catalog = models.map((model) => (model.id === id ? { ...model, voices: names } : model));
|
|
314
|
+
}
|
|
315
|
+
} catch (error) {
|
|
316
|
+
this.log.debug('音色表读取失败,保留模型表里的音色', {
|
|
317
|
+
err: error instanceof Error ? error.message : String(error),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
this.catalog = catalog;
|
|
323
|
+
this.catalogError = undefined;
|
|
324
|
+
this.markService({ ok: true });
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** 错误要回面板,但不能把配置里的服务地址(可能带令牌)一起带出去。 */
|
|
328
|
+
private catalogFailure(error: unknown): string {
|
|
329
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
330
|
+
const url = this.cfg.serviceUrl.trim();
|
|
331
|
+
return url && message.includes(url) ? message.split(url).join('[service]') : message;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
private saveModelPanel(language: Language, args: unknown[]): PanelModelState {
|
|
335
|
+
const t = voiceText(language);
|
|
336
|
+
const raw = args[0];
|
|
337
|
+
if (!isPlainObject(raw)) throw new Error(t.modelSaveShape);
|
|
338
|
+
const model = typeof raw.model === 'string' ? raw.model.trim() : '';
|
|
339
|
+
if (!model) throw new Error(t.modelSaveModel);
|
|
340
|
+
if (typeof raw.voice !== 'string') throw new Error(t.modelSaveVoice);
|
|
341
|
+
if (!isPlainObject(raw.options)) throw new Error(t.modelSaveOptions);
|
|
342
|
+
|
|
343
|
+
const next: VoiceConfigSection['modelOptions'] = { ...(this.cfg.modelOptions ?? {}) };
|
|
344
|
+
// 空对象表示删掉这个模型的条目,回到服务的默认参数。
|
|
345
|
+
if (Object.keys(raw.options).length === 0) delete next[model];
|
|
346
|
+
else next[model] = { ...raw.options };
|
|
347
|
+
// 配置层对对象做深合并,删掉的键不会随 patch 传播;先让活引用指向新对象再写盘。
|
|
348
|
+
this.cfg.modelOptions = next;
|
|
349
|
+
this.opts.persist({ model, voice: raw.voice.trim(), modelOptions: next });
|
|
350
|
+
return this.panelSnapshot();
|
|
351
|
+
}
|
|
352
|
+
|
|
227
353
|
outputTap() {
|
|
228
354
|
const cfg = this.cfg;
|
|
229
355
|
if (!cfg.autoSpeak) return undefined;
|
|
@@ -235,7 +361,7 @@ export class VoiceWorld implements World {
|
|
|
235
361
|
this.host = host;
|
|
236
362
|
this.started = true;
|
|
237
363
|
this.log.info('语音 World 启动', {
|
|
238
|
-
|
|
364
|
+
service: serviceHost(this.cfg.serviceUrl),
|
|
239
365
|
playback: this.cfg.playback,
|
|
240
366
|
auto: this.cfg.autoSpeak ? this.cfg.autoProbability : 'off',
|
|
241
367
|
});
|
|
@@ -244,10 +370,9 @@ export class VoiceWorld implements World {
|
|
|
244
370
|
this.log.warn('播放器预热失败', { err: error instanceof Error ? error.message : String(error) });
|
|
245
371
|
});
|
|
246
372
|
}
|
|
247
|
-
if (this.cfg.httpApi) await this.startHttp();
|
|
248
373
|
this.syncPoller();
|
|
249
374
|
void this.probe();
|
|
250
|
-
this.healthTimer = setInterval(() => { void this.probe(); },
|
|
375
|
+
this.healthTimer = setInterval(() => { void this.probe(); }, HEALTH_INTERVAL_MS);
|
|
251
376
|
this.healthTimer.unref?.();
|
|
252
377
|
}
|
|
253
378
|
|
|
@@ -257,9 +382,6 @@ export class VoiceWorld implements World {
|
|
|
257
382
|
this.healthTimer = null;
|
|
258
383
|
this.poller?.stop();
|
|
259
384
|
this.poller = null;
|
|
260
|
-
const http = this.http;
|
|
261
|
-
this.http = null;
|
|
262
|
-
if (http) await http.close().catch(() => {});
|
|
263
385
|
await this.queue.close();
|
|
264
386
|
if (this.tempDir) {
|
|
265
387
|
try { rmSync(this.tempDir, { recursive: true, force: true }); } catch { /* 临时目录 */ }
|
|
@@ -286,13 +408,6 @@ export class VoiceWorld implements World {
|
|
|
286
408
|
const cfg = this.cfg;
|
|
287
409
|
if (!cfg.autoSpeak) return;
|
|
288
410
|
|
|
289
|
-
// 唱歌标记是显式的,不掷概率。没配唱歌服务时按普通文本念。
|
|
290
|
-
const sing = singRequestOf(raw);
|
|
291
|
-
if (sing.marked && cfg.singUrl.trim()) {
|
|
292
|
-
if (sing.lyrics && this.fresh(sing.lyrics)) this.enqueueSing(sing.lyrics, source);
|
|
293
|
-
return;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
411
|
const filtered = filterSpeech(raw, cfg);
|
|
297
412
|
if (!filtered.text) return;
|
|
298
413
|
|
|
@@ -310,31 +425,11 @@ export class VoiceWorld implements World {
|
|
|
310
425
|
this.log.debug('自动朗读', { source, explicit, chars: [...filtered.text].length });
|
|
311
426
|
}
|
|
312
427
|
|
|
313
|
-
private enqueueSing(lyrics: string, source: string): void {
|
|
314
|
-
const cfg = this.cfg;
|
|
315
|
-
if (!cfg.singUrl.trim()) return;
|
|
316
|
-
this.queue.enqueue({
|
|
317
|
-
text: lyrics,
|
|
318
|
-
chunks: [lyrics],
|
|
319
|
-
silent: cfg.playback === 'none',
|
|
320
|
-
source,
|
|
321
|
-
produce: (chunk, signal) => singRemote({
|
|
322
|
-
url: cfg.singUrl,
|
|
323
|
-
token: cfg.singToken,
|
|
324
|
-
lyrics: chunk,
|
|
325
|
-
timeoutMs: Math.max(cfg.timeoutMs, 180_000),
|
|
326
|
-
fetch: this.deps.fetch,
|
|
327
|
-
signal,
|
|
328
|
-
}),
|
|
329
|
-
});
|
|
330
|
-
this.log.info('自动唱歌', { source, chars: [...lyrics].length });
|
|
331
|
-
}
|
|
332
|
-
|
|
333
428
|
private enqueue(text: string, opts: EnqueueOptions): ReturnType<SpeechQueue['enqueue']> | null {
|
|
334
429
|
const cfg = this.cfg;
|
|
335
430
|
const clipped = truncateSpeech(text, cfg.maxChars);
|
|
336
431
|
if ([...clipped].length < Math.max(1, cfg.minChars)) return null;
|
|
337
|
-
const chunks =
|
|
432
|
+
const chunks = splitSpeech(clipped, cfg.chunkMax);
|
|
338
433
|
if (chunks.length === 0) return null;
|
|
339
434
|
const job: SpeechJob = {
|
|
340
435
|
text: clipped,
|
|
@@ -403,73 +498,6 @@ export class VoiceWorld implements World {
|
|
|
403
498
|
return this.spokenOutcome(undefined, 0, false);
|
|
404
499
|
}
|
|
405
500
|
|
|
406
|
-
private async handleSing(args: Record<string, unknown>): Promise<ToolOutcome> {
|
|
407
|
-
const { lyrics } = singRequestOf(asString(args.lyrics));
|
|
408
|
-
if (!lyrics) return { text: 'voice_sing failed: there are no lyrics to sing.', failed: true };
|
|
409
|
-
if (!this.cfg.singUrl.trim()) {
|
|
410
|
-
return { text: 'voice_sing failed: no singing service is configured (worlds.voice.singUrl).', failed: true };
|
|
411
|
-
}
|
|
412
|
-
const silent = this.cfg.playback === 'none';
|
|
413
|
-
const share = this.cfg.shareAudio;
|
|
414
|
-
const timeoutMs = Math.max(this.cfg.timeoutMs, 180_000);
|
|
415
|
-
const produce = (text: string, signal: AbortSignal) => singRemote({
|
|
416
|
-
url: this.cfg.singUrl,
|
|
417
|
-
token: this.cfg.singToken,
|
|
418
|
-
lyrics: text,
|
|
419
|
-
timeoutMs,
|
|
420
|
-
fetch: this.deps.fetch,
|
|
421
|
-
signal,
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
// 要交附件:合成完整一首就回执,播放交给队列,不等播完。
|
|
425
|
-
if (share) {
|
|
426
|
-
const controller = new AbortController();
|
|
427
|
-
const timer = setTimeout(() => controller.abort(new Error('timeout')), timeoutMs);
|
|
428
|
-
try {
|
|
429
|
-
const audio = await produce(lyrics, controller.signal);
|
|
430
|
-
if (!silent) {
|
|
431
|
-
this.queue.enqueue({
|
|
432
|
-
text: lyrics,
|
|
433
|
-
chunks: [lyrics],
|
|
434
|
-
silent: false,
|
|
435
|
-
source: 'tool:voice_sing',
|
|
436
|
-
produce: async () => audio,
|
|
437
|
-
});
|
|
438
|
-
}
|
|
439
|
-
const outcome = this.spokenOutcome(audio, audio.seconds ?? 0, silent);
|
|
440
|
-
return { ...outcome, text: outcome.text.replace('one line', 'one sung line') };
|
|
441
|
-
} catch (error) {
|
|
442
|
-
return { text: `voice_sing failed: ${error instanceof Error ? error.message : String(error)}`, failed: true };
|
|
443
|
-
} finally {
|
|
444
|
-
clearTimeout(timer);
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
const job: SpeechJob = {
|
|
449
|
-
text: lyrics,
|
|
450
|
-
chunks: [lyrics],
|
|
451
|
-
silent,
|
|
452
|
-
source: 'tool:voice_sing',
|
|
453
|
-
produce,
|
|
454
|
-
};
|
|
455
|
-
const enqueued = this.queue.enqueue(job);
|
|
456
|
-
if (!silent) {
|
|
457
|
-
const first = await within(Promise.race([
|
|
458
|
-
enqueued.started.then(() => 'started' as const),
|
|
459
|
-
enqueued.done.then((r) => r),
|
|
460
|
-
]), timeoutMs);
|
|
461
|
-
if (first === 'timeout') return { text: 'The singing service is still working; the song will play when ready.' };
|
|
462
|
-
if (first !== 'started' && !first.ok) {
|
|
463
|
-
return { text: `voice_sing failed: ${first.error ?? 'cancelled'}`, failed: true };
|
|
464
|
-
}
|
|
465
|
-
return { text: 'Singing now.' };
|
|
466
|
-
}
|
|
467
|
-
const result = await within(enqueued.done, timeoutMs + 60_000);
|
|
468
|
-
if (result === 'timeout') return { text: 'voice_sing failed: the singing service timed out.', failed: true };
|
|
469
|
-
if (!result.ok) return { text: `voice_sing failed: ${result.error ?? 'cancelled'}`, failed: true };
|
|
470
|
-
return { text: 'Sang one line without playing.' };
|
|
471
|
-
}
|
|
472
|
-
|
|
473
501
|
private async handleStyle(args: Record<string, unknown>): Promise<ToolOutcome> {
|
|
474
502
|
const has = (key: string): boolean => Object.prototype.hasOwnProperty.call(args, key);
|
|
475
503
|
const patch: Partial<VoiceConfigSection> = {};
|
|
@@ -482,7 +510,6 @@ export class VoiceWorld implements World {
|
|
|
482
510
|
}
|
|
483
511
|
if (Object.keys(patch).length > 0) {
|
|
484
512
|
this.opts.persist(patch);
|
|
485
|
-
if (patch.voice !== undefined) void this.refreshVoices(true);
|
|
486
513
|
return { text: `Voice style updated. ${this.styleSummary()}` };
|
|
487
514
|
}
|
|
488
515
|
return { text: this.styleSummary() };
|
|
@@ -492,8 +519,7 @@ export class VoiceWorld implements World {
|
|
|
492
519
|
const cfg = this.cfg;
|
|
493
520
|
return `Voice: ${cfg.voice || 'default'}; emotion: ${cfg.defaultEmotion || 'neutral'}; `
|
|
494
521
|
+ `direction: ${cfg.defaultInstruct || 'none'}; speed: ${cfg.speed}; `
|
|
495
|
-
+ `automatic: ${cfg.autoSpeak ? `${Math.round(cfg.autoProbability * 100)}%` : 'off'}
|
|
496
|
-
+ `singing: ${cfg.singUrl ? 'configured' : 'not configured'}.`;
|
|
522
|
+
+ `automatic: ${cfg.autoSpeak ? `${Math.round(cfg.autoProbability * 100)}%` : 'off'}.`;
|
|
497
523
|
}
|
|
498
524
|
|
|
499
525
|
private spokenOutcome(audio: TtsAudio | undefined, seconds: number, silent: boolean): ToolOutcome {
|
|
@@ -518,26 +544,45 @@ export class VoiceWorld implements World {
|
|
|
518
544
|
|
|
519
545
|
// ———————— 合成与出口 ————————
|
|
520
546
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
private async synthesize(text: string, style: { emotion?: string; instruct?: string; voice?: string; speed?: number }, signal: AbortSignal): Promise<TtsAudio> {
|
|
547
|
+
/**
|
|
548
|
+
* 语义字段按 emotionMode 只发一种:instruct 发指令,categorical 发情绪,
|
|
549
|
+
* basic 发语速,tags/none 不发;voice/model/format 每次都带。
|
|
550
|
+
*/
|
|
551
|
+
private async synthesize(
|
|
552
|
+
text: string,
|
|
553
|
+
style: { emotion?: string; instruct?: string; voice?: string; speed?: number },
|
|
554
|
+
signal: AbortSignal,
|
|
555
|
+
): Promise<TtsAudio> {
|
|
531
556
|
const cfg = this.cfg;
|
|
532
|
-
const
|
|
533
|
-
const request: TtsRequest = {
|
|
557
|
+
const resolved = resolveStyle(cfg, style);
|
|
558
|
+
const request: TtsRequest = {
|
|
559
|
+
text,
|
|
560
|
+
format: cfg.format,
|
|
561
|
+
signal,
|
|
562
|
+
...(style.voice ? { voice: style.voice } : {}),
|
|
563
|
+
};
|
|
564
|
+
switch (cfg.emotionMode) {
|
|
565
|
+
case 'instruct': {
|
|
566
|
+
const instruct = buildInstruct(resolved);
|
|
567
|
+
if (instruct) request.instruct = instruct;
|
|
568
|
+
break;
|
|
569
|
+
}
|
|
570
|
+
case 'categorical':
|
|
571
|
+
request.emotion = resolved.emotion || '平静';
|
|
572
|
+
break;
|
|
573
|
+
case 'basic':
|
|
574
|
+
request.speed = resolved.speed;
|
|
575
|
+
break;
|
|
576
|
+
default:
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
534
579
|
try {
|
|
535
|
-
const audio = await
|
|
536
|
-
this.
|
|
580
|
+
const audio = await this.service.synthesize(request, signal);
|
|
581
|
+
this.markService({ ok: true });
|
|
537
582
|
return audio;
|
|
538
583
|
} catch (error) {
|
|
539
584
|
const detail = error instanceof Error ? error.message : String(error);
|
|
540
|
-
this.
|
|
585
|
+
this.markService({ ok: false, detail });
|
|
541
586
|
void this.pushFailure(detail, { source: 'synthesize', chars: [...text].length });
|
|
542
587
|
throw error;
|
|
543
588
|
}
|
|
@@ -570,7 +615,7 @@ export class VoiceWorld implements World {
|
|
|
570
615
|
private async pushFailure(detail: string, meta: Record<string, unknown>): Promise<void> {
|
|
571
616
|
const host = this.host;
|
|
572
617
|
if (!host) return;
|
|
573
|
-
//
|
|
618
|
+
// 服务长时间不可用时,自动朗读会每句都失败;事件留一条,日志照常。
|
|
574
619
|
const now = this.deps.now();
|
|
575
620
|
if (now - this.lastFailureAt < 30_000) return;
|
|
576
621
|
this.lastFailureAt = now;
|
|
@@ -587,11 +632,11 @@ export class VoiceWorld implements World {
|
|
|
587
632
|
} catch { /* 只归档,失败就只留日志 */ }
|
|
588
633
|
}
|
|
589
634
|
|
|
590
|
-
// ————————
|
|
635
|
+
// ———————— 状态 ————————
|
|
591
636
|
|
|
592
|
-
private
|
|
593
|
-
const previous = this.
|
|
594
|
-
this.
|
|
637
|
+
private markService(state: Pick<ServiceState, 'ok' | 'detail'>): void {
|
|
638
|
+
const previous = this.serviceState;
|
|
639
|
+
this.serviceState = { ...state, at: this.deps.now() };
|
|
595
640
|
if (previous?.ok !== state.ok) {
|
|
596
641
|
if (state.ok) this.log.info('语音服务可用');
|
|
597
642
|
else this.log.warn('语音服务不可用', { detail: state.detail ?? '' });
|
|
@@ -602,18 +647,13 @@ export class VoiceWorld implements World {
|
|
|
602
647
|
if (this.probing) return;
|
|
603
648
|
this.probing = true;
|
|
604
649
|
try {
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
url: this.cfg.singUrl,
|
|
610
|
-
token: this.cfg.singToken || undefined,
|
|
611
|
-
fetch: this.deps.fetch,
|
|
650
|
+
const health: ServiceHealth = await this.service.health();
|
|
651
|
+
this.markService({
|
|
652
|
+
ok: health.ok && health.ready,
|
|
653
|
+
detail: health.detail ?? health.status,
|
|
612
654
|
});
|
|
613
|
-
else this.singReady = false;
|
|
614
|
-
await this.refreshVoices();
|
|
615
655
|
} catch (error) {
|
|
616
|
-
this.
|
|
656
|
+
this.markService({ ok: false, detail: error instanceof Error ? error.message : String(error) });
|
|
617
657
|
} finally {
|
|
618
658
|
this.probing = false;
|
|
619
659
|
}
|
|
@@ -622,97 +662,9 @@ export class VoiceWorld implements World {
|
|
|
622
662
|
private maybeRefresh(): void {
|
|
623
663
|
if (!this.started) return;
|
|
624
664
|
const now = this.deps.now();
|
|
625
|
-
if (!this.probing && (this.
|
|
665
|
+
if (!this.probing && (this.serviceState === null || now - this.serviceState.at > HEALTH_TTL_MS)) {
|
|
626
666
|
void this.probe();
|
|
627
667
|
}
|
|
628
|
-
if (!this.voicesLoading && now - this.voicesAt > VOICES_TTL_MS) void this.refreshVoices();
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
private async refreshVoices(force = false): Promise<void> {
|
|
632
|
-
const cfg = this.cfg;
|
|
633
|
-
const signature = `${cfg.backend}|${cfg.baseUrl}|${cfg.model}`;
|
|
634
|
-
if (this.voicesLoading && !force) return;
|
|
635
|
-
if (!force && signature === this.voicesSignature && this.deps.now() - this.voicesAt < VOICES_TTL_MS) return;
|
|
636
|
-
this.voicesLoading = true;
|
|
637
|
-
try {
|
|
638
|
-
const backend = this.deps.backend ?? backendOf(cfg.backend);
|
|
639
|
-
const list = await backend.listVoices(this.backendContext());
|
|
640
|
-
if (signature !== `${this.cfg.backend}|${this.cfg.baseUrl}|${this.cfg.model}`) return;
|
|
641
|
-
this.voices = list;
|
|
642
|
-
this.voicesSignature = signature;
|
|
643
|
-
this.voicesAt = this.deps.now();
|
|
644
|
-
} catch (error) {
|
|
645
|
-
this.log.debug('音色表读取失败', { err: error instanceof Error ? error.message : String(error) });
|
|
646
|
-
} finally {
|
|
647
|
-
this.voicesLoading = false;
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
private voiceOptions(): VoiceOption[] {
|
|
652
|
-
const current = this.cfg.voice.trim();
|
|
653
|
-
const out = [...this.voices];
|
|
654
|
-
if (current && !out.some((v) => v.value === current)) out.unshift({ value: current, label: current });
|
|
655
|
-
return out;
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
private async startHttp(): Promise<void> {
|
|
659
|
-
if (this.http) return;
|
|
660
|
-
try {
|
|
661
|
-
this.http = await startVoiceHttp({
|
|
662
|
-
port: this.cfg.httpPort,
|
|
663
|
-
deps: {
|
|
664
|
-
token: () => this.cfg.httpToken,
|
|
665
|
-
status: () => ({
|
|
666
|
-
ready: this.backendState?.ok === true,
|
|
667
|
-
state: this.backendState?.detail ?? (this.backendState?.ok ? 'ready' : 'unknown'),
|
|
668
|
-
backend: backendOf(this.cfg.backend).label,
|
|
669
|
-
model: this.cfg.model,
|
|
670
|
-
voice: this.cfg.voice,
|
|
671
|
-
queue: this.queue.stats().queued,
|
|
672
|
-
sing: !!this.cfg.singUrl.trim(),
|
|
673
|
-
}),
|
|
674
|
-
synthesize: async ({ request, play }, signal) => {
|
|
675
|
-
const audio = await this.synthesize(request.text, {
|
|
676
|
-
...(request.emotion ? { emotion: request.emotion } : {}),
|
|
677
|
-
...(request.instruct ? { instruct: request.instruct } : {}),
|
|
678
|
-
...(request.voice ? { voice: request.voice } : {}),
|
|
679
|
-
...(request.speed !== undefined ? { speed: request.speed } : {}),
|
|
680
|
-
}, signal);
|
|
681
|
-
if (play || this.cfg.httpPlay) {
|
|
682
|
-
this.queue.enqueue({
|
|
683
|
-
text: request.text,
|
|
684
|
-
chunks: [request.text],
|
|
685
|
-
silent: false,
|
|
686
|
-
source: 'http',
|
|
687
|
-
produce: async () => audio,
|
|
688
|
-
});
|
|
689
|
-
}
|
|
690
|
-
return audio;
|
|
691
|
-
},
|
|
692
|
-
sing: async (lyrics, signal) => {
|
|
693
|
-
if (!this.cfg.singUrl.trim()) throw new Error('no singing service configured');
|
|
694
|
-
return singRemote({
|
|
695
|
-
url: this.cfg.singUrl,
|
|
696
|
-
token: this.cfg.singToken,
|
|
697
|
-
lyrics,
|
|
698
|
-
timeoutMs: Math.max(this.cfg.timeoutMs, 180_000),
|
|
699
|
-
fetch: this.deps.fetch,
|
|
700
|
-
signal,
|
|
701
|
-
});
|
|
702
|
-
},
|
|
703
|
-
voices: async () => {
|
|
704
|
-
await this.refreshVoices(true);
|
|
705
|
-
return this.voiceOptions();
|
|
706
|
-
},
|
|
707
|
-
log: this.log,
|
|
708
|
-
},
|
|
709
|
-
});
|
|
710
|
-
this.httpError = '';
|
|
711
|
-
this.log.info('本地语音接口已就绪', { port: this.http.port });
|
|
712
|
-
} catch (error) {
|
|
713
|
-
this.httpError = error instanceof Error ? error.message : String(error);
|
|
714
|
-
this.log.error('本地语音接口启动失败', { err: this.httpError });
|
|
715
|
-
}
|
|
716
668
|
}
|
|
717
669
|
|
|
718
670
|
private syncPoller(): void {
|
|
@@ -753,4 +705,4 @@ export class VoiceWorld implements World {
|
|
|
753
705
|
}
|
|
754
706
|
}
|
|
755
707
|
|
|
756
|
-
export { VOICE_DEFAULTS, VOICE_ID
|
|
708
|
+
export { VOICE_DEFAULTS, VOICE_ID };
|