arona-agent 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/package.json +1 -1
- package/pet/renderer/style.css +16 -2
- package/python/__pycache__/tts_say.cpython-314.pyc +0 -0
- package/python/tts_say.py +66 -10
- package/src/agent.ts +2 -1
- package/src/gesture_context.ts +48 -0
- package/src/memory.ts +3 -10
- package/src/renderer.ts +11 -0
- package/src/repl.ts +7 -26
- package/src/tts_stream.ts +197 -21
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
## 安装
|
|
16
16
|
|
|
17
17
|
```bash
|
|
18
|
-
npm install -g
|
|
18
|
+
npm install -g arona-agent
|
|
19
19
|
|
|
20
20
|
# 初始化配置文件
|
|
21
21
|
arona setup
|
|
@@ -23,6 +23,9 @@ arona setup
|
|
|
23
23
|
# 初始化后可直接启动
|
|
24
24
|
arona
|
|
25
25
|
|
|
26
|
+
# 更新
|
|
27
|
+
npm update -g arona-agent
|
|
28
|
+
|
|
26
29
|
# 禁用TTS+STT并启动
|
|
27
30
|
arona --no-voice
|
|
28
31
|
|
|
@@ -146,4 +149,4 @@ arona voice add [<角色名>] # 不带角色名则进入 TUI 选择未补全
|
|
|
146
149
|
- 本项目不得用于影响版权方权益之分发,否则自行承担相关侵权责任。
|
|
147
150
|
|
|
148
151
|
此类内容的知识产权归Nexon Games Co., Ltd.及YOSTAR LIMITED所有,其使用、复制及分发规则应严格遵循上述权利方的官方条款及适用法律法规。
|
|
149
|
-
若相关权利人认为本声明或项目使用方式存在不当,可与我联系,将在核实后第一时间调整或移除相关内容。
|
|
152
|
+
若相关权利人认为本声明或项目使用方式存在不当,可与我联系,将在核实后第一时间调整或移除相关内容。
|
package/package.json
CHANGED
package/pet/renderer/style.css
CHANGED
|
@@ -34,11 +34,12 @@ body:active {
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
/* 桌宠文字气泡:定位到角色头部右侧,贴近角色(不遮脸、不高高飘在右上);
|
|
37
|
-
top 取头部高度(头 CSS y 40~195),left
|
|
37
|
+
top 取头部高度(头 CSS y 40~195),left 270px 贴近头部右缘(≈250)留 ~20px 缝隙,
|
|
38
|
+
由 ::after 小尾巴向左伸进缝隙指向角色(视觉上"连到"角色,不显远);
|
|
38
39
|
高于 Spine、鼠标穿透;只显示 mid/final 短消息 */
|
|
39
40
|
#pet-bubble {
|
|
40
41
|
position: absolute;
|
|
41
|
-
left:
|
|
42
|
+
left: 270px;
|
|
42
43
|
top: 90px;
|
|
43
44
|
transform: none;
|
|
44
45
|
max-width: 220px;
|
|
@@ -58,6 +59,19 @@ body:active {
|
|
|
58
59
|
transition: opacity 0.25s ease;
|
|
59
60
|
}
|
|
60
61
|
|
|
62
|
+
/* 左指小三角尾巴:从气泡左缘向左伸进气泡与角色头部的缝隙里,指向角色。
|
|
63
|
+
随父级 hidden / opacity 过渡一起隐藏。top 取气泡中部即可,位置可按需微调 */
|
|
64
|
+
#pet-bubble::after {
|
|
65
|
+
content: "";
|
|
66
|
+
position: absolute;
|
|
67
|
+
left: -12px;
|
|
68
|
+
top: 50%;
|
|
69
|
+
transform: translateY(-50%);
|
|
70
|
+
border-top: 8px solid transparent;
|
|
71
|
+
border-bottom: 8px solid transparent;
|
|
72
|
+
border-right: 12px solid rgba(20, 22, 34, 0.82);
|
|
73
|
+
}
|
|
74
|
+
|
|
61
75
|
#pet-bubble.hidden {
|
|
62
76
|
opacity: 0;
|
|
63
77
|
}
|
|
Binary file
|
package/python/tts_say.py
CHANGED
|
@@ -145,6 +145,65 @@ def play_wav(data):
|
|
|
145
145
|
_play_fallback(data)
|
|
146
146
|
|
|
147
147
|
|
|
148
|
+
def _write_temp_wav(data):
|
|
149
|
+
"""写入临时 wav,返回路径(供 synth_only 模式:合成后暂存,由 play 模式进程读取并删除)。"""
|
|
150
|
+
tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
|
|
151
|
+
tmp.write(data)
|
|
152
|
+
tmp.close()
|
|
153
|
+
return tmp.name
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _synth_and_play(cmd):
|
|
157
|
+
"""默认模式:合成 + 播放(向后兼容,预合成流水线回退路径)。"""
|
|
158
|
+
text = (cmd.get("text") or "").strip()
|
|
159
|
+
voice = cmd.get("voice") or ""
|
|
160
|
+
if not text:
|
|
161
|
+
emit("error", message=t("TTS: 空文本", "TTS: empty text"))
|
|
162
|
+
sys.stdout.flush()
|
|
163
|
+
return
|
|
164
|
+
audio = synthesize(text, voice)
|
|
165
|
+
emit("play_start")
|
|
166
|
+
play_wav(audio)
|
|
167
|
+
emit("play_end")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _synth_only(cmd):
|
|
171
|
+
"""预合成模式:只合成,写临时 wav 后发 synth_done(带路径)退出,不播放。
|
|
172
|
+
Node 侧在上一句播放期间调用本模式预合成下一句,消除句间 HTTP 合成停顿。"""
|
|
173
|
+
text = (cmd.get("text") or "").strip()
|
|
174
|
+
voice = cmd.get("voice") or ""
|
|
175
|
+
if not text:
|
|
176
|
+
emit("error", message=t("TTS: 空文本", "TTS: empty text"))
|
|
177
|
+
sys.stdout.flush()
|
|
178
|
+
return
|
|
179
|
+
audio = synthesize(text, voice)
|
|
180
|
+
path = _write_temp_wav(audio)
|
|
181
|
+
emit("synth_done", path=path)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _play_path(cmd):
|
|
185
|
+
"""播放模式:读已合成的临时 wav 播放(无 HTTP),播完删除临时文件。"""
|
|
186
|
+
path = cmd.get("path") or ""
|
|
187
|
+
if not path or not os.path.exists(path):
|
|
188
|
+
emit("error", message=t("TTS: 无效的音频路径", "TTS: invalid audio path"))
|
|
189
|
+
sys.stdout.flush()
|
|
190
|
+
return
|
|
191
|
+
try:
|
|
192
|
+
with open(path, "rb") as f:
|
|
193
|
+
data = f.read()
|
|
194
|
+
except Exception as e:
|
|
195
|
+
emit("error", message=str(e))
|
|
196
|
+
sys.stdout.flush()
|
|
197
|
+
return
|
|
198
|
+
try:
|
|
199
|
+
os.unlink(path)
|
|
200
|
+
except Exception:
|
|
201
|
+
pass
|
|
202
|
+
emit("play_start")
|
|
203
|
+
play_wav(data)
|
|
204
|
+
emit("play_end")
|
|
205
|
+
|
|
206
|
+
|
|
148
207
|
def main():
|
|
149
208
|
emit("ready")
|
|
150
209
|
line = sys.stdin.readline()
|
|
@@ -156,17 +215,14 @@ def main():
|
|
|
156
215
|
emit("error", message=t("TTS: 无效的 stdin JSON", "TTS: invalid stdin JSON"))
|
|
157
216
|
sys.stdout.flush()
|
|
158
217
|
return
|
|
159
|
-
|
|
160
|
-
voice = cmd.get("voice") or ""
|
|
161
|
-
if not text:
|
|
162
|
-
emit("error", message=t("TTS: 空文本", "TTS: empty text"))
|
|
163
|
-
sys.stdout.flush()
|
|
164
|
-
return
|
|
218
|
+
mode = cmd.get("mode") or "synth_play"
|
|
165
219
|
try:
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
220
|
+
if mode == "synth_only":
|
|
221
|
+
_synth_only(cmd)
|
|
222
|
+
elif mode == "play":
|
|
223
|
+
_play_path(cmd)
|
|
224
|
+
else:
|
|
225
|
+
_synth_and_play(cmd)
|
|
170
226
|
sys.stdout.flush()
|
|
171
227
|
except Exception as e:
|
|
172
228
|
emit("error", message=str(e))
|
package/src/agent.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { connectMcpServers } from "./mcp.ts";
|
|
|
21
21
|
import { InMemoryCredentialStore } from "./in_memory_credentials.ts";
|
|
22
22
|
import { getMainAgent, type SubAgentId, type AgentId } from "./agent_registry.ts";
|
|
23
23
|
import { speakerContextExtension } from "./speaker_context.ts";
|
|
24
|
+
import { gestureContextExtension } from "./gesture_context.ts";
|
|
24
25
|
import { t, getLang } from "./locale.ts";
|
|
25
26
|
|
|
26
27
|
// Asia/Shanghai 当前时间,注入到 system prompt 供情境台词使用;语言随界面
|
|
@@ -571,7 +572,7 @@ export async function initAgent(): Promise<{
|
|
|
571
572
|
appendSystemPromptOverride: () => [],
|
|
572
573
|
// 群聊发言者标注:发送边界给带 speaker 的历史 assistant 消息加「角色名:」前缀,
|
|
573
574
|
// 让模型区分谁说的(speaker 字段不会发给模型,必须编码进文本)
|
|
574
|
-
extensionFactories: [speakerContextExtension],
|
|
575
|
+
extensionFactories: [speakerContextExtension, gestureContextExtension],
|
|
575
576
|
});
|
|
576
577
|
await loader.reload();
|
|
577
578
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// 桌宠手势上下文注入扩展:摸头/dizzy 触发后,在**主 Agent** 的发送边界(context 事件)
|
|
2
|
+
// 追加一条 user 消息,描述"刚才发生了什么",引导主 Agent 在回复中自然流露出确实感受到了。
|
|
3
|
+
//
|
|
4
|
+
// 为什么用发送边界而不是拼进用户输入:repl.ts 的 processInput 会把文本写进主 session 的
|
|
5
|
+
// state.messages,而子 Agent 每轮复制主 session 全量历史(repl.ts runOneAgent)——那样手势
|
|
6
|
+
// 提示词会下放给子 Agent、还会污染会话命名/存储。发送边界注入只改本次请求的消息数组、
|
|
7
|
+
// 不进 state.messages → 仅主 Agent 看到、零存储污染、会话命名天然干净。
|
|
8
|
+
//
|
|
9
|
+
// 只注册在**主 Agent** 的 loader 扩展列表(agent.ts initAgent),子 Agent(initSubAgent)
|
|
10
|
+
// 的 loader 不含本扩展 → 不会收到。消费经 pet.takeGesture()(消费即清空,只注入最近一次)。
|
|
11
|
+
|
|
12
|
+
import type { ContextEvent, ExtensionAPI, InlineExtension } from "@earendil-works/pi-coding-agent";
|
|
13
|
+
import { pet } from "./pet.ts";
|
|
14
|
+
import { t } from "./locale.ts";
|
|
15
|
+
|
|
16
|
+
/** 桌宠手势 → user 消息的上下文场景文本(双语)。 */
|
|
17
|
+
function petGestureScene(type: "pat" | "dizzy"): string {
|
|
18
|
+
if (type === "dizzy") {
|
|
19
|
+
return t(
|
|
20
|
+
"(Sensei刚才拖着你到处晃来晃去,你感觉有些头晕。请在回复里自然地体现出这份晕眩感。)",
|
|
21
|
+
"(Sensei just swung you around, and you feel a bit dizzy. Let that dizziness show naturally in your reply.)",
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
return t(
|
|
25
|
+
"(Sensei刚才摸了摸你的头。请在回复里自然地体现出被摸头的感受。)",
|
|
26
|
+
"(Sensei just petted your head. Let that show naturally in your reply.)",
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const gestureContextExtension: InlineExtension = {
|
|
31
|
+
name: "arona-gesture-context",
|
|
32
|
+
hidden: true, // 不显示在启动扩展列表
|
|
33
|
+
factory: (pi: ExtensionAPI) => {
|
|
34
|
+
pi.on("context", (event: ContextEvent) => {
|
|
35
|
+
// 有手势时才注入;takeGesture 消费即清空 → 一次 LLM 调用只注入一次、无手势时原样返回
|
|
36
|
+
const gesture = pet.takeGesture();
|
|
37
|
+
if (!gesture) return;
|
|
38
|
+
const scene = petGestureScene(gesture);
|
|
39
|
+
if (!scene) return;
|
|
40
|
+
return {
|
|
41
|
+
messages: [
|
|
42
|
+
...event.messages,
|
|
43
|
+
{ role: "user", content: [{ type: "text", text: scene }] },
|
|
44
|
+
],
|
|
45
|
+
};
|
|
46
|
+
});
|
|
47
|
+
},
|
|
48
|
+
};
|
package/src/memory.ts
CHANGED
|
@@ -171,12 +171,7 @@ interface SessionHeader {
|
|
|
171
171
|
preview: string;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
/**
|
|
175
|
-
* 从首条 user 消息提取会话预览。
|
|
176
|
-
* 剥离 repl.ts 注入的桌宠手势场景块(全角括号 `(…)` 包裹、后随空行),
|
|
177
|
-
* 避免"摸头/dizzy 提示词"污染会话命名(用户首条消息常是手势触发)。
|
|
178
|
-
* 仅影响预览命名,不改动存储内容。
|
|
179
|
-
*/
|
|
174
|
+
/** 从首条 user 消息提取会话预览。 */
|
|
180
175
|
function firstUserPreview(messages: any[]): string {
|
|
181
176
|
const firstUserMsg = messages.find((m) => m.role === "user");
|
|
182
177
|
if (!firstUserMsg) return "(empty)";
|
|
@@ -189,10 +184,8 @@ function firstUserPreview(messages: any[]): string {
|
|
|
189
184
|
.map((c: any) => c.text)
|
|
190
185
|
.join(" ")
|
|
191
186
|
: "";
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
let preview = cleaned.slice(0, 50).replace(/\n/g, " ");
|
|
195
|
-
if (cleaned.length > 50) preview += "...";
|
|
187
|
+
let preview = content.slice(0, 50).replace(/\n/g, " ");
|
|
188
|
+
if (content.length > 50) preview += "...";
|
|
196
189
|
return preview;
|
|
197
190
|
}
|
|
198
191
|
|
package/src/renderer.ts
CHANGED
|
@@ -161,6 +161,17 @@ export function createRenderer(
|
|
|
161
161
|
setSpeakerLabel(label: string | undefined) {
|
|
162
162
|
speakerLabel = label;
|
|
163
163
|
},
|
|
164
|
+
// 切 session(setActiveAgent)时显式复位回合状态:消除跨会话 curMsgText/lastText 残留,
|
|
165
|
+
// 防止被新 session 的 agent_end 误读上一角色文本。
|
|
166
|
+
resetTurn() {
|
|
167
|
+
curMsgText = "";
|
|
168
|
+
lastText = "";
|
|
169
|
+
thinkingBuffer = "";
|
|
170
|
+
drawnThinkingLines = 0;
|
|
171
|
+
inThinking = false;
|
|
172
|
+
inText = false;
|
|
173
|
+
textPrefixWritten = false;
|
|
174
|
+
},
|
|
164
175
|
subscribe: (session: any) => {
|
|
165
176
|
return session.subscribe((event: any) => {
|
|
166
177
|
switch (event.type) {
|
package/src/repl.ts
CHANGED
|
@@ -12,7 +12,7 @@ import * as voice from "./voice.ts";
|
|
|
12
12
|
import { TtsStream } from "./tts_stream.ts";
|
|
13
13
|
import { stopComputerUse } from "./tools/computer_use.ts";
|
|
14
14
|
import { disconnectAllMcp } from "./mcp.ts";
|
|
15
|
-
import { pet, stopPet
|
|
15
|
+
import { pet, stopPet } from "./pet.ts";
|
|
16
16
|
import { SlashMenu } from "./slash_menu.ts";
|
|
17
17
|
import { printLogo } from "./logo.ts";
|
|
18
18
|
import { PYTHON_DIR } from "./config.ts";
|
|
@@ -25,20 +25,6 @@ import { getMainAgent, getSubAgents, getAgentLabel, type AgentId, type SubAgentI
|
|
|
25
25
|
// STT 长按阈值:按下录音热键持续 ≥ 该毫秒数并在释放时才触发录音;提前松开视为误触
|
|
26
26
|
const STT_HOLD_MS = 2000;
|
|
27
27
|
|
|
28
|
-
/** 桌宠手势 → 注入用户消息头部的上下文场景行(双语)。引导 Agent 在回复中自然流露出被摸头/被摇晃的感受。 */
|
|
29
|
-
function petGestureScene(type: PetGestureType): string {
|
|
30
|
-
if (type === "dizzy") {
|
|
31
|
-
return t(
|
|
32
|
-
"(Sensei刚才拖着你到处晃来晃去,你感觉有些头晕。请在回复里自然地体现出这份晕眩感。)",
|
|
33
|
-
"(Sensei just swung you around, and you feel a bit dizzy. Let that dizziness show naturally in your reply.)",
|
|
34
|
-
);
|
|
35
|
-
}
|
|
36
|
-
return t(
|
|
37
|
-
"(Sensei刚才摸了摸你的头。请在回复里自然地体现出被摸头的感受。)",
|
|
38
|
-
"(Sensei just petted your head. Let that show naturally in your reply.)",
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
28
|
export class Repl {
|
|
43
29
|
private rl: readline.Interface;
|
|
44
30
|
private session: AgentSession;
|
|
@@ -630,18 +616,11 @@ export class Repl {
|
|
|
630
616
|
}
|
|
631
617
|
|
|
632
618
|
private async processInput(input: string) {
|
|
633
|
-
// 桌宠手势(摸头/dizzy
|
|
634
|
-
//
|
|
635
|
-
// takeGesture
|
|
636
|
-
// processInput 不会被调用(见 line handler 的 / 分支),手势保留到下一条真实消息。
|
|
637
|
-
const gesture = pet.takeGesture();
|
|
638
|
-
let effective = input;
|
|
639
|
-
if (gesture) {
|
|
640
|
-
const scene = petGestureScene(gesture);
|
|
641
|
-
if (scene) effective = `${scene}\n\n${input}`;
|
|
642
|
-
}
|
|
619
|
+
// 桌宠手势(摸头/dizzy)不再拼进用户消息:落到主 Agent 发送边界注入(gesture_context.ts),
|
|
620
|
+
// 不进 state.messages → 子 Agent 复制主 session 历史时看不到、会话命名/存储零污染。
|
|
621
|
+
// takeGesture 消费即清空由发送边界扩展完成,只注入最近一次。
|
|
643
622
|
// 展开 @文件 / !命令 后走完整回合生命周期
|
|
644
|
-
await this.runRawTurn(this.parseInput(
|
|
623
|
+
await this.runRawTurn(this.parseInput(input));
|
|
645
624
|
}
|
|
646
625
|
|
|
647
626
|
/** 把 renderer 订阅切到指定角色 session,并记录当前发言者。 */
|
|
@@ -649,6 +628,8 @@ export class Repl {
|
|
|
649
628
|
this.activeAgentId = agentId;
|
|
650
629
|
this.activeSession = session;
|
|
651
630
|
this.renderer.setSpeakerLabel(getAgentLabel(agentId));
|
|
631
|
+
// 显式复位回合状态,杜绝跨 session 残留 curMsgText/lastText 被误读
|
|
632
|
+
this.renderer.resetTurn();
|
|
652
633
|
this.rendererUnsub?.();
|
|
653
634
|
this.rendererUnsub = this.renderer.subscribe(session);
|
|
654
635
|
}
|
package/src/tts_stream.ts
CHANGED
|
@@ -21,13 +21,27 @@ import { splitStreamedText, countTextUnits } from "./text_split.ts";
|
|
|
21
21
|
* 过程性发言不朗读)——避免被 change_emotion 等工具调用切成多段的句子被拆读
|
|
22
22
|
* - 整段字数判断:countTextUnits(整段) >= MAX_TURN_LEN 则整段跳过不朗读(长回复静音)
|
|
23
23
|
* - 围栏代码块(```...```)整块剔除;句内 markdown 成句后剔除
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
* -
|
|
24
|
+
* - 句子进入串行播放队列 + 单槽预合成流水线:上一句播放期间预合成下一句(synth_only 写临时
|
|
25
|
+
* wav),句间只付一次无 HTTP 的 play 进程启动,消除句间合成停顿;预合成失败回退合成+播放
|
|
26
|
+
* - 音色按句入队时固化发音人 {text, agent},spawn 用该角色 voice_id(不受后续 setVoice 影响)
|
|
27
|
+
* - 每句带 TTS_SENTENCE_TIMEOUT_MS 保险丝:卡死的句子被跳过,busy 必然复位,整体不静音
|
|
28
|
+
* - 覆盖主 + 子 Agent:同一实例通过 setVoice 切当前角色判断开关,队列句音色已各自绑定
|
|
27
29
|
*/
|
|
28
30
|
// 整段字数阈值(countTextUnits 口径):>= 50 的回复不朗读
|
|
29
31
|
const MAX_SENTENCE_LEN = 50;
|
|
30
32
|
const TTS_FORCE_SPLIT_LEN = 9999; // 仅标点切句、不强切
|
|
33
|
+
// 每句合成+播放的时长保险丝:到点仍未 play_end/close 则主动杀进程、按"正常结束"推进下一句。
|
|
34
|
+
// 防止 busy 被卡死的句子永久钉死整条队列(多 Agent 紧邻回合会把单句故障放大成整体静音)。
|
|
35
|
+
const TTS_SENTENCE_TIMEOUT_MS = 30_000;
|
|
36
|
+
|
|
37
|
+
/** 队列项:入队即固化发音人,之后任何 setVoice/cancel 都不影响本句音色选择。
|
|
38
|
+
audioPath:undefined=未尝试预合成;null=预合成失败(回退合成+播放);string=预合成好的 wav 路径 */
|
|
39
|
+
interface PendingSentence {
|
|
40
|
+
text: string;
|
|
41
|
+
agent: AgentId;
|
|
42
|
+
audioPath?: string | null;
|
|
43
|
+
prefetchPromise?: Promise<void>;
|
|
44
|
+
}
|
|
31
45
|
|
|
32
46
|
/** 剔除整段文本中的 ``` 代码围栏(文本已完整,无需跨 delta 状态机)。 */
|
|
33
47
|
function stripCodeBlocks(s: string): string {
|
|
@@ -38,10 +52,12 @@ function stripCodeBlocks(s: string): string {
|
|
|
38
52
|
export class TtsStream {
|
|
39
53
|
private proc: ChildProcessWithoutNullStreams | null = null;
|
|
40
54
|
private pendingCount = 0; // 活跃播放段数(play_start +1 / play_end -1)
|
|
41
|
-
private queue:
|
|
55
|
+
private queue: PendingSentence[] = []; // 待合成句子队列(按句绑定发音人)
|
|
42
56
|
private busy = false; // 当前是否有句子在合成/播放
|
|
43
57
|
private shuttingDown = false;
|
|
44
58
|
private _killRequested = false; // 本次是否因 cancel 主动杀进程(区分正常结束与打断)
|
|
59
|
+
private generation = 0; // 打断纪元:cancel 自增以作废在途 drain,杜绝双 drain 竞态
|
|
60
|
+
private prefetchProc: ChildProcessWithoutNullStreams | null = null; // 在途预合成进程(cancel 时一并杀)
|
|
45
61
|
private currentAgent: AgentId = getMainAgent();
|
|
46
62
|
|
|
47
63
|
constructor(
|
|
@@ -87,18 +103,22 @@ export class TtsStream {
|
|
|
87
103
|
return;
|
|
88
104
|
}
|
|
89
105
|
const { sentences, rest } = splitStreamedText(full, TTS_FORCE_SPLIT_LEN);
|
|
90
|
-
for (const s of sentences) this.emitSentence(s);
|
|
106
|
+
for (const s of sentences) this.emitSentence(s, this.currentAgent);
|
|
91
107
|
const tail = rest.trim();
|
|
92
|
-
if (tail) this.emitSentence(tail);
|
|
108
|
+
if (tail) this.emitSentence(tail, this.currentAgent);
|
|
93
109
|
void this.drain();
|
|
94
110
|
}
|
|
95
111
|
|
|
96
112
|
/** 打断当前合成与队列(新输入 / Esc / Ctrl+C / STT 触发时调用)。 */
|
|
97
113
|
cancel(): void {
|
|
114
|
+
// 自增纪元作废任何在途 drain,并复位 busy:即使被杀进程不 close,下一回合也不会被残余 busy 吞掉
|
|
115
|
+
this.generation++;
|
|
116
|
+
this.busy = false;
|
|
117
|
+
if (verbose) console.error("[tts] cancel (generation++ queue cleared busy=0)");
|
|
98
118
|
this.queue = [];
|
|
99
119
|
this.pendingCount = 0;
|
|
100
120
|
this.killProc();
|
|
101
|
-
|
|
121
|
+
this.killPrefetchProc();
|
|
102
122
|
}
|
|
103
123
|
|
|
104
124
|
/** 切换角色后清空残余(非流式音色动态取,无需杀进程重拉)。 */
|
|
@@ -121,7 +141,7 @@ export class TtsStream {
|
|
|
121
141
|
* 符号不计(countTextUnits);字数为 0(纯符号)的句子不朗读。
|
|
122
142
|
* (整段已 < MAX_SENTENCE_LEN,单句必然 < 阈值,此处过滤为纯防御)
|
|
123
143
|
*/
|
|
124
|
-
private emitSentence(sentence: string): void {
|
|
144
|
+
private emitSentence(sentence: string, agent: AgentId): void {
|
|
125
145
|
const clean = stripMarkdown(sentence).trim();
|
|
126
146
|
if (!clean) return;
|
|
127
147
|
const units = countTextUnits(clean);
|
|
@@ -129,49 +149,185 @@ export class TtsStream {
|
|
|
129
149
|
if (verbose) console.error(`[tts] skip sentence units=${units} "${clean.slice(0, 40)}"`);
|
|
130
150
|
return;
|
|
131
151
|
}
|
|
132
|
-
if (verbose) console.error(`[tts] queue push units=${units} "${clean.slice(0, 40)}"`);
|
|
133
|
-
|
|
152
|
+
if (verbose) console.error(`[tts] queue push agent=${agent} units=${units} "${clean.slice(0, 40)}"`);
|
|
153
|
+
// 入队即固化发音人:之后任何 setVoice/cancel 都不影响本句音色选择
|
|
154
|
+
this.queue.push({ text: clean, agent });
|
|
134
155
|
void this.drain();
|
|
135
156
|
}
|
|
136
157
|
|
|
137
|
-
/** 串行排空队列:合成 +
|
|
158
|
+
/** 串行排空队列:合成 + 播放一句再下一句;下一句在上一句播放期间预合成(单槽流水线)。
|
|
159
|
+
全部播完且未被打断时触发 onIdle。 */
|
|
138
160
|
private async drain(): Promise<void> {
|
|
139
161
|
if (this.busy || this.shuttingDown) return;
|
|
140
162
|
if (this.queue.length === 0) return;
|
|
163
|
+
const gen = this.generation;
|
|
141
164
|
this.busy = true;
|
|
142
165
|
let aborted = false;
|
|
143
166
|
try {
|
|
144
167
|
while (this.queue.length > 0 && !this.shuttingDown) {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
168
|
+
if (this.generation !== gen) return; // 已被 cancel 作废,停止处理
|
|
169
|
+
const item = this.queue.shift()!;
|
|
170
|
+
// 单槽预合成:当前句播放期间预合成下一句(queue[0]),消除句间 HTTP 合成停顿
|
|
171
|
+
const next = this.queue[0];
|
|
172
|
+
if (next && next.audioPath === undefined && !next.prefetchPromise) {
|
|
173
|
+
next.prefetchPromise = this.prefetch(next).finally(() => {
|
|
174
|
+
next.prefetchPromise = undefined;
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
// 短句兜底:上一句太短时预合成可能未完成,等其收尾(已从"从头合成"缩短为"等尾巴")
|
|
178
|
+
if (item.prefetchPromise) {
|
|
179
|
+
if (verbose) console.error(`[tts] await prefetch "${item.text.slice(0, 40)}"`);
|
|
180
|
+
await item.prefetchPromise;
|
|
181
|
+
if (this.generation !== gen) return; // 等待期间被 cancel 作废
|
|
182
|
+
}
|
|
183
|
+
if (verbose) console.error(`[tts] speakOne start agent=${item.agent} "${item.text.slice(0, 40)}"`);
|
|
184
|
+
const ok = await this.speakOne(item);
|
|
185
|
+
if (verbose) console.error(`[tts] speakOne end agent=${item.agent} ok=${ok} remaining=${this.queue.length}`);
|
|
149
186
|
if (!ok) {
|
|
150
187
|
aborted = true;
|
|
151
188
|
break;
|
|
152
189
|
}
|
|
153
190
|
}
|
|
154
191
|
} finally {
|
|
155
|
-
|
|
192
|
+
// 仅最年轻的 drain 复位 busy;被 cancel 作废的旧 drain 不再动它,避免清掉新一轮 drain 的 busy
|
|
193
|
+
if (this.generation === gen && this.busy) this.busy = false;
|
|
156
194
|
}
|
|
157
|
-
if (!aborted && !this.shuttingDown && this.pendingCount === 0) {
|
|
195
|
+
if (this.generation === gen && !aborted && !this.shuttingDown && this.pendingCount === 0) {
|
|
158
196
|
this.onIdle?.();
|
|
159
197
|
}
|
|
160
198
|
}
|
|
161
199
|
|
|
200
|
+
/**
|
|
201
|
+
* 预合成下一句:spawn 一次性 synth_only 进程,把 wav 写到临时文件,供播放时直接读(无 HTTP)。
|
|
202
|
+
* 成功(synth_done)→ item.audioPath = 路径(仅当 generation 未变);失败/超时/被杀 → null(回退合成+播放)。
|
|
203
|
+
* 在上一句播放期间调用,消除句间 HTTP 合成停顿。带 TTS_SENTENCE_TIMEOUT_MS 保险丝。
|
|
204
|
+
*/
|
|
205
|
+
private prefetch(item: PendingSentence): Promise<void> {
|
|
206
|
+
return new Promise((resolve) => {
|
|
207
|
+
if (verbose) console.error(`[tts] prefetch start agent=${item.agent} "${item.text.slice(0, 40)}"`);
|
|
208
|
+
let settled = false;
|
|
209
|
+
let timer: NodeJS.Timeout | undefined;
|
|
210
|
+
const gen = this.generation;
|
|
211
|
+
const doResolve = () => {
|
|
212
|
+
if (settled) return;
|
|
213
|
+
if (timer) clearTimeout(timer);
|
|
214
|
+
settled = true;
|
|
215
|
+
resolve();
|
|
216
|
+
};
|
|
217
|
+
const voiceId = getVoiceId(item.agent);
|
|
218
|
+
let stdoutBuffer = "";
|
|
219
|
+
const proc = spawnCompat(config.pythonPath, ["-u", join(PYTHON_DIR, "tts_say.py")], {
|
|
220
|
+
env: stripProxyEnv({
|
|
221
|
+
...process.env,
|
|
222
|
+
PYTHONUTF8: "1",
|
|
223
|
+
QWEN_WORKSPACE_ID: config.workspaceId,
|
|
224
|
+
QWEN_TTS_API_KEY: config.ttsApiKey,
|
|
225
|
+
QWEN_TTS_MODEL: config.ttsModel,
|
|
226
|
+
QWEN_TTS_VOICE: voiceId,
|
|
227
|
+
}),
|
|
228
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
229
|
+
});
|
|
230
|
+
this.prefetchProc = proc;
|
|
231
|
+
|
|
232
|
+
// 保险丝:预合成最多等 TTS_SENTENCE_TIMEOUT_MS,超时杀进程 + null 回退(busy 必然复位、整体不静音)
|
|
233
|
+
timer = setTimeout(() => {
|
|
234
|
+
if (settled) return;
|
|
235
|
+
if (verbose) console.error(`[tts] prefetch TIMEOUT (${TTS_SENTENCE_TIMEOUT_MS}ms) agent=${item.agent} "${item.text.slice(0, 40)}"`);
|
|
236
|
+
if (this.prefetchProc === proc) this.prefetchProc = null;
|
|
237
|
+
if (!proc.killed) {
|
|
238
|
+
try {
|
|
239
|
+
proc.kill("SIGTERM");
|
|
240
|
+
} catch {
|
|
241
|
+
// 忽略
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (this.generation === gen) item.audioPath = null;
|
|
245
|
+
doResolve();
|
|
246
|
+
}, TTS_SENTENCE_TIMEOUT_MS);
|
|
247
|
+
|
|
248
|
+
proc.stdout.on("data", (data) => {
|
|
249
|
+
stdoutBuffer += data.toString();
|
|
250
|
+
const lines = stdoutBuffer.split("\n");
|
|
251
|
+
stdoutBuffer = lines.pop() || "";
|
|
252
|
+
for (const line of lines) {
|
|
253
|
+
const trimmed = line.trim();
|
|
254
|
+
if (!trimmed) continue;
|
|
255
|
+
let evt: { event?: string; message?: string; path?: string };
|
|
256
|
+
try {
|
|
257
|
+
evt = JSON.parse(trimmed);
|
|
258
|
+
} catch {
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
if (evt.event === "synth_done" && evt.path) {
|
|
262
|
+
if (verbose) console.error(`[tts] prefetch synth_done agent=${item.agent} "${item.text.slice(0, 40)}"`);
|
|
263
|
+
if (this.generation === gen) item.audioPath = evt.path;
|
|
264
|
+
doResolve();
|
|
265
|
+
} else if (evt.event === "error") {
|
|
266
|
+
console.warn(t(`TTS: ${evt.message ?? "unknown error"}`, `TTS: ${evt.message ?? "unknown error"}`));
|
|
267
|
+
if (verbose) console.error(`[tts] prefetch error: ${evt.message}`);
|
|
268
|
+
if (this.generation === gen) item.audioPath = null;
|
|
269
|
+
doResolve();
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
proc.stderr.on("data", (data) => {
|
|
275
|
+
if (!verbose) return;
|
|
276
|
+
const msg = data.toString().trim();
|
|
277
|
+
if (msg) console.error(`[python:tts_say]`, msg);
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
proc.on("close", () => {
|
|
281
|
+
if (this.prefetchProc === proc) this.prefetchProc = null;
|
|
282
|
+
if (!settled) {
|
|
283
|
+
// 进程退出但没等到 synth_done(被杀 / 异常):按失败回退
|
|
284
|
+
if (this.generation === gen) item.audioPath = null;
|
|
285
|
+
doResolve();
|
|
286
|
+
}
|
|
287
|
+
});
|
|
288
|
+
proc.on("error", () => {
|
|
289
|
+
if (this.prefetchProc === proc) this.prefetchProc = null;
|
|
290
|
+
if (!settled) {
|
|
291
|
+
if (this.generation === gen) item.audioPath = null;
|
|
292
|
+
doResolve();
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
try {
|
|
297
|
+
proc.stdin.write(JSON.stringify({ mode: "synth_only", text: item.text, voice: voiceId }) + "\n");
|
|
298
|
+
proc.stdin.end();
|
|
299
|
+
} catch {
|
|
300
|
+
// 忽略
|
|
301
|
+
}
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
|
|
162
305
|
/**
|
|
163
306
|
* 合成并播放一句话:spawn 一次性 python 进程,等待其播完(play_end)或退出。
|
|
164
|
-
*
|
|
307
|
+
* 已预合成(item.audioPath 为 string)走 play 模式(无 HTTP,直接播临时 wav);
|
|
308
|
+
* 未预合成(含预合成失败回退)走默认合成+播放模式。
|
|
309
|
+
* 音色按句固定:环境变量与 stdin 都用该句所属角色的 voice_id,不受后续 setVoice 影响。
|
|
310
|
+
* 带 TTS_SENTENCE_TIMEOUT_MS 保险丝:到点仍未 play_end/close → 杀进程 + 按正常结束继续下一句,
|
|
311
|
+
* 保证 drain 必然推进、busy 必然复位,杜绝"卡死一句 → 整条 TTS 永久静音"。
|
|
312
|
+
* @returns true = 正常结束(含合成失败/超时,继续下一句);false = 被 cancel 打断
|
|
165
313
|
*/
|
|
166
|
-
private speakOne(
|
|
314
|
+
private speakOne(item: PendingSentence): Promise<boolean> {
|
|
167
315
|
return new Promise((resolve) => {
|
|
168
316
|
this._killRequested = false;
|
|
169
317
|
let settled = false;
|
|
318
|
+
let timer: NodeJS.Timeout | undefined; // 时长保险丝句柄(TDZ 安全:先声明,后赋值)
|
|
170
319
|
const doResolve = (val: boolean) => {
|
|
171
320
|
if (settled) return;
|
|
321
|
+
if (timer) clearTimeout(timer);
|
|
172
322
|
settled = true;
|
|
173
323
|
resolve(val);
|
|
174
324
|
};
|
|
325
|
+
const { text, agent } = item;
|
|
326
|
+
const voiceId = getVoiceId(agent);
|
|
327
|
+
const payload =
|
|
328
|
+
typeof item.audioPath === "string"
|
|
329
|
+
? { mode: "play", path: item.audioPath }
|
|
330
|
+
: { text, voice: voiceId };
|
|
175
331
|
let stdoutBuffer = "";
|
|
176
332
|
const proc = spawnCompat(config.pythonPath, ["-u", join(PYTHON_DIR, "tts_say.py")], {
|
|
177
333
|
env: stripProxyEnv({
|
|
@@ -180,12 +336,20 @@ export class TtsStream {
|
|
|
180
336
|
QWEN_WORKSPACE_ID: config.workspaceId,
|
|
181
337
|
QWEN_TTS_API_KEY: config.ttsApiKey,
|
|
182
338
|
QWEN_TTS_MODEL: config.ttsModel,
|
|
183
|
-
QWEN_TTS_VOICE:
|
|
339
|
+
QWEN_TTS_VOICE: voiceId,
|
|
184
340
|
}),
|
|
185
341
|
stdio: ["pipe", "pipe", "pipe"],
|
|
186
342
|
});
|
|
187
343
|
this.proc = proc;
|
|
188
344
|
|
|
345
|
+
// 时长保险丝:一句合成+播放最多等 TTS_SENTENCE_TIMEOUT_MS,超时主动杀进程并推进一步
|
|
346
|
+
timer = setTimeout(() => {
|
|
347
|
+
if (settled) return;
|
|
348
|
+
if (verbose) console.error(`[tts] speakOne TIMEOUT (${TTS_SENTENCE_TIMEOUT_MS}ms) agent=${agent} "${text.slice(0, 40)}"`);
|
|
349
|
+
if (this.proc === proc) this.killProc();
|
|
350
|
+
doResolve(true);
|
|
351
|
+
}, TTS_SENTENCE_TIMEOUT_MS);
|
|
352
|
+
|
|
189
353
|
proc.stdout.on("data", (data) => {
|
|
190
354
|
stdoutBuffer += data.toString();
|
|
191
355
|
const lines = stdoutBuffer.split("\n");
|
|
@@ -237,7 +401,7 @@ export class TtsStream {
|
|
|
237
401
|
});
|
|
238
402
|
|
|
239
403
|
try {
|
|
240
|
-
proc.stdin.write(JSON.stringify(
|
|
404
|
+
proc.stdin.write(JSON.stringify(payload) + "\n");
|
|
241
405
|
proc.stdin.end();
|
|
242
406
|
} catch {
|
|
243
407
|
// 忽略
|
|
@@ -245,6 +409,18 @@ export class TtsStream {
|
|
|
245
409
|
});
|
|
246
410
|
}
|
|
247
411
|
|
|
412
|
+
private killPrefetchProc(): void {
|
|
413
|
+
const proc = this.prefetchProc;
|
|
414
|
+
this.prefetchProc = null;
|
|
415
|
+
if (proc && !proc.killed) {
|
|
416
|
+
try {
|
|
417
|
+
proc.kill("SIGTERM");
|
|
418
|
+
} catch {
|
|
419
|
+
// 忽略
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
248
424
|
private killProc(): void {
|
|
249
425
|
this._killRequested = true;
|
|
250
426
|
const proc = this.proc;
|