agent-loft 0.2.113 → 0.2.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -1
- package/src/agents/codex/session.ts +31 -3
- package/src/channels/web/server.ts +69 -11
- package/src/core/hub.ts +3 -2
- package/src/core/realtime-relay.ts +142 -0
- package/src/core/types.ts +4 -1
- package/web/css/composer.css +4 -159
- package/web/css/voice-call.css +196 -0
- package/web/index.html +1 -0
- package/web/lib/rt-worklet.js +110 -0
- package/web/lib/ws.js +5 -2
- package/web/views/composer.js +10 -4
- package/web/views/voice-call.js +343 -83
package/README.md
CHANGED
|
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
|
|
|
54
54
|
- **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
|
|
55
55
|
- 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
|
|
56
56
|
- 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
|
|
57
|
-
- 语音对话(只有 Codex,包括 Codex
|
|
57
|
+
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话界面的齿轮里可以选音色和回答用的语言(记在这台设备上),切换设置后重连会保留静音状态和收起状态。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
|
|
58
58
|
- 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
|
|
59
59
|
- 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
|
|
60
60
|
- 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-loft",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.115",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
|
|
6
6
|
"keywords": [
|
|
@@ -58,9 +58,11 @@
|
|
|
58
58
|
"echarts": "^6.1.0",
|
|
59
59
|
"marked": "^16.0.0",
|
|
60
60
|
"mermaid": "^12.2.0",
|
|
61
|
+
"opusscript": "^0.1.1",
|
|
61
62
|
"qrcode": "^1.5.4",
|
|
62
63
|
"sharp": "^0.35.5",
|
|
63
64
|
"tsx": "^4.20.6",
|
|
65
|
+
"werift": "^0.27.0",
|
|
64
66
|
"ws": "^8.18.0"
|
|
65
67
|
},
|
|
66
68
|
"devDependencies": {
|
|
@@ -20,6 +20,7 @@ import type {
|
|
|
20
20
|
InteractionSpec,
|
|
21
21
|
InterruptOptions,
|
|
22
22
|
QuestionSpec,
|
|
23
|
+
RealtimeOptions,
|
|
23
24
|
RuntimeInfo,
|
|
24
25
|
SavedAttachment,
|
|
25
26
|
SessionOptions,
|
|
@@ -816,12 +817,12 @@ export class CodexSession implements AgentSession {
|
|
|
816
817
|
// ---------- 实时语音对话(thread/realtime/*,实验接口) ----------
|
|
817
818
|
|
|
818
819
|
/**
|
|
819
|
-
*
|
|
820
|
-
* 对端 answer 从 thread/realtime/sdp
|
|
820
|
+
* offer 是服务器这边的 WebRTC 端点(core/realtime-relay.ts)建的,app-server 去 OpenAI 建通话,
|
|
821
|
+
* 对端 answer 从 thread/realtime/sdp 通知回来。声音经服务器中转,浏览器那头走 WebSocket。
|
|
821
822
|
* 实测:ChatGPT 账号登录只能走 WebRTC(WebSocket 要 API key),协议版本要 v3。
|
|
822
823
|
* 语音模型把要干的活转交给这条 thread,app-server 自己开一轮(见 turn/started 里的 externalPending)。
|
|
823
824
|
*/
|
|
824
|
-
async startRealtime(offer: string): Promise<string> {
|
|
825
|
+
async startRealtime(offer: string, opts: RealtimeOptions = {}): Promise<string> {
|
|
825
826
|
if (this.disposed) throw new Error("会话已关闭");
|
|
826
827
|
if (this.rt) await this.stopRealtime();
|
|
827
828
|
const threadId = await this.open();
|
|
@@ -853,6 +854,7 @@ export class CodexSession implements AgentSession {
|
|
|
853
854
|
// 启动背景里有这台机器上最近的会话和目录结构:只给内置专家的管理员
|
|
854
855
|
includeStartupContext: !expert && this.opts.skills?.admin === true,
|
|
855
856
|
...(context ? { realtimeStartInstructions: context } : {}),
|
|
857
|
+
...realtimeVoiceParams(opts),
|
|
856
858
|
});
|
|
857
859
|
return await answer;
|
|
858
860
|
} catch (err) {
|
|
@@ -1336,6 +1338,32 @@ function kindMark(kind: unknown): string {
|
|
|
1336
1338
|
return k === "add" ? "+" : k === "delete" ? "-" : "~";
|
|
1337
1339
|
}
|
|
1338
1340
|
|
|
1341
|
+
/** v3 实时语音能用的音色(实测是 thread/realtime/listVoices 里 v1 那组,给 v2 的会报不支持;默认 cove) */
|
|
1342
|
+
export const REALTIME_VOICES = ["cove", "juniper", "maple", "spruce", "ember", "vale", "breeze", "arbor", "sol"] as const;
|
|
1343
|
+
|
|
1344
|
+
/**
|
|
1345
|
+
* 语音模型说哪种话:协议里没有语言参数,用 v3 的 initialItems 放一条 developer 指令。
|
|
1346
|
+
* 不在表里(或「自动」)就不加,跟着用户说的语言走
|
|
1347
|
+
*/
|
|
1348
|
+
export const REALTIME_LANGUAGES: Record<string, string> = {
|
|
1349
|
+
"zh-CN": "Always speak Mandarin Chinese (普通话) and use Simplified Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1350
|
+
"zh-TW": "Always speak Mandarin Chinese (國語) and use Traditional Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1351
|
+
yue: "Always speak Cantonese (粵語/廣東話), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1352
|
+
en: "Always speak English, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1353
|
+
ja: "Always speak Japanese (日本語), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1354
|
+
ko: "Always speak Korean (한국어), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1355
|
+
};
|
|
1356
|
+
|
|
1357
|
+
/** 页面选的音色、语言 → thread/realtime/start 的参数;认不出的忽略(用 Codex 默认) */
|
|
1358
|
+
export function realtimeVoiceParams(opts: RealtimeOptions): Record<string, unknown> {
|
|
1359
|
+
const voice = (REALTIME_VOICES as readonly string[]).includes(opts.voice ?? "") ? opts.voice : undefined;
|
|
1360
|
+
const language = opts.language ? REALTIME_LANGUAGES[opts.language] : undefined;
|
|
1361
|
+
return {
|
|
1362
|
+
...(voice ? { voice } : {}),
|
|
1363
|
+
...(language ? { initialItems: [{ role: "developer", text: language }] } : {}),
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1339
1367
|
/** 语音转交的一轮没拿到用户说的话时显示的占位 */
|
|
1340
1368
|
const VOICE_FALLBACK = `${VOICE_MARK}(语音)`;
|
|
1341
1369
|
|
|
@@ -19,6 +19,7 @@ import { skillEvents } from "../../core/skills.js";
|
|
|
19
19
|
import { expertEvents } from "../../core/experts.js";
|
|
20
20
|
import { enforceTerminalAccess, killAllTerminals, terminalEvents, watchTerminalAccess } from "../../core/terminals.js";
|
|
21
21
|
import { RUNNING_VERSION } from "../../core/upgrade.js";
|
|
22
|
+
import { RealtimeRelay, RELAY_RATE } from "../../core/realtime-relay.js";
|
|
22
23
|
import type { RealtimeEvent, Settlement } from "../../core/types.js";
|
|
23
24
|
import { getUser, isAdmin, legacyOwnerId, userEvents, type User } from "../../core/users.js";
|
|
24
25
|
import { currentUser, handleApi, takeUploads } from "./api.js";
|
|
@@ -268,8 +269,24 @@ type Client = {
|
|
|
268
269
|
pushEndpoint: string | null;
|
|
269
270
|
/** 最后一次收到这个页面的消息(含 ping) */
|
|
270
271
|
seen: number;
|
|
272
|
+
/** 这个页面正在打的语音对话:声音经这里中转(relay 建好之前是 undefined) */
|
|
273
|
+
rt?: { key: string; relay?: RealtimeRelay };
|
|
271
274
|
};
|
|
272
275
|
|
|
276
|
+
/** 一块麦克风 PCM 的上限(正常是 20ms ≈ 1KB) */
|
|
277
|
+
const MAX_AUDIO_CHUNK = 64 * 1024;
|
|
278
|
+
/** 发给页面的声音积压超过这么多就丢(手机网络卡住时别越攒越多、越放越晚) */
|
|
279
|
+
const MAX_AUDIO_BACKLOG = 256 * 1024;
|
|
280
|
+
|
|
281
|
+
/** 收掉这个页面的语音中转;stopCall:顺带让 Codex 挂断 */
|
|
282
|
+
function endRelay(client: Client, stopCall: boolean): void {
|
|
283
|
+
const rt = client.rt;
|
|
284
|
+
if (!rt) return;
|
|
285
|
+
client.rt = undefined;
|
|
286
|
+
rt.relay?.close();
|
|
287
|
+
if (stopCall) void hub.realtimeStop(rt.key).catch(() => {});
|
|
288
|
+
}
|
|
289
|
+
|
|
273
290
|
/** 每次进程启动换一个:客户端发现变了,就知道 seq 已经从头计数,得整页重拉 */
|
|
274
291
|
const EPOCH = Math.random().toString(36).slice(2, 10);
|
|
275
292
|
|
|
@@ -305,7 +322,10 @@ function broadcastEvent(env: EventEnvelope): void {
|
|
|
305
322
|
|
|
306
323
|
/** 语音对话的字幕和状态:只推给订阅了这条对话的页面(不进缓冲区,刷新后不重放) */
|
|
307
324
|
function broadcastRealtime({ key, ev }: { key: string; ev: RealtimeEvent }): void {
|
|
308
|
-
for (const client of clients)
|
|
325
|
+
for (const client of clients) {
|
|
326
|
+
if (ev.type === "closed" && client.rt?.key === key && client.rt.relay) endRelay(client, false);
|
|
327
|
+
if (client.subs.has(key) && canSeeKey(client.user, key)) send(client, { t: "rt", key, ev });
|
|
328
|
+
}
|
|
309
329
|
}
|
|
310
330
|
|
|
311
331
|
function broadcastInteraction(i: Interaction): void {
|
|
@@ -533,19 +553,45 @@ function handleClientMessage(client: Client, raw: string): void {
|
|
|
533
553
|
}
|
|
534
554
|
|
|
535
555
|
case "rt": {
|
|
536
|
-
//
|
|
556
|
+
// 实时语音对话:服务器自己和 OpenAI 建 WebRTC,声音在这条 WebSocket 上中转(二进制帧,24kHz 16 位 PCM)。
|
|
557
|
+
// start 等连上了才 ack;stop 挂断
|
|
537
558
|
if (!getConversation(key) || !canSeeKey(user, key)) return ack(false, { error: "会话不存在" });
|
|
538
559
|
if (msg.op === "start") {
|
|
539
|
-
|
|
540
|
-
|
|
560
|
+
endRelay(client, false);
|
|
561
|
+
const rt: NonNullable<Client["rt"]> = { key };
|
|
562
|
+
client.rt = rt;
|
|
541
563
|
client.subs.add(key);
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
564
|
+
const opts = {
|
|
565
|
+
...(typeof msg.voice === "string" && msg.voice ? { voice: msg.voice } : {}),
|
|
566
|
+
...(typeof msg.language === "string" && msg.language ? { language: msg.language } : {}),
|
|
567
|
+
};
|
|
568
|
+
void (async () => {
|
|
569
|
+
const { relay, offer } = await RealtimeRelay.create();
|
|
570
|
+
if (client.rt !== rt) {
|
|
571
|
+
relay.close();
|
|
572
|
+
throw new Error("已挂断");
|
|
573
|
+
}
|
|
574
|
+
rt.relay = relay;
|
|
575
|
+
relay.onAudio = (pcm) => {
|
|
576
|
+
if (client.ws.readyState === client.ws.OPEN && client.ws.bufferedAmount < MAX_AUDIO_BACKLOG) client.ws.send(pcm, { binary: true });
|
|
577
|
+
};
|
|
578
|
+
relay.onFailed = (reason) => {
|
|
579
|
+
if (client.rt !== rt) return;
|
|
580
|
+
endRelay(client, true);
|
|
581
|
+
send(client, { t: "rt", key, ev: { type: "closed", reason } });
|
|
582
|
+
};
|
|
583
|
+
const answer = await hub.realtimeStart(key, offer, opts);
|
|
584
|
+
if (client.rt !== rt) throw new Error("已挂断");
|
|
585
|
+
await relay.accept(answer);
|
|
586
|
+
ack(true, { rate: RELAY_RATE });
|
|
587
|
+
})().catch((err) => {
|
|
588
|
+
if (client.rt === rt) endRelay(client, true);
|
|
589
|
+
ack(false, { error: err instanceof Error ? err.message : String(err) });
|
|
590
|
+
});
|
|
546
591
|
return;
|
|
547
592
|
}
|
|
548
593
|
if (msg.op === "stop") {
|
|
594
|
+
if (client.rt?.key === key) endRelay(client, false);
|
|
549
595
|
void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
|
|
550
596
|
return;
|
|
551
597
|
}
|
|
@@ -653,15 +699,27 @@ export async function startWeb(): Promise<void> {
|
|
|
653
699
|
const client: Client = { ws, subs: new Set(), alive: true, token, user, visible: false, viewing: null, foreground: false, pushEndpoint: null, seen: Date.now() };
|
|
654
700
|
clients.add(client);
|
|
655
701
|
ws.on("pong", () => (client.alive = true));
|
|
656
|
-
ws.on("message", (data) => {
|
|
702
|
+
ws.on("message", (data, isBinary) => {
|
|
657
703
|
try {
|
|
704
|
+
// 二进制帧只有一种:语音对话里麦克风的 PCM
|
|
705
|
+
if (isBinary) {
|
|
706
|
+
const chunk = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer);
|
|
707
|
+
if (chunk.length <= MAX_AUDIO_CHUNK) client.rt?.relay?.pushAudio(chunk);
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
658
710
|
handleClientMessage(client, data.toString());
|
|
659
711
|
} catch (err) {
|
|
660
712
|
console.error("[web] 处理 WebSocket 消息出错:", err);
|
|
661
713
|
}
|
|
662
714
|
});
|
|
663
|
-
ws.on("close", () =>
|
|
664
|
-
|
|
715
|
+
ws.on("close", () => {
|
|
716
|
+
clients.delete(client);
|
|
717
|
+
endRelay(client, true);
|
|
718
|
+
});
|
|
719
|
+
ws.on("error", () => {
|
|
720
|
+
clients.delete(client);
|
|
721
|
+
endRelay(client, true);
|
|
722
|
+
});
|
|
665
723
|
send(client, { t: "hello", seq: hub.currentSeq(), epoch: EPOCH, version: RUNNING_VERSION });
|
|
666
724
|
});
|
|
667
725
|
} catch (err) {
|
package/src/core/hub.ts
CHANGED
|
@@ -50,6 +50,7 @@ import type {
|
|
|
50
50
|
InteractionSpec,
|
|
51
51
|
ModelInfo,
|
|
52
52
|
RealtimeEvent,
|
|
53
|
+
RealtimeOptions,
|
|
53
54
|
RuntimeInfo,
|
|
54
55
|
SessionSummary,
|
|
55
56
|
StepKind,
|
|
@@ -2057,7 +2058,7 @@ class Hub extends EventEmitter<{
|
|
|
2057
2058
|
// ---------- 实时语音对话(只有 Codex) ----------
|
|
2058
2059
|
|
|
2059
2060
|
/** 开始通话:offer 是浏览器的 WebRTC SDP,返回对端 answer。会话没拉起来就按绑定拉起 */
|
|
2060
|
-
async realtimeStart(key: string, offer: string): Promise<string> {
|
|
2061
|
+
async realtimeStart(key: string, offer: string, opts?: RealtimeOptions): Promise<string> {
|
|
2061
2062
|
const conv = getConversation(key);
|
|
2062
2063
|
if (!conv) throw new Error("会话不存在");
|
|
2063
2064
|
if (conv.agent !== "codex") throw new Error("只有 Codex 的对话能语音对话");
|
|
@@ -2065,7 +2066,7 @@ class Hub extends EventEmitter<{
|
|
|
2065
2066
|
const session = this.session(key);
|
|
2066
2067
|
if (!session.startRealtime) throw new Error("这个 agent 不支持语音对话");
|
|
2067
2068
|
touchConversation(key);
|
|
2068
|
-
return session.startRealtime(offer);
|
|
2069
|
+
return session.startRealtime(offer, opts);
|
|
2069
2070
|
}
|
|
2070
2071
|
|
|
2071
2072
|
async realtimeStop(key: string): Promise<void> {
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 实时语音的中转:服务器自己当 WebRTC 端点连 OpenAI,浏览器那头走现有的 WebSocket 收发 PCM。
|
|
3
|
+
* 这样声音和网页走同一条路(反向代理、Cloudflare Tunnel 都能过),浏览器不用直连 OpenAI。
|
|
4
|
+
*
|
|
5
|
+
* 浏览器 ⇄ 这里:16 位小端、单声道、24kHz 的 PCM(每块多长都行)。
|
|
6
|
+
* 这里 ⇄ OpenAI:Opus(RTP 时钟 48kHz,每包 20ms),加上 OpenAI 要求的 oai-events 数据通道。
|
|
7
|
+
*/
|
|
8
|
+
import OpusScript from "opusscript";
|
|
9
|
+
import { MediaStreamTrack, RTCPeerConnection, RTCRtpCodecParameters, RtpHeader, RtpPacket } from "werift";
|
|
10
|
+
|
|
11
|
+
export const RELAY_RATE = 24000;
|
|
12
|
+
/** 20ms 一帧:24kHz 下 480 个采样;RTP 时间戳按 48kHz 走,一帧加 960 */
|
|
13
|
+
const FRAME = 480;
|
|
14
|
+
const FRAME_BYTES = FRAME * 2;
|
|
15
|
+
const RTP_STEP = 960;
|
|
16
|
+
/** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
|
|
17
|
+
const MAX_PENDING = RELAY_RATE * 2;
|
|
18
|
+
|
|
19
|
+
export class RealtimeRelay {
|
|
20
|
+
/** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
|
|
21
|
+
onAudio: (pcm: Buffer) => void = () => {};
|
|
22
|
+
/** 和 OpenAI 的连接断了(不是 close() 主动关的) */
|
|
23
|
+
onFailed: (reason: string) => void = () => {};
|
|
24
|
+
|
|
25
|
+
private readonly pc: RTCPeerConnection;
|
|
26
|
+
private readonly track = new MediaStreamTrack({ kind: "audio" });
|
|
27
|
+
private readonly encoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
|
|
28
|
+
private readonly decoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
|
|
29
|
+
private pending = Buffer.alloc(0);
|
|
30
|
+
private seq = Math.floor(Math.random() * 0xffff);
|
|
31
|
+
private ts = Math.floor(Math.random() * 0xffffffff);
|
|
32
|
+
private closed = false;
|
|
33
|
+
private dropTimer: NodeJS.Timeout | undefined;
|
|
34
|
+
|
|
35
|
+
private constructor() {
|
|
36
|
+
this.pc = new RTCPeerConnection({
|
|
37
|
+
codecs: { audio: [new RTCRtpCodecParameters({ mimeType: "audio/opus", clockRate: 48000, channels: 2 })] },
|
|
38
|
+
});
|
|
39
|
+
const transceiver = this.pc.addTransceiver(this.track, { direction: "sendrecv" });
|
|
40
|
+
transceiver.onTrack.subscribe((remote) => {
|
|
41
|
+
remote.onReceiveRtp.subscribe((rtp) => {
|
|
42
|
+
if (this.closed || !rtp.payload.length) return;
|
|
43
|
+
try {
|
|
44
|
+
this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
|
|
45
|
+
} catch {
|
|
46
|
+
// 坏包丢掉
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
// OpenAI 要求 offer 里有这个数据通道(事件我们走 Codex 的通知,用不上它)
|
|
51
|
+
this.pc.createDataChannel("oai-events");
|
|
52
|
+
this.pc.connectionStateChange.subscribe((state) => {
|
|
53
|
+
if (this.closed) return;
|
|
54
|
+
clearTimeout(this.dropTimer);
|
|
55
|
+
if (state === "failed" || state === "closed") this.fail("和语音服务的连接断开了");
|
|
56
|
+
// 网络抖一下会短暂 disconnected,几秒内没恢复才算断
|
|
57
|
+
else if (state === "disconnected") this.dropTimer = setTimeout(() => this.fail("和语音服务的连接断开了"), 5000);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 建好连接对象和 offer(ICE 候选收齐了写在里面),offer 交给 Codex 去 OpenAI 建通话 */
|
|
62
|
+
static async create(): Promise<{ relay: RealtimeRelay; offer: string }> {
|
|
63
|
+
const relay = new RealtimeRelay();
|
|
64
|
+
await relay.pc.setLocalDescription(await relay.pc.createOffer());
|
|
65
|
+
await relay.gathered(1500);
|
|
66
|
+
const offer = relay.pc.localDescription?.sdp;
|
|
67
|
+
if (!offer) {
|
|
68
|
+
relay.close();
|
|
69
|
+
throw new Error("没能建立语音连接");
|
|
70
|
+
}
|
|
71
|
+
return { relay, offer };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** OpenAI 的 answer 回来了:连上 */
|
|
75
|
+
async accept(answer: string): Promise<void> {
|
|
76
|
+
if (this.closed) throw new Error("已挂断");
|
|
77
|
+
await this.pc.setRemoteDescription({ type: "answer", sdp: answer });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 浏览器发来的麦克风 PCM:凑够 20ms 就编码发出去 */
|
|
81
|
+
pushAudio(pcm: Buffer): void {
|
|
82
|
+
if (this.closed || !pcm.length) return;
|
|
83
|
+
this.pending = this.pending.length ? Buffer.concat([this.pending, pcm]) : Buffer.from(pcm);
|
|
84
|
+
if (this.pending.length > MAX_PENDING) this.pending = this.pending.subarray(this.pending.length - MAX_PENDING);
|
|
85
|
+
while (this.pending.length >= FRAME_BYTES) {
|
|
86
|
+
const frame = this.pending.subarray(0, FRAME_BYTES);
|
|
87
|
+
this.pending = this.pending.subarray(FRAME_BYTES);
|
|
88
|
+
this.sendFrame(frame);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
close(): void {
|
|
93
|
+
if (this.closed) return;
|
|
94
|
+
this.closed = true;
|
|
95
|
+
clearTimeout(this.dropTimer);
|
|
96
|
+
this.pending = Buffer.alloc(0);
|
|
97
|
+
void this.pc.close().catch(() => {});
|
|
98
|
+
for (const codec of [this.encoder, this.decoder]) {
|
|
99
|
+
try {
|
|
100
|
+
codec.delete();
|
|
101
|
+
} catch {
|
|
102
|
+
// 已经释放
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private sendFrame(frame: Buffer): void {
|
|
108
|
+
let payload: Buffer;
|
|
109
|
+
try {
|
|
110
|
+
payload = this.encoder.encode(frame, FRAME);
|
|
111
|
+
} catch {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const header = new RtpHeader({ sequenceNumber: this.seq, timestamp: this.ts, marker: false, payloadType: 111 });
|
|
115
|
+
this.seq = (this.seq + 1) & 0xffff;
|
|
116
|
+
this.ts = (this.ts + RTP_STEP) >>> 0;
|
|
117
|
+
try {
|
|
118
|
+
this.track.writeRtp(new RtpPacket(header, Buffer.from(payload)));
|
|
119
|
+
} catch {
|
|
120
|
+
// 还没连上 / 已经关了
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private fail(reason: string): void {
|
|
125
|
+
if (this.closed) return;
|
|
126
|
+
this.close();
|
|
127
|
+
this.onFailed(reason);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private gathered(ms: number): Promise<void> {
|
|
131
|
+
if (this.pc.iceGatheringState === "complete") return Promise.resolve();
|
|
132
|
+
return new Promise((resolve) => {
|
|
133
|
+
const timer = setTimeout(done, ms);
|
|
134
|
+
const sub = this.pc.iceGatheringStateChange.subscribe((state) => state === "complete" && done());
|
|
135
|
+
function done(): void {
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
sub.unSubscribe();
|
|
138
|
+
resolve();
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
package/src/core/types.ts
CHANGED
|
@@ -144,6 +144,9 @@ export type AgentEvents = {
|
|
|
144
144
|
onRealtime?: (ev: RealtimeEvent) => void;
|
|
145
145
|
};
|
|
146
146
|
|
|
147
|
+
/** 实时语音对话的音色和语言(页面上选的,没选就用 Codex 默认) */
|
|
148
|
+
export type RealtimeOptions = { voice?: string; language?: string };
|
|
149
|
+
|
|
147
150
|
/** 实时语音对话的事件(Codex 的 thread/realtime/*) */
|
|
148
151
|
export type RealtimeEvent =
|
|
149
152
|
| { type: "started" }
|
|
@@ -315,7 +318,7 @@ export interface AgentSession {
|
|
|
315
318
|
/** 开关计划模式。Claude 立即生效,Codex 下一轮生效 */
|
|
316
319
|
setPlan?(on: boolean): Promise<void>;
|
|
317
320
|
/** 开始实时语音对话(只有 Codex):offer 是浏览器的 WebRTC SDP,返回对端的 answer */
|
|
318
|
-
startRealtime?(offer: string): Promise<string>;
|
|
321
|
+
startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
|
|
319
322
|
/** 挂断实时语音对话;没在通话就什么都不做 */
|
|
320
323
|
stopRealtime?(): Promise<void>;
|
|
321
324
|
}
|
package/web/css/composer.css
CHANGED
|
@@ -317,6 +317,10 @@ button.ctx-item:hover {
|
|
|
317
317
|
margin-left: 6px;
|
|
318
318
|
}
|
|
319
319
|
|
|
320
|
+
.send-btn[hidden] {
|
|
321
|
+
display: none;
|
|
322
|
+
}
|
|
323
|
+
|
|
320
324
|
.send-btn:disabled {
|
|
321
325
|
opacity: 0.4;
|
|
322
326
|
}
|
|
@@ -542,165 +546,6 @@ textarea.sending {
|
|
|
542
546
|
}
|
|
543
547
|
}
|
|
544
548
|
|
|
545
|
-
/* ---- 语音对话:通话时输入框换成通话条(views/voice-call.js) ---- */
|
|
546
|
-
|
|
547
|
-
.call-btn[hidden] {
|
|
548
|
-
display: none;
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
.call-panel {
|
|
552
|
-
background: var(--bg-input);
|
|
553
|
-
border: 1px solid color-mix(in srgb, var(--border) 85%, transparent);
|
|
554
|
-
border-radius: 22px;
|
|
555
|
-
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.03), 0 4px 16px rgba(0, 0, 0, 0.05);
|
|
556
|
-
padding: 10px 10px 10px 16px;
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
.call-main {
|
|
560
|
-
display: flex;
|
|
561
|
-
align-items: center;
|
|
562
|
-
gap: 14px;
|
|
563
|
-
min-width: 0;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
.call-wave {
|
|
567
|
-
display: flex;
|
|
568
|
-
align-items: center;
|
|
569
|
-
gap: 3px;
|
|
570
|
-
height: 28px;
|
|
571
|
-
flex-shrink: 0;
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
.call-bar {
|
|
575
|
-
width: 4px;
|
|
576
|
-
height: 100%;
|
|
577
|
-
border-radius: 2px;
|
|
578
|
-
background: var(--fg);
|
|
579
|
-
transform: scaleY(0.2);
|
|
580
|
-
transition: transform 80ms linear;
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
.call-panel.speaking .call-bar {
|
|
584
|
-
background: var(--accent);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
.call-panel.connecting .call-bar {
|
|
588
|
-
opacity: 0.45;
|
|
589
|
-
animation: call-wait 1.1s ease-in-out infinite;
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
.call-panel.connecting .call-bar:nth-child(2) { animation-delay: 0.1s; }
|
|
593
|
-
.call-panel.connecting .call-bar:nth-child(3) { animation-delay: 0.2s; }
|
|
594
|
-
.call-panel.connecting .call-bar:nth-child(4) { animation-delay: 0.3s; }
|
|
595
|
-
.call-panel.connecting .call-bar:nth-child(5) { animation-delay: 0.4s; }
|
|
596
|
-
|
|
597
|
-
@keyframes call-wait {
|
|
598
|
-
0%, 100% { transform: scaleY(0.2); }
|
|
599
|
-
50% { transform: scaleY(0.6); }
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
.call-info {
|
|
603
|
-
flex: 1;
|
|
604
|
-
min-width: 0;
|
|
605
|
-
display: flex;
|
|
606
|
-
flex-direction: column;
|
|
607
|
-
gap: 2px;
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
.call-line {
|
|
611
|
-
display: flex;
|
|
612
|
-
align-items: baseline;
|
|
613
|
-
gap: 8px;
|
|
614
|
-
font-size: 14px;
|
|
615
|
-
font-weight: 500;
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
.call-time {
|
|
619
|
-
color: var(--fg-faint);
|
|
620
|
-
font-size: 12px;
|
|
621
|
-
font-variant-numeric: tabular-nums;
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
.call-caption {
|
|
625
|
-
display: flex;
|
|
626
|
-
gap: 6px;
|
|
627
|
-
min-width: 0;
|
|
628
|
-
font-size: 13px;
|
|
629
|
-
color: var(--fg-faint);
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
.call-panel.has-caption .call-caption {
|
|
633
|
-
color: var(--fg);
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
.call-who {
|
|
637
|
-
flex-shrink: 0;
|
|
638
|
-
color: var(--fg-faint);
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
.call-who:empty {
|
|
642
|
-
display: none;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
/* 字幕只留最近两行:长句子往上滚走 */
|
|
646
|
-
.call-text {
|
|
647
|
-
min-width: 0;
|
|
648
|
-
display: -webkit-box;
|
|
649
|
-
-webkit-box-orient: vertical;
|
|
650
|
-
-webkit-line-clamp: 2;
|
|
651
|
-
overflow: hidden;
|
|
652
|
-
line-height: 1.45;
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
.call-ctls {
|
|
656
|
-
display: flex;
|
|
657
|
-
align-items: center;
|
|
658
|
-
gap: 8px;
|
|
659
|
-
flex-shrink: 0;
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
.call-ctl {
|
|
663
|
-
height: 36px;
|
|
664
|
-
min-width: 36px;
|
|
665
|
-
border-radius: 18px;
|
|
666
|
-
display: inline-grid;
|
|
667
|
-
place-items: center;
|
|
668
|
-
background: var(--bg-soft);
|
|
669
|
-
color: var(--fg);
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
.call-ctl svg {
|
|
673
|
-
width: 18px;
|
|
674
|
-
height: 18px;
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
.call-ctl.on {
|
|
678
|
-
color: var(--danger);
|
|
679
|
-
background: color-mix(in srgb, var(--danger) 12%, transparent);
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
.call-ctl.call-end {
|
|
683
|
-
padding: 0 16px;
|
|
684
|
-
font-size: 14px;
|
|
685
|
-
font-weight: 500;
|
|
686
|
-
color: #fff;
|
|
687
|
-
background: var(--danger);
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
@media (max-width: 640px) {
|
|
691
|
-
.call-panel {
|
|
692
|
-
padding-left: 12px;
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
.call-main {
|
|
696
|
-
gap: 10px;
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
.call-ctl.call-end {
|
|
700
|
-
padding: 0 12px;
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
|
|
704
549
|
/* 通话中输入框收起来,通话条顶上 */
|
|
705
550
|
.composer.in-call {
|
|
706
551
|
display: none;
|