agent-loft 0.2.114 → 0.2.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/agents/codex/session.ts +4 -0
- package/src/core/hub.ts +22 -4
- package/src/core/realtime-relay.ts +25 -0
- package/src/core/types.ts +2 -0
- package/web/css/composer.css +4 -197
- package/web/css/voice-call.css +196 -0
- package/web/index.html +1 -0
- package/web/lib/rt-worklet.js +42 -5
- package/web/views/composer.js +10 -4
- package/web/views/voice-call.js +99 -31
package/README.md
CHANGED
|
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
|
|
|
54
54
|
- **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
|
|
55
55
|
- 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
|
|
56
56
|
- 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
|
|
57
|
-
- 语音对话(只有 Codex,包括 Codex
|
|
57
|
+
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话界面的齿轮里可以选音色和回答用的语言(记在这台设备上),切换设置后重连会保留静音状态和收起状态。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
|
|
58
58
|
- 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
|
|
59
59
|
- 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
|
|
60
60
|
- 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
|
package/package.json
CHANGED
package/src/core/hub.ts
CHANGED
|
@@ -846,7 +846,10 @@ class Hub extends EventEmitter<{
|
|
|
846
846
|
this.externalTurn(key, text, anchor);
|
|
847
847
|
},
|
|
848
848
|
// 不看 alive:会话被换下去时发的「通话结束」也要让页面知道
|
|
849
|
-
onRealtime: (ev) =>
|
|
849
|
+
onRealtime: (ev) => {
|
|
850
|
+
this.emit("realtime", { key, ev });
|
|
851
|
+
if (ev.type === "closed") this.realtimeEnded(key);
|
|
852
|
+
},
|
|
850
853
|
onStep: (kind, text) => alive() && this.emitEvent(key, { type: "step", kind, text }),
|
|
851
854
|
onTodos: (todos) => alive() && this.emitEvent(key, { type: "todos", todos }),
|
|
852
855
|
onTextDelta: (text) => alive() && this.emitEvent(key, { type: "delta", text }),
|
|
@@ -1847,7 +1850,21 @@ class Hub extends EventEmitter<{
|
|
|
1847
1850
|
|| (this.pendingOps.get(key) ?? 0) > 0
|
|
1848
1851
|
|| this.isolatedRuns.has(key)
|
|
1849
1852
|
|| this.rewinding.has(key)
|
|
1850
|
-
|| interactions.listPending(key).length > 0
|
|
1853
|
+
|| interactions.listPending(key).length > 0
|
|
1854
|
+
|| this.inCall(key);
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
/** 正在实时语音通话:通话本身不走轮次,不算进 isBusy,但进程不能闲置回收、也不能因为记忆更新被换掉 */
|
|
1858
|
+
private inCall(key: string): boolean {
|
|
1859
|
+
return this.live.get(key)?.session.inRealtime?.() === true;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
/** 通话结束:从这时起重新算闲置;通话期间攒下的「记忆已更新」这时再换进程 */
|
|
1863
|
+
private realtimeEnded(key: string): void {
|
|
1864
|
+
const live = this.live.get(key);
|
|
1865
|
+
if (!live) return;
|
|
1866
|
+
this.liveUsedAt.set(live.id, Date.now());
|
|
1867
|
+
this.dropIfStale(key);
|
|
1851
1868
|
}
|
|
1852
1869
|
|
|
1853
1870
|
/** 关掉空闲太久的会话进程(只关进程,不动绑定),见 LIVE_IDLE_MS */
|
|
@@ -1877,13 +1894,13 @@ class Hub extends EventEmitter<{
|
|
|
1877
1894
|
|
|
1878
1895
|
private dropIfStale(key: string): void {
|
|
1879
1896
|
const live = this.live.get(key);
|
|
1880
|
-
if (!live || !this.staleLive.has(live.id) || this.isBusy(key)) return;
|
|
1897
|
+
if (!live || !this.staleLive.has(live.id) || this.isBusy(key) || this.inCall(key)) return;
|
|
1881
1898
|
this.staleLive.delete(live.id);
|
|
1882
1899
|
void this.serial(key, async () => {
|
|
1883
1900
|
// 排队期间可能已经换了进程、或者又开始说话了:那就留到下一轮结束
|
|
1884
1901
|
const now = this.live.get(key);
|
|
1885
1902
|
if (now?.id !== live.id) return;
|
|
1886
|
-
if (this.isBusy(key)) {
|
|
1903
|
+
if (this.isBusy(key) || this.inCall(key)) {
|
|
1887
1904
|
this.staleLive.add(live.id);
|
|
1888
1905
|
return;
|
|
1889
1906
|
}
|
|
@@ -2071,6 +2088,7 @@ class Hub extends EventEmitter<{
|
|
|
2071
2088
|
|
|
2072
2089
|
async realtimeStop(key: string): Promise<void> {
|
|
2073
2090
|
await this.live.get(key)?.session.stopRealtime?.();
|
|
2091
|
+
this.realtimeEnded(key);
|
|
2074
2092
|
}
|
|
2075
2093
|
|
|
2076
2094
|
/**
|
|
@@ -15,6 +15,11 @@ const FRAME_BYTES = FRAME * 2;
|
|
|
15
15
|
const RTP_STEP = 960;
|
|
16
16
|
/** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
|
|
17
17
|
const MAX_PENDING = RELAY_RATE * 2;
|
|
18
|
+
/**
|
|
19
|
+
* 收的方向丢了几个包以内用 Opus 的丢包补偿填上(解码空包 = PLC,一次给 120ms,按丢的帧数截)。
|
|
20
|
+
* 服务器到 OpenAI 这段常过代理,实测丢 2~3%:不填的话浏览器那头的缓冲每丢一包就少 20ms,过一阵就放空、一卡一卡
|
|
21
|
+
*/
|
|
22
|
+
const MAX_CONCEAL = 6;
|
|
18
23
|
|
|
19
24
|
export class RealtimeRelay {
|
|
20
25
|
/** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
|
|
@@ -30,6 +35,8 @@ export class RealtimeRelay {
|
|
|
30
35
|
private seq = Math.floor(Math.random() * 0xffff);
|
|
31
36
|
private ts = Math.floor(Math.random() * 0xffffffff);
|
|
32
37
|
private closed = false;
|
|
38
|
+
/** 收到的上一个 RTP 包的序号 */
|
|
39
|
+
private lastSeq: number | undefined;
|
|
33
40
|
private dropTimer: NodeJS.Timeout | undefined;
|
|
34
41
|
|
|
35
42
|
private constructor() {
|
|
@@ -40,6 +47,14 @@ export class RealtimeRelay {
|
|
|
40
47
|
transceiver.onTrack.subscribe((remote) => {
|
|
41
48
|
remote.onReceiveRtp.subscribe((rtp) => {
|
|
42
49
|
if (this.closed || !rtp.payload.length) return;
|
|
50
|
+
const seq = rtp.header.sequenceNumber;
|
|
51
|
+
if (this.lastSeq !== undefined) {
|
|
52
|
+
const gap = (seq - this.lastSeq) & 0xffff;
|
|
53
|
+
// 重复的、迟到的(已经补过或跳过了):丢掉,别打乱顺序
|
|
54
|
+
if (gap === 0 || gap > 0x8000) return;
|
|
55
|
+
if (gap > 1 && gap - 1 <= MAX_CONCEAL) this.conceal(gap - 1);
|
|
56
|
+
}
|
|
57
|
+
this.lastSeq = seq;
|
|
43
58
|
try {
|
|
44
59
|
this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
|
|
45
60
|
} catch {
|
|
@@ -121,6 +136,16 @@ export class RealtimeRelay {
|
|
|
121
136
|
}
|
|
122
137
|
}
|
|
123
138
|
|
|
139
|
+
/** 丢了 frames 个 20ms 的包:用解码器的丢包补偿顶上同样长的声音 */
|
|
140
|
+
private conceal(frames: number): void {
|
|
141
|
+
try {
|
|
142
|
+
const pcm = this.decoder.decode(Buffer.alloc(0));
|
|
143
|
+
this.onAudio(Buffer.from(pcm.subarray(0, Math.min(pcm.length, frames * FRAME_BYTES))));
|
|
144
|
+
} catch {
|
|
145
|
+
// 补不了就算了
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
124
149
|
private fail(reason: string): void {
|
|
125
150
|
if (this.closed) return;
|
|
126
151
|
this.close();
|
package/src/core/types.ts
CHANGED
|
@@ -321,6 +321,8 @@ export interface AgentSession {
|
|
|
321
321
|
startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
|
|
322
322
|
/** 挂断实时语音对话;没在通话就什么都不做 */
|
|
323
323
|
stopRealtime?(): Promise<void>;
|
|
324
|
+
/** 正在实时语音通话(含建连中):这期间进程不能当闲置回收 */
|
|
325
|
+
inRealtime?(): boolean;
|
|
324
326
|
}
|
|
325
327
|
|
|
326
328
|
export type Diagnosis = {
|
package/web/css/composer.css
CHANGED
|
@@ -317,6 +317,10 @@ button.ctx-item:hover {
|
|
|
317
317
|
margin-left: 6px;
|
|
318
318
|
}
|
|
319
319
|
|
|
320
|
+
.send-btn[hidden] {
|
|
321
|
+
display: none;
|
|
322
|
+
}
|
|
323
|
+
|
|
320
324
|
.send-btn:disabled {
|
|
321
325
|
opacity: 0.4;
|
|
322
326
|
}
|
|
@@ -542,203 +546,6 @@ textarea.sending {
|
|
|
542
546
|
}
|
|
543
547
|
}
|
|
544
548
|
|
|
545
|
-
/* ---- 语音对话:通话时输入框换成通话条(views/voice-call.js) ---- */
|
|
546
|
-
|
|
547
|
-
.call-btn[hidden] {
|
|
548
|
-
display: none;
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
.call-panel {
|
|
552
|
-
background: var(--bg-input);
|
|
553
|
-
border: 1px solid color-mix(in srgb, var(--border) 85%, transparent);
|
|
554
|
-
border-radius: 22px;
|
|
555
|
-
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.03), 0 4px 16px rgba(0, 0, 0, 0.05);
|
|
556
|
-
padding: 10px 10px 10px 16px;
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
.call-main {
|
|
560
|
-
display: flex;
|
|
561
|
-
align-items: center;
|
|
562
|
-
gap: 14px;
|
|
563
|
-
min-width: 0;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
.call-wave {
|
|
567
|
-
display: flex;
|
|
568
|
-
align-items: center;
|
|
569
|
-
gap: 3px;
|
|
570
|
-
height: 28px;
|
|
571
|
-
flex-shrink: 0;
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
.call-bar {
|
|
575
|
-
width: 4px;
|
|
576
|
-
height: 100%;
|
|
577
|
-
border-radius: 2px;
|
|
578
|
-
background: var(--fg);
|
|
579
|
-
transform: scaleY(0.2);
|
|
580
|
-
transition: transform 80ms linear;
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
.call-panel.speaking .call-bar {
|
|
584
|
-
background: var(--accent);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
.call-panel.connecting .call-bar {
|
|
588
|
-
opacity: 0.45;
|
|
589
|
-
animation: call-wait 1.1s ease-in-out infinite;
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
.call-panel.connecting .call-bar:nth-child(2) { animation-delay: 0.1s; }
|
|
593
|
-
.call-panel.connecting .call-bar:nth-child(3) { animation-delay: 0.2s; }
|
|
594
|
-
.call-panel.connecting .call-bar:nth-child(4) { animation-delay: 0.3s; }
|
|
595
|
-
.call-panel.connecting .call-bar:nth-child(5) { animation-delay: 0.4s; }
|
|
596
|
-
|
|
597
|
-
@keyframes call-wait {
|
|
598
|
-
0%, 100% { transform: scaleY(0.2); }
|
|
599
|
-
50% { transform: scaleY(0.6); }
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
.call-info {
|
|
603
|
-
flex: 1;
|
|
604
|
-
min-width: 0;
|
|
605
|
-
display: flex;
|
|
606
|
-
flex-direction: column;
|
|
607
|
-
gap: 2px;
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
.call-line {
|
|
611
|
-
display: flex;
|
|
612
|
-
align-items: baseline;
|
|
613
|
-
gap: 8px;
|
|
614
|
-
font-size: 14px;
|
|
615
|
-
font-weight: 500;
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
.call-time {
|
|
619
|
-
color: var(--fg-faint);
|
|
620
|
-
font-size: 12px;
|
|
621
|
-
font-variant-numeric: tabular-nums;
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
.call-caption {
|
|
625
|
-
display: flex;
|
|
626
|
-
gap: 6px;
|
|
627
|
-
min-width: 0;
|
|
628
|
-
font-size: 13px;
|
|
629
|
-
color: var(--fg-faint);
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
.call-panel.has-caption .call-caption {
|
|
633
|
-
color: var(--fg);
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
.call-who {
|
|
637
|
-
flex-shrink: 0;
|
|
638
|
-
color: var(--fg-faint);
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
.call-who:empty {
|
|
642
|
-
display: none;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
/* 字幕只留最近两行:长句子往上滚走 */
|
|
646
|
-
.call-text {
|
|
647
|
-
min-width: 0;
|
|
648
|
-
display: -webkit-box;
|
|
649
|
-
-webkit-box-orient: vertical;
|
|
650
|
-
-webkit-line-clamp: 2;
|
|
651
|
-
overflow: hidden;
|
|
652
|
-
line-height: 1.45;
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
.call-ctls {
|
|
656
|
-
display: flex;
|
|
657
|
-
align-items: center;
|
|
658
|
-
gap: 8px;
|
|
659
|
-
flex-shrink: 0;
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
.call-ctl {
|
|
663
|
-
height: 36px;
|
|
664
|
-
min-width: 36px;
|
|
665
|
-
border-radius: 18px;
|
|
666
|
-
display: inline-grid;
|
|
667
|
-
place-items: center;
|
|
668
|
-
background: var(--bg-soft);
|
|
669
|
-
color: var(--fg);
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
.call-ctl svg {
|
|
673
|
-
width: 18px;
|
|
674
|
-
height: 18px;
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
.call-ctl.on {
|
|
678
|
-
color: var(--danger);
|
|
679
|
-
background: color-mix(in srgb, var(--danger) 12%, transparent);
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
.call-ctl.active {
|
|
683
|
-
color: var(--accent);
|
|
684
|
-
background: color-mix(in srgb, var(--accent) 12%, transparent);
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
.call-opts {
|
|
688
|
-
display: flex;
|
|
689
|
-
flex-wrap: wrap;
|
|
690
|
-
align-items: center;
|
|
691
|
-
gap: 8px 16px;
|
|
692
|
-
margin-top: 10px;
|
|
693
|
-
padding-top: 10px;
|
|
694
|
-
border-top: 1px solid color-mix(in srgb, var(--border) 70%, transparent);
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
.call-opts[hidden] {
|
|
698
|
-
display: none;
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
.call-opt {
|
|
702
|
-
display: inline-flex;
|
|
703
|
-
align-items: center;
|
|
704
|
-
gap: 6px;
|
|
705
|
-
font-size: 13px;
|
|
706
|
-
color: var(--fg-soft);
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
.call-opt select {
|
|
710
|
-
font-size: 13px;
|
|
711
|
-
max-width: 180px;
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
.call-opts-hint {
|
|
715
|
-
flex-basis: 100%;
|
|
716
|
-
font-size: 12px;
|
|
717
|
-
color: var(--fg-faint);
|
|
718
|
-
}
|
|
719
|
-
|
|
720
|
-
.call-ctl.call-end {
|
|
721
|
-
padding: 0 16px;
|
|
722
|
-
font-size: 14px;
|
|
723
|
-
font-weight: 500;
|
|
724
|
-
color: #fff;
|
|
725
|
-
background: var(--danger);
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
@media (max-width: 640px) {
|
|
729
|
-
.call-panel {
|
|
730
|
-
padding-left: 12px;
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
.call-main {
|
|
734
|
-
gap: 10px;
|
|
735
|
-
}
|
|
736
|
-
|
|
737
|
-
.call-ctl.call-end {
|
|
738
|
-
padding: 0 12px;
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
|
|
742
549
|
/* 通话中输入框收起来,通话条顶上 */
|
|
743
550
|
.composer.in-call {
|
|
744
551
|
display: none;
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/* 语音对话:沉浸式光球与收起后的通话条。音频逻辑共用 views/voice-call.js。 */
|
|
2
|
+
.call-btn[hidden] { display: none; }
|
|
3
|
+
|
|
4
|
+
.call-dialog {
|
|
5
|
+
position: fixed;
|
|
6
|
+
inset: 0;
|
|
7
|
+
width: 100%;
|
|
8
|
+
max-width: none;
|
|
9
|
+
height: 100%;
|
|
10
|
+
height: 100dvh;
|
|
11
|
+
max-height: none;
|
|
12
|
+
margin: 0;
|
|
13
|
+
padding: 0;
|
|
14
|
+
border: 0;
|
|
15
|
+
background: var(--bg);
|
|
16
|
+
color: var(--fg);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.call-dialog::backdrop { background: var(--bg); }
|
|
20
|
+
|
|
21
|
+
.call-panel {
|
|
22
|
+
--call-blue: #2686ec;
|
|
23
|
+
position: relative;
|
|
24
|
+
display: flex;
|
|
25
|
+
flex-direction: column;
|
|
26
|
+
align-items: center;
|
|
27
|
+
width: 100%;
|
|
28
|
+
height: 100%;
|
|
29
|
+
overflow-y: auto;
|
|
30
|
+
overscroll-behavior: contain;
|
|
31
|
+
padding: max(24px, env(safe-area-inset-top)) 40px max(28px, env(safe-area-inset-bottom));
|
|
32
|
+
background: radial-gradient(ellipse at 50% 40%, color-mix(in srgb, #8bc8ff 7%, transparent), transparent 60%), var(--bg);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
.call-header {
|
|
36
|
+
display: flex;
|
|
37
|
+
align-items: center;
|
|
38
|
+
gap: 16px;
|
|
39
|
+
width: 100%;
|
|
40
|
+
flex-shrink: 0;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
.call-brand { display: flex; align-items: center; gap: 12px; font-size: 17px; font-weight: 600; line-height: 1.3; }
|
|
44
|
+
.call-brand small { display: block; margin-top: 4px; font-size: 11px; font-weight: 400; color: var(--fg-faint); letter-spacing: 0.12em; }
|
|
45
|
+
.call-brand-icon { display: grid; place-items: center; width: 40px; height: 40px; background: var(--bg-soft); border: 1px solid var(--border); border-radius: 14px; }
|
|
46
|
+
.call-brand-icon svg { width: 22px; height: 22px; }
|
|
47
|
+
.call-session { display: flex; align-items: center; gap: 7px; margin-left: auto; color: var(--fg-soft); font-size: 12px; font-variant-numeric: tabular-nums; }
|
|
48
|
+
.call-live-dot { width: 6px; height: 6px; border-radius: 50%; background: var(--call-blue); }
|
|
49
|
+
.connecting .call-live-dot { animation: call-dot 1.4s ease-in-out infinite; }
|
|
50
|
+
.call-minimize { border: 1px solid var(--border); border-radius: 50%; width: 40px; height: 40px; }
|
|
51
|
+
|
|
52
|
+
.call-main { flex: 1 0 auto; display: flex; flex-direction: column; align-items: center; justify-content: center; width: 100%; min-height: 310px; padding: 24px 0; }
|
|
53
|
+
.call-stage { display: grid; place-items: center; width: 100%; padding: 30px 0 42px; }
|
|
54
|
+
|
|
55
|
+
/* 叠加的云层在球面内流动,用 transform 合成动画,实际音量控制整个球的缩放。 */
|
|
56
|
+
.call-orb {
|
|
57
|
+
--voice-scale: 1;
|
|
58
|
+
position: relative;
|
|
59
|
+
width: clamp(170px, 28vh, 280px);
|
|
60
|
+
aspect-ratio: 1;
|
|
61
|
+
flex-shrink: 0;
|
|
62
|
+
transform: scale(var(--voice-scale));
|
|
63
|
+
transition: transform 160ms ease-out, opacity 0.5s, filter 0.5s;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
.call-orb::before {
|
|
67
|
+
content: "";
|
|
68
|
+
position: absolute;
|
|
69
|
+
inset: -12%;
|
|
70
|
+
border-radius: 50%;
|
|
71
|
+
background: radial-gradient(circle, #56b7ff33, #80c8ff0d 60%, transparent 72%);
|
|
72
|
+
filter: blur(12px);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
.call-orb-surface {
|
|
76
|
+
position: absolute;
|
|
77
|
+
inset: 0;
|
|
78
|
+
isolation: isolate;
|
|
79
|
+
overflow: hidden;
|
|
80
|
+
border-radius: 50%;
|
|
81
|
+
background: radial-gradient(circle at 30% 20%, #f7fcff 5%, #dbf4ff 22%, #79c9ff 52%, #2588ed 78%, #1b68c5 100%);
|
|
82
|
+
box-shadow: inset 0 0 28px #ffffff80, inset -10px -16px 30px #276bd833, 0 10px 60px #3697ef1c;
|
|
83
|
+
animation: call-float 7s ease-in-out infinite;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
.call-orb-surface::after { content: ""; position: absolute; inset: 0; border-radius: inherit; background: radial-gradient(ellipse at 32% 12%, #fff9, transparent 55%); box-shadow: inset 0 0 3px #fff8; }
|
|
87
|
+
.call-orb-cloud { position: absolute; inset: -35%; border-radius: 44% 56% 62% 38%; filter: blur(16px); will-change: transform; }
|
|
88
|
+
.cloud-a { background: conic-gradient(from 40deg, transparent 8%, #f8fdff 28%, #dbf5ffb3 38%, transparent 55%, #1976e8cc 76%, transparent); animation: call-flow 14s linear infinite; }
|
|
89
|
+
.cloud-b { inset: -25%; background: conic-gradient(from 160deg, #fff0 10%, #e4faffcc 22%, #fff0 40%, #2787e9b3 60%, #fff0 82%); animation: call-flow 19s linear infinite reverse; }
|
|
90
|
+
.cloud-c { inset: 14% -35% -40%; background: radial-gradient(ellipse, #006ce7b3 10%, #3da7ff80 35%, transparent 64%); animation: call-drift 8s ease-in-out infinite alternate; }
|
|
91
|
+
.speaking .cloud-a { animation-duration: 5s; }
|
|
92
|
+
.speaking .cloud-b { animation-duration: 7s; }
|
|
93
|
+
.connecting .call-orb { opacity: 0.55; }
|
|
94
|
+
.muted .call-orb { filter: saturate(0.3); }
|
|
95
|
+
.processing .call-orb-surface { animation-duration: 3s; }
|
|
96
|
+
|
|
97
|
+
.call-info { text-align: center; }
|
|
98
|
+
.call-status { display: block; font-size: 23px; font-weight: 500; letter-spacing: -0.02em; }
|
|
99
|
+
.call-hint { margin: 8px 0 0; color: var(--fg-faint); font-size: 13px; }
|
|
100
|
+
.call-caption { width: min(620px, 100%); min-height: 88px; max-height: min(22vh, 180px); overflow-y: auto; text-align: center; padding: 8px 12px 20px; scrollbar-width: thin; flex-shrink: 0; }
|
|
101
|
+
.call-who { display: block; color: var(--fg-faint); font-size: 12px; margin-bottom: 8px; }
|
|
102
|
+
.call-text { font-size: clamp(17px, 2.3vw, 22px); line-height: 1.7; overflow-wrap: anywhere; white-space: pre-wrap; }
|
|
103
|
+
.call-footer { display: flex; flex-direction: column; align-items: center; gap: 22px; padding-top: 22px; flex-shrink: 0; }
|
|
104
|
+
.call-ctls { display: flex; align-items: flex-start; justify-content: center; gap: 24px; }
|
|
105
|
+
.call-control { display: flex; flex-direction: column; align-items: center; gap: 9px; }
|
|
106
|
+
.call-control > span { font-size: 11px; color: var(--fg-faint); }
|
|
107
|
+
.call-ctl { display: inline-grid; place-items: center; width: 56px; height: 56px; flex-shrink: 0; border: 1px solid var(--border); border-radius: 50%; background: var(--bg-soft); color: var(--fg); transition: background 0.18s, transform 0.18s; }
|
|
108
|
+
.call-ctl:hover { background: var(--bg-hover); transform: translateY(-2px); }
|
|
109
|
+
.call-ctl svg { width: 22px; height: 22px; }
|
|
110
|
+
.call-ctl.on { color: var(--danger); background: color-mix(in srgb, var(--danger) 10%, var(--bg)); border-color: color-mix(in srgb, var(--danger) 20%, var(--border)); }
|
|
111
|
+
.call-ctl.active { color: var(--call-blue); border-color: var(--call-blue); }
|
|
112
|
+
.call-end { background: color-mix(in srgb, var(--danger) 10%, var(--bg)); color: var(--danger); border-color: transparent; }
|
|
113
|
+
.call-end:hover { background: color-mix(in srgb, var(--danger) 18%, var(--bg)); }
|
|
114
|
+
.call-panel :is(button, select):focus-visible { outline: 2px solid var(--call-blue); outline-offset: 4px; }
|
|
115
|
+
.call-footer-note { color: var(--fg-faint); font-size: 11px; }
|
|
116
|
+
.call-expand { display: none; }
|
|
117
|
+
|
|
118
|
+
.call-opts { display: flex; flex-wrap: wrap; justify-content: center; gap: 12px 20px; width: min(620px, 100%); padding: 16px; flex-shrink: 0; border: 1px solid var(--border); background: var(--bg-soft); border-radius: 18px; }
|
|
119
|
+
.call-opts[hidden] { display: none; }
|
|
120
|
+
.call-opt { display: flex; align-items: center; gap: 10px; min-width: 0; font-size: 13px; color: var(--fg-soft); }
|
|
121
|
+
.call-opt > span { flex-shrink: 0; }
|
|
122
|
+
.call-opt select { max-width: 205px; min-width: 0; border: 1px solid var(--border); border-radius: 9px; padding: 7px; background: var(--bg-input); font-size: 13px; }
|
|
123
|
+
.call-opts-hint { width: 100%; text-align: center; font-size: 11px; color: var(--fg-faint); }
|
|
124
|
+
|
|
125
|
+
/* 展开设置时给字幕和操作留出空间,小屏上挂断按钮始终留在视口内。 */
|
|
126
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-main { min-height: 210px; padding: 12px 0; }
|
|
127
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-stage { padding: 8px 0 20px; }
|
|
128
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-orb { width: clamp(100px, 16vh, 160px); }
|
|
129
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-caption { min-height: 48px; max-height: 100px; }
|
|
130
|
+
|
|
131
|
+
/* 收起时继续通话,保留光球、状态、字幕和全部操作。 */
|
|
132
|
+
.call-panel.compact { display: grid; grid-template-columns: minmax(0, 1fr) auto; height: auto; overflow: visible; padding: 14px 16px; border: 1px solid var(--border); border-radius: 24px; background: var(--bg-input); box-shadow: var(--shadow); }
|
|
133
|
+
.compact .call-header, .compact .call-hint, .compact .call-footer-note, .compact .call-control > span { display: none; }
|
|
134
|
+
.compact .call-main { grid-column: 1; grid-row: 1; min-height: 0; padding: 0; flex-direction: row; justify-content: flex-start; gap: 14px; }
|
|
135
|
+
.compact .call-stage { width: auto; padding: 0; }
|
|
136
|
+
.compact .call-orb { width: 42px; }
|
|
137
|
+
.compact .call-orb-cloud { filter: blur(4px); }
|
|
138
|
+
.compact .call-info { min-width: 0; text-align: left; }
|
|
139
|
+
.compact .call-status { font-size: 13px; }
|
|
140
|
+
.compact .call-footer { grid-column: 2; grid-row: 1; padding: 0; }
|
|
141
|
+
.compact .call-ctls { gap: 6px; align-items: center; }
|
|
142
|
+
.compact .call-ctl { width: 36px; height: 36px; }
|
|
143
|
+
.compact .call-ctl svg { width: 17px; height: 17px; }
|
|
144
|
+
.compact .call-expand { display: inline-grid; }
|
|
145
|
+
.compact .call-caption { display: none; grid-column: 1 / -1; padding: 12px 0 0; margin-top: 12px; min-height: 0; max-height: 100px; border-top: 1px solid var(--border); text-align: left; }
|
|
146
|
+
.compact.has-caption .call-caption { display: block; }
|
|
147
|
+
.compact .call-who { display: inline; margin-right: 8px; }
|
|
148
|
+
.compact .call-text { font-size: 13px; }
|
|
149
|
+
.compact .call-opts { grid-column: 1 / -1; grid-row: 3; width: 100%; margin-top: 12px; }
|
|
150
|
+
|
|
151
|
+
@keyframes call-flow { to { transform: rotate(360deg); } }
|
|
152
|
+
@keyframes call-float { 0%, 100% { transform: translateY(0) rotate(-4deg); } 50% { transform: translateY(-8px) rotate(4deg); } }
|
|
153
|
+
@keyframes call-drift { from { transform: translate(-10%, 6%) rotate(-20deg); } to { transform: translate(16%, -15%) rotate(30deg); } }
|
|
154
|
+
@keyframes call-dot { 50% { opacity: 0.3; } }
|
|
155
|
+
|
|
156
|
+
@media (prefers-color-scheme: dark) {
|
|
157
|
+
.call-panel { background: radial-gradient(ellipse at 50% 40%, #123e6526, transparent 60%), var(--bg); }
|
|
158
|
+
.call-orb-surface { box-shadow: inset 0 0 28px #ffffff70, inset -10px -16px 30px #276bd855, 0 0 80px #3697ef26; }
|
|
159
|
+
.call-panel.compact { background: var(--bg-input); }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
@media (max-width: 640px) {
|
|
163
|
+
.call-panel { padding: max(18px, env(safe-area-inset-top)) 20px max(22px, env(safe-area-inset-bottom)); }
|
|
164
|
+
.call-main { min-height: 270px; }
|
|
165
|
+
.call-stage { padding: 20px 0 36px; }
|
|
166
|
+
.call-status { font-size: 21px; }
|
|
167
|
+
.call-caption { min-height: 90px; }
|
|
168
|
+
.call-ctls { gap: 22px; }
|
|
169
|
+
.call-opts { gap: 12px; }
|
|
170
|
+
.call-opt { width: 100%; justify-content: space-between; }
|
|
171
|
+
.call-opt select { flex: 1; max-width: 240px; }
|
|
172
|
+
.call-panel.compact { padding: 12px; }
|
|
173
|
+
.compact .call-main { gap: 10px; }
|
|
174
|
+
.compact .call-status { font-size: 12px; }
|
|
175
|
+
.compact .call-orb { width: 30px; }
|
|
176
|
+
.compact .call-ctls { gap: 4px; }
|
|
177
|
+
.compact .call-ctl { width: 36px; height: 36px; }
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
@media (max-height: 600px) {
|
|
181
|
+
.call-panel:not(.compact) { padding-top: 12px; padding-bottom: 12px; }
|
|
182
|
+
.call-main { min-height: 170px; padding: 8px 0; }
|
|
183
|
+
.call-orb { width: 110px; }
|
|
184
|
+
.call-stage { padding: 12px 0 16px; }
|
|
185
|
+
.call-caption { min-height: 40px; max-height: 80px; padding-bottom: 8px; }
|
|
186
|
+
.call-footer { padding-top: 8px; gap: 8px; }
|
|
187
|
+
.call-ctl { width: 44px; height: 44px; }
|
|
188
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-main { display: none; }
|
|
189
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-caption { min-height: 0; max-height: 50px; padding: 4px; }
|
|
190
|
+
.call-panel:not(.compact):has(.call-opts:not([hidden])) .call-opts { margin-top: auto; padding: 10px; gap: 8px 16px; }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
@media (prefers-reduced-motion: reduce) {
|
|
194
|
+
.call-panel *, .call-panel *::before, .call-panel *::after { animation: none !important; transition: none !important; }
|
|
195
|
+
.call-orb { transform: none; }
|
|
196
|
+
}
|
package/web/index.html
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
<link rel="stylesheet" href="/css/albums.css" />
|
|
23
23
|
<link rel="stylesheet" href="/css/loader.css" />
|
|
24
24
|
<link rel="stylesheet" href="/css/composer.css" />
|
|
25
|
+
<link rel="stylesheet" href="/css/voice-call.css" />
|
|
25
26
|
<link rel="stylesheet" href="/css/overlay.css" />
|
|
26
27
|
<link rel="stylesheet" href="/css/search.css" />
|
|
27
28
|
<link rel="stylesheet" href="/css/settings.css" />
|
package/web/lib/rt-worklet.js
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
// 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
|
|
2
2
|
// rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
|
|
3
3
|
// rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
|
|
4
|
+
// 放空之后很快(JB_JITTER_SEC 内)又来了声音 = 说到一半网络抖了:把缓冲加长一点;隔得久的是正常停顿,不算。
|
|
5
|
+
// 一阵子没再放空就慢慢缩回去;攒得比目标多太多(卡了一阵又一下子涌来)就跳回目标,免得越放越晚
|
|
4
6
|
|
|
5
7
|
const RATE = 24000;
|
|
6
8
|
const CHUNK = 480; // 20ms
|
|
9
|
+
/** 播放缓冲(24kHz 采样数):起步、最短、最长、每次放空加多少 */
|
|
10
|
+
const JB_START = RATE * 0.1;
|
|
11
|
+
const JB_MIN = RATE * 0.06;
|
|
12
|
+
const JB_MAX = RATE * 0.4;
|
|
13
|
+
const JB_STEP = RATE * 0.06;
|
|
14
|
+
/** 这么久(秒)没放空,缓冲缩短 20ms */
|
|
15
|
+
const JB_DECAY_SEC = 8;
|
|
16
|
+
/** 放空后这么久(秒)内又来了声音,算网络抖动 */
|
|
17
|
+
const JB_JITTER_SEC = 0.3;
|
|
18
|
+
/** 比目标多出这么多就跳回目标 */
|
|
19
|
+
const JB_SLACK = RATE * 0.3;
|
|
20
|
+
/** 没攒够目标、但这么久(秒)没再来声音:先放掉(很短的一句别卡在缓冲里) */
|
|
21
|
+
const JB_FLUSH_SEC = 0.1;
|
|
7
22
|
|
|
8
23
|
class Capture extends AudioWorkletProcessor {
|
|
9
24
|
constructor() {
|
|
@@ -52,13 +67,27 @@ class Play extends AudioWorkletProcessor {
|
|
|
52
67
|
this.avail = 0; // 还没放的采样数
|
|
53
68
|
this.write = 0;
|
|
54
69
|
this.playing = false;
|
|
70
|
+
this.target = JB_START;
|
|
71
|
+
/** 上次放空 / 缩短缓冲以来放了多少个输出采样 */
|
|
72
|
+
this.smooth = 0;
|
|
73
|
+
/** 说着说着放空的时刻(currentFrame),等下一块声音来了判断是抖动还是停顿 */
|
|
74
|
+
this.starvedAt = -1;
|
|
75
|
+
/** 最近一次收到声音的时刻(currentFrame) */
|
|
76
|
+
this.lastData = 0;
|
|
55
77
|
this.port.onmessage = (e) => {
|
|
56
78
|
if (e.data === "flush") {
|
|
57
79
|
this.avail = 0;
|
|
58
80
|
this.playing = false;
|
|
81
|
+
this.target = JB_START;
|
|
82
|
+
this.starvedAt = -1;
|
|
59
83
|
return;
|
|
60
84
|
}
|
|
61
85
|
const pcm = new Int16Array(e.data);
|
|
86
|
+
this.lastData = currentFrame;
|
|
87
|
+
if (this.starvedAt >= 0) {
|
|
88
|
+
if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC) this.target = Math.min(JB_MAX, this.target + JB_STEP);
|
|
89
|
+
this.starvedAt = -1;
|
|
90
|
+
}
|
|
62
91
|
// 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
|
|
63
92
|
const over = this.avail + pcm.length - this.size;
|
|
64
93
|
if (over > 0) {
|
|
@@ -70,9 +99,9 @@ class Play extends AudioWorkletProcessor {
|
|
|
70
99
|
this.write = (this.write + 1) % this.size;
|
|
71
100
|
}
|
|
72
101
|
this.avail += pcm.length;
|
|
73
|
-
//
|
|
74
|
-
if (this.avail >
|
|
75
|
-
const skip = this.avail -
|
|
102
|
+
// 太多了说明延迟越积越大,追回到目标
|
|
103
|
+
if (this.avail > this.target + JB_SLACK) {
|
|
104
|
+
const skip = this.avail - this.target;
|
|
76
105
|
this.read = (this.read + skip) % this.size;
|
|
77
106
|
this.avail -= skip;
|
|
78
107
|
}
|
|
@@ -83,10 +112,14 @@ class Play extends AudioWorkletProcessor {
|
|
|
83
112
|
const out = outputs[0];
|
|
84
113
|
const ch = out[0];
|
|
85
114
|
if (!ch) return true;
|
|
86
|
-
//
|
|
87
|
-
if (!this.playing && this.avail >=
|
|
115
|
+
// 先攒够目标再开始放,扛网络抖动;放空了就停下来重新攒,下次多攒一点
|
|
116
|
+
if (!this.playing && this.avail >= 2 && (this.avail >= this.target || currentFrame - this.lastData > sampleRate * JB_FLUSH_SEC)) {
|
|
117
|
+
this.playing = true;
|
|
118
|
+
this.smooth = 0;
|
|
119
|
+
}
|
|
88
120
|
for (let i = 0; i < ch.length; i++) {
|
|
89
121
|
if (!this.playing || this.avail < 2) {
|
|
122
|
+
if (this.playing) this.starvedAt = currentFrame + i;
|
|
90
123
|
ch[i] = 0;
|
|
91
124
|
this.playing = false;
|
|
92
125
|
continue;
|
|
@@ -101,6 +134,10 @@ class Play extends AudioWorkletProcessor {
|
|
|
101
134
|
this.avail -= used;
|
|
102
135
|
this.read = next % this.size;
|
|
103
136
|
}
|
|
137
|
+
if (this.playing && (this.smooth += ch.length) > sampleRate * JB_DECAY_SEC) {
|
|
138
|
+
this.smooth = 0;
|
|
139
|
+
this.target = Math.max(JB_MIN, this.target - RATE * 0.02);
|
|
140
|
+
}
|
|
104
141
|
for (let c = 1; c < out.length; c++) out[c].set(ch);
|
|
105
142
|
return true;
|
|
106
143
|
}
|
package/web/views/composer.js
CHANGED
|
@@ -8,7 +8,7 @@ import { addItem, openConversation, openLightbox, renderEmpty, scrollToBottom }
|
|
|
8
8
|
import { renderScheduled } from "./schedule.js";
|
|
9
9
|
import { renderTodos } from "./todos.js";
|
|
10
10
|
import { cancelVoice, SpeechRec, toggleVoice } from "./voice.js";
|
|
11
|
-
import { callButton } from "./voice-call.js";
|
|
11
|
+
import { callButton, canCall } from "./voice-call.js";
|
|
12
12
|
import { closeSlashMenu, commandUsage, oneLine, slashKeydown, updateSlashMenu } from "./slash.js";
|
|
13
13
|
import { doAction, openModelPicker, openSessions } from "./conv-actions.js";
|
|
14
14
|
|
|
@@ -155,7 +155,13 @@ function hasDraft() {
|
|
|
155
155
|
|
|
156
156
|
export function updateSendBtn() {
|
|
157
157
|
if (!ui.sendBtn) return;
|
|
158
|
-
|
|
158
|
+
// 最右侧共用一个位置:空闲且没有草稿时通话,有文字/附件时发送;回复中仍保留停止。
|
|
159
|
+
const draft = hasDraft();
|
|
160
|
+
const editing = S.editingMessage === S.current && !!S.current;
|
|
161
|
+
const showCall = canCall() && !draft && !S.busy && !S.sending && !editing;
|
|
162
|
+
if (ui.callBtn) ui.callBtn.hidden = !showCall;
|
|
163
|
+
ui.sendBtn.hidden = showCall;
|
|
164
|
+
if (editing) {
|
|
159
165
|
ui.sendBtn.disabled = true;
|
|
160
166
|
ui.sendBtn.title = "正在恢复消息…";
|
|
161
167
|
return;
|
|
@@ -167,10 +173,10 @@ export function updateSendBtn() {
|
|
|
167
173
|
ui.sendBtn.disabled = true;
|
|
168
174
|
return;
|
|
169
175
|
}
|
|
170
|
-
const stop = S.busy && !
|
|
176
|
+
const stop = S.busy && !draft;
|
|
171
177
|
ui.sendBtn.innerHTML = stop ? ICON.stop : ICON.send;
|
|
172
178
|
ui.sendBtn.title = stop ? "停止" : "发送";
|
|
173
|
-
ui.sendBtn.disabled = !stop && !
|
|
179
|
+
ui.sendBtn.disabled = !stop && !draft;
|
|
174
180
|
}
|
|
175
181
|
|
|
176
182
|
export function setBusy(busy) {
|
package/web/views/voice-call.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { h, icon, toast } from "../lib/dom.js";
|
|
2
2
|
import { desktop, S, ui } from "../state.js";
|
|
3
3
|
import { WS } from "../lib/ws.js";
|
|
4
|
-
import { ensureConversation } from "./composer.js";
|
|
4
|
+
import { ensureConversation, updateSendBtn } from "./composer.js";
|
|
5
5
|
import { currentAgentId } from "./slash.js";
|
|
6
6
|
import { cancelVoice } from "./voice.js";
|
|
7
7
|
|
|
@@ -10,7 +10,7 @@ import { cancelVoice } from "./voice.js";
|
|
|
10
10
|
// 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
|
|
11
11
|
// 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
|
|
12
12
|
// 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
|
|
13
|
-
//
|
|
13
|
+
// 通话默认打开沉浸式光球界面,也可以收起成输入框位置的通话条。
|
|
14
14
|
|
|
15
15
|
/** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
|
|
16
16
|
const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
|
|
@@ -42,6 +42,9 @@ const call = {
|
|
|
42
42
|
/** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
|
|
43
43
|
caption: { role: null, text: "", done: false },
|
|
44
44
|
panel: null,
|
|
45
|
+
dialog: null,
|
|
46
|
+
compact: false,
|
|
47
|
+
session: 0,
|
|
45
48
|
els: null,
|
|
46
49
|
};
|
|
47
50
|
|
|
@@ -95,8 +98,12 @@ const ICON_MIC_OFF =
|
|
|
95
98
|
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 3l18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9.3V5a3 3 0 0 0-5.9-.6"/><path d="M5 10a7 7 0 0 0 11.7 5.2M19 10a7 7 0 0 1-.6 2.8M12 17v5"/></svg>';
|
|
96
99
|
|
|
97
100
|
/** 输入框里的「语音对话」按钮 */
|
|
101
|
+
export function canCall() {
|
|
102
|
+
return supported && currentAgentId() === "codex";
|
|
103
|
+
}
|
|
104
|
+
|
|
98
105
|
export function callButton() {
|
|
99
|
-
ui.callBtn = h("button", { class: "
|
|
106
|
+
ui.callBtn = h("button", { class: "send-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
|
|
100
107
|
syncCall();
|
|
101
108
|
return ui.callBtn;
|
|
102
109
|
}
|
|
@@ -110,11 +117,14 @@ export function syncCall() {
|
|
|
110
117
|
if (call.state !== "idle") {
|
|
111
118
|
if (call.key && S.current !== call.key) {
|
|
112
119
|
endCall("切换了对话,语音对话已结束");
|
|
113
|
-
} else if (call.panel
|
|
114
|
-
|
|
120
|
+
} else if (call.panel) {
|
|
121
|
+
ui.composer?.classList.add("in-call");
|
|
122
|
+
if (!call.panel.isConnected || (call.compact && call.panel.previousElementSibling !== ui.composer)) {
|
|
123
|
+
mountPanel();
|
|
124
|
+
}
|
|
115
125
|
}
|
|
116
126
|
}
|
|
117
|
-
|
|
127
|
+
updateSendBtn();
|
|
118
128
|
}
|
|
119
129
|
|
|
120
130
|
async function startCall() {
|
|
@@ -124,7 +134,9 @@ async function startCall() {
|
|
|
124
134
|
return;
|
|
125
135
|
}
|
|
126
136
|
cancelVoice();
|
|
137
|
+
const session = ++call.session;
|
|
127
138
|
call.state = "connecting";
|
|
139
|
+
call.compact = false;
|
|
128
140
|
call.muted = false;
|
|
129
141
|
call.caption = { role: null, text: "", done: false };
|
|
130
142
|
// 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
|
|
@@ -135,18 +147,24 @@ async function startCall() {
|
|
|
135
147
|
mountPanel();
|
|
136
148
|
try {
|
|
137
149
|
const key = await ensureConversation();
|
|
138
|
-
if (call.
|
|
150
|
+
if (session !== call.session) return;
|
|
139
151
|
call.key = key;
|
|
140
|
-
|
|
141
|
-
if (call.
|
|
152
|
+
const mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
153
|
+
if (session !== call.session) {
|
|
154
|
+
for (const track of mic.getTracks()) track.stop();
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
call.mic = mic;
|
|
142
158
|
// iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
|
|
143
159
|
const track = call.mic.getAudioTracks()[0];
|
|
144
160
|
if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
|
|
161
|
+
track.enabled = !call.muted;
|
|
145
162
|
if (track.muted) await waitUnmute(track, 1500);
|
|
163
|
+
if (session !== call.session) return;
|
|
146
164
|
const ctx = audioCtx();
|
|
147
165
|
if (!ctx) throw new Error("浏览器不支持音频处理");
|
|
148
166
|
await loadWorklet(ctx);
|
|
149
|
-
if (call.
|
|
167
|
+
if (session !== call.session) return;
|
|
150
168
|
// 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
|
|
151
169
|
const src = ctx.createMediaStreamSource(call.mic);
|
|
152
170
|
const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
@@ -174,22 +192,26 @@ async function startCall() {
|
|
|
174
192
|
play.connect(dest);
|
|
175
193
|
call.nodes.push(dest);
|
|
176
194
|
try {
|
|
177
|
-
|
|
195
|
+
const loop = await loopback(dest.stream);
|
|
196
|
+
if (session !== call.session) return loop.close();
|
|
197
|
+
call.loop = loop;
|
|
178
198
|
call.audio.srcObject = call.loop.stream;
|
|
179
199
|
call.audio.play().catch(() => {});
|
|
180
200
|
} catch {
|
|
181
201
|
// 环回建不起来:直接放(可能有回声)
|
|
202
|
+
if (session !== call.session) return;
|
|
182
203
|
play.connect(ctx.destination);
|
|
183
204
|
}
|
|
184
205
|
}
|
|
185
|
-
if (call.
|
|
206
|
+
if (session !== call.session) return;
|
|
186
207
|
const prefs = loadPrefs();
|
|
187
208
|
await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
|
|
188
|
-
if (call.
|
|
209
|
+
if (session !== call.session) return;
|
|
189
210
|
call.state = "live";
|
|
190
211
|
call.startedAt = Date.now();
|
|
191
212
|
render();
|
|
192
213
|
} catch (err) {
|
|
214
|
+
if (session !== call.session) return;
|
|
193
215
|
const message = err?.name === "NotAllowedError" ? "没有麦克风权限,请在浏览器里允许使用麦克风" : err?.name === "NotFoundError" ? "没找到可用的麦克风" : err?.message ?? String(err);
|
|
194
216
|
endCall(`语音对话没能开始:${message}`);
|
|
195
217
|
}
|
|
@@ -199,6 +221,7 @@ async function startCall() {
|
|
|
199
221
|
export function endCall(reason, fromServer = false) {
|
|
200
222
|
if (call.state === "idle") return;
|
|
201
223
|
const key = call.key;
|
|
224
|
+
++call.session;
|
|
202
225
|
call.state = "idle";
|
|
203
226
|
if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
|
|
204
227
|
cancelAnimationFrame(call.raf);
|
|
@@ -224,9 +247,13 @@ export function endCall(reason, fromServer = false) {
|
|
|
224
247
|
call.micLevel = call.outLevel = null;
|
|
225
248
|
call.key = null;
|
|
226
249
|
call.panel?.remove();
|
|
250
|
+
call.dialog?.close();
|
|
251
|
+
call.dialog?.remove();
|
|
252
|
+
call.dialog = null;
|
|
227
253
|
call.panel = call.els = null;
|
|
228
254
|
ui.composer?.classList.remove("in-call");
|
|
229
255
|
syncCall();
|
|
256
|
+
(ui.callBtn && !ui.callBtn.hidden ? ui.callBtn : ui.sendBtn)?.focus({ preventScroll: true });
|
|
230
257
|
if (reason) toast(reason);
|
|
231
258
|
}
|
|
232
259
|
|
|
@@ -261,41 +288,71 @@ export function onRealtimeEvent(msg) {
|
|
|
261
288
|
renderCaption();
|
|
262
289
|
}
|
|
263
290
|
|
|
264
|
-
// ----------
|
|
291
|
+
// ---------- 通话界面 ----------
|
|
265
292
|
|
|
266
293
|
function mountPanel() {
|
|
267
294
|
if (!ui.composer) return;
|
|
268
295
|
if (!call.panel) {
|
|
269
|
-
const
|
|
296
|
+
const orb = h("div", { class: "call-orb", "aria-hidden": "true" },
|
|
297
|
+
h("div", { class: "call-orb-surface" }, h("span", { class: "call-orb-cloud cloud-a" }), h("span", { class: "call-orb-cloud cloud-b" }), h("span", { class: "call-orb-cloud cloud-c" })),
|
|
298
|
+
);
|
|
270
299
|
const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
|
|
271
300
|
const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
|
|
272
|
-
const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "
|
|
273
|
-
const
|
|
301
|
+
const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, icon("x"));
|
|
302
|
+
const minimize = h("button", { class: "call-minimize icon-btn", title: "收起通话", "aria-label": "收起通话", onclick: () => setCompact(true) }, icon("down"));
|
|
303
|
+
const expand = h("button", { class: "call-expand call-ctl", title: "展开通话", "aria-label": "展开通话", onclick: () => setCompact(false) }, icon("expand"));
|
|
304
|
+
const status = h("span", { class: "call-status", role: "status" });
|
|
274
305
|
const time = h("span", { class: "call-time" });
|
|
275
306
|
const who = h("span", { class: "call-who" });
|
|
276
307
|
const text = h("span", { class: "call-text" });
|
|
277
308
|
const opts = optsRow();
|
|
278
|
-
|
|
309
|
+
const hint = h("p", { class: "call-hint" }, "直接开口,随时打断");
|
|
310
|
+
call.els = { orb, mute, gear, opts, status, time, who, text, hint, expand, minimize };
|
|
279
311
|
call.panel = h(
|
|
280
312
|
"div",
|
|
281
313
|
{ class: "call-panel", role: "region", "aria-label": "语音对话" },
|
|
314
|
+
h("header", { class: "call-header" },
|
|
315
|
+
h("div", { class: "call-brand" }, h("span", { class: "call-brand-icon", html: ICON_WAVE }), h("span", {}, "Codex", h("small", {}, "语音对话"))),
|
|
316
|
+
h("div", { class: "call-session" }, h("span", { class: "call-live-dot" }), time), minimize,
|
|
317
|
+
),
|
|
282
318
|
h(
|
|
283
319
|
"div",
|
|
284
320
|
{ class: "call-main" },
|
|
285
|
-
h("div", { class: "call-
|
|
286
|
-
h("div", { class: "call-info" },
|
|
287
|
-
h("div", { class: "call-ctls" }, mute, gear, end),
|
|
321
|
+
h("div", { class: "call-stage" }, orb),
|
|
322
|
+
h("div", { class: "call-info" }, status, hint),
|
|
288
323
|
),
|
|
324
|
+
h("div", { class: "call-caption", "aria-live": "polite", "aria-atomic": "true" }, who, text),
|
|
289
325
|
opts,
|
|
326
|
+
h("footer", { class: "call-footer" }, h("div", { class: "call-ctls" }, expand,
|
|
327
|
+
h("div", { class: "call-control" }, mute, h("span", {}, "麦克风")),
|
|
328
|
+
h("div", { class: "call-control" }, gear, h("span", {}, "音色与语言")),
|
|
329
|
+
h("div", { class: "call-control" }, end, h("span", {}, "结束")),
|
|
330
|
+
), h("span", { class: "call-footer-note" }, "可以收起通话,查看聊天")),
|
|
290
331
|
);
|
|
332
|
+
call.dialog = h("dialog", { class: "call-dialog", "aria-label": "Codex 语音对话", oncancel: (e) => { e.preventDefault(); setCompact(true); } });
|
|
291
333
|
}
|
|
292
334
|
ui.composer.classList.add("in-call");
|
|
293
|
-
ui.composer.after(call.panel);
|
|
335
|
+
if (call.compact) ui.composer.after(call.panel);
|
|
336
|
+
else {
|
|
337
|
+
if (!call.dialog.isConnected) document.body.append(call.dialog);
|
|
338
|
+
call.dialog.append(call.panel);
|
|
339
|
+
if (!call.dialog.open) call.dialog.showModal();
|
|
340
|
+
}
|
|
294
341
|
render();
|
|
295
342
|
cancelAnimationFrame(call.raf);
|
|
296
343
|
call.raf = requestAnimationFrame(tick);
|
|
297
344
|
}
|
|
298
345
|
|
|
346
|
+
/** 收起只切换视图,麦克风和播放保持连接。Esc 也收起,避免误挂断。 */
|
|
347
|
+
function setCompact(compact) {
|
|
348
|
+
if (!call.panel || call.compact === compact) return;
|
|
349
|
+
call.compact = compact;
|
|
350
|
+
call.panel.classList.toggle("compact", compact);
|
|
351
|
+
if (compact) call.dialog.close();
|
|
352
|
+
mountPanel();
|
|
353
|
+
(compact ? call.els.expand : call.els.minimize).focus({ preventScroll: true });
|
|
354
|
+
}
|
|
355
|
+
|
|
299
356
|
/** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
|
|
300
357
|
function optsRow() {
|
|
301
358
|
const prefs = loadPrefs();
|
|
@@ -327,8 +384,11 @@ function toggleOpts() {
|
|
|
327
384
|
/** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
|
|
328
385
|
function restartCall() {
|
|
329
386
|
if (call.state === "idle") return;
|
|
387
|
+
const { compact, muted } = call;
|
|
330
388
|
endCall("");
|
|
331
389
|
void startCall();
|
|
390
|
+
if (muted) toggleMute();
|
|
391
|
+
if (compact) setCompact(true);
|
|
332
392
|
// startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
|
|
333
393
|
if (call.els) toggleOpts();
|
|
334
394
|
}
|
|
@@ -345,8 +405,10 @@ function render() {
|
|
|
345
405
|
mute.innerHTML = call.muted ? ICON_MIC_OFF : icon("mic").innerHTML;
|
|
346
406
|
mute.title = call.muted ? "取消静音" : "静音";
|
|
347
407
|
mute.setAttribute("aria-label", mute.title);
|
|
408
|
+
mute.setAttribute("aria-pressed", String(call.muted));
|
|
348
409
|
mute.classList.toggle("on", call.muted);
|
|
349
410
|
call.panel.classList.toggle("connecting", call.state === "connecting");
|
|
411
|
+
call.panel.classList.toggle("muted", call.muted);
|
|
350
412
|
renderCaption();
|
|
351
413
|
}
|
|
352
414
|
|
|
@@ -354,28 +416,34 @@ function renderCaption() {
|
|
|
354
416
|
if (!call.els) return;
|
|
355
417
|
const { who, text } = call.els;
|
|
356
418
|
const c = call.caption;
|
|
419
|
+
const caption = text.parentElement;
|
|
420
|
+
const follow = caption.scrollHeight - caption.scrollTop - caption.clientHeight < 32;
|
|
357
421
|
who.textContent = c.text ? (c.role === "user" ? "你" : "Codex") : "";
|
|
358
|
-
text.textContent
|
|
422
|
+
if (text.textContent !== c.text) {
|
|
423
|
+
text.textContent = c.text || "";
|
|
424
|
+
if (follow) requestAnimationFrame(() => { if (caption.isConnected) caption.scrollTop = caption.scrollHeight; });
|
|
425
|
+
}
|
|
359
426
|
call.panel.classList.toggle("has-caption", !!c.text);
|
|
360
427
|
}
|
|
361
428
|
|
|
362
|
-
/**
|
|
429
|
+
/** 每帧读取实际音量;光球平滑响应声音,状态和时长仅在变化时写入 DOM。 */
|
|
363
430
|
function tick() {
|
|
364
431
|
if (!call.els) return;
|
|
365
432
|
const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
|
|
366
433
|
const out = call.outLevel?.() ?? 0;
|
|
367
434
|
const speaking = out > 0.02;
|
|
368
435
|
const level = Math.min(1, (speaking ? out : mic) * 6);
|
|
369
|
-
call.els.
|
|
370
|
-
const shape = [0.45, 0.75, 1, 0.75, 0.45][i];
|
|
371
|
-
const wobble = call.state === "live" ? 0.15 + 0.85 * level * shape * (0.8 + 0.2 * Math.sin(Date.now() / 120 + i)) : 0.2;
|
|
372
|
-
bar.style.transform = `scaleY(${Math.max(0.15, wobble).toFixed(3)})`;
|
|
373
|
-
});
|
|
436
|
+
call.els.orb.style.setProperty("--voice-scale", (1 + level * 0.14).toFixed(3));
|
|
374
437
|
call.panel.classList.toggle("speaking", speaking);
|
|
375
438
|
const busy = S.busy && S.current === call.key;
|
|
376
|
-
call.
|
|
439
|
+
call.panel.classList.toggle("processing", !!busy && !speaking);
|
|
440
|
+
const status =
|
|
377
441
|
call.state === "connecting" ? "正在连接…" : speaking ? "Codex 正在说" : busy ? "Codex 正在处理…" : call.muted ? "已静音" : "正在听";
|
|
378
|
-
|
|
442
|
+
const hint = call.state === "connecting" ? "正在为你接通 Codex" : call.muted ? "麦克风已关闭,仍可听到回复" : busy && !speaking ? "完成后会接着告诉你" : "直接开口,随时打断";
|
|
443
|
+
const time = call.state === "live" ? clock(Date.now() - call.startedAt) : "连接中";
|
|
444
|
+
for (const [el, value] of [[call.els.status, status], [call.els.time, time], [call.els.hint, hint]]) {
|
|
445
|
+
if (el.textContent !== value) el.textContent = value;
|
|
446
|
+
}
|
|
379
447
|
call.raf = requestAnimationFrame(tick);
|
|
380
448
|
}
|
|
381
449
|
|