agent-loft 0.2.112 → 0.2.114

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +4 -3
  2. package/package.json +3 -1
  3. package/src/agents/codex/session.ts +31 -3
  4. package/src/agents/dsh/acp.ts +193 -0
  5. package/src/agents/dsh/driver.ts +126 -0
  6. package/src/agents/dsh/errors.ts +43 -0
  7. package/src/agents/dsh/history.ts +346 -0
  8. package/src/agents/dsh/sdk.ts +191 -0
  9. package/src/agents/dsh/session.ts +454 -0
  10. package/src/agents/dsh/steps.ts +22 -0
  11. package/src/agents/dsh/which.ts +23 -0
  12. package/src/agents/health.ts +51 -2
  13. package/src/channels/feishu/bot.ts +1 -1
  14. package/src/channels/web/api.ts +14 -0
  15. package/src/channels/web/server.ts +69 -11
  16. package/src/core/builtin-agents.ts +1 -1
  17. package/src/core/command-catalog.ts +2 -1
  18. package/src/core/commands.ts +2 -1
  19. package/src/core/config.ts +23 -3
  20. package/src/core/experts.ts +3 -3
  21. package/src/core/hub.ts +4 -3
  22. package/src/core/memory-curator.ts +2 -2
  23. package/src/core/memory.ts +2 -2
  24. package/src/core/permission.ts +1 -1
  25. package/src/core/realtime-relay.ts +142 -0
  26. package/src/core/types.ts +4 -1
  27. package/src/core/users.ts +1 -1
  28. package/src/main.ts +5 -1
  29. package/web/css/composer.css +38 -0
  30. package/web/lib/dom.js +2 -0
  31. package/web/lib/rt-worklet.js +110 -0
  32. package/web/lib/ws.js +5 -2
  33. package/web/settings/agent-defaults.js +2 -2
  34. package/web/settings/agents.js +15 -1
  35. package/web/settings/commands.js +1 -1
  36. package/web/settings/experts.js +1 -1
  37. package/web/settings/memory.js +2 -1
  38. package/web/views/composer.js +1 -1
  39. package/web/views/sidebar.js +1 -1
  40. package/web/views/topbar.js +1 -1
  41. package/web/views/voice-call.js +248 -56
@@ -6,35 +6,89 @@ import { currentAgentId } from "./slash.js";
6
6
  import { cancelVoice } from "./voice.js";
7
7
 
8
8
  // ---------- 实时语音对话(只有 Codex) ----------
9
- // 浏览器建 RTCPeerConnection(麦克风 + oai-events 数据通道),offer 交给服务端,服务端让 Codex app-server 去 OpenAI 建通话,
10
- // answer 拿回来连上。声音在浏览器和 OpenAI 之间直连,不经过 agent-loft。
9
+ // 声音经服务器中转:服务器自己和 OpenAI 建 WebRTC(core/realtime-relay.ts),这里和服务器之间走现有的 WebSocket,
10
+ // 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
11
+ // 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
11
12
  // 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
12
13
  // 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
13
14
 
14
15
  /** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
15
- const supported = !desktop && !!window.RTCPeerConnection && !!navigator.mediaDevices?.getUserMedia;
16
+ const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
17
+
18
+ /**
19
+ * 回复的声音直接从 AudioContext 放出来,iOS / Safari 的回声消除照样管得到;Chrome 等的回声消除只认 WebRTC 放出来的声音,
20
+ * 不绕一下模型会听到自己说话、自己打断自己:在本页里用一对 RTCPeerConnection 环回一下,再交给 <audio> 放
21
+ */
22
+ const ua = navigator.userAgent;
23
+ const appleAudio = /iPad|iPhone|iPod/.test(ua) || (/Macintosh/.test(ua) && navigator.maxTouchPoints > 1) || (/Safari\//.test(ua) && !/Chrome|Chromium|Edg|OPR|Firefox/.test(ua));
16
24
 
17
25
  const call = {
18
26
  /** idle / connecting / live */
19
27
  state: "idle",
20
28
  key: null,
21
- pc: null,
22
29
  mic: null,
30
+ /** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
31
+ play: null,
32
+ /** Chrome 等的回声消除环回(见 appleAudio) */
33
+ loop: null,
23
34
  audio: null,
24
35
  muted: false,
25
36
  startedAt: 0,
26
37
  raf: 0,
27
- ctx: null,
38
+ /** 音量表接上的节点,挂断时断开(AudioContext 本身留着,见 audioCtx) */
39
+ nodes: [],
28
40
  micLevel: null,
29
41
  outLevel: null,
30
42
  /** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
31
43
  caption: { role: null, text: "", done: false },
32
44
  panel: null,
33
45
  els: null,
34
- /** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
35
- dropTimer: 0,
36
46
  };
37
47
 
48
+ // ---------- 音色和语言(按设备记在浏览器里) ----------
49
+
50
+ const PREFS_KEY = "agent-loft.voiceCall";
51
+ /** 和服务端 REALTIME_VOICES 一致(就是 ChatGPT 语音模式那几个);空 = Codex 默认(cove) */
52
+ const VOICES = [
53
+ ["", "默认(Cove)"],
54
+ ["cove", "Cove · 男声,沉着直接"],
55
+ ["juniper", "Juniper · 女声,开朗"],
56
+ ["maple", "Maple · 女声,爽朗坦率"],
57
+ ["vale", "Vale · 女声,明亮好奇"],
58
+ ["sol", "Sol · 女声,从容干练"],
59
+ ["spruce", "Spruce · 男声,平和"],
60
+ ["ember", "Ember · 男声,自信乐观"],
61
+ ["arbor", "Arbor · 男声,随和"],
62
+ ["breeze", "Breeze · 活泼真诚"],
63
+ ];
64
+ /** 和服务端 REALTIME_LANGUAGES 一致;空 = 跟着你说的语言 */
65
+ const LANGUAGES = [
66
+ ["", "跟着我说的语言"],
67
+ ["zh-CN", "普通话(简体)"],
68
+ ["zh-TW", "国语(繁体)"],
69
+ ["yue", "粤语"],
70
+ ["en", "English"],
71
+ ["ja", "日本語"],
72
+ ["ko", "한국어"],
73
+ ];
74
+
75
+ function loadPrefs() {
76
+ try {
77
+ const p = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "{}");
78
+ return { voice: typeof p.voice === "string" ? p.voice : "", language: typeof p.language === "string" ? p.language : "" };
79
+ } catch {
80
+ return { voice: "", language: "" };
81
+ }
82
+ }
83
+
84
+ function savePrefs(p) {
85
+ try {
86
+ localStorage.setItem(PREFS_KEY, JSON.stringify(p));
87
+ } catch {
88
+ // 隐私模式等存不了:这次通话照样用
89
+ }
90
+ }
91
+
38
92
  const ICON_WAVE =
39
93
  '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
40
94
  const ICON_MIC_OFF =
@@ -76,6 +130,8 @@ async function startCall() {
76
130
  // 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
77
131
  call.audio = h("audio", { autoplay: true, playsinline: true });
78
132
  call.audio.play().catch(() => {});
133
+ // AudioContext 也要在点击里建 / 恢复:iOS 上点击之外起的一直是 suspended,第二通起就没声音、录不到
134
+ audioCtx();
79
135
  mountPanel();
80
136
  try {
81
137
  const key = await ensureConversation();
@@ -83,31 +139,53 @@ async function startCall() {
83
139
  call.key = key;
84
140
  call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
85
141
  if (call.state === "idle") return stopTracks();
86
- const pc = new RTCPeerConnection();
87
- call.pc = pc;
88
- for (const track of call.mic.getTracks()) pc.addTrack(track, call.mic);
89
- // OpenAI 要求 offer 里有这个数据通道(事件走这里;字幕我们用服务端转来的)
90
- pc.createDataChannel("oai-events");
91
- pc.ontrack = (e) => {
92
- const stream = e.streams[0] ?? new MediaStream([e.track]);
93
- call.audio.srcObject = stream;
94
- call.audio.play().catch(() => {});
95
- call.outLevel = meter(stream);
96
- };
97
- pc.onconnectionstatechange = () => {
98
- const st = pc.connectionState;
99
- clearTimeout(call.dropTimer);
100
- if (st === "failed" || st === "closed") endCall("语音连接断开了");
101
- // 网络抖一下会短暂 disconnected,几秒内没恢复才算断
102
- else if (st === "disconnected") call.dropTimer = setTimeout(() => endCall("语音连接断开了"), 5000);
142
+ // iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
143
+ const track = call.mic.getAudioTracks()[0];
144
+ if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
145
+ if (track.muted) await waitUnmute(track, 1500);
146
+ const ctx = audioCtx();
147
+ if (!ctx) throw new Error("浏览器不支持音频处理");
148
+ await loadWorklet(ctx);
149
+ if (call.state === "idle") return stopTracks();
150
+ // 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
151
+ const src = ctx.createMediaStreamSource(call.mic);
152
+ const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
153
+ // 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
154
+ const sink = ctx.createGain();
155
+ sink.gain.value = 0;
156
+ src.connect(capture);
157
+ capture.connect(sink);
158
+ sink.connect(ctx.destination);
159
+ capture.port.onmessage = (e) => {
160
+ const sock = WS.sock;
161
+ // 网络卡住时别越攒越多(攒下的说出去也晚了)
162
+ if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
103
163
  };
104
- call.micLevel = meter(call.mic);
105
- await pc.setLocalDescription(await pc.createOffer());
106
- await iceGathered(pc, 1500);
164
+ // 服务器发来的 → 播放 → 扬声器
165
+ const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
166
+ call.play = play;
167
+ call.nodes.push(src, capture, sink, play);
168
+ call.micLevel = meter(src);
169
+ call.outLevel = meter(play);
170
+ if (appleAudio) {
171
+ play.connect(ctx.destination);
172
+ } else {
173
+ const dest = ctx.createMediaStreamDestination();
174
+ play.connect(dest);
175
+ call.nodes.push(dest);
176
+ try {
177
+ call.loop = await loopback(dest.stream);
178
+ call.audio.srcObject = call.loop.stream;
179
+ call.audio.play().catch(() => {});
180
+ } catch {
181
+ // 环回建不起来:直接放(可能有回声)
182
+ play.connect(ctx.destination);
183
+ }
184
+ }
107
185
  if (call.state === "idle") return;
108
- const res = await WS.request({ t: "rt", op: "start", key, sdp: pc.localDescription.sdp }, 30_000);
186
+ const prefs = loadPrefs();
187
+ await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
109
188
  if (call.state === "idle") return;
110
- await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
111
189
  call.state = "live";
112
190
  call.startedAt = Date.now();
113
191
  render();
@@ -123,15 +201,26 @@ export function endCall(reason, fromServer = false) {
123
201
  const key = call.key;
124
202
  call.state = "idle";
125
203
  if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
126
- clearTimeout(call.dropTimer);
127
204
  cancelAnimationFrame(call.raf);
128
- call.pc?.close();
129
- call.pc = null;
130
205
  stopTracks();
131
- if (call.audio) call.audio.srcObject = null;
206
+ call.loop?.close();
207
+ call.loop = null;
208
+ call.play = null;
209
+ if (call.audio) {
210
+ call.audio.pause();
211
+ call.audio.srcObject = null;
212
+ }
132
213
  call.audio = null;
133
- call.ctx?.close().catch(() => {});
134
- call.ctx = null;
214
+ // 不 close:iOS 上关掉接过麦克风的 AudioContext,之后再开麦克风常常是静音的。断开节点、先暂停,下次点击时恢复
215
+ for (const n of call.nodes) {
216
+ try {
217
+ n.disconnect();
218
+ } catch {
219
+ // 已经断开
220
+ }
221
+ }
222
+ call.nodes = [];
223
+ if (sharedCtx?.state === "running") sharedCtx.suspend().catch(() => {});
135
224
  call.micLevel = call.outLevel = null;
136
225
  call.key = null;
137
226
  call.panel?.remove();
@@ -179,12 +268,14 @@ function mountPanel() {
179
268
  if (!call.panel) {
180
269
  const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
181
270
  const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
271
+ const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
182
272
  const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
183
273
  const status = h("span", { class: "call-status" });
184
274
  const time = h("span", { class: "call-time" });
185
275
  const who = h("span", { class: "call-who" });
186
276
  const text = h("span", { class: "call-text" });
187
- call.els = { bars, mute, status, time, who, text };
277
+ const opts = optsRow();
278
+ call.els = { bars, mute, gear, opts, status, time, who, text };
188
279
  call.panel = h(
189
280
  "div",
190
281
  { class: "call-panel", role: "region", "aria-label": "语音对话" },
@@ -193,8 +284,9 @@ function mountPanel() {
193
284
  { class: "call-main" },
194
285
  h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
195
286
  h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
196
- h("div", { class: "call-ctls" }, mute, end),
287
+ h("div", { class: "call-ctls" }, mute, gear, end),
197
288
  ),
289
+ opts,
198
290
  );
199
291
  }
200
292
  ui.composer.classList.add("in-call");
@@ -204,6 +296,43 @@ function mountPanel() {
204
296
  call.raf = requestAnimationFrame(tick);
205
297
  }
206
298
 
299
+ /** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
300
+ function optsRow() {
301
+ const prefs = loadPrefs();
302
+ const pick = (label, list, value, name) => {
303
+ const sel = h("select", { "aria-label": label }, list.map(([v, t]) => h("option", { value: v }, t)));
304
+ sel.value = list.some(([v]) => v === value) ? value : "";
305
+ sel.addEventListener("change", () => {
306
+ savePrefs({ ...loadPrefs(), [name]: sel.value });
307
+ restartCall();
308
+ });
309
+ return h("label", { class: "call-opt" }, h("span", {}, label), sel);
310
+ };
311
+ return h(
312
+ "div",
313
+ { class: "call-opts", hidden: true },
314
+ pick("音色", VOICES, prefs.voice, "voice"),
315
+ pick("语言", LANGUAGES, prefs.language, "language"),
316
+ h("div", { class: "call-opts-hint" }, "记在这台设备上。通话中改了会重新连接,对话内容不受影响"),
317
+ );
318
+ }
319
+
320
+ function toggleOpts() {
321
+ const { opts, gear } = call.els;
322
+ opts.hidden = !opts.hidden;
323
+ gear.classList.toggle("active", !opts.hidden);
324
+ gear.setAttribute("aria-expanded", String(!opts.hidden));
325
+ }
326
+
327
+ /** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
328
+ function restartCall() {
329
+ if (call.state === "idle") return;
330
+ endCall("");
331
+ void startCall();
332
+ // startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
333
+ if (call.els) toggleOpts();
334
+ }
335
+
207
336
  function toggleMute() {
208
337
  call.muted = !call.muted;
209
338
  for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
@@ -255,15 +384,42 @@ function clock(ms) {
255
384
  return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
256
385
  }
257
386
 
258
- /** 一路声音的音量(0~1 的均方根),给动画用 */
259
- function meter(stream) {
387
+ /** 整个页面共用一个 AudioContext(iOS 上反复新建、关闭会把麦克风搞坏),在点击里调用来建 / 恢复 */
388
+ let sharedCtx = null;
389
+ function audioCtx() {
260
390
  try {
261
- call.ctx ??= new AudioContext();
262
- void call.ctx.resume();
263
- const src = call.ctx.createMediaStreamSource(stream);
264
- const an = call.ctx.createAnalyser();
391
+ if (!sharedCtx || sharedCtx.state === "closed") sharedCtx = new (window.AudioContext ?? window.webkitAudioContext)();
392
+ if (sharedCtx.state !== "running") sharedCtx.resume().catch(() => {});
393
+ } catch {
394
+ sharedCtx = null;
395
+ }
396
+ return sharedCtx;
397
+ }
398
+
399
+ /** 轨道 muted 时等它恢复一会儿;还是不行就报错 */
400
+ function waitUnmute(track, ms) {
401
+ return new Promise((resolve, reject) => {
402
+ const timer = setTimeout(() => {
403
+ track.removeEventListener("unmute", ok);
404
+ reject(new Error("麦克风被系统静音了,请检查是否有其他应用在用麦克风,或刷新页面后重试"));
405
+ }, ms);
406
+ const ok = () => {
407
+ clearTimeout(timer);
408
+ resolve();
409
+ };
410
+ track.addEventListener("unmute", ok, { once: true });
411
+ });
412
+ }
413
+
414
+ /** 一个节点的音量(0~1 的均方根),给动画用 */
415
+ function meter(node) {
416
+ try {
417
+ const ctx = audioCtx();
418
+ if (!ctx) return null;
419
+ const an = ctx.createAnalyser();
265
420
  an.fftSize = 512;
266
- src.connect(an);
421
+ node.connect(an);
422
+ call.nodes.push(an);
267
423
  const buf = new Float32Array(an.fftSize);
268
424
  return () => {
269
425
  an.getFloatTimeDomainData(buf);
@@ -276,18 +432,54 @@ function meter(stream) {
276
432
  }
277
433
  }
278
434
 
279
- /** 等 ICE 候选收集完(最多 ms 毫秒):offer 里带上候选,不用另外传 */
280
- function iceGathered(pc, ms) {
281
- if (pc.iceGatheringState === "complete") return Promise.resolve();
282
- return new Promise((resolve) => {
283
- const done = () => {
284
- pc.removeEventListener("icegatheringstatechange", check);
285
- resolve();
286
- };
287
- const check = () => pc.iceGatheringState === "complete" && done();
288
- pc.addEventListener("icegatheringstatechange", check);
289
- setTimeout(done, ms);
290
- });
435
+ /** 每个 AudioContext 加载一次 worklet */
436
+ const workletLoaded = new WeakMap();
437
+ function loadWorklet(ctx) {
438
+ let p = workletLoaded.get(ctx);
439
+ if (!p) {
440
+ p = ctx.audioWorklet.addModule(new URL("../lib/rt-worklet.js", import.meta.url));
441
+ p.catch(() => workletLoaded.delete(ctx));
442
+ workletLoaded.set(ctx, p);
443
+ }
444
+ return p;
445
+ }
446
+
447
+ /** 本页里一对 RTCPeerConnection 把声音环回一下,让浏览器的回声消除认得它(见 appleAudio) */
448
+ async function loopback(stream) {
449
+ const a = new RTCPeerConnection();
450
+ const b = new RTCPeerConnection();
451
+ const close = () => {
452
+ a.close();
453
+ b.close();
454
+ };
455
+ try {
456
+ a.onicecandidate = (e) => e.candidate && b.addIceCandidate(e.candidate).catch(() => {});
457
+ b.onicecandidate = (e) => e.candidate && a.addIceCandidate(e.candidate).catch(() => {});
458
+ const got = new Promise((resolve, reject) => {
459
+ b.ontrack = (e) => resolve(e.streams[0] ?? new MediaStream([e.track]));
460
+ setTimeout(() => reject(new Error("环回超时")), 3000);
461
+ });
462
+ for (const t of stream.getTracks()) a.addTrack(t, stream);
463
+ await a.setLocalDescription(await a.createOffer());
464
+ await b.setRemoteDescription(a.localDescription);
465
+ await b.setLocalDescription(await b.createAnswer());
466
+ await a.setRemoteDescription(b.localDescription);
467
+ return { stream: await got, close };
468
+ } catch (err) {
469
+ close();
470
+ throw err;
471
+ }
472
+ }
473
+
474
+ /** 服务器转来的回复声音(WebSocket 二进制帧:24kHz 16 位 PCM) */
475
+ export function onRealtimeAudio(buf) {
476
+ if (call.state === "idle" || !call.play) return;
477
+ call.play.port.postMessage(buf, [buf]);
478
+ }
479
+
480
+ /** WebSocket 断了:声音走不了了,挂断 */
481
+ export function onSocketLost() {
482
+ endCall("和服务器的连接断开了,语音对话已结束", true);
291
483
  }
292
484
 
293
485
  window.addEventListener("pagehide", () => endCall());