agent-loft 0.2.111 → 0.2.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +6 -3
  2. package/package.json +1 -1
  3. package/src/agents/codex/protocol.ts +10 -0
  4. package/src/agents/codex/session.ts +201 -5
  5. package/src/agents/dsh/acp.ts +193 -0
  6. package/src/agents/dsh/driver.ts +126 -0
  7. package/src/agents/dsh/errors.ts +43 -0
  8. package/src/agents/dsh/history.ts +346 -0
  9. package/src/agents/dsh/sdk.ts +191 -0
  10. package/src/agents/dsh/session.ts +454 -0
  11. package/src/agents/dsh/steps.ts +22 -0
  12. package/src/agents/dsh/which.ts +23 -0
  13. package/src/agents/health.ts +51 -2
  14. package/src/channels/feishu/bot.ts +1 -1
  15. package/src/channels/web/api.ts +14 -0
  16. package/src/channels/web/experts-api.ts +1 -0
  17. package/src/channels/web/server.ts +28 -1
  18. package/src/core/builtin-agents.ts +1 -1
  19. package/src/core/command-catalog.ts +2 -1
  20. package/src/core/commands.ts +2 -1
  21. package/src/core/config.ts +23 -3
  22. package/src/core/expert-prompt.ts +15 -0
  23. package/src/core/experts.ts +22 -3
  24. package/src/core/hub.ts +60 -3
  25. package/src/core/memory-curator.ts +2 -2
  26. package/src/core/memory.ts +2 -2
  27. package/src/core/permission.ts +1 -1
  28. package/src/core/types.ts +19 -0
  29. package/src/core/users.ts +1 -1
  30. package/src/main.ts +5 -1
  31. package/web/css/composer.css +164 -0
  32. package/web/css/experts.css +66 -0
  33. package/web/lib/dom.js +2 -0
  34. package/web/lib/ws.js +4 -0
  35. package/web/settings/agent-defaults.js +2 -2
  36. package/web/settings/agents.js +15 -1
  37. package/web/settings/commands.js +1 -1
  38. package/web/settings/experts.js +86 -17
  39. package/web/settings/memory.js +2 -1
  40. package/web/views/composer.js +3 -1
  41. package/web/views/messages.js +1 -1
  42. package/web/views/sidebar.js +1 -1
  43. package/web/views/topbar.js +3 -1
  44. package/web/views/voice-call.js +293 -0
@@ -0,0 +1,293 @@
1
+ import { h, icon, toast } from "../lib/dom.js";
2
+ import { desktop, S, ui } from "../state.js";
3
+ import { WS } from "../lib/ws.js";
4
+ import { ensureConversation } from "./composer.js";
5
+ import { currentAgentId } from "./slash.js";
6
+ import { cancelVoice } from "./voice.js";
7
+
8
+ // ---------- 实时语音对话(只有 Codex) ----------
9
+ // 浏览器建 RTCPeerConnection(麦克风 + oai-events 数据通道),offer 交给服务端,服务端让 Codex app-server 去 OpenAI 建通话,
10
+ // answer 拿回来连上。声音在浏览器和 OpenAI 之间直连,不经过 agent-loft。
11
+ // 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
12
+ // 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
13
+
14
+ /** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
15
+ const supported = !desktop && !!window.RTCPeerConnection && !!navigator.mediaDevices?.getUserMedia;
16
+
17
+ const call = {
18
+ /** idle / connecting / live */
19
+ state: "idle",
20
+ key: null,
21
+ pc: null,
22
+ mic: null,
23
+ audio: null,
24
+ muted: false,
25
+ startedAt: 0,
26
+ raf: 0,
27
+ ctx: null,
28
+ micLevel: null,
29
+ outLevel: null,
30
+ /** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
31
+ caption: { role: null, text: "", done: false },
32
+ panel: null,
33
+ els: null,
34
+ /** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
35
+ dropTimer: 0,
36
+ };
37
+
38
+ const ICON_WAVE =
39
+ '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
40
+ const ICON_MIC_OFF =
41
+ '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 3l18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9.3V5a3 3 0 0 0-5.9-.6"/><path d="M5 10a7 7 0 0 0 11.7 5.2M19 10a7 7 0 0 1-.6 2.8M12 17v5"/></svg>';
42
+
43
+ /** 输入框里的「语音对话」按钮 */
44
+ export function callButton() {
45
+ ui.callBtn = h("button", { class: "icon-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
46
+ syncCall();
47
+ return ui.callBtn;
48
+ }
49
+
50
+ /**
51
+ * 换了对话 / 重画了输入框之后对齐:只有 Codex 的对话显示按钮;通话中切到别的对话就挂断,
52
+ * 输入框重画了就把通话条重新放回去
53
+ */
54
+ export function syncCall() {
55
+ // 新对话里点的:会话还在建(call.key 还没有),建好时会重画一次,不算切走
56
+ if (call.state !== "idle") {
57
+ if (call.key && S.current !== call.key) {
58
+ endCall("切换了对话,语音对话已结束");
59
+ } else if (call.panel && !call.panel.isConnected) {
60
+ mountPanel();
61
+ }
62
+ }
63
+ if (ui.callBtn) ui.callBtn.hidden = !supported || currentAgentId() !== "codex";
64
+ }
65
+
66
+ async function startCall() {
67
+ if (call.state !== "idle") return;
68
+ if (!window.isSecureContext) {
69
+ toast("语音对话要用麦克风,需要 HTTPS 访问(或者本机 localhost)");
70
+ return;
71
+ }
72
+ cancelVoice();
73
+ call.state = "connecting";
74
+ call.muted = false;
75
+ call.caption = { role: null, text: "", done: false };
76
+ // 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
77
+ call.audio = h("audio", { autoplay: true, playsinline: true });
78
+ call.audio.play().catch(() => {});
79
+ mountPanel();
80
+ try {
81
+ const key = await ensureConversation();
82
+ if (call.state === "idle") return;
83
+ call.key = key;
84
+ call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
85
+ if (call.state === "idle") return stopTracks();
86
+ const pc = new RTCPeerConnection();
87
+ call.pc = pc;
88
+ for (const track of call.mic.getTracks()) pc.addTrack(track, call.mic);
89
+ // OpenAI 要求 offer 里有这个数据通道(事件走这里;字幕我们用服务端转来的)
90
+ pc.createDataChannel("oai-events");
91
+ pc.ontrack = (e) => {
92
+ const stream = e.streams[0] ?? new MediaStream([e.track]);
93
+ call.audio.srcObject = stream;
94
+ call.audio.play().catch(() => {});
95
+ call.outLevel = meter(stream);
96
+ };
97
+ pc.onconnectionstatechange = () => {
98
+ const st = pc.connectionState;
99
+ clearTimeout(call.dropTimer);
100
+ if (st === "failed" || st === "closed") endCall("语音连接断开了");
101
+ // 网络抖一下会短暂 disconnected,几秒内没恢复才算断
102
+ else if (st === "disconnected") call.dropTimer = setTimeout(() => endCall("语音连接断开了"), 5000);
103
+ };
104
+ call.micLevel = meter(call.mic);
105
+ await pc.setLocalDescription(await pc.createOffer());
106
+ await iceGathered(pc, 1500);
107
+ if (call.state === "idle") return;
108
+ const res = await WS.request({ t: "rt", op: "start", key, sdp: pc.localDescription.sdp }, 30_000);
109
+ if (call.state === "idle") return;
110
+ await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
111
+ call.state = "live";
112
+ call.startedAt = Date.now();
113
+ render();
114
+ } catch (err) {
115
+ const message = err?.name === "NotAllowedError" ? "没有麦克风权限,请在浏览器里允许使用麦克风" : err?.name === "NotFoundError" ? "没找到可用的麦克风" : err?.message ?? String(err);
116
+ endCall(`语音对话没能开始:${message}`);
117
+ }
118
+ }
119
+
120
+ /** 挂断。fromServer:服务端已经说结束了,不用再通知它 */
121
+ export function endCall(reason, fromServer = false) {
122
+ if (call.state === "idle") return;
123
+ const key = call.key;
124
+ call.state = "idle";
125
+ if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
126
+ clearTimeout(call.dropTimer);
127
+ cancelAnimationFrame(call.raf);
128
+ call.pc?.close();
129
+ call.pc = null;
130
+ stopTracks();
131
+ if (call.audio) call.audio.srcObject = null;
132
+ call.audio = null;
133
+ call.ctx?.close().catch(() => {});
134
+ call.ctx = null;
135
+ call.micLevel = call.outLevel = null;
136
+ call.key = null;
137
+ call.panel?.remove();
138
+ call.panel = call.els = null;
139
+ ui.composer?.classList.remove("in-call");
140
+ syncCall();
141
+ if (reason) toast(reason);
142
+ }
143
+
144
+ function stopTracks() {
145
+ for (const t of call.mic?.getTracks() ?? []) t.stop();
146
+ call.mic = null;
147
+ }
148
+
149
+ /** 服务端转来的通话事件(ws 的 t: "rt") */
150
+ export function onRealtimeEvent(msg) {
151
+ if (call.state === "idle" || msg.key !== call.key) return;
152
+ const ev = msg.ev ?? {};
153
+ switch (ev.type) {
154
+ case "transcript": {
155
+ const c = call.caption;
156
+ if (c.role !== ev.role || c.done) call.caption = { role: ev.role, text: ev.delta, done: false };
157
+ else c.text += ev.delta;
158
+ break;
159
+ }
160
+ case "transcript_done":
161
+ call.caption = { role: ev.role, text: ev.text, done: true };
162
+ break;
163
+ case "started":
164
+ break;
165
+ case "error":
166
+ toast(`语音对话出错:${ev.message}`, 4000);
167
+ break;
168
+ case "closed":
169
+ endCall(ev.reason && ev.reason !== "requested" ? `语音对话已结束(${ev.reason})` : "语音对话已结束", true);
170
+ return;
171
+ }
172
+ renderCaption();
173
+ }
174
+
175
+ // ---------- 通话条 ----------
176
+
177
+ function mountPanel() {
178
+ if (!ui.composer) return;
179
+ if (!call.panel) {
180
+ const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
181
+ const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
182
+ const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
183
+ const status = h("span", { class: "call-status" });
184
+ const time = h("span", { class: "call-time" });
185
+ const who = h("span", { class: "call-who" });
186
+ const text = h("span", { class: "call-text" });
187
+ call.els = { bars, mute, status, time, who, text };
188
+ call.panel = h(
189
+ "div",
190
+ { class: "call-panel", role: "region", "aria-label": "语音对话" },
191
+ h(
192
+ "div",
193
+ { class: "call-main" },
194
+ h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
195
+ h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
196
+ h("div", { class: "call-ctls" }, mute, end),
197
+ ),
198
+ );
199
+ }
200
+ ui.composer.classList.add("in-call");
201
+ ui.composer.after(call.panel);
202
+ render();
203
+ cancelAnimationFrame(call.raf);
204
+ call.raf = requestAnimationFrame(tick);
205
+ }
206
+
207
+ function toggleMute() {
208
+ call.muted = !call.muted;
209
+ for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
210
+ render();
211
+ }
212
+
213
+ function render() {
214
+ if (!call.els) return;
215
+ const { mute } = call.els;
216
+ mute.innerHTML = call.muted ? ICON_MIC_OFF : icon("mic").innerHTML;
217
+ mute.title = call.muted ? "取消静音" : "静音";
218
+ mute.setAttribute("aria-label", mute.title);
219
+ mute.classList.toggle("on", call.muted);
220
+ call.panel.classList.toggle("connecting", call.state === "connecting");
221
+ renderCaption();
222
+ }
223
+
224
+ function renderCaption() {
225
+ if (!call.els) return;
226
+ const { who, text } = call.els;
227
+ const c = call.caption;
228
+ who.textContent = c.text ? (c.role === "user" ? "你" : "Codex") : "";
229
+ text.textContent = c.text || (call.state === "connecting" ? "" : "说点什么吧,可以随时打断它");
230
+ call.panel.classList.toggle("has-caption", !!c.text);
231
+ }
232
+
233
+ /** 每帧:音量条、状态、时长 */
234
+ function tick() {
235
+ if (!call.els) return;
236
+ const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
237
+ const out = call.outLevel?.() ?? 0;
238
+ const speaking = out > 0.02;
239
+ const level = Math.min(1, (speaking ? out : mic) * 6);
240
+ call.els.bars.forEach((bar, i) => {
241
+ const shape = [0.45, 0.75, 1, 0.75, 0.45][i];
242
+ const wobble = call.state === "live" ? 0.15 + 0.85 * level * shape * (0.8 + 0.2 * Math.sin(Date.now() / 120 + i)) : 0.2;
243
+ bar.style.transform = `scaleY(${Math.max(0.15, wobble).toFixed(3)})`;
244
+ });
245
+ call.panel.classList.toggle("speaking", speaking);
246
+ const busy = S.busy && S.current === call.key;
247
+ call.els.status.textContent =
248
+ call.state === "connecting" ? "正在连接…" : speaking ? "Codex 正在说" : busy ? "Codex 正在处理…" : call.muted ? "已静音" : "正在听";
249
+ call.els.time.textContent = call.state === "live" ? clock(Date.now() - call.startedAt) : "";
250
+ call.raf = requestAnimationFrame(tick);
251
+ }
252
+
253
+ function clock(ms) {
254
+ const s = Math.floor(ms / 1000);
255
+ return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
256
+ }
257
+
258
+ /** 一路声音的音量(0~1 的均方根),给动画用 */
259
+ function meter(stream) {
260
+ try {
261
+ call.ctx ??= new AudioContext();
262
+ void call.ctx.resume();
263
+ const src = call.ctx.createMediaStreamSource(stream);
264
+ const an = call.ctx.createAnalyser();
265
+ an.fftSize = 512;
266
+ src.connect(an);
267
+ const buf = new Float32Array(an.fftSize);
268
+ return () => {
269
+ an.getFloatTimeDomainData(buf);
270
+ let sum = 0;
271
+ for (const v of buf) sum += v * v;
272
+ return Math.sqrt(sum / buf.length);
273
+ };
274
+ } catch {
275
+ return null;
276
+ }
277
+ }
278
+
279
+ /** 等 ICE 候选收集完(最多 ms 毫秒):offer 里带上候选,不用另外传 */
280
+ function iceGathered(pc, ms) {
281
+ if (pc.iceGatheringState === "complete") return Promise.resolve();
282
+ return new Promise((resolve) => {
283
+ const done = () => {
284
+ pc.removeEventListener("icegatheringstatechange", check);
285
+ resolve();
286
+ };
287
+ const check = () => pc.iceGatheringState === "complete" && done();
288
+ pc.addEventListener("icegatheringstatechange", check);
289
+ setTimeout(done, ms);
290
+ });
291
+ }
292
+
293
+ window.addEventListener("pagehide", () => endCall());