agent-loft 0.2.110 → 0.2.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/agents/codex/protocol.ts +10 -0
- package/src/agents/codex/session.ts +201 -5
- package/src/channels/feishu/client.ts +3 -2
- package/src/channels/feishu/index.ts +26 -0
- package/src/channels/web/experts-api.ts +1 -0
- package/src/channels/web/server.ts +33 -4
- package/src/channels/web/tasks-api.ts +4 -2
- package/src/channels/weixin/bot.ts +6 -0
- package/src/channels/weixin/index.ts +19 -0
- package/src/core/expert-prompt.ts +15 -0
- package/src/core/experts.ts +19 -0
- package/src/core/hub.ts +59 -2
- package/src/core/tasks-bridge.ts +19 -2
- package/src/core/tasks.ts +53 -0
- package/src/core/types.ts +19 -0
- package/src/mcp/tasks-server.ts +2 -0
- package/web/css/composer.css +164 -0
- package/web/css/experts.css +66 -0
- package/web/css/tasks.css +5 -0
- package/web/lib/ws.js +4 -0
- package/web/settings/experts.js +85 -16
- package/web/views/composer.js +2 -0
- package/web/views/messages.js +1 -1
- package/web/views/tasks.js +53 -1
- package/web/views/topbar.js +2 -0
- package/web/views/voice-call.js +293 -0
package/web/settings/experts.js
CHANGED
|
@@ -22,6 +22,15 @@ const PERMS = [
|
|
|
22
22
|
{ id: "exec", label: "跑命令", hint: "执行命令,等于能改任何文件;需要授权时会弹卡" },
|
|
23
23
|
];
|
|
24
24
|
const FILE_LEVELS = ["read", "edit", "exec"];
|
|
25
|
+
/** Codex 自带、专家默认关掉的能力(core/experts.ts 的 CODEX_TOOLS)。admin:用这台机器上的东西,只有管理员能开 */
|
|
26
|
+
const CODEX_TOOLS = [
|
|
27
|
+
{ id: "image_generation", label: "生图", hint: "Codex 自带的生图工具,图片显示在回复里,占 ChatGPT 订阅额度" },
|
|
28
|
+
{ id: "multi_agent", label: "子智能体", hint: "把子任务派给 Codex 子智能体并行做,更费额度" },
|
|
29
|
+
{ id: "browser", label: "操作浏览器", hint: "打开网页、点击、填表,用这台机器上 Codex 的浏览器和登录状态", admin: true },
|
|
30
|
+
{ id: "computer_use", label: "操作桌面", hint: "截屏、点鼠标、敲键盘,控制这台机器的桌面(要有图形桌面)", admin: true },
|
|
31
|
+
{ id: "apps", label: "插件和应用", hint: "Codex 插件,和 Codex 登录的 ChatGPT 账号连接的应用(邮箱、网盘……)", admin: true },
|
|
32
|
+
];
|
|
33
|
+
const CODEX_TOOL_LABEL = Object.fromEntries(CODEX_TOOLS.map((t) => [t.id, t.label]));
|
|
25
34
|
let refreshExpertsPage = () => {};
|
|
26
35
|
document.addEventListener("aim:experts", () => refreshExpertsPage());
|
|
27
36
|
|
|
@@ -162,6 +171,7 @@ function expertCard(data, e, reload) {
|
|
|
162
171
|
e.skills.length ? `${e.skills.length} 个技能` : null,
|
|
163
172
|
e.mcp.length ? `${e.mcp.length} 个 MCP` : null,
|
|
164
173
|
e.subagents.length ? `${e.subagents.length} 个子智能体` : null,
|
|
174
|
+
...(e.engine === "codex" ? (e.codexTools ?? []).map((t) => CODEX_TOOL_LABEL[t]).filter(Boolean) : []),
|
|
165
175
|
e.memory ? null : "不带记忆",
|
|
166
176
|
].filter(Boolean);
|
|
167
177
|
return h(
|
|
@@ -187,6 +197,9 @@ function viewSharedExpert(e) {
|
|
|
187
197
|
h("p", { class: "muted" }, "这是共享专家,设定由创建者维护。你的记忆和工具独立,工具可在「我的工具」里配置。"),
|
|
188
198
|
field("说明", h("p", {}, e.description || "暂无说明")),
|
|
189
199
|
field("引擎", h("p", {}, ENGINE_LABEL[e.engine])),
|
|
200
|
+
e.engine === "codex" && e.codexTools?.length
|
|
201
|
+
? field("Codex 扩展能力", h("p", {}, e.codexTools.map((t) => CODEX_TOOL_LABEL[t] ?? t).join("、")), CODEX_TOOLS.some((t) => t.admin && e.codexTools.includes(t.id)) ? "操作浏览器、操作桌面、插件和应用只在管理员使用时打开" : null)
|
|
202
|
+
: null,
|
|
190
203
|
field("提示词", prompt),
|
|
191
204
|
) });
|
|
192
205
|
}
|
|
@@ -236,6 +249,38 @@ function field(label, input, hint) {
|
|
|
236
249
|
return h("div", { class: "field" }, h("span", {}, label), input, hint ? h("small", {}, hint) : null);
|
|
237
250
|
}
|
|
238
251
|
|
|
252
|
+
/** 编辑框里的一组字段,带个小标题。所有字段都藏起来时整组跟着藏(见 syncSections) */
|
|
253
|
+
function section(title, ...fields) {
|
|
254
|
+
return h("section", { class: "form-section" }, h("h3", { class: "form-section-title" }, title), ...fields);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function syncSections(body) {
|
|
258
|
+
for (const sec of body.querySelectorAll(".form-section")) {
|
|
259
|
+
sec.hidden = [...sec.children].filter((c) => !c.classList.contains("form-section-title")).every((c) => c.hidden);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Codex 扩展能力的勾选:本机能力只有管理员能勾,成员看得到但勾不了 */
|
|
264
|
+
function codexToolChecks(picked) {
|
|
265
|
+
const admin = isAdmin();
|
|
266
|
+
const boxes = CODEX_TOOLS.map((t) => {
|
|
267
|
+
const locked = !!t.admin && !admin;
|
|
268
|
+
const box = h("input", { type: "checkbox", checked: picked.includes(t.id) && !locked, disabled: locked });
|
|
269
|
+
const el = h(
|
|
270
|
+
"label",
|
|
271
|
+
{ class: `check-item${locked ? " disabled" : ""}`, title: locked ? "只有管理员能开" : t.hint },
|
|
272
|
+
box,
|
|
273
|
+
h("span", {}, t.label, t.admin ? h("em", { class: "check-tag" }, "管理员") : null),
|
|
274
|
+
h("small", {}, t.hint),
|
|
275
|
+
);
|
|
276
|
+
return { id: t.id, box, el };
|
|
277
|
+
});
|
|
278
|
+
return {
|
|
279
|
+
el: h("div", { class: "check-list grid perm-list" }, boxes.map((b) => b.el)),
|
|
280
|
+
value: () => boxes.filter((b) => b.box.checked).map((b) => b.id),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
239
284
|
/** 一组勾选框:items [{ id, label, hint }],返回 { el, value() } */
|
|
240
285
|
function checkList(items, picked, empty) {
|
|
241
286
|
const boxes = items.map((it) => {
|
|
@@ -274,6 +319,7 @@ function openEditor(data, e, reload, preset) {
|
|
|
274
319
|
pi: "追加在 pi 原装提示词后面(带上 pi 的工具用法和规则,每轮多约 600 token)",
|
|
275
320
|
};
|
|
276
321
|
const perms = permChecks(src);
|
|
322
|
+
const codexTools = codexToolChecks(src.codexTools ?? []);
|
|
277
323
|
let cwd = src.cwd ?? "";
|
|
278
324
|
const cwdBtn = h("button", { type: "button", class: "btn task-cwd" });
|
|
279
325
|
const showCwd = () => {
|
|
@@ -323,35 +369,57 @@ function openEditor(data, e, reload, preset) {
|
|
|
323
369
|
);
|
|
324
370
|
fill(subBox, subs.el);
|
|
325
371
|
subField.hidden = eng !== "claude";
|
|
372
|
+
codexField.hidden = eng !== "codex";
|
|
326
373
|
baseField.hidden = eng !== "claude" && eng !== "pi";
|
|
327
374
|
if (BASE_LABEL[eng]) basePrompt.options[1].textContent = BASE_LABEL[eng];
|
|
328
375
|
perms.setEngine(eng);
|
|
376
|
+
syncSections(body);
|
|
329
377
|
void models.load(eng);
|
|
330
378
|
};
|
|
331
|
-
const sync = () =>
|
|
379
|
+
const sync = () => {
|
|
380
|
+
cwdField.hidden = perms.access() === "web";
|
|
381
|
+
syncSections(body);
|
|
382
|
+
};
|
|
332
383
|
const cwdField = field("默认工作目录", cwdBtn, "新建对话、定时任务时默认在这个目录里(可以再改)。点击选择目录,默认使用工作区根目录");
|
|
333
384
|
const baseField = field("基础提示词", basePrompt);
|
|
334
385
|
const subField = field("子智能体", subBox, "主专家可以把子任务派给它们(Agent 工具)。子智能体的权限不能超过主专家,只支持 Claude 专家");
|
|
386
|
+
const codexField = field(
|
|
387
|
+
"Codex 扩展能力",
|
|
388
|
+
codexTools.el,
|
|
389
|
+
"Codex 自带、专家默认关掉的能力(关着省 token)。标「管理员」的用的是这台机器上的浏览器、桌面和账号,只有管理员能开;共享给别人后,别人用时也不开",
|
|
390
|
+
);
|
|
335
391
|
engine.onchange = renderEngineParts;
|
|
336
392
|
perms.onchange = sync;
|
|
337
393
|
|
|
338
394
|
const body = h(
|
|
339
395
|
"div",
|
|
340
396
|
{ class: "modal-body task-form expert-form" },
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
397
|
+
section(
|
|
398
|
+
"基本信息",
|
|
399
|
+
h("div", { class: "row-fields avatar-row" }, field("头像", avatar.el), field("名字", name)),
|
|
400
|
+
field("说明", desc),
|
|
401
|
+
field("共享", h("label", { class: "check-item" }, shared, h("span", {}, "共享给所有账号")), "其他账号可在网页、飞书私聊和定时任务中使用;只有你能修改设定。取消共享后,其他人的旧对话保留但不能继续调用。子智能体也需要先共享。"),
|
|
402
|
+
),
|
|
403
|
+
section(
|
|
404
|
+
"模型和提示词",
|
|
405
|
+
h("div", { class: "row-fields" }, field("引擎", engine, e ? "建好后不能换" : null), field("模型", models.model), field("思考程度", models.effort)),
|
|
406
|
+
field("提示词", prompt, "agent-loft 会在后面自动补上:这个专家能做什么、不能做什么,今天的日期,记忆"),
|
|
407
|
+
baseField,
|
|
408
|
+
),
|
|
409
|
+
section(
|
|
410
|
+
"权限和能力",
|
|
411
|
+
field("权限", perms.el, "改文件包含读文件,跑命令包含改文件。什么都不勾就是只能聊天"),
|
|
412
|
+
cwdField,
|
|
413
|
+
codexField,
|
|
414
|
+
),
|
|
415
|
+
section(
|
|
416
|
+
"工具",
|
|
417
|
+
field("我的技能", skills.el, "只影响你的工具选择;共享后其他账号单独选择自己的技能"),
|
|
418
|
+
field("我的 MCP", mcpBox, "只使用你的 MCP;共享后不会把你的 MCP 或凭据交给其他账号"),
|
|
419
|
+
subField,
|
|
420
|
+
field("记忆", h("label", { class: "check-item" }, memory, h("span", {}, "使用聊天者自己的个人记忆(需在设置 → 记忆开启)"))),
|
|
421
|
+
),
|
|
422
|
+
section("新对话页", field("欢迎语", welcome), field("快捷提问", quick)),
|
|
355
423
|
);
|
|
356
424
|
renderEngineParts();
|
|
357
425
|
sync();
|
|
@@ -384,6 +452,7 @@ function openEditor(data, e, reload, preset) {
|
|
|
384
452
|
skills: skills.value(),
|
|
385
453
|
mcp: mcp.value(),
|
|
386
454
|
subagents: engine.value === "claude" ? subs.value() : [],
|
|
455
|
+
codexTools: engine.value === "codex" ? codexTools.value() : [],
|
|
387
456
|
memory: memory.checked,
|
|
388
457
|
welcome: welcome.value,
|
|
389
458
|
quickPrompts,
|
|
@@ -429,7 +498,7 @@ function permChecks(src) {
|
|
|
429
498
|
return { p, el };
|
|
430
499
|
});
|
|
431
500
|
return Object.assign(api, {
|
|
432
|
-
el: h("div", { class: "check-list perm-list" }, items.map((x) => x.el)),
|
|
501
|
+
el: h("div", { class: "check-list grid perm-list" }, items.map((x) => x.el)),
|
|
433
502
|
// 藏起来的(Codex 的读网页)照旧按勾选状态存,和以前一样
|
|
434
503
|
checked: (id) => boxes.get(id).checked,
|
|
435
504
|
access: () => [...FILE_LEVELS].reverse().find((id) => boxes.get(id).checked) ?? "web",
|
package/web/views/composer.js
CHANGED
|
@@ -8,6 +8,7 @@ import { addItem, openConversation, openLightbox, renderEmpty, scrollToBottom }
|
|
|
8
8
|
import { renderScheduled } from "./schedule.js";
|
|
9
9
|
import { renderTodos } from "./todos.js";
|
|
10
10
|
import { cancelVoice, SpeechRec, toggleVoice } from "./voice.js";
|
|
11
|
+
import { callButton } from "./voice-call.js";
|
|
11
12
|
import { closeSlashMenu, commandUsage, oneLine, slashKeydown, updateSlashMenu } from "./slash.js";
|
|
12
13
|
import { doAction, openModelPicker, openSessions } from "./conv-actions.js";
|
|
13
14
|
|
|
@@ -103,6 +104,7 @@ export function renderComposer() {
|
|
|
103
104
|
ui.ctxRing,
|
|
104
105
|
ui.modelSlot,
|
|
105
106
|
ui.micBtn,
|
|
107
|
+
callButton(),
|
|
106
108
|
ui.sendBtn,
|
|
107
109
|
),
|
|
108
110
|
);
|
package/web/views/messages.js
CHANGED
|
@@ -612,7 +612,7 @@ function setQueueState(item, state) {
|
|
|
612
612
|
function renderItem(item) {
|
|
613
613
|
switch (item.type) {
|
|
614
614
|
case "user": {
|
|
615
|
-
const origin = item.origin && item.origin.channel !== "web" ? `来自${item.origin.name ?? item.origin.channel}` : "";
|
|
615
|
+
const origin = item.origin && item.origin.channel !== "web" && item.origin.channel !== "voice" ? `来自${item.origin.name ?? item.origin.channel}` : "";
|
|
616
616
|
return h(
|
|
617
617
|
"div",
|
|
618
618
|
{ class: "msg-user-group" },
|
package/web/views/tasks.js
CHANGED
|
@@ -16,7 +16,16 @@ import { capOfAccess, FULL_WARNING, minPermission, PERMISSIONS, permissionsUpTo
|
|
|
16
16
|
// 到点让 agent 按提示词干活,结果回到一条对话里;「有需要才提醒」时由 agent 判断这次要不要打扰你。
|
|
17
17
|
// 列表只有自己的任务(服务端按人过滤)。任务有变化时服务端推 { t: "tasks" },这里重新拉一遍。
|
|
18
18
|
|
|
19
|
-
const T = { list: [], tz: "", token: 0, reloadTimer: 0 };
|
|
19
|
+
const T = { list: [], tz: "", token: 0, reloadTimer: 0, channels: null };
|
|
20
|
+
|
|
21
|
+
/** 结果推送的渠道。飞书 / 微信要先在设置里绑定 */
|
|
22
|
+
const PUSH = [
|
|
23
|
+
["web", "Web"],
|
|
24
|
+
["feishu", "飞书"],
|
|
25
|
+
["weixin", "微信"],
|
|
26
|
+
];
|
|
27
|
+
const pushOf = (t) => t.push ?? ["web"];
|
|
28
|
+
const pushLabel = (push) => (push.length ? `推送到${push.map((c) => PUSH.find(([v]) => v === c)?.[1] ?? c).join("、")}` : "不推送");
|
|
20
29
|
|
|
21
30
|
const STATUS = {
|
|
22
31
|
running: { text: "运行中", cls: "run" },
|
|
@@ -111,6 +120,7 @@ async function reload() {
|
|
|
111
120
|
if (my !== T.token || S.view !== "tasks") return;
|
|
112
121
|
T.list = data.tasks;
|
|
113
122
|
T.tz = data.tz;
|
|
123
|
+
T.channels = data.channels ?? null;
|
|
114
124
|
render();
|
|
115
125
|
} catch (err) {
|
|
116
126
|
if (my === T.token && S.view === "tasks") fill(ui.msgInner, h("div", { class: "empty" }, err.message));
|
|
@@ -238,6 +248,7 @@ function taskCard(t) {
|
|
|
238
248
|
h("span", {}, t.summary),
|
|
239
249
|
h("span", {}, state),
|
|
240
250
|
h("span", {}, t.notify === "smart" ? "有需要才提醒" : "每次都提醒"),
|
|
251
|
+
h("span", {}, pushLabel(pushOf(t))),
|
|
241
252
|
h("span", {}, where),
|
|
242
253
|
t.freshContext ? h("span", {}, "不带上下文") : null,
|
|
243
254
|
t.target.kind === "dedicated" && t.permission ? h("span", { class: t.permission === "full" ? "task-perm-full" : "" }, PERMISSIONS[t.permission].label) : null,
|
|
@@ -416,6 +427,42 @@ export function openTaskForm(task, preset) {
|
|
|
416
427
|
h("option", { value: "always", selected: src.notify === "always" }, "每次都提醒"),
|
|
417
428
|
);
|
|
418
429
|
|
|
430
|
+
// ---- 推送渠道(可多选,默认 Web)。飞书 / 微信没绑定就不列出来 ----
|
|
431
|
+
let push = [...pushOf(src)];
|
|
432
|
+
let feishuBot = src.feishuBot ?? "";
|
|
433
|
+
const pushBox = h("div", { class: "task-weekdays task-push", role: "group", "aria-label": "推送渠道(可多选)" });
|
|
434
|
+
const botSelect = h("select", { class: "task-feishu-bot", "aria-label": "推送用的飞书机器人" });
|
|
435
|
+
botSelect.onchange = () => (feishuBot = botSelect.value);
|
|
436
|
+
const botField = field("飞书机器人", botSelect, "你绑定了多个飞书机器人,选一个来推送。选定的机器人不在线时不会改用别的。");
|
|
437
|
+
const pushHint = h("small", {});
|
|
438
|
+
const bound = (c) => c === "web" || (c === "feishu" ? !!T.channels?.feishu?.length : !!T.channels?.weixin);
|
|
439
|
+
const renderPush = () => {
|
|
440
|
+
const shown = PUSH.filter(([c]) => bound(c));
|
|
441
|
+
pushBox.style.gridTemplateColumns = `repeat(${shown.length}, minmax(0, 1fr))`;
|
|
442
|
+
fill(
|
|
443
|
+
pushBox,
|
|
444
|
+
shown.map(([c, label]) => {
|
|
445
|
+
const b = h("button", { type: "button", class: "task-weekday", "aria-pressed": String(push.includes(c)) }, label);
|
|
446
|
+
b.onclick = () => {
|
|
447
|
+
push = push.includes(c) ? push.filter((x) => x !== c) : [...push, c];
|
|
448
|
+
b.setAttribute("aria-pressed", String(push.includes(c)));
|
|
449
|
+
syncBot();
|
|
450
|
+
};
|
|
451
|
+
return b;
|
|
452
|
+
}),
|
|
453
|
+
);
|
|
454
|
+
const missing = PUSH.filter(([c]) => !bound(c)).map(([, label]) => label);
|
|
455
|
+
pushHint.textContent = `要提醒时推到哪里,都不选就只留在对话里。${missing.length ? `${missing.join("、")}要先在设置里绑定才能选。` : "飞书、微信发到你绑定的私聊。"}`;
|
|
456
|
+
const bots = T.channels?.feishu ?? [];
|
|
457
|
+
if (feishuBot && !bots.some((b) => b.id === feishuBot)) feishuBot = "";
|
|
458
|
+
fill(botSelect, h("option", { value: "", selected: !feishuBot }, "自动(第一个在线的)"), bots.map((b) => h("option", { value: b.id, selected: b.id === feishuBot }, b.name)));
|
|
459
|
+
syncBot();
|
|
460
|
+
};
|
|
461
|
+
const syncBot = () => (botField.hidden = !(push.includes("feishu") && (T.channels?.feishu?.length ?? 0) > 1));
|
|
462
|
+
renderPush();
|
|
463
|
+
// 不是从定时任务页打开的:还不知道绑定了什么,拉一下
|
|
464
|
+
if (!T.channels) void api("/api/tasks").then((d) => ((T.channels = d.channels ?? null), renderPush()), () => {});
|
|
465
|
+
|
|
419
466
|
// ---- 结果发到哪 ----
|
|
420
467
|
const currentTarget = task?.target?.kind === "conv" ? task.target.key : "dedicated";
|
|
421
468
|
const target = h(
|
|
@@ -537,6 +584,8 @@ export function openTaskForm(task, preset) {
|
|
|
537
584
|
field("频率", freq),
|
|
538
585
|
field("执行时间", freqBox, freqHint),
|
|
539
586
|
field("提醒", notify),
|
|
587
|
+
h("div", { class: "field" }, h("span", {}, "推送渠道"), pushBox, pushHint),
|
|
588
|
+
botField,
|
|
540
589
|
field("结果发到", target),
|
|
541
590
|
convBox,
|
|
542
591
|
dedicatedBox,
|
|
@@ -594,6 +643,9 @@ export function openTaskForm(task, preset) {
|
|
|
594
643
|
prompt: prompt.value,
|
|
595
644
|
schedule,
|
|
596
645
|
notify: notify.value,
|
|
646
|
+
// 之前勾过、后来解绑了的渠道不再提交(还没拿到绑定情况时原样提交,由服务端核)
|
|
647
|
+
push: PUSH.map(([c]) => c).filter((c) => push.includes(c) && (bound(c) || !T.channels)),
|
|
648
|
+
feishuBot: push.includes("feishu") ? feishuBot : "",
|
|
597
649
|
target: dedicated ? { kind: "dedicated" } : { kind: "conv", key: target.value },
|
|
598
650
|
freshContext: fresh,
|
|
599
651
|
...(dedicated ? { expert: expert.value, cwd: findExpert(expert.value)?.access === "web" ? "" : cwd, permission: level, model: models.model.value, effort: models.effort.value } : {}),
|
package/web/views/topbar.js
CHANGED
|
@@ -10,6 +10,7 @@ import { openSchedule } from "./schedule.js";
|
|
|
10
10
|
import { changeCwd, deleteConv, doAction, forkConversation, handoffConversation, openModelPicker, openPermissionPicker, renameConv, switchAgent } from "./conv-actions.js";
|
|
11
11
|
import { capOfAccess, defaultPermissionFor, minPermission, permissionsUpTo, PERMISSIONS } from "../lib/permission.js";
|
|
12
12
|
import { canSeeChanges, syncWorkspace, toggleWorkspace, workspaceButtons, workspaceStats } from "./workspace-entry.js";
|
|
13
|
+
import { syncCall } from "./voice-call.js";
|
|
13
14
|
|
|
14
15
|
// ---------- 顶栏 ----------
|
|
15
16
|
|
|
@@ -398,6 +399,7 @@ export function renderContext() {
|
|
|
398
399
|
agentEl,
|
|
399
400
|
);
|
|
400
401
|
updateCtxVisibility();
|
|
402
|
+
syncCall();
|
|
401
403
|
|
|
402
404
|
// 直接显示实际的模型名和思考程度(和选择列表里一样的写法),不显示「默认模型」
|
|
403
405
|
let model;
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { h, icon, toast } from "../lib/dom.js";
|
|
2
|
+
import { desktop, S, ui } from "../state.js";
|
|
3
|
+
import { WS } from "../lib/ws.js";
|
|
4
|
+
import { ensureConversation } from "./composer.js";
|
|
5
|
+
import { currentAgentId } from "./slash.js";
|
|
6
|
+
import { cancelVoice } from "./voice.js";
|
|
7
|
+
|
|
8
|
+
// ---------- 实时语音对话(只有 Codex) ----------
|
|
9
|
+
// 浏览器建 RTCPeerConnection(麦克风 + oai-events 数据通道),offer 交给服务端,服务端让 Codex app-server 去 OpenAI 建通话,
|
|
10
|
+
// answer 拿回来连上。声音在浏览器和 OpenAI 之间直连,不经过 agent-loft。
|
|
11
|
+
// 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
|
|
12
|
+
// 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
|
|
13
|
+
|
|
14
|
+
/** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
|
|
15
|
+
const supported = !desktop && !!window.RTCPeerConnection && !!navigator.mediaDevices?.getUserMedia;
|
|
16
|
+
|
|
17
|
+
const call = {
|
|
18
|
+
/** idle / connecting / live */
|
|
19
|
+
state: "idle",
|
|
20
|
+
key: null,
|
|
21
|
+
pc: null,
|
|
22
|
+
mic: null,
|
|
23
|
+
audio: null,
|
|
24
|
+
muted: false,
|
|
25
|
+
startedAt: 0,
|
|
26
|
+
raf: 0,
|
|
27
|
+
ctx: null,
|
|
28
|
+
micLevel: null,
|
|
29
|
+
outLevel: null,
|
|
30
|
+
/** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
|
|
31
|
+
caption: { role: null, text: "", done: false },
|
|
32
|
+
panel: null,
|
|
33
|
+
els: null,
|
|
34
|
+
/** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
|
|
35
|
+
dropTimer: 0,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const ICON_WAVE =
|
|
39
|
+
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
|
|
40
|
+
const ICON_MIC_OFF =
|
|
41
|
+
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 3l18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9.3V5a3 3 0 0 0-5.9-.6"/><path d="M5 10a7 7 0 0 0 11.7 5.2M19 10a7 7 0 0 1-.6 2.8M12 17v5"/></svg>';
|
|
42
|
+
|
|
43
|
+
/** 输入框里的「语音对话」按钮 */
|
|
44
|
+
export function callButton() {
|
|
45
|
+
ui.callBtn = h("button", { class: "icon-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
|
|
46
|
+
syncCall();
|
|
47
|
+
return ui.callBtn;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* 换了对话 / 重画了输入框之后对齐:只有 Codex 的对话显示按钮;通话中切到别的对话就挂断,
|
|
52
|
+
* 输入框重画了就把通话条重新放回去
|
|
53
|
+
*/
|
|
54
|
+
export function syncCall() {
|
|
55
|
+
// 新对话里点的:会话还在建(call.key 还没有),建好时会重画一次,不算切走
|
|
56
|
+
if (call.state !== "idle") {
|
|
57
|
+
if (call.key && S.current !== call.key) {
|
|
58
|
+
endCall("切换了对话,语音对话已结束");
|
|
59
|
+
} else if (call.panel && !call.panel.isConnected) {
|
|
60
|
+
mountPanel();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (ui.callBtn) ui.callBtn.hidden = !supported || currentAgentId() !== "codex";
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function startCall() {
|
|
67
|
+
if (call.state !== "idle") return;
|
|
68
|
+
if (!window.isSecureContext) {
|
|
69
|
+
toast("语音对话要用麦克风,需要 HTTPS 访问(或者本机 localhost)");
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
cancelVoice();
|
|
73
|
+
call.state = "connecting";
|
|
74
|
+
call.muted = false;
|
|
75
|
+
call.caption = { role: null, text: "", done: false };
|
|
76
|
+
// 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
|
|
77
|
+
call.audio = h("audio", { autoplay: true, playsinline: true });
|
|
78
|
+
call.audio.play().catch(() => {});
|
|
79
|
+
mountPanel();
|
|
80
|
+
try {
|
|
81
|
+
const key = await ensureConversation();
|
|
82
|
+
if (call.state === "idle") return;
|
|
83
|
+
call.key = key;
|
|
84
|
+
call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
85
|
+
if (call.state === "idle") return stopTracks();
|
|
86
|
+
const pc = new RTCPeerConnection();
|
|
87
|
+
call.pc = pc;
|
|
88
|
+
for (const track of call.mic.getTracks()) pc.addTrack(track, call.mic);
|
|
89
|
+
// OpenAI 要求 offer 里有这个数据通道(事件走这里;字幕我们用服务端转来的)
|
|
90
|
+
pc.createDataChannel("oai-events");
|
|
91
|
+
pc.ontrack = (e) => {
|
|
92
|
+
const stream = e.streams[0] ?? new MediaStream([e.track]);
|
|
93
|
+
call.audio.srcObject = stream;
|
|
94
|
+
call.audio.play().catch(() => {});
|
|
95
|
+
call.outLevel = meter(stream);
|
|
96
|
+
};
|
|
97
|
+
pc.onconnectionstatechange = () => {
|
|
98
|
+
const st = pc.connectionState;
|
|
99
|
+
clearTimeout(call.dropTimer);
|
|
100
|
+
if (st === "failed" || st === "closed") endCall("语音连接断开了");
|
|
101
|
+
// 网络抖一下会短暂 disconnected,几秒内没恢复才算断
|
|
102
|
+
else if (st === "disconnected") call.dropTimer = setTimeout(() => endCall("语音连接断开了"), 5000);
|
|
103
|
+
};
|
|
104
|
+
call.micLevel = meter(call.mic);
|
|
105
|
+
await pc.setLocalDescription(await pc.createOffer());
|
|
106
|
+
await iceGathered(pc, 1500);
|
|
107
|
+
if (call.state === "idle") return;
|
|
108
|
+
const res = await WS.request({ t: "rt", op: "start", key, sdp: pc.localDescription.sdp }, 30_000);
|
|
109
|
+
if (call.state === "idle") return;
|
|
110
|
+
await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
|
|
111
|
+
call.state = "live";
|
|
112
|
+
call.startedAt = Date.now();
|
|
113
|
+
render();
|
|
114
|
+
} catch (err) {
|
|
115
|
+
const message = err?.name === "NotAllowedError" ? "没有麦克风权限,请在浏览器里允许使用麦克风" : err?.name === "NotFoundError" ? "没找到可用的麦克风" : err?.message ?? String(err);
|
|
116
|
+
endCall(`语音对话没能开始:${message}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 挂断。fromServer:服务端已经说结束了,不用再通知它 */
|
|
121
|
+
export function endCall(reason, fromServer = false) {
|
|
122
|
+
if (call.state === "idle") return;
|
|
123
|
+
const key = call.key;
|
|
124
|
+
call.state = "idle";
|
|
125
|
+
if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
|
|
126
|
+
clearTimeout(call.dropTimer);
|
|
127
|
+
cancelAnimationFrame(call.raf);
|
|
128
|
+
call.pc?.close();
|
|
129
|
+
call.pc = null;
|
|
130
|
+
stopTracks();
|
|
131
|
+
if (call.audio) call.audio.srcObject = null;
|
|
132
|
+
call.audio = null;
|
|
133
|
+
call.ctx?.close().catch(() => {});
|
|
134
|
+
call.ctx = null;
|
|
135
|
+
call.micLevel = call.outLevel = null;
|
|
136
|
+
call.key = null;
|
|
137
|
+
call.panel?.remove();
|
|
138
|
+
call.panel = call.els = null;
|
|
139
|
+
ui.composer?.classList.remove("in-call");
|
|
140
|
+
syncCall();
|
|
141
|
+
if (reason) toast(reason);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function stopTracks() {
|
|
145
|
+
for (const t of call.mic?.getTracks() ?? []) t.stop();
|
|
146
|
+
call.mic = null;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** 服务端转来的通话事件(ws 的 t: "rt") */
|
|
150
|
+
export function onRealtimeEvent(msg) {
|
|
151
|
+
if (call.state === "idle" || msg.key !== call.key) return;
|
|
152
|
+
const ev = msg.ev ?? {};
|
|
153
|
+
switch (ev.type) {
|
|
154
|
+
case "transcript": {
|
|
155
|
+
const c = call.caption;
|
|
156
|
+
if (c.role !== ev.role || c.done) call.caption = { role: ev.role, text: ev.delta, done: false };
|
|
157
|
+
else c.text += ev.delta;
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
case "transcript_done":
|
|
161
|
+
call.caption = { role: ev.role, text: ev.text, done: true };
|
|
162
|
+
break;
|
|
163
|
+
case "started":
|
|
164
|
+
break;
|
|
165
|
+
case "error":
|
|
166
|
+
toast(`语音对话出错:${ev.message}`, 4000);
|
|
167
|
+
break;
|
|
168
|
+
case "closed":
|
|
169
|
+
endCall(ev.reason && ev.reason !== "requested" ? `语音对话已结束(${ev.reason})` : "语音对话已结束", true);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
renderCaption();
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ---------- 通话条 ----------
|
|
176
|
+
|
|
177
|
+
function mountPanel() {
|
|
178
|
+
if (!ui.composer) return;
|
|
179
|
+
if (!call.panel) {
|
|
180
|
+
const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
|
|
181
|
+
const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
|
|
182
|
+
const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
|
|
183
|
+
const status = h("span", { class: "call-status" });
|
|
184
|
+
const time = h("span", { class: "call-time" });
|
|
185
|
+
const who = h("span", { class: "call-who" });
|
|
186
|
+
const text = h("span", { class: "call-text" });
|
|
187
|
+
call.els = { bars, mute, status, time, who, text };
|
|
188
|
+
call.panel = h(
|
|
189
|
+
"div",
|
|
190
|
+
{ class: "call-panel", role: "region", "aria-label": "语音对话" },
|
|
191
|
+
h(
|
|
192
|
+
"div",
|
|
193
|
+
{ class: "call-main" },
|
|
194
|
+
h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
|
|
195
|
+
h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
|
|
196
|
+
h("div", { class: "call-ctls" }, mute, end),
|
|
197
|
+
),
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
ui.composer.classList.add("in-call");
|
|
201
|
+
ui.composer.after(call.panel);
|
|
202
|
+
render();
|
|
203
|
+
cancelAnimationFrame(call.raf);
|
|
204
|
+
call.raf = requestAnimationFrame(tick);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function toggleMute() {
|
|
208
|
+
call.muted = !call.muted;
|
|
209
|
+
for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
|
|
210
|
+
render();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function render() {
|
|
214
|
+
if (!call.els) return;
|
|
215
|
+
const { mute } = call.els;
|
|
216
|
+
mute.innerHTML = call.muted ? ICON_MIC_OFF : icon("mic").innerHTML;
|
|
217
|
+
mute.title = call.muted ? "取消静音" : "静音";
|
|
218
|
+
mute.setAttribute("aria-label", mute.title);
|
|
219
|
+
mute.classList.toggle("on", call.muted);
|
|
220
|
+
call.panel.classList.toggle("connecting", call.state === "connecting");
|
|
221
|
+
renderCaption();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function renderCaption() {
|
|
225
|
+
if (!call.els) return;
|
|
226
|
+
const { who, text } = call.els;
|
|
227
|
+
const c = call.caption;
|
|
228
|
+
who.textContent = c.text ? (c.role === "user" ? "你" : "Codex") : "";
|
|
229
|
+
text.textContent = c.text || (call.state === "connecting" ? "" : "说点什么吧,可以随时打断它");
|
|
230
|
+
call.panel.classList.toggle("has-caption", !!c.text);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** 每帧:音量条、状态、时长 */
|
|
234
|
+
function tick() {
|
|
235
|
+
if (!call.els) return;
|
|
236
|
+
const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
|
|
237
|
+
const out = call.outLevel?.() ?? 0;
|
|
238
|
+
const speaking = out > 0.02;
|
|
239
|
+
const level = Math.min(1, (speaking ? out : mic) * 6);
|
|
240
|
+
call.els.bars.forEach((bar, i) => {
|
|
241
|
+
const shape = [0.45, 0.75, 1, 0.75, 0.45][i];
|
|
242
|
+
const wobble = call.state === "live" ? 0.15 + 0.85 * level * shape * (0.8 + 0.2 * Math.sin(Date.now() / 120 + i)) : 0.2;
|
|
243
|
+
bar.style.transform = `scaleY(${Math.max(0.15, wobble).toFixed(3)})`;
|
|
244
|
+
});
|
|
245
|
+
call.panel.classList.toggle("speaking", speaking);
|
|
246
|
+
const busy = S.busy && S.current === call.key;
|
|
247
|
+
call.els.status.textContent =
|
|
248
|
+
call.state === "connecting" ? "正在连接…" : speaking ? "Codex 正在说" : busy ? "Codex 正在处理…" : call.muted ? "已静音" : "正在听";
|
|
249
|
+
call.els.time.textContent = call.state === "live" ? clock(Date.now() - call.startedAt) : "";
|
|
250
|
+
call.raf = requestAnimationFrame(tick);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function clock(ms) {
|
|
254
|
+
const s = Math.floor(ms / 1000);
|
|
255
|
+
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** 一路声音的音量(0~1 的均方根),给动画用 */
|
|
259
|
+
function meter(stream) {
|
|
260
|
+
try {
|
|
261
|
+
call.ctx ??= new AudioContext();
|
|
262
|
+
void call.ctx.resume();
|
|
263
|
+
const src = call.ctx.createMediaStreamSource(stream);
|
|
264
|
+
const an = call.ctx.createAnalyser();
|
|
265
|
+
an.fftSize = 512;
|
|
266
|
+
src.connect(an);
|
|
267
|
+
const buf = new Float32Array(an.fftSize);
|
|
268
|
+
return () => {
|
|
269
|
+
an.getFloatTimeDomainData(buf);
|
|
270
|
+
let sum = 0;
|
|
271
|
+
for (const v of buf) sum += v * v;
|
|
272
|
+
return Math.sqrt(sum / buf.length);
|
|
273
|
+
};
|
|
274
|
+
} catch {
|
|
275
|
+
return null;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** 等 ICE 候选收集完(最多 ms 毫秒):offer 里带上候选,不用另外传 */
|
|
280
|
+
function iceGathered(pc, ms) {
|
|
281
|
+
if (pc.iceGatheringState === "complete") return Promise.resolve();
|
|
282
|
+
return new Promise((resolve) => {
|
|
283
|
+
const done = () => {
|
|
284
|
+
pc.removeEventListener("icegatheringstatechange", check);
|
|
285
|
+
resolve();
|
|
286
|
+
};
|
|
287
|
+
const check = () => pc.iceGatheringState === "complete" && done();
|
|
288
|
+
pc.addEventListener("icegatheringstatechange", check);
|
|
289
|
+
setTimeout(done, ms);
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
window.addEventListener("pagehide", () => endCall());
|