agent-loft 0.2.110 → 0.2.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -30,7 +30,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
30
30
  | 🔀 | **飞书机器人灵活绑定 agent** | 机器人数量不限,每个机器人可以直接勾选内置或共享专家,并选择默认专家(可在会话里切换)。网页上扫码就能创建,改完立即生效;由服务端主动连向飞书,家里没有公网 IP 也能在外面用 |
31
31
  | 🔗 | **飞书续接 Web 任务** | 飞书里发 `/web` 接上一条 Web 会话,或者在网页聊天右上角「更多」或侧栏会话右键菜单里选「在飞书继续」。之后回复、授权、提问都发到飞书,发 `/detach` 回来 |
32
32
  | 💬 | **微信 ClawBot** | 设置 → 账号里用微信扫码,就能在微信里和 agent 聊天、回数字处理授权和提问,网页上是同一条对话。每个人连自己的微信。微信限制较多,适合发起任务和接收回复,详见下面的说明 |
33
- | 🖥️ | **完整的 Web 工作台** | 流式回复、附件、语音输入、模型和思考程度切换、上下文占用、从任意回复分叉、按项目分组的侧栏、搜索会话、全部设置,电脑和手机(PWA)都能用 |
33
+ | 🖥️ | **完整的 Web 工作台** | 流式回复、附件、语音输入、Codex 语音对话、模型和思考程度切换、上下文占用、从任意回复分叉、按项目分组的侧栏、搜索会话、全部设置,电脑和手机(PWA)都能用 |
34
34
  | 🔔 | **飞书 + 苹果系统推送** | 除了飞书消息,Web 会话还能推送到 iPhone、iPad 锁屏和 Mac 通知中心(标准 Web Push,不需要开发者账号),图标上显示未读数 |
35
35
  | 🗂️ | **桌面端同时接入多台设备** | 桌面端(macOS / Windows / Linux)同时连多台服务器,菜单栏 / 托盘一眼看完各台的额度、待处理和运行中的会话 |
36
36
 
@@ -53,6 +53,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
53
53
  - **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
54
54
  - 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
55
55
  - 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
56
+ - 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框里点声波按钮,和 Codex 桌面端的语音模式一样边说边聊,可以随时打断。输入框换成通话条(音量动画、状态、实时字幕、静音、结束);要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3、WebRTC),声音在浏览器和 OpenAI 之间直连,不经过服务器;用的是 Codex 登录的 ChatGPT 账号。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
56
57
  - 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
57
58
  - 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
58
59
  - 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
@@ -89,6 +90,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
89
90
  - 技能:只给选中的(Claude 用技能白名单,没选的调不了;没选技能就不挂 Skill 工具,省 token);
90
91
  - MCP:从「设置 → MCP」里勾自己的或别人共享的(管理员还能勾机器上配的);
91
92
  - 子智能体(只有 Claude):别的 Claude 专家,主专家用 Agent 工具把子任务派给它。子智能体的权限不能超过主专家(运行时也封顶在主专家的权限)。
93
+ - Codex 扩展能力(只有 Codex):Codex 自带、专家默认关掉的能力(关着省 token),勾了才开:生图(图片显示在回复里,占 ChatGPT 订阅额度)、子智能体(Codex 自己的)、操作浏览器、操作桌面、插件和应用(Codex 登录的 ChatGPT 账号连接的应用)。后三项用的是这台机器上的浏览器、桌面和账号,只有管理员能勾,共享出去后别人用时也不开。内置的 Codex 专家不受这些限制,按 Codex 自己的配置来。
92
94
 
93
95
  只勾联网搜索、读网页的 Claude 专家就是以前聊天模式的效果。专家改了,用它的对话下一条消息起生效;专家删了,用它的对话还能看、不能再发消息,用它的定时任务暂停。以前的聊天模式取消了:升级后聊天模式的旧对话不再保留,聊天模式的定时任务暂停,选个专家再启用。
94
96
  - **MCP**:设置 → MCP,每人管自己的:HTTP 地址(+ 请求头),或者本机进程(命令 + 参数 + 环境变量,会在服务器上直接执行命令、不受专家权限约束,所以只有管理员能添加;成员以前建的不再启动)。值可以标成密钥(网页上不回显,只交给 MCP 进程或请求头,不进提示词)。可以设「调用前询问」,不设就直接用。**共享**:勾「共享给所有账号」后,别人能在专家和内置 Agent 里选它;共享的是命令、地址和普通的环境变量 / 请求头,标成密钥的项每人在「填写我的密钥」里填自己的值,作者的不会给出去;用别人共享的 MCP 时,调用工具前一律先弹授权卡;取消共享后别人填的值一起删掉。自定义专家加载各自勾选的服务;内置专家在「设置 → Agent → 对应 Agent 卡片 → 默认加载」里选择,支持个人 MCP、管理员可选的机器 MCP,也可全部取消;不指定时管理员沿用机器配置。
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-loft",
3
- "version": "0.2.110",
3
+ "version": "0.2.112",
4
4
  "type": "module",
5
5
  "description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
6
6
  "keywords": [
@@ -249,6 +249,16 @@ export type ThreadItem =
249
249
  status: "inProgress" | "completed" | "failed";
250
250
  }
251
251
  | { type: "webSearch"; id: string; query?: string }
252
+ | {
253
+ type: "imageGeneration";
254
+ id: string;
255
+ status: string;
256
+ revisedPrompt: string | null;
257
+ failure: { type: string } | null;
258
+ /** Codex 存下来的图片(默认在 CODEX_HOME/generated_images 下) */
259
+ savedPath?: string;
260
+ }
261
+ | { type: "collabAgentToolCall"; id: string; tool: string; status: string; prompt: string | null }
252
262
  | { type: "error"; id: string; message: string }
253
263
  | { type: string; id: string };
254
264
 
@@ -5,8 +5,8 @@
5
5
  import { appServer } from "./app-server.js";
6
6
  import { refusalFor, UnsupportedRequest } from "./refusals.js";
7
7
  import { config } from "../../core/config.js";
8
- import { contextBlock } from "../../core/expert-prompt.js";
9
- import { ACCESS_LEVELS, type ExpertAccess, type ExpertRuntime } from "../../core/experts.js";
8
+ import { contextBlock, stripContext, VOICE_MARK } from "../../core/expert-prompt.js";
9
+ import { ACCESS_LEVELS, type CodexTool, type ExpertAccess, type ExpertRuntime } from "../../core/experts.js";
10
10
  import { MEMORY_MCP_NAME, memoryMcpSpawn } from "../../core/memory-mcp.js";
11
11
  import { SKILLS_MCP_NAME, skillsMcpSpawn, TASK_TOOL_TIMEOUT_SEC, TASKS_MCP_NAME, tasksMcpSpawn } from "../../core/tasks-mcp.js";
12
12
  import type { PermissionLevel } from "../../core/permission.js";
@@ -123,11 +123,19 @@ export const CHAT_CONFIG: Record<string, unknown> = {
123
123
  "workspace_dependencies",
124
124
  "in_app_browser",
125
125
  "in_app_local_automation",
126
- "realtime_conversation",
127
126
  ].map((f) => [`features.${f}`, false]),
128
127
  ),
129
128
  };
130
129
 
130
+ /** 专家勾了的 Codex 自带能力在 CHAT_CONFIG 上打开哪些(core/experts.ts 的 CODEX_TOOLS) */
131
+ const CODEX_TOOL_CONFIG: Record<CodexTool, Record<string, unknown>> = {
132
+ image_generation: { "features.image_generation": true },
133
+ multi_agent: { "features.multi_agent": true },
134
+ browser: { "features.browser_use": true, "features.in_app_browser": true },
135
+ computer_use: { "features.computer_use": true },
136
+ apps: { "features.plugins": true, "features.apps": true, include_apps_instructions: true },
137
+ };
138
+
131
139
  /**
132
140
  * 按权限在 CHAT_CONFIG 上打开的东西。能读文件就要 shell(Codex 读文件也是跑命令,沙箱只读管着它改不了);
133
141
  * 能改文件再给 apply_patch。
@@ -141,6 +149,7 @@ export function expertConfig(e: ExpertRuntime): Record<string, unknown> {
141
149
  ? { include_permissions_instructions: true, "features.shell_tool": true, "features.unified_exec": true, "features.view_image": true }
142
150
  : {}),
143
151
  ...(rank >= 2 ? { include_apply_patch_tool: true } : {}),
152
+ ...Object.assign({}, ...(e.codexTools ?? []).map((t) => CODEX_TOOL_CONFIG[t] ?? {})),
144
153
  };
145
154
  }
146
155
 
@@ -340,6 +349,10 @@ export class CodexSession implements AgentSession {
340
349
  private disposed = false;
341
350
  /** 轮次还在 starting 时点了停止:拿到轮次 id 立刻中断 */
342
351
  private stopRequested = false;
352
+ /** 实时语音对话:在通话中;answer / fail 是 start 还在等对端 SDP */
353
+ private rt: { answer?: (sdp: string) => void; fail?: (err: Error) => void } | undefined;
354
+ /** 这一轮不是这边开的(语音对话转交给 Codex 的):还没把用户说的话报上去 */
355
+ private externalPending = false;
343
356
 
344
357
  constructor(
345
358
  private readonly opts: SessionOptions,
@@ -780,6 +793,10 @@ export class CodexSession implements AgentSession {
780
793
  }
781
794
 
782
795
  async dispose(): Promise<void> {
796
+ if (this.rt) {
797
+ await this.stopRealtime();
798
+ this.events.onRealtime?.({ type: "closed", reason: "会话已切换" });
799
+ }
783
800
  this.disposed = true;
784
801
  this.queue = [];
785
802
  await this.interrupt();
@@ -796,6 +813,128 @@ export class CodexSession implements AgentSession {
796
813
  this.endTurn();
797
814
  }
798
815
 
816
+ // ---------- 实时语音对话(thread/realtime/*,实验接口) ----------
817
+
818
+ /**
819
+ * 浏览器建好 RTCPeerConnection(麦克风 + oai-events 数据通道)把 offer 给过来,app-server 去 OpenAI 建通话,
820
+ * 对端 answer 从 thread/realtime/sdp 通知回来。声音在浏览器和 OpenAI 之间直连,不经过这里。
821
+ * 实测:ChatGPT 账号登录只能走 WebRTC(WebSocket 要 API key),协议版本要 v3。
822
+ * 语音模型把要干的活转交给这条 thread,app-server 自己开一轮(见 turn/started 里的 externalPending)。
823
+ */
824
+ async startRealtime(offer: string): Promise<string> {
825
+ if (this.disposed) throw new Error("会话已关闭");
826
+ if (this.rt) await this.stopRealtime();
827
+ const threadId = await this.open();
828
+ // 转交出来的轮次不经过 turn/start:先把这条对话现在的工作目录、沙箱、模型写到 thread 上,权限才对得上
829
+ await appServer.call("thread/settings/update", {
830
+ threadId,
831
+ cwd: this.opts.cwd,
832
+ approvalPolicy: "on-request",
833
+ sandboxPolicy: sandboxPolicyFor(this.sandboxMode, this.opts.cwd),
834
+ ...(this.opts.model ? { model: this.opts.model } : {}),
835
+ ...(this.opts.effort ? { effort: this.opts.effort } : {}),
836
+ });
837
+ if (this.disposed) throw new DisposedError();
838
+ // 专家、记忆、技能清单:打字的第一轮会附在消息后面;语音先开口的话,在这里交给背后的 Codex
839
+ const expert = this.opts.expert;
840
+ const context = this.contextPending
841
+ ? contextBlock(expert ? { prompt: expert, cwd: this.opts.cwd } : undefined, this.opts.memory?.prompt, codexSkillLines(this.opts.skills))
842
+ : "";
843
+ const answer = new Promise<string>((resolve, reject) => {
844
+ this.rt = { answer: resolve, fail: reject };
845
+ });
846
+ const timer = setTimeout(() => this.rt?.fail?.(new Error("连接语音服务超时")), 20_000);
847
+ try {
848
+ await appServer.call("thread/realtime/start", {
849
+ threadId,
850
+ outputModality: "audio",
851
+ version: "v3",
852
+ transport: { type: "webrtc", sdp: offer },
853
+ // 启动背景里有这台机器上最近的会话和目录结构:只给内置专家的管理员
854
+ includeStartupContext: !expert && this.opts.skills?.admin === true,
855
+ ...(context ? { realtimeStartInstructions: context } : {}),
856
+ });
857
+ return await answer;
858
+ } catch (err) {
859
+ this.rt = undefined;
860
+ void appServer.call("thread/realtime/stop", { threadId }).catch(() => {});
861
+ throw err;
862
+ } finally {
863
+ clearTimeout(timer);
864
+ if (this.rt) this.rt = {};
865
+ }
866
+ }
867
+
868
+ async stopRealtime(): Promise<void> {
869
+ if (!this.rt) return;
870
+ this.rt.fail?.(new Error("已挂断"));
871
+ this.rt = undefined;
872
+ if (!this.threadId) return;
873
+ try {
874
+ await appServer.call("thread/realtime/stop", { threadId: this.threadId });
875
+ } catch (err) {
876
+ console.warn("[codex] 挂断语音对话失败:", err instanceof Error ? err.message : err);
877
+ }
878
+ }
879
+
880
+ /** 实时语音的通知:字幕、建连、结束。返回 false = 不是这类通知 */
881
+ private onRealtime(method: string, params: unknown): boolean {
882
+ if (!method.startsWith("thread/realtime/")) return false;
883
+ const p = params as { sdp?: string; role?: string; delta?: string; text?: string; message?: string; reason?: string | null };
884
+ const role = p.role === "user" ? "user" : "assistant";
885
+ switch (method) {
886
+ case "thread/realtime/sdp":
887
+ if (p.sdp) this.rt?.answer?.(p.sdp);
888
+ return true;
889
+ case "thread/realtime/started":
890
+ this.events.onRealtime?.({ type: "started" });
891
+ return true;
892
+ case "thread/realtime/transcript/delta":
893
+ if (p.delta) this.events.onRealtime?.({ type: "transcript", role, delta: p.delta });
894
+ return true;
895
+ case "thread/realtime/transcript/done":
896
+ if (p.text) this.events.onRealtime?.({ type: "transcript_done", role, text: p.text });
897
+ return true;
898
+ case "thread/realtime/error": {
899
+ const message = realtimeError(p.message);
900
+ if (this.rt?.fail) this.rt.fail(new Error(message));
901
+ else this.events.onRealtime?.({ type: "error", message });
902
+ return true;
903
+ }
904
+ case "thread/realtime/closed":
905
+ if (this.rt) {
906
+ this.rt = undefined;
907
+ this.events.onRealtime?.({ type: "closed", ...(p.reason ? { reason: p.reason } : {}) });
908
+ }
909
+ return true;
910
+ default:
911
+ return true;
912
+ }
913
+ }
914
+
915
+ /** 语音转交出来的一轮开始了:当成这边的一轮接住(回复、审批、用量照常),用户说的话等 userMessage 到了再报 */
916
+ private adoptExternalTurn(id: string): void {
917
+ this.currentInput = undefined;
918
+ this.replyText = "";
919
+ this.planText = "";
920
+ this.turnPlan = false;
921
+ this.turnTokens = undefined;
922
+ this.turnSeq += 1;
923
+ this.phase = "running";
924
+ this.turnId = id;
925
+ this.stopRequested = false;
926
+ this.commentaryItems.clear();
927
+ this.turnSeen = true;
928
+ this.externalPending = true;
929
+ }
930
+
931
+ private announceExternal(text: string): void {
932
+ if (!this.externalPending) return;
933
+ this.externalPending = false;
934
+ this.events.onExternalTurn?.(text, this.turnId);
935
+ this.events.onTurnStart?.();
936
+ }
937
+
799
938
  // ---------- 反向请求:审批与提问 ----------
800
939
 
801
940
  /** 审批都走这里:带上「本会话全部允许」按钮,点了就记住。 */
@@ -934,13 +1073,19 @@ export class CodexSession implements AgentSession {
934
1073
  // ---------- 通知 ----------
935
1074
 
936
1075
  handleNotification(method: string, params: unknown): void {
1076
+ if (this.onRealtime(method, params)) return;
937
1077
 
938
1078
  switch (method) {
939
1079
  case "turn/started": {
940
1080
  // 轮次 id 在 params.turn.id,不在顶层
941
1081
  const id = (params as { turn?: { id?: string } }).turn?.id;
942
- // 只认「正在等它开始」的那一轮;收过尾的旧轮次晚到的通知忽略
943
- if (this.phase === "idle" || (id && this.endedTurns.has(id))) return;
1082
+ if (id && this.endedTurns.has(id)) return;
1083
+ // 空闲时冒出来的一轮不是这边开的:语音对话把事情转交给了 Codex(这个 app-server 里只有这边订阅着这条 thread)
1084
+ if (this.phase === "idle") {
1085
+ if (id && !this.disposed) this.adoptExternalTurn(id);
1086
+ return;
1087
+ }
1088
+ // 只认「正在等它开始」的那一轮
944
1089
  if (id) this.onTurnId(id);
945
1090
  this.commentaryItems.clear();
946
1091
  this.turnSeen = true;
@@ -1011,6 +1156,7 @@ export class CodexSession implements AgentSession {
1011
1156
  // 已经收过尾的、或者不是当前这一轮的,不处理
1012
1157
  if (this.phase === "idle" || (turn.id && this.endedTurns.has(turn.id))) return;
1013
1158
  if (this.turnId && turn.id && turn.id !== this.turnId) return;
1159
+ this.announceExternal(VOICE_FALLBACK);
1014
1160
  const text = this.replyText;
1015
1161
  this.replyText = "";
1016
1162
  this.endTurn();
@@ -1043,6 +1189,7 @@ export class CodexSession implements AgentSession {
1043
1189
  return;
1044
1190
  }
1045
1191
  if (this.phase === "idle") return;
1192
+ this.announceExternal(VOICE_FALLBACK);
1046
1193
  this.endTurn();
1047
1194
  this.events.onError(new Error(error?.message ?? "未知错误"));
1048
1195
  this.drainQueue();
@@ -1066,7 +1213,15 @@ export class CodexSession implements AgentSession {
1066
1213
  }
1067
1214
 
1068
1215
  private onItem(item: ThreadItem, phase: "started" | "completed"): void {
1216
+ if (this.externalPending && item.type !== "userMessage") this.announceExternal(VOICE_FALLBACK);
1069
1217
  switch (item.type) {
1218
+ case "userMessage": {
1219
+ if (!this.externalPending || phase !== "completed") return;
1220
+ const content = (item as { content?: { type?: string; text?: string }[] }).content ?? [];
1221
+ const raw = content.map((c) => (c.type === "text" ? (c.text ?? "") : "")).join("\n");
1222
+ this.announceExternal(stripContext(raw).trim() || VOICE_FALLBACK);
1223
+ return;
1224
+ }
1070
1225
  case "agentMessage": {
1071
1226
  const message = item as Extract<ThreadItem, { type: "agentMessage" }>;
1072
1227
  if (message.phase === "commentary") this.commentaryItems.add(message.id);
@@ -1138,6 +1293,32 @@ export class CodexSession implements AgentSession {
1138
1293
  this.events.onStep("note", `📋 ${summarize(plan.text, 90)}`);
1139
1294
  return;
1140
1295
  }
1296
+ case "imageGeneration": {
1297
+ if (phase === "started") {
1298
+ this.events.onStep("tool", "🎨 生成图片");
1299
+ return;
1300
+ }
1301
+ const image = item as Extract<ThreadItem, { type: "imageGeneration" }>;
1302
+ if (image.failure?.type === "usageLimitExceeded") {
1303
+ this.events.onStep("warning", "⚠️ 生图额度用完了");
1304
+ return;
1305
+ }
1306
+ // 图片 Codex 自己存好了(savedPath),写进回复里网页才显示得出来(只认回复里写过的路径)
1307
+ if (!image.savedPath) return;
1308
+ const alt = summarize(image.revisedPrompt ?? "", 40).replace(/[[\]\n]/g, " ").trim() || "生成的图片";
1309
+ // 尖括号包住:路径里有空格也能认;回复原文里是原样的路径,服务端按它放行
1310
+ const md = `![${alt}](<${image.savedPath}>)`;
1311
+ this.replyText += (this.replyText ? "\n\n" : "") + md;
1312
+ this.events.onText(md);
1313
+ return;
1314
+ }
1315
+ case "collabAgentToolCall": {
1316
+ if (phase !== "started") return;
1317
+ const call = item as Extract<ThreadItem, { type: "collabAgentToolCall" }>;
1318
+ const what = call.tool === "spawnAgent" ? `派出子智能体${call.prompt ? `:${summarize(call.prompt, 60)}` : ""}` : `子智能体 ${call.tool}`;
1319
+ this.events.onStep("tool", `🤝 ${what}`);
1320
+ return;
1321
+ }
1141
1322
  case "error": {
1142
1323
  if (phase !== "completed") return;
1143
1324
  const err = item as Extract<ThreadItem, { type: "error" }>;
@@ -1155,6 +1336,21 @@ function kindMark(kind: unknown): string {
1155
1336
  return k === "add" ? "+" : k === "delete" ? "-" : "~";
1156
1337
  }
1157
1338
 
1339
+ /** 语音转交的一轮没拿到用户说的话时显示的占位 */
1340
+ const VOICE_FALLBACK = `${VOICE_MARK}(语音)`;
1341
+
1342
+ /** OpenAI 的报错常常是一整段 JSON:取里面的 message */
1343
+ function realtimeError(raw: string | undefined): string {
1344
+ if (!raw) return "语音对话出错了";
1345
+ try {
1346
+ const msg = (JSON.parse(raw) as { error?: { message?: unknown } }).error?.message;
1347
+ if (typeof msg === "string" && msg) return msg;
1348
+ } catch {
1349
+ // 不是 JSON,原样给
1350
+ }
1351
+ return raw;
1352
+ }
1353
+
1158
1354
  export function summarize(text: string, max = 80): string {
1159
1355
  const line = text.replace(/\s+/g, " ").trim();
1160
1356
  return line.length > max ? `${line.slice(0, max)}…` : line;
@@ -47,10 +47,11 @@ export class FeishuClient {
47
47
  }
48
48
  }
49
49
 
50
- async sendCard(chatId: string, card: unknown): Promise<string | undefined> {
50
+ /** idType 是 open_id 时 chatId 填用户的 open_id:直接发到和这个人的私聊 */
51
+ async sendCard(chatId: string, card: unknown, idType: "chat_id" | "open_id" = "chat_id"): Promise<string | undefined> {
51
52
  try {
52
53
  const res = await this.lark.im.v1.message.create({
53
- params: { receive_id_type: "chat_id" },
54
+ params: { receive_id_type: idType },
54
55
  data: { receive_id: chatId, msg_type: "interactive", content: JSON.stringify(card) },
55
56
  });
56
57
  const messageId = res.data?.message_id;
@@ -1,7 +1,12 @@
1
1
  // 飞书渠道:按配置拉起 / 停掉机器人。Web 上改了机器人配置,这里就地对账,不用重启进程。
2
2
  import { config, onConfigChange, type FeishuBotConfig } from "../../core/config.js";
3
+ import { listLinks } from "../../core/feishu-links.js";
4
+ import { hub, type TaskTurnEnd } from "../../core/hub.js";
3
5
  import { watchPairing } from "../../core/pairing.js";
6
+ import { getConversation } from "../../core/store.js";
7
+ import { findTask, taskPush } from "../../core/tasks.js";
4
8
  import { FeishuBot } from "./bot.js";
9
+ import { replyCard } from "./cards.js";
5
10
 
6
11
  const bots = new Map<string, FeishuBot>();
7
12
 
@@ -35,7 +40,27 @@ function reconcile(list: FeishuBotConfig[]): void {
35
40
  }
36
41
  }
37
42
 
43
+ /**
44
+ * 定时任务选了推送到飞书:把结果发到主人绑定的飞书私聊。
45
+ * 任务本来就发在飞书对话里的不用再发(那边已经显示了)。绑了多个机器人的,用任务指定的那个,没指定发第一个在线的。
46
+ */
47
+ function onTaskResult(e: TaskTurnEnd): void {
48
+ if (e.status !== "notified") return;
49
+ const task = findTask(e.taskId);
50
+ if (!task || !taskPush(task).includes("feishu")) return;
51
+ if (getConversation(e.key)?.channel === "feishu") return;
52
+ // 指定了机器人就只用它(它不在线就不发,不悄悄换成别的机器人);没指定用第一个在线的
53
+ const link = listLinks(task.owner).find((l) => bots.has(l.botId) && (!task.feishuBot || l.botId === task.feishuBot));
54
+ if (!link) {
55
+ console.warn(`[飞书] 定时任务「${task.title}」要推送到飞书,但${task.feishuBot ? "指定的机器人不在线或已解绑" : "主人没有绑定在线的飞书机器人"}`);
56
+ return;
57
+ }
58
+ void bots.get(link.botId)!.client.sendCard(link.openId, replyCard(`⏰ **${e.title}**\n\n${e.text}`), "open_id");
59
+ }
60
+
38
61
  export function startFeishu(): void {
62
+ hub.off("task", onTaskResult);
63
+ hub.on("task", onTaskResult);
39
64
  reconcile(config().feishu.bots);
40
65
  onConfigChange((next, prev) => {
41
66
  reconcile(next.feishu.bots);
@@ -55,6 +80,7 @@ export function startFeishu(): void {
55
80
  }
56
81
 
57
82
  export function stopFeishu(): void {
83
+ hub.off("task", onTaskResult);
58
84
  for (const bot of bots.values()) bot.stop();
59
85
  bots.clear();
60
86
  }
@@ -57,6 +57,7 @@ function expertView(e: Expert, viewer: string) {
57
57
  skills: tools.skills,
58
58
  mcp: tools.mcp,
59
59
  subagents: e.subagents.filter((id) => canUseExpert(viewer, id)),
60
+ codexTools: e.codexTools ?? [],
60
61
  memory: e.memory,
61
62
  updatedAt: e.updatedAt,
62
63
  };
@@ -14,12 +14,12 @@ import { GLOBAL_ALWAYS_DENIED, hub, type EventEnvelope } from "../../core/hub.js
14
14
  import { interactions, type Interaction } from "../../core/interactions.js";
15
15
  import { weixinEvents } from "../weixin/events.js";
16
16
  import { getConversation } from "../../core/store.js";
17
- import { taskEvents } from "../../core/tasks.js";
17
+ import { findTask, taskEvents, taskPush } from "../../core/tasks.js";
18
18
  import { skillEvents } from "../../core/skills.js";
19
19
  import { expertEvents } from "../../core/experts.js";
20
20
  import { enforceTerminalAccess, killAllTerminals, terminalEvents, watchTerminalAccess } from "../../core/terminals.js";
21
21
  import { RUNNING_VERSION } from "../../core/upgrade.js";
22
- import type { Settlement } from "../../core/types.js";
22
+ import type { RealtimeEvent, Settlement } from "../../core/types.js";
23
23
  import { getUser, isAdmin, legacyOwnerId, userEvents, type User } from "../../core/users.js";
24
24
  import { currentUser, handleApi, takeUploads } from "./api.js";
25
25
  import { authEvents, ensureSetupToken, sessionUser } from "./auth.js";
@@ -303,6 +303,11 @@ function broadcastEvent(env: EventEnvelope): void {
303
303
  }
304
304
  }
305
305
 
306
+ /** 语音对话的字幕和状态:只推给订阅了这条对话的页面(不进缓冲区,刷新后不重放) */
307
+ function broadcastRealtime({ key, ev }: { key: string; ev: RealtimeEvent }): void {
308
+ for (const client of clients) if (client.subs.has(key) && canSeeKey(client.user, key)) send(client, { t: "rt", key, ev });
309
+ }
310
+
306
311
  function broadcastInteraction(i: Interaction): void {
307
312
  for (const client of clients) if (canSeeKey(client.user, i.convKey)) send(client, { t: "ix", interaction: i });
308
313
  }
@@ -414,9 +419,11 @@ function onWeixinUndelivered({ key, body }: { key: string; body: string }): void
414
419
  function pushEvent(env: EventEnvelope): void {
415
420
  const ev = env.ev;
416
421
  if (ev.type !== "result" && ev.type !== "error") return;
417
- if (!isWebConv(env.key)) return;
418
422
  // 定时任务这次判断不用提醒
419
423
  if (ev.type === "result" && ev.silent) return;
424
+ // 定时任务的结果按任务选的推送渠道:没选 Web 就不推;选了的话发到飞书 / 微信对话里的也推
425
+ const task = ev.type === "result" && ev.task ? findTask(ev.task.id) : undefined;
426
+ if (task ? !taskPush(task).includes("web") : !isWebConv(env.key)) return;
420
427
  const name = convName(env.key);
421
428
  const payload =
422
429
  ev.type === "result"
@@ -424,7 +431,7 @@ function pushEvent(env: EventEnvelope): void {
424
431
  ? { title: `⏰ ${ev.task.title}`, body: plainPreview(ev.text) || "(没有文字回复)" }
425
432
  : { title: `${name} · 回复完成`, body: plainPreview(ev.text) || "(没有文字回复)" }
426
433
  : { title: `${name} · 出错了`, body: plainPreview(ev.message) };
427
- notify(env.key, { ...payload, tag: `reply:${env.key}` });
434
+ deliverNotice(env.key, { ...payload, tag: `reply:${env.key}` });
428
435
  }
429
436
 
430
437
  function pushInteraction(i: Interaction): void {
@@ -525,6 +532,26 @@ function handleClientMessage(client: Client, raw: string): void {
525
532
  return ack(true);
526
533
  }
527
534
 
535
+ case "rt": {
536
+ // 实时语音对话:start 带浏览器的 SDP offer,ack 回对端 answer;stop 挂断
537
+ if (!getConversation(key) || !canSeeKey(user, key)) return ack(false, { error: "会话不存在" });
538
+ if (msg.op === "start") {
539
+ const sdp = typeof msg.sdp === "string" ? msg.sdp : "";
540
+ if (!sdp || sdp.length > 50_000) return ack(false, { error: "SDP 不对" });
541
+ client.subs.add(key);
542
+ void hub
543
+ .realtimeStart(key, sdp)
544
+ .then((answer) => ack(true, { sdp: answer }))
545
+ .catch((err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
546
+ return;
547
+ }
548
+ if (msg.op === "stop") {
549
+ void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
550
+ return;
551
+ }
552
+ return ack(false, { error: "未知操作" });
553
+ }
554
+
528
555
  case "ix": {
529
556
  const id = String(msg.id ?? "");
530
557
  const target = interactions.get(id);
@@ -663,6 +690,7 @@ export async function startWeb(): Promise<void> {
663
690
  heartbeat.unref();
664
691
 
665
692
  hub.on("event", broadcastEvent);
693
+ hub.on("realtime", broadcastRealtime);
666
694
  hub.on("project-deleted", broadcastProjectDeleted);
667
695
  hub.on("event", pushEvent);
668
696
  interactions.on("created", broadcastInteraction);
@@ -715,6 +743,7 @@ function onRevoked(): void {
715
743
  export async function stopWeb(): Promise<void> {
716
744
  authEvents.off("revoked", onRevoked);
717
745
  hub.off("event", broadcastEvent);
746
+ hub.off("realtime", broadcastRealtime);
718
747
  hub.off("project-deleted", broadcastProjectDeleted);
719
748
  hub.off("event", pushEvent);
720
749
  interactions.off("created", broadcastInteraction);
@@ -8,7 +8,7 @@ import { memoryOwnerOf } from "../../core/personal.js";
8
8
  import { expertIdOf, getConversation } from "../../core/store.js";
9
9
  import { forgetTask, isTaskRunning, releaseTaskConv, runTaskNow } from "../../core/task-runner.js";
10
10
  import { serverTimeZone } from "../../core/task-schedule.js";
11
- import { createTask, deleteTask, getTask, listTasks, updateTask, viewOf, type Task, type TaskInput } from "../../core/tasks.js";
11
+ import { createTask, deleteTask, getTask, listTasks, pushOptions, updateTask, viewOf, type Task, type TaskInput } from "../../core/tasks.js";
12
12
  import type { User } from "../../core/users.js";
13
13
  import { HttpError, readJsonBody, sendJson } from "./http.js";
14
14
 
@@ -91,7 +91,9 @@ const bad = (err: unknown) => (err instanceof HttpError ? err : new HttpError(40
91
91
  export async function handleTasksApi(req: IncomingMessage, res: ServerResponse, p: string, user: User): Promise<boolean> {
92
92
  const method = req.method ?? "GET";
93
93
  if (p === "/api/tasks" && method === "GET") {
94
- sendJson(res, 200, { tasks: listTasks(user.id).map(view), tz: serverTimeZone(), now: Date.now() });
94
+ // 能推送到哪些渠道:飞书 / 微信要先绑定(设置里)
95
+ const channels = pushOptions(user.id);
96
+ sendJson(res, 200, { tasks: listTasks(user.id).map(view), tz: serverTimeZone(), now: Date.now(), channels });
95
97
  return true;
96
98
  }
97
99
  if (p === "/api/tasks" && method === "POST") {
@@ -409,6 +409,12 @@ export class WeixinBot {
409
409
 
410
410
  // ---------- 发消息 ----------
411
411
 
412
+ /** 别的对话里的定时任务选了推送到微信:把结果发过来 */
413
+ pushTaskResult(title: string, text: string): void {
414
+ const body = `⏰ ${title}\n\n${text}`;
415
+ if (text.trim()) this.reply(body);
416
+ }
417
+
412
418
  private remaining(): number {
413
419
  return SEND_LIMIT - this.account.sentSinceInbound;
414
420
  }
@@ -1,4 +1,6 @@
1
1
  // 微信渠道:按绑定拉起 / 停掉长轮询。扫码、解绑、停用用户后就地对账,不用重启进程。
2
+ import { hub, type TaskTurnEnd } from "../../core/hub.js";
3
+ import { findTask, taskPush } from "../../core/tasks.js";
2
4
  import { getUser, userEvents } from "../../core/users.js";
3
5
  import { listWeixinAccounts, weixinAccountEvents, type WeixinAccount } from "../../core/weixin-accounts.js";
4
6
  import { WeixinBot, type WeixinStatus } from "./bot.js";
@@ -30,9 +32,25 @@ function reconcile(): void {
30
32
  }
31
33
  }
32
34
 
35
+ /** 定时任务选了推送到微信:发到主人绑定的微信 */
36
+ function onTaskResult(e: TaskTurnEnd): void {
37
+ if (e.status !== "notified") return;
38
+ const task = findTask(e.taskId);
39
+ if (!task || !taskPush(task).includes("weixin")) return;
40
+ const bot = bots.get(task.owner);
41
+ if (!bot) {
42
+ console.warn(`[微信] 定时任务「${task.title}」要推送到微信,但主人没有绑定微信`);
43
+ return;
44
+ }
45
+ // 发在微信正接着的那条对话里:结果已经发过去了
46
+ if (e.key === bot.key) return;
47
+ bot.pushTaskResult(e.title, e.text);
48
+ }
49
+
33
50
  export function startWeixin(): void {
34
51
  if (started) return;
35
52
  started = true;
53
+ hub.on("task", onTaskResult);
36
54
  reconcile();
37
55
  weixinAccountEvents.on("changed", reconcile);
38
56
  userEvents.on("changed", reconcile);
@@ -40,6 +58,7 @@ export function startWeixin(): void {
40
58
 
41
59
  export function stopWeixin(): void {
42
60
  started = false;
61
+ hub.off("task", onTaskResult);
43
62
  weixinAccountEvents.off("changed", reconcile);
44
63
  userEvents.off("changed", reconcile);
45
64
  stopWeixinLogins();
@@ -140,7 +140,22 @@ export function wrapContext(lines: string[]): string {
140
140
  }
141
141
 
142
142
  /** 剥掉 contextBlock(读历史、列会话标题时用)。一条消息里可能有好几段(定时任务的说明 + Codex 的专家说明) */
143
+ /** 语音对话转交给 Codex 的那一轮,用户消息长这样:<realtime_delegation><input>说的话</input>…</realtime_delegation> */
144
+ const REALTIME_RE = /^\s*<realtime_delegation>[\s\S]*?<input>([\s\S]*?)<\/input>[\s\S]*<\/realtime_delegation>\s*$/;
145
+ const XML_ENTITIES: Record<string, string> = { "&lt;": "<", "&gt;": ">", "&quot;": '"', "&apos;": "'", "&amp;": "&" };
146
+
147
+ /** 语音转交的那一轮用户说了什么;不是语音转交的返回 undefined */
148
+ export function realtimeInput(text: string): string | undefined {
149
+ const m = REALTIME_RE.exec(text);
150
+ return m ? m[1]!.replace(/&(lt|gt|quot|apos|amp);/g, (e) => XML_ENTITIES[e] ?? e).trim() : undefined;
151
+ }
152
+
153
+ /** 显示时语音说的话前面加个麦克风,和打字发的区分开 */
154
+ export const VOICE_MARK = "🎙️ ";
155
+
143
156
  export function stripContext(text: string): string {
157
+ const voice = realtimeInput(text);
158
+ if (voice !== undefined) return `${VOICE_MARK}${voice}`;
144
159
  let out = text;
145
160
  for (let start = out.indexOf(OPEN); start >= 0; start = out.indexOf(OPEN)) {
146
161
  const end = out.indexOf(CLOSE, start);