hob-bot 1.0.3 → 1.1.0

package/server/src/delegations.ts

@@ -1,6 +1,6 @@
-import { readFileSync } from "node:fs";
 import type { Delegation } from "../../shared/types.ts";
 import { newId } from "./ids.ts";
+import { jobResultEvent } from "./jobs.ts";
 import type { HobRuntime } from "./runtime.ts";
 
 /** 转交状态由目标回合、任务和审批共同决定；判定和标记回流不跨异步边界。 */
@@ -19,25 +19,40 @@ export class DelegationManager {
     });
   }
 
-  async start(to: string, text: string): Promise<Delegation> {
+  async start(to: string, text: string, files: string[] = []): Promise<Delegation> {
     const target = this.runtime.store.assistant(to);
     if (!target || to === "hob") throw new Error("目标助手不存在或不能转交给自己。");
     const delegation: Delegation = { id: newId("dg-"), from: "hob", to, text, createdAt: new Date().toISOString(), status: "running", jobIds: [], approvalIds: [] };
     this.runtime.store.saveDelegation(delegation);
     await this.runtime.router.addItem("hob", { kind: "delegation", delegationId: delegation.id });
+    let sourceUserId: string;
     try {
-      await this.runtime.router.addForwarded(to, text, delegation.id);
+      const forwarded = await this.runtime.router.addForwarded(to, text, delegation.id, "hob", files);
+      sourceUserId = forwarded.id;
     } catch (error) {
       this.turnEnded(delegation.id, error instanceof Error ? error.message : String(error), true);
       throw error;
     }
     this.runtime.bus.emit({ type: "delegation", delegation });
-    void this.runtime.router.relay(to, text, delegation.id).catch((error: unknown) => {
+    void this.runtime.router.relay(to, `转自 Hob：${text}`, delegation.id, { sourceUserId }, text).catch((error: unknown) => {
       this.turnEnded(delegation.id, error instanceof Error ? error.message : String(error), true);
     });
     return delegation;
   }
 
+  takeOver(to: string): void {
+    const { store, bus, router } = this.runtime;
+    for (const record of store.delegations()) {
+      if (record.to !== to || record.reportedAt || record.status === "interrupted") continue;
+      const now = new Date().toISOString();
+      const delegation: Delegation = { ...record, status: "takenOver", endedAt: now, reportedAt: now };
+      store.saveDelegation(delegation);
+      bus.emit({ type: "delegation", delegation });
+      const name = store.assistant(to)?.name ?? to;
+      void router.event(record.from, `转交给「${name}」的事已被用户接手。原请求：${record.text}`).catch(() => undefined);
+    }
+  }
+
   addJob(id: string, jobId: string): void {
     const record = this.runtime.store.delegation(id);
     if (!record || record.jobIds.includes(jobId)) return;
@@ -54,10 +69,26 @@ export class DelegationManager {
   turnEnded(id: string, reply: string, failed = false): void {
     const record = this.runtime.store.delegation(id);
     if (!record || record.turnFinished) return;
-    this.runtime.store.saveDelegation({ ...record, reply, turnFinished: true, ...(failed ? { turnFailed: true } : {}) });
+    const replies = [record.reply, reply].filter(Boolean).join("\n");
+    this.runtime.store.saveDelegation({ ...record, reply: replies, turnFinished: true, ...(failed ? { turnFailed: true } : {}) });
     this.update(id);
   }
 
+  /**
+   * 转交里派的任务结束了、或用户答了转交里的提问：结果先回到目标助手，它这一轮可能接着再派任务、再问用户。
+   * 这一轮跑完、名下没有在跑的任务，转交才算结束，Hob 才收到结果；
+   * 否则目标助手还在处理时 Hob 就会收到「已结束」，另开任务和它冲突。
+   * 调用方要在发出 job/approval 事件之前调用，再用 router.relay 把结果送给目标助手。
+   * 返回 false 表示转交已经回流过（或已退回），结果按普通事件处理。
+   */
+  resume(id: string): boolean {
+    const record = this.runtime.store.delegation(id);
+    if (!record || record.reportedAt || record.status === "returned" || record.status === "interrupted") return false;
+    const { turnFailed: _failed, ...rest } = record;
+    this.runtime.store.saveDelegation({ ...rest, turnFinished: false });
+    return true;
+  }
+
   returnToHob(id: string, text: string, reason: string): void {
     const record = this.runtime.store.delegation(id);
     if (!record || record.turnFinished) throw new Error("转交已结束，无法退回");
@@ -82,16 +113,11 @@ export class DelegationManager {
     if (complete) {
       const reports = jobs.map((job) => {
         if (!job) return "任务记录丢失";
-        let report = job.summary ?? job.statusReason ?? "无报告";
-        if (job.reportPath) {
-          try { report = readFileSync(job.reportPath, "utf8"); }
-          catch (error) { report = `报告读取失败：${error instanceof Error ? error.message : String(error)}；摘要：${report}`; }
-        }
-        return `任务「${job.title}」${job.status}：${report.length > 6000 ? `${report.slice(0, 6000)}…（报告过长已截断，完整内容在任务详情）` : report}`;
+        return jobResultEvent(job);
       });
       const name = store.assistant(record.to)?.name ?? record.to;
       const message = record.status === "returned"
-        ? `转交给「${name}」的事已退回：${record.reply}。原请求：${record.text}\n交回内容：${record.returnedText ?? record.text}\n「${name}」已退回过一次，别再转给它。`
+        ? `转交给「${name}」的事已退回：${record.reply}。原请求：${record.text}\n交回内容：${record.returnedText ?? record.text}\n「${name}」已退回过一次。`
         : `转交给「${name}」的事已结束：${record.reply || "（没有回复）"}`;
       void router.event(record.from, `${message}\n${reports.join("\n")}`).catch(() => undefined);
     }
@@ -102,6 +128,8 @@ export class DelegationManager {
     for (const record of this.runtime.store.delegations()) {
       if (record.reportedAt || !["running", "waiting", "returned"].includes(record.status)) continue;
       if (record.jobIds.some((id) => { const job = this.runtime.store.job(id); return job && ["running", "queued", "waiting"].includes(job.status); })) continue;
+      // 目标助手在等用户选：审批卡重启后还在，答了照常继续，不算中断
+      if (record.approvalIds?.some((id) => this.runtime.store.approval(id)?.status === "pending")) continue;
       const now = new Date().toISOString();
       const next: Delegation = { ...record, status: "interrupted", endedAt: now, reportedAt: now };
       this.runtime.store.saveDelegation(next);

package/server/src/desktop-control.ts

@@ -0,0 +1,152 @@
+import { WebSocket, WebSocketServer } from "ws";
+import type { IncomingMessage } from "node:http";
+import type { serve } from "@hono/node-server";
+import type { Duplex } from "node:stream";
+import type { HobRuntime } from "./runtime.ts";
+import { validCookie } from "./http.ts";
+import { desktopProfile } from "./desktop-tools.ts";
+
+const CLIENT = /^[a-f0-9-]{36}$/;
+
+export class DesktopControl {
+  private screens = new Map<string, { owner: string | null; connections: Map<string, number>; timer?: NodeJS.Timeout }>();
+  constructor(private runtime: HobRuntime) {}
+  private screen(id: string) {
+    let screen = this.screens.get(id);
+    if (!screen) { screen = { owner: null, connections: new Map() }; this.screens.set(id, screen); }
+    return screen;
+  }
+  state(client: string, id = "hob") { const screen = this.screen(id); return { taken: screen.owner !== null, mine: screen.owner === client }; }
+  private emit(id: string) { this.runtime.bus.emit({ type: "desktop-control", screen: id, taken: this.screen(id).owner !== null }); }
+  take(client: string, id = "hob") {
+    const screen = this.screen(id);
+    if (!CLIENT.test(client) || !screen.connections.has(client)) throw new Error("请先打开云桌面。");
+    if (screen.owner && screen.owner !== client) throw new Error("另一位用户正在操作云桌面。");
+    screen.owner = client;
+    if (screen.timer) clearTimeout(screen.timer);
+    screen.timer = undefined;
+    this.emit(id);
+  }
+  release(client: string, id = "hob") {
+    const screen = this.screen(id);
+    if (screen.owner !== client) throw new Error("当前设备没有接管云桌面。");
+    screen.owner = null;
+    if (screen.timer) clearTimeout(screen.timer);
+    screen.timer = undefined;
+    this.emit(id);
+  }
+  connected(client: string, id = "hob") {
+    const screen = this.screen(id);
+    screen.connections.set(client, (screen.connections.get(client) ?? 0) + 1);
+    if (screen.owner === client && screen.timer) { clearTimeout(screen.timer); screen.timer = undefined; }
+  }
+  disconnected(client: string, id = "hob") {
+    const screen = this.screen(id);
+    const count = (screen.connections.get(client) ?? 1) - 1;
+    if (count > 0) screen.connections.set(client, count);
+    else {
+      screen.connections.delete(client);
+      if (screen.owner === client) {
+        screen.timer = setTimeout(() => { if (screen.owner === client && !screen.connections.has(client)) { screen.owner = null; this.emit(id); } }, 60_000);
+        screen.timer.unref();
+      }
+    }
+  }
+  get taken() { return this.screen("hob").owner !== null; }
+  dispose() { for (const screen of this.screens.values()) if (screen.timer) clearTimeout(screen.timer); this.screens.clear(); }
+}
+
+// RFB 3.8: version, security type and ClientInit, then length-delimited client messages.
+// Parse the stream rather than trusting WebSocket frame boundaries so view-only cannot inject input.
+export class RfbClientFilter {
+  private data = Buffer.alloc(0);
+  private phase = 0;
+  constructor(private allowed: () => boolean) {}
+  push(input: Buffer): Buffer[] {
+    this.data = Buffer.concat([this.data, input]);
+    if (this.data.length > 1_048_576) throw new Error("RFB 消息过大");
+    const output: Buffer[] = [];
+    while (this.data.length) {
+      let size: number;
+      let isInput = false;
+      if (this.phase === 0) {
+        size = 12;
+        if (this.data.length < size) break;
+        if (this.data.subarray(0, 12).toString() !== "RFB 003.008\n") throw new Error("不支持的 RFB 协议");
+      } else if (this.phase === 1 || this.phase === 2) {
+        size = 1;
+        if (this.phase === 1 && this.data[0] !== 1) throw new Error("不支持的 RFB 认证方式");
+      } else {
+        const type = this.data[0];
+        switch (type) {
+          case 0: size = 20; break;
+          case 2: if (this.data.length < 4) return output; size = 4 + this.data.readUInt16BE(2) * 4; break;
+          case 3: size = 10; break;
+          case 4: size = 8; isInput = true; break;
+          case 5: if (this.data.length < 2) return output; size = (this.data[1] & 0x80) ? 7 : 6; isInput = true; break;
+          case 6: if (this.data.length < 8) return output; size = 8 + Math.abs(this.data.readInt32BE(4)); isInput = true; break;
+          case 150: size = 10; break;
+          case 248: if (this.data.length < 9) return output; size = 9 + this.data[8]; break;
+          case 251: if (this.data.length < 8) return output; size = 8 + this.data[6] * 16; isInput = true; break;
+          case 255: size = 12; isInput = true; break;
+          default: throw new Error(`不支持的 RFB 消息 ${type}`);
+        }
+      }
+      if (size > 1_048_576) throw new Error("RFB 消息过大");
+      if (this.data.length < size) break;
+      const message = this.data.subarray(0, size);
+      this.data = this.data.subarray(size);
+      if (this.phase < 3) this.phase++;
+      if (!isInput || this.allowed()) output.push(message);
+    }
+    return output;
+  }
+}
+
+export function attachDesktopWebSocket(server: ReturnType<typeof serve>, runtime: HobRuntime): () => void {
+  const wss = new WebSocketServer({ noServer: true, maxPayload: 1_048_576 });
+  const upgrade = (req: IncomingMessage, socket: Duplex, head: Buffer) => {
+    const requestUrl = new URL(req.url ?? "/", "http://localhost");
+    if (requestUrl.pathname !== "/api/desktop/vnc") return;
+    const reject = (status: string) => { socket.write(`HTTP/1.1 ${status}\r\nConnection: close\r\n\r\n`); socket.destroy(); };
+    const cookie = req.headers.cookie?.split(";").map((s) => s.trim()).find((s) => s.startsWith("hob="))?.slice(4);
+    if (!validCookie(runtime.config, cookie)) return reject("401 Unauthorized");
+    const origin = req.headers.origin;
+    const forwardedHeader = req.headers["x-forwarded-host"];
+    const forwardedHost = (Array.isArray(forwardedHeader) ? forwardedHeader[0] : forwardedHeader)?.split(",")[0].trim();
+    try { if (origin && ![req.headers.host, forwardedHost].includes(new URL(origin).host)) return reject("403 Forbidden"); }
+    catch { return reject("403 Forbidden"); }
+    const client = requestUrl.searchParams.get("client") ?? "";
+    const screen = requestUrl.searchParams.get("screen") ?? "";
+    if (!CLIENT.test(client) || !/^[a-z0-9-]{1,64}$/.test(screen)) return reject("400 Bad Request");
+    if (screen !== "public" && !runtime.store.assistants().some((assistant) => desktopProfile(assistant.id) === screen)) return reject("404 Not Found");
+    let connection: { url: string; token: string };
+    try { connection = runtime.desktop.connection(); } catch { return reject("503 Service Unavailable"); }
+    const target = new URL(`/vnc/${screen}`, connection.url);
+    target.protocol = target.protocol === "https:" ? "wss:" : "ws:";
+    const upstream = new WebSocket(target, { headers: { Authorization: `Bearer ${connection.token}` }, perMessageDeflate: false });
+    const failed = () => reject("502 Bad Gateway");
+    upstream.once("error", failed);
+    socket.once("close", () => { if (upstream.readyState === WebSocket.CONNECTING) upstream.terminate(); });
+    upstream.once("open", () => {
+      upstream.off("error", failed);
+      wss.handleUpgrade(req, socket, head, (ws) => {
+        runtime.desktopControl.connected(client, screen);
+        const filter = new RfbClientFilter(() => runtime.desktopControl.state(client, screen).mine);
+        upstream.on("message", (data, binary) => { if (ws.readyState === WebSocket.OPEN) ws.send(data, { binary }); });
+        ws.on("message", (data, binary) => {
+          try {
+            if (!binary) throw new Error("RFB 需要二进制数据");
+            for (const message of filter.push(Buffer.from(data as Buffer))) if (upstream.readyState === WebSocket.OPEN) upstream.send(message);
+          } catch (error) { console.error("云桌面协议错误：", error instanceof Error ? error.message : String(error)); ws.close(1002, "RFB 协议错误"); }
+        });
+        ws.on("error", () => upstream.close());
+        upstream.on("error", () => ws.close());
+        ws.on("close", () => { runtime.desktopControl.disconnected(client, screen); upstream.close(); });
+        upstream.on("close", () => ws.close());
+      });
+    });
+  };
+  server.on("upgrade", upgrade);
+  return () => { server.off("upgrade", upgrade); for (const client of wss.clients) client.terminate(); wss.close(); };
+}

package/server/src/desktop-events.ts

@@ -0,0 +1,63 @@
+import { WebSocket } from "ws";
+import type { DesktopScreen } from "../../shared/types.ts";
+import type { HobRuntime } from "./runtime.ts";
+
+/** 监听 desktopd 的屏幕快照，连接中断时清空并退避重连。 */
+export class DesktopEvents {
+  private socket?: WebSocket;
+  private timer?: NodeJS.Timeout;
+  private generation = 0;
+  private delay = 1000;
+  screens: DesktopScreen[] = [];
+  constructor(private runtime: HobRuntime) {}
+
+  restart() {
+    this.stop();
+    this.delay = 1000;
+    this.connect(this.generation);
+  }
+
+  stop() {
+    this.generation++;
+    if (this.timer) clearTimeout(this.timer);
+    this.timer = undefined;
+    this.socket?.terminate();
+    this.socket = undefined;
+    this.publish([]);
+  }
+
+  private publish(screens: DesktopScreen[]) {
+    this.screens = screens;
+    this.runtime.bus.emit({ type: "desktop-screens", screens });
+  }
+
+  private connect(generation: number) {
+    let connection: { url: string; token: string };
+    try { connection = this.runtime.desktop.connection(); } catch { this.retry(generation); return; }
+    const url = new URL("/api/events", connection.url);
+    url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+    const socket = new WebSocket(url, { headers: { Authorization: `Bearer ${connection.token}` } });
+    this.socket = socket;
+    socket.on("message", (data) => {
+      if (generation !== this.generation) return;
+      try {
+        const value = JSON.parse(data.toString()) as { type?: string; screens?: DesktopScreen[] };
+        if (value.type === "screens" && Array.isArray(value.screens)) { this.delay = 1000; this.publish(value.screens); }
+      } catch { /* 忽略无效事件，保留上次快照 */ }
+    });
+    socket.on("error", () => {});
+    socket.on("close", () => {
+      if (generation !== this.generation) return;
+      this.socket = undefined;
+      this.publish([]);
+      this.retry(generation);
+    });
+  }
+
+  private retry(generation: number) {
+    if (generation !== this.generation) return;
+    this.timer = setTimeout(() => this.connect(generation), this.delay);
+    this.timer.unref();
+    this.delay = Math.min(this.delay * 2, 30_000);
+  }
+}

package/server/src/desktop-public.ts

@@ -0,0 +1,43 @@
+interface Waiter {
+  owner: object;
+  name: string;
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+}
+
+/** 公共屏幕由任务持有至结束；其他任务按到达顺序等待。 */
+export class DesktopPublicLock {
+  private current?: { owner: object; name: string };
+  private waiters: Waiter[] = [];
+  constructor(private timeoutMs = 10 * 60_000) {}
+
+  acquire(owner: object, name: string): Promise<void> {
+    if (this.current?.owner === owner) return Promise.resolve();
+    if (!this.current) { this.current = { owner, name }; return Promise.resolve(); }
+    return new Promise((resolve, reject) => {
+      const waiter: Waiter = { owner, name, resolve, reject, timer: setTimeout(() => {
+        this.waiters = this.waiters.filter((entry) => entry !== waiter);
+        reject(new Error(`公共屏幕正被『${this.current?.name ?? "其他助手"}』的任务使用`));
+      }, this.timeoutMs) };
+      waiter.timer.unref();
+      this.waiters.push(waiter);
+    });
+  }
+
+  cancelWait(owner: object) {
+    for (const waiter of this.waiters.filter((entry) => entry.owner === owner)) {
+      clearTimeout(waiter.timer);
+      waiter.reject(new Error("任务已结束。"));
+    }
+    this.waiters = this.waiters.filter((entry) => entry.owner !== owner);
+  }
+
+  release(owner: object) {
+    this.cancelWait(owner);
+    if (this.current?.owner !== owner) return;
+    this.current = undefined;
+    const next = this.waiters.shift();
+    if (next) { clearTimeout(next.timer); this.current = { owner: next.owner, name: next.name }; next.resolve(); }
+  }
+}

package/server/src/desktop-tools.ts

@@ -0,0 +1,161 @@
+import { createHash } from "node:crypto";
+import { Type } from "@earendil-works/pi-ai";
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { chromium, type Browser, type Page } from "playwright-core";
+import type { HobRuntime } from "./runtime.ts";
+
+const LOGIN = "需要登录的页面不要自己登录，用 ask_user 请用户打开云桌面登录后回复。";
+const TAKEN = "用户正在操作云桌面，请稍后再试或用 ask_user 询问";
+const point = Type.Object({ x: Type.Integer(), y: Type.Integer() });
+const action = Type.Object({
+  type: Type.Union([Type.Literal("click"), Type.Literal("move"), Type.Literal("drag"), Type.Literal("type"), Type.Literal("key"), Type.Literal("scroll"), Type.Literal("wait")]),
+  x: Type.Optional(Type.Integer()), y: Type.Optional(Type.Integer()),
+  from: Type.Optional(point), to: Type.Optional(point),
+  button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])),
+  count: Type.Optional(Type.Integer()), text: Type.Optional(Type.String()), keys: Type.Optional(Type.String()),
+  dy: Type.Optional(Type.Integer()), dx: Type.Optional(Type.Integer()), ms: Type.Optional(Type.Integer()),
+});
+
+type PageConnection = { browser?: Browser; page?: Page; opening?: Promise<Page> };
+
+export class DesktopToolSession {
+  private connections = new Map<string, PageConnection>();
+  private publicWait?: Promise<void>;
+  private active = new Set<Promise<unknown>>();
+  private disposed = false;
+  private profile: string;
+  constructor(private runtime: HobRuntime, private assistantId: string, private canSeeImages: boolean) {
+    this.profile = desktopProfile(assistantId);
+  }
+
+  private check(screen: string) {
+    if (this.disposed) throw new Error("任务已结束。");
+    if (this.runtime.desktopControl.state("", screen).taken) throw new Error(TAKEN);
+  }
+  private async screen(value: unknown): Promise<string> {
+    if (value !== undefined && value !== "own" && value !== "public") throw new Error("屏幕参数无效。");
+    const screen = value === "public" ? "public" : this.profile;
+    this.check(screen);
+    if (screen === "public") {
+      if (!this.runtime.settings.settings.desktop?.publicAssistants?.includes(this.assistantId)) throw new Error("这个助手没有使用公共屏幕的权限。");
+      if (!this.publicWait) {
+        const name = this.runtime.store.assistant(this.assistantId)?.name ?? this.assistantId;
+        this.publicWait = this.runtime.desktopPublic.acquire(this, name).catch((error) => { this.publicWait = undefined; throw error; });
+      }
+      await this.publicWait;
+      this.check(screen);
+      if (!this.runtime.settings.settings.desktop?.publicAssistants?.includes(this.assistantId)) throw new Error("这个助手没有使用公共屏幕的权限。");
+    }
+    return screen;
+  }
+  private async getPage(screen: string): Promise<Page> {
+    this.check(screen);
+    let connection = this.connections.get(screen);
+    if (!connection) { connection = {}; this.connections.set(screen, connection); }
+    if (connection.page && !connection.page.isClosed() && connection.browser?.isConnected()) return connection.page;
+    if (!connection.opening) connection.opening = this.openPage(screen, connection).finally(() => { connection!.opening = undefined; });
+    return connection.opening;
+  }
+  private async openPage(screen: string, connection: PageConnection): Promise<Page> {
+    const result = await (await this.runtime.desktop.request(`/api/screens/${encodeURIComponent(screen)}/browser/start`, { method: "POST" })).json() as { cdp: string };
+    this.check(screen);
+    if (result.cdp !== `/cdp/${screen}`) throw new Error("云桌面返回了无效的浏览器地址。");
+    const { url, token } = this.runtime.desktop.connection();
+    const endpoint = new URL(result.cdp, url);
+    endpoint.protocol = endpoint.protocol === "https:" ? "wss:" : "ws:";
+    const browser = await chromium.connectOverCDP(endpoint.toString(), { headers: { Authorization: `Bearer ${token}` }, timeout: 15_000 });
+    try {
+      this.check(screen);
+      const context = browser.contexts()[0];
+      if (!context) throw new Error("云桌面浏览器没有可用的窗口。");
+      const page = await context.newPage();
+      this.check(screen);
+      page.setDefaultTimeout(15_000);
+      connection.browser = browser;
+      connection.page = page;
+      return page;
+    } catch (error) { await browser.close().catch(() => undefined); throw error; }
+  }
+  async dispose() {
+    this.disposed = true;
+    this.runtime.desktopPublic.cancelWait(this);
+    await Promise.allSettled([...this.active]);
+    await Promise.all([...this.connections.values()].map(async (connection) => {
+      await connection.opening?.catch(() => undefined);
+      await connection.browser?.close().catch(() => undefined);
+    }));
+    this.runtime.desktopPublic.release(this);
+    this.connections.clear();
+  }
+  tools(): ToolDefinition<any, any>[] {
+    const allowPublic = this.runtime.settings.settings.desktop?.publicAssistants?.includes(this.assistantId) ?? false;
+    const make = (name: string, label: string, description: string, parameters: any, run: (params: any, screen: string) => Promise<string | Buffer>): ToolDefinition<any, any> => ({
+      name, label, description: `${description} ${LOGIN}`,
+      parameters: Type.Object({ ...parameters.properties, ...(allowPublic ? { screen: Type.Optional(Type.Union([Type.Literal("own"), Type.Literal("public")])) } : {}) }),
+      execute: async (_id, params) => {
+        const operation = (async () => {
+          try {
+            const values = (params ?? {}) as Record<string, unknown>;
+            const screen = await this.screen(values.screen);
+            const value = await run(values, screen);
+            if (Buffer.isBuffer(value)) {
+              if (!this.canSeeImages) return { content: [{ type: "text" as const, text: "当前模型不能看图，请使用页面快照或请用户查看云桌面。" }], details: undefined };
+              return { content: [{ type: "image" as const, data: value.toString("base64"), mimeType: "image/png" }], details: undefined };
+            }
+            return { content: [{ type: "text" as const, text: value }], details: undefined };
+          } catch (error) {
+            return { content: [{ type: "text" as const, text: error instanceof Error ? error.message : String(error) }], details: undefined, isError: true };
+          }
+        })();
+        this.active.add(operation);
+        try { return await operation; } finally { this.active.delete(operation); }
+      },
+    });
+    return [
+      make("browser_open", "打开网页", "在助手的浏览器中打开 HTTP(S) 网页。", Type.Object({ url: Type.String() }), async ({ url }, screen) => {
+        const target = new URL(url);
+        if (target.protocol !== "http:" && target.protocol !== "https:") throw new Error("只支持 HTTP(S) 网页。");
+        const page = await this.getPage(screen); this.check(screen); await page.goto(target.toString(), { waitUntil: "domcontentloaded", timeout: 30_000 }); return `已打开 ${page.url()}`;
+      }),
+      make("browser_snapshot", "读取页面", "读取当前页面的无障碍快照；引用用于点击或输入。", Type.Object({}), async (_, screen) => { const page = await this.getPage(screen); this.check(screen); return page.ariaSnapshot({ mode: "ai" }); }),
+      make("browser_click", "点击网页", "点击最新页面快照中的引用。", Type.Object({ ref: Type.String() }), async ({ ref }, screen) => { const page = await this.getPage(screen); this.check(screen); await page.getByRef(ref).click(); return "已点击。"; }),
+      make("browser_type", "输入文字", "向最新页面快照中的引用输入文字。", Type.Object({ ref: Type.String(), text: Type.String(), submit: Type.Optional(Type.Boolean()) }), async ({ ref, text, submit }, screen) => { const page = await this.getPage(screen); this.check(screen); const target = page.getByRef(ref); await target.fill(text); if (submit) { this.check(screen); await target.press("Enter"); } return "已输入。"; }),
+      make("browser_press", "按键", "向当前网页发送按键，如 Enter、Control+L。", Type.Object({ key: Type.String() }), async ({ key }, screen) => { const page = await this.getPage(screen); this.check(screen); await page.keyboard.press(key); return "已按键。"; }),
+      make("browser_back", "返回网页", "返回浏览器上一页。", Type.Object({}), async (_, screen) => { const page = await this.getPage(screen); this.check(screen); await page.goBack({ waitUntil: "domcontentloaded" }); return `当前页面：${page.url()}`; }),
+      make("browser_screenshot", "网页截图", "截取当前浏览器页面。", Type.Object({}), async (_, screen) => { const page = await this.getPage(screen); this.check(screen); return page.screenshot(); }),
+      make("desktop_screenshot", "桌面截图", "截取整个云桌面。", Type.Object({}), async (_, screen) => {
+        await this.runtime.desktop.request(`/api/screens/${screen}/start`, { method: "POST" });
+        this.check(screen);
+        return Buffer.from(await (await this.runtime.desktop.request(`/api/screens/${screen}/screenshot`)).arrayBuffer());
+      }),
+      make("desktop_action", "操作桌面", "按顺序执行桌面动作：click、move、drag、type、key、scroll、wait。", Type.Object({ actions: Type.Array(action, { minItems: 1, maxItems: 100 }) }), async ({ actions }, screen) => {
+        await this.runtime.desktop.request(`/api/screens/${screen}/start`, { method: "POST" });
+        const health = await (await this.runtime.desktop.request("/api/health")).json() as { screens: { id: string; browserRunning: boolean }[] };
+        this.check(screen);
+        if (health.screens.some((item) => item.id === screen && item.browserRunning)) {
+          await this.runtime.desktop.request(`/api/screens/${screen}/browser/focus`, { method: "POST" });
+        }
+        this.check(screen);
+        await this.runtime.desktop.request(`/api/screens/${screen}/input`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ actions }), signal: AbortSignal.timeout(60_000) });
+        return "桌面操作已完成。";
+      }),
+    ];
+  }
+}
+
+/** 公共屏幕保留 public；无效或冲突的助手 id 用稳定哈希。 */
+export function desktopProfile(assistantId: string): string {
+  return assistantId !== "public" && /^[a-z0-9-]{1,64}$/.test(assistantId) ? assistantId : `p-${createHash("sha256").update(assistantId).digest("hex").slice(0, 62)}`;
+}
+
+/** 在云桌面里打开（或提到最前）这个助手的浏览器：用户接管后自己登录网站用。 */
+export async function openDesktopBrowser(runtime: HobRuntime, assistantId: string): Promise<void> {
+  const profile = encodeURIComponent(desktopProfile(assistantId));
+  await runtime.desktop.request(`/api/screens/${profile}/browser/start`, { method: "POST" });
+  await runtime.desktop.request(`/api/screens/${profile}/browser/focus`, { method: "POST" });
+}
+
+export async function availableDesktopTools(runtime: HobRuntime, profile: string, canSeeImages: boolean): Promise<DesktopToolSession | undefined> {
+  try { await runtime.desktop.request("/api/health"); return new DesktopToolSession(runtime, profile, canSeeImages); }
+  catch { return undefined; }
+}

package/server/src/desktop.ts

@@ -0,0 +1,76 @@
+import { readFileSync } from "node:fs";
+import { readSecrets } from "./secrets.ts";
+import type { SettingsStore } from "./settings.ts";
+
+export class DesktopService {
+  constructor(private home: string, private settings: SettingsStore, private env: NodeJS.ProcessEnv = process.env) {}
+
+  get deployed(): boolean {
+    return !!(this.env.HOB_DESKTOP_URL && this.deploymentToken());
+  }
+
+  token(): string {
+    return this.configuration().token;
+  }
+
+  private deploymentToken(): string {
+    const token = this.env.HOB_DESKTOP_TOKEN;
+    if (token) return token;
+    const file = this.env.HOB_DESKTOP_TOKEN_FILE;
+    if (!file) return "";
+    try { return readFileSync(file, "utf8").trim(); } catch { return ""; }
+  }
+
+  private configuration() {
+    const saved = this.settings.settings.desktop ?? { url: "", enabled: false, publicAssistants: [] };
+    const deploymentToken = this.env.HOB_DESKTOP_URL ? this.deploymentToken() : "";
+    const deployed = !!deploymentToken;
+    const token = deployed ? deploymentToken : readSecrets(this.home)["desktop:token"] ?? "";
+    return {
+      revision: this.settings.get().revision,
+      url: deployed ? this.env.HOB_DESKTOP_URL! : saved.url,
+      enabled: deployed ? true : saved.enabled,
+      deployed,
+      tokenConfigured: !!token,
+      publicAssistants: saved.publicAssistants ?? [],
+      token,
+    };
+  }
+
+  view() {
+    const { token: _token, ...view } = this.configuration();
+    return view;
+  }
+
+  connection() {
+    const { enabled, url, token } = this.configuration();
+    if (!enabled || !url || !token) throw new Error("云桌面未配置或尚未启用。");
+    return { url: checkDesktopUrl(url), token };
+  }
+
+  async request(path: string, init: RequestInit = {}): Promise<Response> {
+    const { url, token } = this.connection();
+    const response = await fetch(new URL(path, url), {
+      ...init,
+      headers: { ...init.headers, Authorization: `Bearer ${token}` },
+      signal: init.signal ?? AbortSignal.timeout(10_000),
+      redirect: "error",
+    });
+    if (!response.ok) {
+      const body = await response.json().catch(() => null) as { error?: { message?: unknown } } | null;
+      const detail = typeof body?.error?.message === "string" ? body.error.message.replaceAll(token, "[已隐藏]") : "";
+      throw new Error(`云桌面请求失败（HTTP ${response.status}）${detail ? `：${detail}` : "。"}`);
+    }
+    return response;
+  }
+}
+
+export function checkDesktopUrl(value: string): string {
+  if (!value.trim()) return "";
+  let url: URL;
+  try { url = new URL(value); } catch { throw new Error("云桌面地址无效。"); }
+  if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.pathname !== '/') {
+    throw new Error("云桌面地址须为不含凭据或路径的 HTTP(S) 地址。");
+  }
+  return url.origin;
+}

package/server/src/embedding.ts

@@ -0,0 +1,52 @@
+import type { EmbeddingSettings } from "../../shared/types.ts";
+import { readSecrets } from "./secrets.ts";
+
+export const EMBEDDING_KEY_SECRET = "embedding";
+
+export function embeddingKey(home: string): string {
+  return readSecrets(home)[EMBEDDING_KEY_SECRET]?.trim() ?? "";
+}
+
+export function checkEmbeddingSettings(raw: unknown): EmbeddingSettings {
+  const input = (raw ?? {}) as Record<string, unknown>;
+  if (input.source === "off") return { source: "off" };
+  if (input.source !== "openai") throw new Error("向量模型来源无效。");
+  const baseUrl = typeof input.baseUrl === "string" ? input.baseUrl.trim().replace(/\/+$/, "") : "";
+  let url: URL;
+  try { url = new URL(baseUrl); } catch { throw new Error("向量服务地址无效，请填写 http:// 或 https:// 地址。"); }
+  if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("向量服务地址仅支持 http:// 或 https://。");
+  const model = typeof input.model === "string" ? input.model.trim() : "";
+  if (!model) throw new Error("请填写向量模型名称。");
+  return { source: "openai", baseUrl, model };
+}
+
+export function embeddingModel(config: EmbeddingSettings): string {
+  return JSON.stringify(config.source === "openai" ? { source: config.source, baseUrl: config.baseUrl, model: config.model } : config);
+}
+
+export function normalizeEmbeddingSettings(raw: unknown, fallback: EmbeddingSettings): EmbeddingSettings {
+  try { return checkEmbeddingSettings(raw); } catch { return fallback; }
+}
+
+export async function embed(config: EmbeddingSettings, key: string, input: string[], fetchImpl: typeof fetch = fetch): Promise<number[][]> {
+  if (config.source !== "openai") throw new Error("尚未配置向量模型。");
+  if (!key.trim()) throw new Error("尚未填写向量模型 API Key。");
+  const vectors: number[][] = [];
+  for (let start = 0; start < input.length; start += 32) {
+    const batch = input.slice(start, start + 32);
+    const response = await fetchImpl(`${config.baseUrl}/embeddings`, {
+      method: "POST", headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify({ model: config.model, input: batch }), signal: AbortSignal.timeout(30_000),
+    });
+    if (!response.ok) throw new Error(`向量接口返回 ${response.status}：${(await response.text()).slice(0, 1000)}`);
+    const result = await response.json() as { data?: { index: number; embedding: unknown }[] };
+    if (!Array.isArray(result.data) || result.data.length !== batch.length) throw new Error("向量接口返回的数量不正确。");
+    const sorted = [...result.data].sort((a, b) => a.index - b.index);
+    for (const [i, item] of sorted.entries()) {
+      if (item.index !== i || !Array.isArray(item.embedding) || !item.embedding.length || !item.embedding.every((v: unknown) => typeof v === "number" && Number.isFinite(v))) throw new Error("向量接口返回的数据无效。");
+      vectors.push(item.embedding as number[]);
+    }
+  }
+  if (vectors.some((v) => v.length !== vectors[0]?.length)) throw new Error("向量接口返回的维数不一致。");
+  return vectors;
+}