@greatlhd/ailo-desktop 1.0.0

package/src/qq-ws.ts

@@ -0,0 +1,223 @@
+import WebSocket from "ws";
+import {
+  type QQConfig,
+  type QQGatewayPayload,
+  OP_DISPATCH,
+  OP_HEARTBEAT,
+  OP_IDENTIFY,
+  OP_RESUME,
+  OP_RECONNECT,
+  OP_INVALID_SESSION,
+  OP_HELLO,
+  OP_HEARTBEAT_ACK,
+  INTENT_PUBLIC_GUILD_MESSAGES,
+  INTENT_DIRECT_MESSAGE,
+  INTENT_GROUP_AND_C2C,
+  DEFAULT_API_BASE,
+  TOKEN_URL,
+  RECONNECT_DELAYS,
+  MAX_RECONNECT_ATTEMPTS,
+} from "./qq-types.js";
+
+type DispatchHandler = (event: string, data: any) => void;
+
+export class QQGatewayClient {
+  private ws: WebSocket | null = null;
+  private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
+  private lastSeq: number | null = null;
+  private sessionId: string | null = null;
+  private reconnectAttempts = 0;
+  private closed = false;
+
+  private accessToken: string = "";
+  private tokenExpiresAt = 0;
+
+  private onDispatch: DispatchHandler;
+  private log: (level: string, msg: string, data?: Record<string, unknown>) => void;
+
+  constructor(
+    private config: QQConfig,
+    onDispatch: DispatchHandler,
+    log?: (level: string, msg: string, data?: Record<string, unknown>) => void,
+  ) {
+    this.onDispatch = onDispatch;
+    this.log = log ?? ((level, msg, data) => console.log(`[qq-ws] [${level}] ${msg}`, data ?? ""));
+  }
+
+  private get apiBase(): string {
+    return (this.config.apiBase ?? DEFAULT_API_BASE).replace(/\/$/, "");
+  }
+
+  async refreshToken(): Promise<string> {
+    if (this.accessToken && Date.now() < this.tokenExpiresAt - 30_000) {
+      return this.accessToken;
+    }
+    const res = await fetch(TOKEN_URL, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ appId: this.config.appId, clientSecret: this.config.appSecret }),
+    });
+    if (!res.ok) throw new Error(`QQ token refresh failed: HTTP ${res.status}`);
+    const body = (await res.json()) as { access_token: string; expires_in: number };
+    this.accessToken = body.access_token;
+    this.tokenExpiresAt = Date.now() + body.expires_in * 1000;
+    this.log("info", "access token refreshed", { expires_in: body.expires_in });
+    return this.accessToken;
+  }
+
+  getAccessToken(): string {
+    return this.accessToken;
+  }
+
+  async connect(): Promise<void> {
+    this.closed = false;
+    await this.refreshToken();
+
+    const gatewayUrl = await this.fetchGatewayUrl();
+    this.log("info", `connecting to gateway: ${gatewayUrl}`);
+    this.createConnection(gatewayUrl);
+  }
+
+  private async fetchGatewayUrl(): Promise<string> {
+    const token = await this.refreshToken();
+    const res = await fetch(`${this.apiBase}/gateway`, {
+      headers: { Authorization: `QQBot ${token}` },
+    });
+    if (!res.ok) throw new Error(`QQ gateway fetch failed: HTTP ${res.status}`);
+    const body = (await res.json()) as { url: string };
+    return body.url;
+  }
+
+  private createConnection(url: string): void {
+    const ws = new WebSocket(url);
+    this.ws = ws;
+
+    ws.on("open", () => {
+      this.log("info", "WebSocket connected");
+      this.reconnectAttempts = 0;
+    });
+
+    ws.on("message", (raw: WebSocket.Data) => {
+      try {
+        const payload = JSON.parse(raw.toString("utf-8")) as QQGatewayPayload;
+        this.handlePayload(payload);
+      } catch (err) {
+        this.log("error", "failed to parse WS message", { err: String(err) });
+      }
+    });
+
+    ws.on("close", (code: number, reason: Buffer) => {
+      this.log("warn", `WebSocket closed: ${code} ${reason.toString("utf-8")}`);
+      this.stopHeartbeat();
+      if (!this.closed) this.scheduleReconnect();
+    });
+
+    ws.on("error", (err: Error) => {
+      this.log("error", "WebSocket error", { err: err.message });
+    });
+  }
+
+  private handlePayload(payload: QQGatewayPayload): void {
+    if (payload.s != null) this.lastSeq = payload.s;
+
+    switch (payload.op) {
+      case OP_HELLO:
+        this.startHeartbeat((payload.d as Record<string, unknown>)?.heartbeat_interval as number ?? 41250);
+        if (this.sessionId) {
+          this.sendResume();
+        } else {
+          this.sendIdentify();
+        }
+        break;
+
+      case OP_DISPATCH:
+        if (payload.t === "READY") {
+          this.sessionId = (payload.d as Record<string, unknown>)?.session_id as string ?? null;
+          this.log("info", "READY", { session_id: this.sessionId });
+        }
+        if (payload.t) {
+          this.onDispatch(payload.t, payload.d);
+        }
+        break;
+
+      case OP_HEARTBEAT_ACK:
+        break;
+
+      case OP_RECONNECT:
+        this.log("info", "server requested reconnect");
+        this.ws?.close(4000, "reconnect");
+        break;
+
+      case OP_INVALID_SESSION:
+        this.log("warn", "invalid session, re-identifying");
+        this.sessionId = null;
+        this.lastSeq = null;
+        setTimeout(() => this.sendIdentify(), 2000);
+        break;
+
+      default:
+        this.log("debug", `unhandled op: ${payload.op}`, { d: payload.d });
+    }
+  }
+
+  private sendIdentify(): void {
+    const intents = INTENT_PUBLIC_GUILD_MESSAGES | INTENT_DIRECT_MESSAGE | INTENT_GROUP_AND_C2C;
+    this.send({
+      op: OP_IDENTIFY,
+      d: { token: `QQBot ${this.accessToken}`, intents, shard: [0, 1] },
+    });
+    this.log("debug", "sent IDENTIFY");
+  }
+
+  private sendResume(): void {
+    this.send({
+      op: OP_RESUME,
+      d: { token: `QQBot ${this.accessToken}`, session_id: this.sessionId, seq: this.lastSeq },
+    });
+    this.log("debug", "sent RESUME");
+  }
+
+  private startHeartbeat(intervalMs: number): void {
+    this.stopHeartbeat();
+    this.heartbeatTimer = setInterval(() => {
+      this.send({ op: OP_HEARTBEAT, d: this.lastSeq });
+    }, intervalMs);
+  }
+
+  private stopHeartbeat(): void {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
+
+  private send(payload: QQGatewayPayload): void {
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(payload));
+    }
+  }
+
+  private scheduleReconnect(): void {
+    if (this.reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {
+      this.log("error", "max reconnect attempts reached, giving up");
+      return;
+    }
+    const delay = RECONNECT_DELAYS[Math.min(this.reconnectAttempts, RECONNECT_DELAYS.length - 1)] * 1000;
+    this.reconnectAttempts++;
+    this.log("info", `reconnecting in ${delay / 1000}s (attempt ${this.reconnectAttempts})`);
+    setTimeout(() => {
+      if (!this.closed) {
+        this.connect().catch((err) => this.log("error", "reconnect failed", { err: String(err) }));
+      }
+    }, delay);
+  }
+
+  close(): void {
+    this.closed = true;
+    this.stopHeartbeat();
+    if (this.ws) {
+      this.ws.close(1000, "shutdown");
+      this.ws = null;
+    }
+  }
+}

package/src/screenshot.ts

@@ -0,0 +1,297 @@
+import { randomUUID } from "crypto";
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import type { ContentPart } from "@greatlhd/ailo-endpoint-sdk";
+import type { CoordinateSpace, DesktopObservation, DesktopScope, Rect } from "./desktop_types.js";
+
+interface ScreenDescriptor extends Rect {
+  index: number;
+  primary: boolean;
+}
+
+export interface ScreenInfo {
+  count: number;
+  screens: ScreenDescriptor[];
+  virtualBounds: Rect;
+}
+
+export interface ScreenshotOptions {
+  capture_window?: boolean;
+  /** 多显示器时：不传或 "all" 截取全部，0/1/2... 截取指定显示器（0-based）*/
+  screen?: number | "all";
+}
+
+export interface ScreenshotCaptureResult {
+  parts: ContentPart[];
+  observation?: DesktopObservation;
+}
+
+function ok(message: string, observation: DesktopObservation): ContentPart[] {
+  return [
+    {
+      type: "text",
+      text: JSON.stringify({
+        ok: true,
+        message,
+        observation: {
+          observation_id: observation.id,
+          timestamp: observation.timestamp,
+          scope: observation.scope,
+          coordinate_space: observation.coordinateSpace,
+          image_width: observation.imageWidth,
+          image_height: observation.imageHeight,
+          image_path: observation.image.path,
+        },
+      }, null, 2),
+    },
+    {
+      type: "image",
+      media: {
+        type: "image",
+        path: observation.image.path,
+        mime: observation.image.mime,
+        name: observation.image.name,
+      },
+    },
+  ];
+}
+
+function fail(error: string): ScreenshotCaptureResult {
+  return {
+    parts: [{ type: "text", text: JSON.stringify({ ok: false, error }, null, 2) }],
+  };
+}
+
+export function getScreenInfo(): ScreenInfo {
+  const platform = os.platform();
+  if (platform === "win32") {
+    const ps = [
+      "Add-Type -AssemblyName System.Windows.Forms",
+      "$screens=[System.Windows.Forms.Screen]::AllScreens",
+      "$primary=[System.Windows.Forms.Screen]::PrimaryScreen",
+      "$vs=[System.Windows.Forms.SystemInformation]::VirtualScreen",
+      "$r=@()",
+      "for($i=0;$i -lt $screens.Count;$i++){",
+      "  $b=$screens[$i].Bounds",
+      "  $r+=@{index=$i;x=$b.X;y=$b.Y;width=$b.Width;height=$b.Height;primary=($screens[$i] -eq $primary)}",
+      "}",
+      "Write-Output (@{count=$screens.Count;screens=$r;virtualBounds=@{x=$vs.X;y=$vs.Y;width=$vs.Width;height=$vs.Height}}|ConvertTo-Json -Compress)",
+    ].join("; ");
+    const r = spawnSync("powershell", ["-Command", ps], { encoding: "utf-8", timeout: 5000 });
+    try {
+      const j = JSON.parse((r.stdout ?? "").trim());
+      return {
+        count: j.count ?? 1,
+        screens: (j.screens ?? []).map((s: any) => ({
+          index: s.index,
+          x: s.x ?? 0,
+          y: s.y ?? 0,
+          width: s.width,
+          height: s.height,
+          primary: !!s.primary,
+        })),
+        virtualBounds: {
+          x: j.virtualBounds?.x ?? 0,
+          y: j.virtualBounds?.y ?? 0,
+          width: j.virtualBounds?.width ?? 1920,
+          height: j.virtualBounds?.height ?? 1080,
+        },
+      };
+    } catch {
+      return {
+        count: 1,
+        screens: [{ index: 0, x: 0, y: 0, width: 1920, height: 1080, primary: true }],
+        virtualBounds: { x: 0, y: 0, width: 1920, height: 1080 },
+      };
+    }
+  } else if (platform === "darwin") {
+    const r = spawnSync("system_profiler", ["SPDisplaysDataType", "-json"], { encoding: "utf-8", timeout: 5000 });
+    try {
+      const j = JSON.parse((r.stdout ?? "").trim());
+      const displays = (j?.SPDisplaysDataType ?? []).flatMap((d: any) => d.spdisplays_ndrvs ?? [d]);
+      const screens = displays
+        .filter((d: any) => d.spdisplays_resolution)
+        .map((d: any, i: number) => {
+          const m = String(d.spdisplays_resolution ?? "").match(/(\d+)\s*x\s*(\d+)/);
+          return {
+            index: i,
+            x: 0,
+            y: 0,
+            width: m ? Number(m[1]) : 1920,
+            height: m ? Number(m[2]) : 1080,
+            primary: i === 0,
+          };
+        });
+      const primary = screens[0] ?? { index: 0, x: 0, y: 0, width: 1920, height: 1080, primary: true };
+      return { count: screens.length || 1, screens: screens.length ? screens : [primary], virtualBounds: { x: 0, y: 0, width: primary.width, height: primary.height } };
+    } catch {
+      return {
+        count: 1,
+        screens: [{ index: 0, x: 0, y: 0, width: 1920, height: 1080, primary: true }],
+        virtualBounds: { x: 0, y: 0, width: 1920, height: 1080 },
+      };
+    }
+  } else {
+    const r = spawnSync("xdpyinfo", [], { encoding: "utf-8", timeout: 5000 });
+    const m = (r.stdout ?? "").match(/dimensions:\s*(\d+)x(\d+)/);
+    const width = m ? Number(m[1]) : 1920;
+    const height = m ? Number(m[2]) : 1080;
+    return {
+      count: 1,
+      screens: [{ index: 0, x: 0, y: 0, width, height, primary: true }],
+      virtualBounds: { x: 0, y: 0, width, height },
+    };
+  }
+}
+
+function buildObservation(args: {
+  imagePath: string;
+  imageWidth: number;
+  imageHeight: number;
+  captureWindow: boolean;
+  screenOpt?: number | "all";
+}): DesktopObservation {
+  const { imagePath, imageWidth, imageHeight, captureWindow, screenOpt } = args;
+  const screenInfo = getScreenInfo();
+  let scope: DesktopScope;
+  let coordinateSpace: CoordinateSpace;
+
+  if (captureWindow) {
+    scope = {
+      kind: "window",
+      bounds: { x: 0, y: 0, width: imageWidth, height: imageHeight },
+    };
+    coordinateSpace = "window_local";
+  } else if (typeof screenOpt === "number") {
+    const screen = screenInfo.screens.find((item) => item.index === screenOpt) ?? screenInfo.screens[0];
+    scope = {
+      kind: "screen",
+      screenIndex: screen.index,
+      bounds: { x: screen.x, y: screen.y, width: screen.width, height: screen.height },
+    };
+    coordinateSpace = "screen_local";
+  } else {
+    scope = {
+      kind: "virtual_screen",
+      bounds: screenInfo.virtualBounds,
+    };
+    coordinateSpace = "virtual_screen";
+  }
+
+  return {
+    id: `obs_${randomUUID()}`,
+    timestamp: Date.now(),
+    scope,
+    coordinateSpace,
+    imageWidth,
+    imageHeight,
+    image: {
+      path: imagePath,
+      mime: "image/png",
+      name: "screenshot.png",
+    },
+  };
+}
+
+export async function captureDesktopObservation(opts: ScreenshotOptions | boolean = false): Promise<ScreenshotCaptureResult> {
+  const captureWindow = typeof opts === "boolean" ? opts : !!opts.capture_window;
+  const screenOpt = typeof opts === "boolean" ? undefined : opts?.screen;
+  const tmpPath = path.join(os.tmpdir(), `ailo_screenshot_${Date.now()}.png`);
+  const platform = os.platform();
+
+  if (platform === "darwin") {
+    if (captureWindow) {
+      spawnSync("screencapture", ["-w", tmpPath]);
+    } else if (typeof screenOpt === "number") {
+      spawnSync("screencapture", ["-x", "-D", String(screenOpt + 1), tmpPath]);
+    } else {
+      spawnSync("screencapture", ["-x", tmpPath]);
+    }
+  } else if (platform === "win32") {
+    const escaped = tmpPath.replace(/'/g, "''").replace(/\\/g, "\\\\");
+    let ps: string;
+    if (typeof screenOpt === "number") {
+      ps = [
+        "Add-Type -AssemblyName System.Windows.Forms",
+        "Add-Type -AssemblyName System.Drawing",
+        "$screens=[System.Windows.Forms.Screen]::AllScreens",
+        `$idx=${screenOpt}`,
+        "if($idx -ge $screens.Count){ $idx=0 }",
+        "$screen=$screens[$idx].Bounds",
+        "$bmp=New-Object System.Drawing.Bitmap($screen.Width,$screen.Height)",
+        "$g=[System.Drawing.Graphics]::FromImage($bmp)",
+        "$g.CopyFromScreen($screen.Location,[System.Drawing.Point]::Empty,$screen.Size)",
+        `$bmp.Save('${escaped}')`,
+      ].join("; ");
+    } else {
+      ps = [
+        "Add-Type -AssemblyName System.Windows.Forms",
+        "Add-Type -AssemblyName System.Drawing",
+        "$vs=[System.Windows.Forms.SystemInformation]::VirtualScreen",
+        "$bmp=New-Object System.Drawing.Bitmap($vs.Width,$vs.Height)",
+        "$g=[System.Drawing.Graphics]::FromImage($bmp)",
+        "$g.CopyFromScreen($vs.X,$vs.Y,0,0,[System.Drawing.Size]::new($vs.Width,$vs.Height))",
+        `$bmp.Save('${escaped}')`,
+      ].join("; ");
+    }
+    spawnSync("powershell", ["-Command", ps]);
+  } else {
+    const r = spawnSync("scrot", [tmpPath]);
+    if (r.status !== 0) spawnSync("import", ["-window", "root", tmpPath]);
+  }
+
+  if (!fs.existsSync(tmpPath)) return fail("截图失败：未能生成截图文件");
+
+  let imageWidth = 0;
+  let imageHeight = 0;
+  try {
+    const screenInfo = getScreenInfo();
+    if (typeof screenOpt === "number") {
+      const screen = screenInfo.screens.find((item) => item.index === screenOpt) ?? screenInfo.screens[0];
+      imageWidth = screen.width;
+      imageHeight = screen.height;
+    } else {
+      imageWidth = screenInfo.virtualBounds.width;
+      imageHeight = screenInfo.virtualBounds.height;
+    }
+  } catch {
+    imageWidth = 1920;
+    imageHeight = 1080;
+  }
+
+  const observation = buildObservation({
+    imagePath: tmpPath,
+    imageWidth,
+    imageHeight,
+    captureWindow,
+    screenOpt,
+  });
+
+  let message: string;
+  if (captureWindow) {
+    message = "窗口截图完成";
+  } else {
+    const screenInfo = getScreenInfo();
+    if (screenInfo.count > 1) {
+      if (typeof screenOpt === "number") {
+        message = `截图完成（第 ${screenOpt + 1} 块/共 ${screenInfo.count} 块屏。可用 screen=0～${screenInfo.count - 1} 分别截取其他屏）`;
+      } else {
+        message = `截图完成（共 ${screenInfo.count} 块屏，已截取全部。可用 screen=0～${screenInfo.count - 1} 分别截取单块）`;
+      }
+    } else {
+      message = typeof screenOpt === "number" ? `显示器 ${screenOpt} 截图完成` : "截图完成";
+    }
+  }
+
+  return {
+    parts: ok(message, observation),
+    observation,
+  };
+}
+
+export async function takeScreenshot(opts: ScreenshotOptions | boolean = false): Promise<ContentPart[]> {
+  const result = await captureDesktopObservation(opts);
+  return result.parts;
+}