@greatlhd/ailo-desktop 1.0.0

package/src/connection_util.ts

@@ -0,0 +1,23 @@
+/**
+ * Re-exports from SDK + config.json loading helper for desktop.
+ */
+
+export {
+  hasValidConfig,
+  backoffDelayMs,
+  readConfig,
+} from "@greatlhd/ailo-endpoint-sdk";
+export type { AiloConnectionConfig } from "@greatlhd/ailo-endpoint-sdk";
+
+import { readConfig } from "@greatlhd/ailo-endpoint-sdk";
+import type { AiloConnectionConfig } from "@greatlhd/ailo-endpoint-sdk";
+
+export function loadConnectionConfig(configPath: string): AiloConnectionConfig {
+  const raw = readConfig(configPath);
+  const ailo = (raw as Record<string, unknown>).ailo as Record<string, unknown> | undefined;
+  return {
+    url: process.env.AILO_WS_URL || (ailo?.wsUrl as string) || "",
+    apiKey: process.env.AILO_API_KEY || (ailo?.apiKey as string) || "",
+    endpointId: process.env.AILO_ENDPOINT_ID || (ailo?.endpointId as string) || "",
+  };
+}

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const CONFIG_FILENAME = "config.json";
+export const NO_SUBJECT = "（无主题）";

package/src/desktop_state_store.ts

@@ -0,0 +1,64 @@
+import type { DesktopActionResult, DesktopObservation, DesktopVerdict } from "./desktop_types.js";
+
+const DEFAULT_OBSERVATION_TTL_MS = 60_000;
+const MAX_OBSERVATIONS = 20;
+
+export class DesktopStateStore {
+  private readonly observations = new Map<string, DesktopObservation>();
+  private lastObservationId: string | null = null;
+  private lastAction: DesktopActionResult | null = null;
+  private lastVerdict: DesktopVerdict | null = null;
+
+  constructor(private readonly observationTtlMs = DEFAULT_OBSERVATION_TTL_MS) {}
+
+  saveObservation(observation: DesktopObservation): void {
+    this.pruneExpired();
+    this.observations.set(observation.id, observation);
+    this.lastObservationId = observation.id;
+    while (this.observations.size > MAX_OBSERVATIONS) {
+      const oldestKey = this.observations.keys().next().value;
+      if (!oldestKey) break;
+      this.observations.delete(oldestKey);
+    }
+  }
+
+  getObservation(id: string): DesktopObservation | null {
+    this.pruneExpired();
+    return this.observations.get(id) ?? null;
+  }
+
+  getLatestObservation(): DesktopObservation | null {
+    this.pruneExpired();
+    if (!this.lastObservationId) return null;
+    return this.observations.get(this.lastObservationId) ?? null;
+  }
+
+  isExpired(observation: DesktopObservation): boolean {
+    return Date.now() - observation.timestamp > this.observationTtlMs;
+  }
+
+  setLastAction(action: DesktopActionResult): void {
+    this.lastAction = action;
+  }
+
+  getLastAction(): DesktopActionResult | null {
+    return this.lastAction;
+  }
+
+  setLastVerdict(verdict: DesktopVerdict): void {
+    this.lastVerdict = verdict;
+  }
+
+  getLastVerdict(): DesktopVerdict | null {
+    return this.lastVerdict;
+  }
+
+  private pruneExpired(): void {
+    for (const [id, observation] of this.observations) {
+      if (this.isExpired(observation)) this.observations.delete(id);
+    }
+    if (this.lastObservationId && !this.observations.has(this.lastObservationId)) {
+      this.lastObservationId = null;
+    }
+  }
+}

package/src/desktop_types.ts

@@ -0,0 +1,44 @@
+export interface Rect {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export type DesktopScopeKind = "virtual_screen" | "screen" | "window" | "region";
+export type CoordinateSpace = "virtual_screen" | "screen_local" | "window_local" | "region_local";
+
+export interface DesktopScope {
+  kind: DesktopScopeKind;
+  bounds: Rect;
+  screenIndex?: number;
+}
+
+export interface DesktopObservation {
+  id: string;
+  timestamp: number;
+  scope: DesktopScope;
+  coordinateSpace: CoordinateSpace;
+  imageWidth: number;
+  imageHeight: number;
+  image: {
+    path: string;
+    mime: string;
+    name: string;
+  };
+}
+
+export interface DesktopActionResult {
+  accepted: boolean;
+  executed: boolean;
+  action: string;
+  timestamp: number;
+  observationId?: string;
+  error?: string;
+  details?: Record<string, unknown>;
+}
+
+export interface DesktopVerdict {
+  status: "success" | "failure" | "uncertain";
+  reason: string;
+}

package/src/desktop_verifier.ts

@@ -0,0 +1,45 @@
+import { readFileSync } from "fs";
+import { PNG } from "pngjs";
+import pixelmatch from "pixelmatch";
+import type { DesktopActionResult, DesktopObservation, DesktopVerdict } from "./desktop_types.js";
+
+function imagesDiffer(beforePath: string, afterPath: string): boolean {
+  const before = PNG.sync.read(readFileSync(beforePath));
+  const after = PNG.sync.read(readFileSync(afterPath));
+  if (before.width !== after.width || before.height !== after.height) return true;
+  const diff = new PNG({ width: before.width, height: before.height });
+  // pixelmatch is exported as CJS export =, use .default in ESM context
+  const pm = (pixelmatch as unknown as { default: typeof pixelmatch }).default ?? pixelmatch;
+  const numDiffPixels = pm(before.data, after.data, diff.data, before.width, before.height, { threshold: 0.1 });
+  // 少于 0.5% 像素变化视为无变化（防误报）
+  const totalPixels = before.width * before.height;
+  return numDiffPixels / totalPixels > 0.005;
+}
+
+export function verifyDesktopAction(args: {
+  beforeObservation: DesktopObservation | null;
+  afterObservation: DesktopObservation | null;
+  actionResult: DesktopActionResult;
+}): DesktopVerdict {
+  const { beforeObservation, afterObservation, actionResult } = args;
+  if (!actionResult.accepted) {
+    return { status: "failure", reason: actionResult.error ?? "动作被拒绝执行" };
+  }
+  if (!actionResult.executed) {
+    return { status: "failure", reason: actionResult.error ?? "动作未执行" };
+  }
+  if (!beforeObservation || !afterObservation) {
+    return { status: "uncertain", reason: "缺少动作前后 observation，无法完成验证" };
+  }
+  try {
+    if (imagesDiffer(beforeObservation.image.path, afterObservation.image.path)) {
+      return { status: "success", reason: "动作后界面发生变化" };
+    }
+    return { status: "uncertain", reason: "动作后未观察到明显界面变化" };
+  } catch (error) {
+    return {
+      status: "uncertain",
+      reason: `无法比较动作前后截图: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}

package/src/dingtalk-types.ts

@@ -0,0 +1,26 @@
+export interface DingTalkConfig {
+  clientId: string;
+  clientSecret: string;
+}
+
+export interface DingTalkBotMessage {
+  conversationId: string;
+  conversationType: "1" | "2";
+  chatbotCorpId?: string;
+  chatbotUserId?: string;
+  msgId: string;
+  msgtype: string;
+  text?: { content: string };
+  senderNick: string;
+  senderStaffId: string;
+  senderCorpId?: string;
+  sessionWebhook: string;
+  sessionWebhookExpiredTime: number;
+  createAt: number;
+  conversationTitle?: string;
+  atUsers?: Array<{ dingtalkId: string; staffId?: string }>;
+  isAdmin?: boolean;
+  isInAtList?: boolean;
+}
+
+export const STALE_MESSAGE_THRESHOLD_MS = 5 * 60 * 1000;

package/src/exec_tool.ts

@@ -0,0 +1,93 @@
+import { spawn } from "child_process";
+import * as os from "os";
+import * as path from "path";
+import { mkdtemp, rm } from "fs/promises";
+import type { EndpointContext } from "@greatlhd/ailo-endpoint-sdk";
+
+const MAX_OUTPUT = 50000;
+const DEFAULT_TIMEOUT_MS = 120_000; // 2 分钟
+
+/**
+ * 异步执行 shell 命令。
+ * 立即返回"已启动"，命令在后台运行，完成后通过 sendSignal("tool_result") 推送结果给 LLM。
+ */
+export async function execTool(ctx: EndpointContext, args: Record<string, unknown>): Promise<string> {
+  const command = String(args.command ?? "").trim();
+  if (!command) throw new Error("command 必填");
+  const cwd = (args.cwd as string) || undefined;
+  const timeoutSec = Math.max(5, Math.min(600, Number(args.timeout) || DEFAULT_TIMEOUT_MS / 1000));
+  const timeoutMs = timeoutSec * 1000;
+
+  const isWin = os.platform() === "win32";
+  const shell = isWin ? "powershell" : "/bin/sh";
+  const wrappedCommand = isWin
+    ? `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; ${command}`
+    : command;
+  const shellArgs = isWin ? ["-Command", wrappedCommand] : ["-c", command];
+  const env = {
+    ...process.env,
+    PYTHONIOENCODING: "utf-8",
+    PYTHONUTF8: "1",
+  };
+
+  // 在临时目录执行命令，避免 cwd 污染
+  const tmpDir = await mkdtemp(path.join(os.tmpdir(), "ailo-exec-"));
+  const proc = spawn(shell, shellArgs, { stdio: ["pipe", "pipe", "pipe"], cwd: cwd || tmpDir, env });
+
+  const output: string[] = [];
+  let sent = false;
+
+  const cleanup = () => {
+    if (timeoutHandle) { clearTimeout(timeoutHandle); timeoutHandle = null; }
+    rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+  };
+
+  const sendResult = (content: string) => {
+    if (sent) return;
+    sent = true;
+    ctx.sendSignal("tool_result", { content });
+    cleanup();
+  };
+
+  let timeoutHandle: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+    timeoutHandle = null;
+    try { proc.kill("SIGTERM"); } catch {}
+    setTimeout(() => { try { proc.kill("SIGKILL"); } catch {} }, 2000);
+    sendResult(JSON.stringify({
+      ok: false,
+      type: "timeout",
+      command,
+      timeoutSec: timeoutMs / 1000,
+      message: `超过 ${timeoutMs / 1000}s 未完成，已终止`,
+    }));
+  }, timeoutMs);
+
+  proc.stdout?.on("data", (d: Buffer) => output.push(d.toString("utf-8")));
+  proc.stderr?.on("data", (d: Buffer) => output.push(d.toString("utf-8")));
+
+  proc.on("close", (code) => {
+    const text = output.join("").trim();
+    const truncated = text.length > MAX_OUTPUT
+      ? text.slice(0, MAX_OUTPUT / 2) + "\n...[截断]...\n" + text.slice(-MAX_OUTPUT / 2)
+      : text;
+
+    sendResult(JSON.stringify({
+      ok: true,
+      type: "completed",
+      command,
+      exitCode: code,
+      output: truncated,
+    }));
+  });
+
+  proc.on("error", (err) => {
+    sendResult(JSON.stringify({
+      ok: false,
+      type: "error",
+      command,
+      message: err.message,
+    }));
+  });
+
+  return JSON.stringify({ ok: true, type: "started", command });
+}