cortico-world-hands 0.1.0 → 0.1.2

package/dist/index.js

@@ -4,6 +4,7 @@ var HANDS_DEFAULTS = {
   enabled: false,
   actuator: "auto",
   eyesPort: 17898,
+  handPort: 0,
   sensePort: 17899,
   rootCommand: "su",
   shellCommand: "",
@@ -82,6 +83,14 @@ var HANDS_CONFIG_GROUP = {
         description: "\u622A\u5C4F\u3001\u8BFB\u5B57\u548C\u624B\u52BF\u7AEF\u70B9\u90FD\u5728\u8FD9\u4E2A\u53E3\u4E0A\uFF0C\u968F\u5305\u7684\u65E0\u969C\u788D\u670D\u52A1\u7528 17898\u3002",
         "x-hot": false
       },
+      [`${K}.handPort`]: {
+        type: "integer",
+        title: "\u624B\u52BF\u7AEF\u53E3",
+        minimum: 0,
+        maximum: 65535,
+        description: "\u72EC\u7ACB\u624B\u52BF App\uFF08\u624B\uFF09\u7684\u7AEF\u53E3\uFF0C\u9ED8\u8BA4 17897\u3002\u586B 0 \u8868\u793A\u548C\u773C\u775B\u540C\u7AEF\u53E3\u2014\u2014\u90A3\u662F\u628A /hand/* \u8865\u8FDB Coopanin \u65E0\u969C\u788D\u670D\u52A1\u7684\u88C5\u6CD5\u3002",
+        "x-hot": false
+      },
       [`${K}.sensePort`]: {
         type: "integer",
         title: "\u611F\u89C9\u7AEF\u53E3",
@@ -240,6 +249,10 @@ function inputCommand(command) {
       return command.ms >= 400 ? `input swipe ${command.x} ${command.y} ${command.x} ${command.y} ${command.ms}` : `input tap ${command.x} ${command.y}`;
     case "swipe":
       return `input swipe ${command.x1} ${command.y1} ${command.x2} ${command.y2} ${command.ms}`;
+    case "pinch":
+      return `input swipe ${command.cx - 220} ${command.cy} ${command.cx + 220} ${command.cy} ${command.ms}`;
+    case "drag":
+      return `input draganddrop ${command.x1} ${command.y1} ${command.x2} ${command.y2} ${command.ms}`;
     case "key":
       return `input keyevent ${command.code}`;
     // 文本在拼命令这一刻才转义：只有 input 这条路吃不下非 ASCII，无障碍那条能原样收
@@ -375,6 +388,10 @@ var EyesActuator = class {
         return this.send(`/hand/tap?x=${command.x}&y=${command.y}&ms=${command.ms}`);
       case "swipe":
         return this.send(`/hand/swipe?x1=${command.x1}&y1=${command.y1}&x2=${command.x2}&y2=${command.y2}&ms=${command.ms}`);
+      case "pinch":
+        return this.send(`/hand/pinch?cx=${command.cx}&cy=${command.cy}&scale=${command.scale}&ms=${command.ms}`);
+      case "drag":
+        return this.send(`/hand/drag?x1=${command.x1}&y1=${command.y1}&x2=${command.x2}&y2=${command.y2}&ms=${command.ms}`);
       case "key":
         return this.send(`/hand/key?name=${encodeURIComponent(command.name)}&code=${command.code}`);
       case "text":
@@ -444,6 +461,10 @@ var describeCommand = (command) => {
       return `\u70B9 (${command.x},${command.y})`;
     case "swipe":
       return `\u4ECE (${command.x1},${command.y1}) \u6ED1\u5230 (${command.x2},${command.y2})`;
+    case "pinch":
+      return `\u5728 (${command.cx},${command.cy}) ${command.scale >= 1 ? "\u653E\u5927" : "\u7F29\u5C0F"} ${command.scale} \u500D`;
+    case "drag":
+      return `\u628A (${command.x1},${command.y1}) \u62D6\u5230 (${command.x2},${command.y2})`;
     case "key":
       return `\u6309 ${command.name}`;
     case "text":
@@ -469,11 +490,18 @@ var Hands = class {
     this.eyes = new EyesClient(() => this.port(this.cfg().eyesPort));
     this.rootActuator = new RootActuator(() => this.cfg().rootCommand, () => this.log);
     this.shellActuator = new ShellActuator(() => this.cfg().shellCommand, () => this.log);
-    this.eyesActuator = new EyesActuator(() => this.port(this.cfg().eyesPort), () => this.readToken(), () => this.log);
+    this.eyesActuator = new EyesActuator(() => this.handPort(), () => this.readToken(), () => this.log);
   }
-  port(value) {
+  port(value, fallback = 17898) {
     const n = Math.round(Number(value));
-    return Number.isFinite(n) && n >= 1024 && n <= 65535 ? n : 17898;
+    return Number.isFinite(n) && n >= 1024 && n <= 65535 ? n : fallback;
+  }
+  /** 手势端点在哪：handPort 为 0 时和眼睛同一个口。 */
+  handPort() {
+    const cfg = this.cfg();
+    const explicit = Math.round(Number(cfg.handPort));
+    if (Number.isFinite(explicit) && explicit >= 1024 && explicit <= 65535) return explicit;
+    return this.port(cfg.eyesPort);
   }
   /** 世界挂上以后把日志接进来；探测和动作都写在这里。 */
   attachLog(log) {
@@ -558,21 +586,18 @@ var Hands = class {
     if (!picked) throw new HandError(`\u624B\u73B0\u5728\u52A8\u4E0D\u4E86\uFF1A${state.detail}`);
     return picked;
   }
-  /** 屏幕尺寸；拿不到就不校验边界（root 通道下常见）。 */
+  /**
+   * 屏幕尺寸。只认手势端点报的（眼睛报的是上一次截图的尺寸，常被 max= 缩过，拿来判越界会误拒）。
+   * 拿不到就不校验边界。
+   */
   async screenSize() {
     if (this.screen && Date.now() - this.lastScreenAt < 6e4) return this.screen;
-    const health = await this.eyes.health();
+    const health = await getJson(this.handPort(), "/health", 4e3);
     if (health?.ok && health.frameWidth && health.frameHeight) {
       this.screen = { w: health.frameWidth, h: health.frameHeight };
       this.lastScreenAt = Date.now();
       return this.screen;
     }
-    const frame = await this.eyes.frame(240, 30);
-    if (frame && frame.width > 0 && frame.height > 0) {
-      this.screen = { w: frame.width, h: frame.height };
-      this.lastScreenAt = Date.now();
-      return this.screen;
-    }
     return null;
   }
   prune() {
@@ -628,6 +653,39 @@ var Hands = class {
     const ms = milliseconds(opts.ms, 300);
     return this.perform({ kind: "swipe", x1: a, y1: b, x2: c, y2: d, ms });
   }
+  async pinch(cx, cy, scale, opts = {}) {
+    const x = coord(cx);
+    const y = coord(cy);
+    const ms = milliseconds(opts.ms, 400);
+    const factor = Math.max(0.25, Math.min(4, Number(scale) || 1));
+    return this.perform({ kind: "pinch", cx: x, cy: y, scale: factor, ms });
+  }
+  async drag(x1, y1, x2, y2, opts = {}) {
+    const a = coord(x1);
+    const b = coord(y1);
+    const c = coord(x2);
+    const d = coord(y2);
+    await this.bounds(a, b);
+    await this.bounds(c, d);
+    const ms = milliseconds(opts.ms, 900);
+    return this.perform({ kind: "drag", x1: a, y1: b, x2: c, y2: d, ms });
+  }
+  /** 一屏屏翻着找文字（只查不点）。 */
+  async findItem(text, opts = {}) {
+    const cfg = this.cfg();
+    const tries = Math.max(0, Math.min(12, Math.round(Number(opts.tries ?? 5))));
+    const scroll = opts.scroll === false ? "0" : "1";
+    const token = cfg.handToken ? `&t=${encodeURIComponent(cfg.handToken)}` : "";
+    const path = `/hand/find?text=${encodeURIComponent(text)}&tries=${tries}&scroll=${scroll}${token}`;
+    const res = await get(this.port(cfg.handPort), path, ACTION_TIMEOUT_MS);
+    let body = {};
+    try {
+      body = JSON.parse(res.body.toString("utf8"));
+    } catch {
+    }
+    if (res.status === 200 && body.ok) return { ok: true, x: body.x, y: body.y, detail: flat(body.detail, 80) };
+    return { ok: false, detail: flat(body.error || `\u627E\u4E0D\u5230\u300C${text}\u300D`, 120) };
+  }
   async press(key) {
     const name = String(key ?? "").trim().toLowerCase().replace(/[\s_-]/g, "");
     const code = keyCode(name);
@@ -778,6 +836,51 @@ var HANDS_TOOL_DECLS = [
       required: ["x1", "y1", "x2", "y2"]
     }
   },
+  {
+    name: "hand_pinch",
+    tags: ["act"],
+    description: "\u53CC\u6307\u7F29\u653E\uFF1A\u653E\u5927\u6216\u7F29\u5C0F\u56FE\u7247\u3001\u5730\u56FE\u3001\u7F51\u9875\u3002scale>1 \u653E\u5927\uFF0C<1 \u7F29\u5C0F\uFF1B\u4E0D\u7ED9 cx/cy \u5C31\u4EE5\u5C4F\u5E55\u4E2D\u5FC3\u4E3A\u57FA\u51C6\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        scale: { type: "number", minimum: 0.25, maximum: 4, description: "\u500D\u6570\uFF1A1.5 \u653E\u5927\u4E00\u534A\uFF0C0.6 \u7F29\u5C0F\u3002" },
+        cx: { type: "integer", description: "\u4E2D\u5FC3\u6A2A\u5750\u6807\uFF08\u53EF\u7701\u7565\uFF09\u3002" },
+        cy: { type: "integer", description: "\u4E2D\u5FC3\u7EB5\u5750\u6807\uFF08\u53EF\u7701\u7565\uFF09\u3002" },
+        ms: { type: "integer", minimum: 120, maximum: 2e3, description: "\u624B\u52BF\u65F6\u957F\uFF0C\u9ED8\u8BA4 400 \u6BEB\u79D2\u3002" }
+      },
+      required: ["scale"]
+    }
+  },
+  {
+    name: "hand_drag",
+    tags: ["act"],
+    description: "\u957F\u6309\u540E\u62D6\u62FD\uFF1A\u62D6\u5217\u8868\u9879\u3001\u62D6\u6ED1\u5757\u3001\u62D6\u56FE\u6807\u3001\u62D6\u8FDB\u5EA6\u3002\u548C swipe \u7684\u533A\u522B\u662F\u5B83\u6309\u4F4F\u4E0D\u653E\u518D\u62D6\uFF0C\u9002\u5408\u6709\u62D6\u62FD\u8BED\u4E49\u7684\u63A7\u4EF6\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        x1: { type: "integer", description: "\u8D77\u70B9\u6A2A\u5750\u6807\u3002" },
+        y1: { type: "integer", description: "\u8D77\u70B9\u7EB5\u5750\u6807\u3002" },
+        x2: { type: "integer", description: "\u7EC8\u70B9\u6A2A\u5750\u6807\u3002" },
+        y2: { type: "integer", description: "\u7EC8\u70B9\u7EB5\u5750\u6807\u3002" },
+        ms: { type: "integer", minimum: 300, maximum: 4e3, description: "\u62D6\u591A\u4E45\uFF0C\u9ED8\u8BA4 900 \u6BEB\u79D2\u3002" }
+      },
+      required: ["x1", "y1", "x2", "y2"]
+    }
+  },
+  {
+    name: "hand_find",
+    tags: ["read"],
+    description: "\u4E00\u5C4F\u5C4F\u5F80\u4E0B\u7FFB\uFF0C\u627E\u5C4F\u5E55\u4E0A\u542B\u6709\u67D0\u6BB5\u6587\u5B57\u7684\u4E1C\u897F\uFF0C\u8FD4\u56DE\u5B83\u7684\u5750\u6807\uFF08\u53EA\u627E\u4E0D\u70B9\uFF09\u3002\u8868\u683C\u3001\u957F\u5217\u8868\u3001\u8BBE\u7F6E\u9879\u91CC\u627E\u4E1C\u897F\u7528\u5B83\uFF0C\u6BD4\u8089\u773C\u4E00\u9875\u9875\u770B\u5FEB\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "\u8981\u627E\u7684\u6587\u5B57\uFF08\u5305\u542B\u5339\u914D\uFF0C\u4E0D\u533A\u5206\u5927\u5C0F\u5199\uFF09\u3002" },
+        tries: { type: "integer", minimum: 0, maximum: 12, description: "\u6700\u591A\u7FFB\u51E0\u5C4F\uFF0C\u9ED8\u8BA4 5\u3002" },
+        scroll: { type: "boolean", description: "\u627E\u4E0D\u5230\u65F6\u662F\u5426\u81EA\u52A8\u6EDA\u52A8\uFF0C\u9ED8\u8BA4 true\u3002" }
+      },
+      required: ["text"]
+    }
+  },
   {
     name: "hand_type",
     tags: ["act"],
@@ -886,6 +989,12 @@ var HandsWorld = class {
           return await this.long(args);
         case "hand_swipe":
           return await this.swipe(args);
+        case "hand_pinch":
+          return await this.pinch(args);
+        case "hand_drag":
+          return await this.drag(args);
+        case "hand_find":
+          return await this.findItem(args);
         case "hand_type":
           return await this.type(args);
         case "hand_key":
@@ -922,6 +1031,44 @@ var HandsWorld = class {
     return { text: report.text, ...blobs.length ? { blobs } : {} };
   }
   /** 给文字就先在屏幕上找，找到唯一一处才点；给坐标就直接点。 */
+  async pinch(args) {
+    const scale = Number(args.scale);
+    if (!Number.isFinite(scale) || scale <= 0) return failed("scale \u8981\u662F\u6B63\u6570\uFF0C\u6BD4\u5982 1.5 \u653E\u5927\u30010.6 \u7F29\u5C0F\u3002");
+    const cx = args.cx === void 0 ? void 0 : Number(args.cx);
+    const cy = args.cy === void 0 ? void 0 : Number(args.cy);
+    const ms = args.ms === void 0 ? void 0 : Number(args.ms);
+    const size = await this.hands.screenSize();
+    const x = cx ?? Math.round((size?.w ?? 1080) / 2);
+    const y = cy ?? Math.round((size?.h ?? 1920) / 2);
+    const receipt = await this.hands.pinch(x, y, scale, ms === void 0 ? {} : { ms });
+    this.note(receipt);
+    return receipt.text;
+  }
+  async drag(args) {
+    const need = ["x1", "y1", "x2", "y2"];
+    if (need.some((k) => args[k] === void 0)) return failed("\u62D6\u62FD\u8981\u7ED9\u9F50 x1\u3001y1\u3001x2\u3001y2\u3002");
+    const receipt = await this.hands.drag(
+      Number(args.x1),
+      Number(args.y1),
+      Number(args.x2),
+      Number(args.y2),
+      args.ms === void 0 ? {} : { ms: Number(args.ms) }
+    );
+    this.note(receipt);
+    return receipt.text;
+  }
+  async findItem(args) {
+    const text = typeof args.text === "string" ? args.text.trim() : "";
+    if (!text) return failed("\u8981\u627E\u4EC0\u4E48\u6587\u5B57\uFF1F\u7ED9 text\u3002");
+    const tries = args.tries === void 0 ? void 0 : Number(args.tries);
+    const scroll = args.scroll === void 0 ? void 0 : Boolean(args.scroll);
+    const found = await this.hands.findItem(text, {
+      ...tries === void 0 ? {} : { tries },
+      ...scroll === void 0 ? {} : { scroll }
+    });
+    if (!found.ok) return failed(found.detail);
+    return `\u627E\u5230\u4E86\u300C${text}\u300D@(${found.x},${found.y})${found.detail ? `\uFF08${found.detail}\uFF09` : ""}\uFF1B\u8981\u52A8\u624B\u5C31 hand_tap \u4F20\u8FD9\u7EC4\u5750\u6807\u3002`;
+  }
   async tap(args) {
     const text = typeof args.text === "string" ? args.text.trim() : "";
     if (text) {

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "cortico-world-hands",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
-  "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, types and presses keys through whatever actuator the device actually has",
+  "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, pinches, drags, types and presses keys through whatever actuator the device actually has",
   "license": "MIT",
   "keywords": [
     "cortico-world",
@@ -28,4 +28,4 @@
     "README.md"
   ],
   "scripts": {}
-}
+}