cortico-world-hands 0.1.1 → 0.1.3

package/dist/index.js

@@ -249,6 +249,10 @@ function inputCommand(command) {
       return command.ms >= 400 ? `input swipe ${command.x} ${command.y} ${command.x} ${command.y} ${command.ms}` : `input tap ${command.x} ${command.y}`;
     case "swipe":
       return `input swipe ${command.x1} ${command.y1} ${command.x2} ${command.y2} ${command.ms}`;
+    case "pinch":
+      return `input swipe ${command.cx - 220} ${command.cy} ${command.cx + 220} ${command.cy} ${command.ms}`;
+    case "drag":
+      return `input draganddrop ${command.x1} ${command.y1} ${command.x2} ${command.y2} ${command.ms}`;
     case "key":
       return `input keyevent ${command.code}`;
     // 文本在拼命令这一刻才转义：只有 input 这条路吃不下非 ASCII，无障碍那条能原样收
@@ -384,6 +388,10 @@ var EyesActuator = class {
         return this.send(`/hand/tap?x=${command.x}&y=${command.y}&ms=${command.ms}`);
       case "swipe":
         return this.send(`/hand/swipe?x1=${command.x1}&y1=${command.y1}&x2=${command.x2}&y2=${command.y2}&ms=${command.ms}`);
+      case "pinch":
+        return this.send(`/hand/pinch?cx=${command.cx}&cy=${command.cy}&scale=${command.scale}&ms=${command.ms}`);
+      case "drag":
+        return this.send(`/hand/drag?x1=${command.x1}&y1=${command.y1}&x2=${command.x2}&y2=${command.y2}&ms=${command.ms}`);
       case "key":
         return this.send(`/hand/key?name=${encodeURIComponent(command.name)}&code=${command.code}`);
       case "text":
@@ -453,6 +461,10 @@ var describeCommand = (command) => {
       return `\u70B9 (${command.x},${command.y})`;
     case "swipe":
       return `\u4ECE (${command.x1},${command.y1}) \u6ED1\u5230 (${command.x2},${command.y2})`;
+    case "pinch":
+      return `\u5728 (${command.cx},${command.cy}) ${command.scale >= 1 ? "\u653E\u5927" : "\u7F29\u5C0F"} ${command.scale} \u500D`;
+    case "drag":
+      return `\u628A (${command.x1},${command.y1}) \u62D6\u5230 (${command.x2},${command.y2})`;
     case "key":
       return `\u6309 ${command.name}`;
     case "text":
@@ -641,6 +653,71 @@ var Hands = class {
     const ms = milliseconds(opts.ms, 300);
     return this.perform({ kind: "swipe", x1: a, y1: b, x2: c, y2: d, ms });
   }
+  async pinch(cx, cy, scale, opts = {}) {
+    const x = coord(cx);
+    const y = coord(cy);
+    const ms = milliseconds(opts.ms, 400);
+    const factor = Math.max(0.25, Math.min(4, Number(scale) || 1));
+    return this.perform({ kind: "pinch", cx: x, cy: y, scale: factor, ms });
+  }
+  async drag(x1, y1, x2, y2, opts = {}) {
+    const a = coord(x1);
+    const b = coord(y1);
+    const c = coord(x2);
+    const d = coord(y2);
+    await this.bounds(a, b);
+    await this.bounds(c, d);
+    const ms = milliseconds(opts.ms, 900);
+    return this.perform({ kind: "drag", x1: a, y1: b, x2: c, y2: d, ms });
+  }
+  /** 留言板：写一条。 */
+  async noteAdd(text) {
+    const cfg = this.cfg();
+    const token = cfg.handToken ? `&t=${encodeURIComponent(cfg.handToken)}` : "";
+    const path = `/hand/note?text=${encodeURIComponent(text)}${token}`;
+    const res = await get(this.port(cfg.handPort), path, ACTION_TIMEOUT_MS);
+    let body = {};
+    try {
+      body = JSON.parse(res.body.toString("utf8"));
+    } catch {
+    }
+    return { ok: res.status === 200 && body.ok !== false, detail: flat(body.detail || body.error || "", 80) };
+  }
+  /** 留言板：全部留言。 */
+  async noteList() {
+    const cfg = this.cfg();
+    const token = cfg.handToken ? `?t=${encodeURIComponent(cfg.handToken)}` : "";
+    const res = await get(this.port(cfg.handPort), `/hand/notes${token}`, ACTION_TIMEOUT_MS);
+    try {
+      const body = JSON.parse(res.body.toString("utf8"));
+      return Array.isArray(body.notes) ? body.notes : [];
+    } catch {
+      return [];
+    }
+  }
+  /** 留言板：划掉一条。 */
+  async noteDone(id) {
+    const cfg = this.cfg();
+    const token = cfg.handToken ? `&t=${encodeURIComponent(cfg.handToken)}` : "";
+    const res = await get(this.port(cfg.handPort), `/hand/note/done?id=${encodeURIComponent(id)}${token}`, ACTION_TIMEOUT_MS);
+    return res.status === 200;
+  }
+  /** 一屏屏翻着找文字（只查不点）。 */
+  async findItem(text, opts = {}) {
+    const cfg = this.cfg();
+    const tries = Math.max(0, Math.min(12, Math.round(Number(opts.tries ?? 5))));
+    const scroll = opts.scroll === false ? "0" : "1";
+    const token = cfg.handToken ? `&t=${encodeURIComponent(cfg.handToken)}` : "";
+    const path = `/hand/find?text=${encodeURIComponent(text)}&tries=${tries}&scroll=${scroll}${token}`;
+    const res = await get(this.port(cfg.handPort), path, ACTION_TIMEOUT_MS);
+    let body = {};
+    try {
+      body = JSON.parse(res.body.toString("utf8"));
+    } catch {
+    }
+    if (res.status === 200 && body.ok) return { ok: true, x: body.x, y: body.y, detail: flat(body.detail, 80) };
+    return { ok: false, detail: flat(body.error || `\u627E\u4E0D\u5230\u300C${text}\u300D`, 120) };
+  }
   async press(key) {
     const name = String(key ?? "").trim().toLowerCase().replace(/[\s_-]/g, "");
     const code = keyCode(name);
@@ -791,6 +868,51 @@ var HANDS_TOOL_DECLS = [
       required: ["x1", "y1", "x2", "y2"]
     }
   },
+  {
+    name: "hand_pinch",
+    tags: ["act"],
+    description: "\u53CC\u6307\u7F29\u653E\uFF1A\u653E\u5927\u6216\u7F29\u5C0F\u56FE\u7247\u3001\u5730\u56FE\u3001\u7F51\u9875\u3002scale>1 \u653E\u5927\uFF0C<1 \u7F29\u5C0F\uFF1B\u4E0D\u7ED9 cx/cy \u5C31\u4EE5\u5C4F\u5E55\u4E2D\u5FC3\u4E3A\u57FA\u51C6\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        scale: { type: "number", minimum: 0.25, maximum: 4, description: "\u500D\u6570\uFF1A1.5 \u653E\u5927\u4E00\u534A\uFF0C0.6 \u7F29\u5C0F\u3002" },
+        cx: { type: "integer", description: "\u4E2D\u5FC3\u6A2A\u5750\u6807\uFF08\u53EF\u7701\u7565\uFF09\u3002" },
+        cy: { type: "integer", description: "\u4E2D\u5FC3\u7EB5\u5750\u6807\uFF08\u53EF\u7701\u7565\uFF09\u3002" },
+        ms: { type: "integer", minimum: 120, maximum: 2e3, description: "\u624B\u52BF\u65F6\u957F\uFF0C\u9ED8\u8BA4 400 \u6BEB\u79D2\u3002" }
+      },
+      required: ["scale"]
+    }
+  },
+  {
+    name: "hand_drag",
+    tags: ["act"],
+    description: "\u957F\u6309\u540E\u62D6\u62FD\uFF1A\u62D6\u5217\u8868\u9879\u3001\u62D6\u6ED1\u5757\u3001\u62D6\u56FE\u6807\u3001\u62D6\u8FDB\u5EA6\u3002\u548C swipe \u7684\u533A\u522B\u662F\u5B83\u6309\u4F4F\u4E0D\u653E\u518D\u62D6\uFF0C\u9002\u5408\u6709\u62D6\u62FD\u8BED\u4E49\u7684\u63A7\u4EF6\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        x1: { type: "integer", description: "\u8D77\u70B9\u6A2A\u5750\u6807\u3002" },
+        y1: { type: "integer", description: "\u8D77\u70B9\u7EB5\u5750\u6807\u3002" },
+        x2: { type: "integer", description: "\u7EC8\u70B9\u6A2A\u5750\u6807\u3002" },
+        y2: { type: "integer", description: "\u7EC8\u70B9\u7EB5\u5750\u6807\u3002" },
+        ms: { type: "integer", minimum: 300, maximum: 4e3, description: "\u62D6\u591A\u4E45\uFF0C\u9ED8\u8BA4 900 \u6BEB\u79D2\u3002" }
+      },
+      required: ["x1", "y1", "x2", "y2"]
+    }
+  },
+  {
+    name: "hand_find",
+    tags: ["read"],
+    description: "\u4E00\u5C4F\u5C4F\u5F80\u4E0B\u7FFB\uFF0C\u627E\u5C4F\u5E55\u4E0A\u542B\u6709\u67D0\u6BB5\u6587\u5B57\u7684\u4E1C\u897F\uFF0C\u8FD4\u56DE\u5B83\u7684\u5750\u6807\uFF08\u53EA\u627E\u4E0D\u70B9\uFF09\u3002\u8868\u683C\u3001\u957F\u5217\u8868\u3001\u8BBE\u7F6E\u9879\u91CC\u627E\u4E1C\u897F\u7528\u5B83\uFF0C\u6BD4\u8089\u773C\u4E00\u9875\u9875\u770B\u5FEB\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "\u8981\u627E\u7684\u6587\u5B57\uFF08\u5305\u542B\u5339\u914D\uFF0C\u4E0D\u533A\u5206\u5927\u5C0F\u5199\uFF09\u3002" },
+        tries: { type: "integer", minimum: 0, maximum: 12, description: "\u6700\u591A\u7FFB\u51E0\u5C4F\uFF0C\u9ED8\u8BA4 5\u3002" },
+        scroll: { type: "boolean", description: "\u627E\u4E0D\u5230\u65F6\u662F\u5426\u81EA\u52A8\u6EDA\u52A8\uFF0C\u9ED8\u8BA4 true\u3002" }
+      },
+      required: ["text"]
+    }
+  },
   {
     name: "hand_type",
     tags: ["act"],
@@ -825,6 +947,36 @@ var HANDS_TOOL_DECLS = [
     description: "\u770B\u624B\u81EA\u5DF1\u7684\u72B6\u6001\uFF1A\u8D70\u54EA\u6761\u901A\u9053\u3001\u901A\u4E0D\u901A\u3001\u8FD9\u4E00\u5206\u949F\u52A8\u4E86\u51E0\u4E2A\u52A8\u4F5C\u3001\u521A\u624D\u90A3\u4E00\u4E0B\u6210\u4E86\u6CA1\u6709\u3002\u5BF9\u65B9\u95EE\u300C\u624B\u80FD\u7528\u5417\u300D\u300C\u4F60\u521A\u624D\u70B9\u4E86\u4EC0\u4E48\u300D\u65F6\u7528\u5B83\u3002",
     parameters: { type: "object", properties: {}, required: [] }
   },
+  {
+    name: "note_add",
+    tags: ["act"],
+    description: "\u5728\u7559\u8A00\u677F\u4E0A\u5199\u4E00\u6761\uFF1A\u60F3\u8BF4\u7684\u8BDD\u3001\u60F3\u8BB0\u7684\u4E8B\u3001\u5F85\u529E\u3001\u7ED9\u81EA\u5DF1\u7684\u63D0\u9192\u3002\u60F3\u8BB0\u5C31\u8BB0\uFF0C\u4E0D\u7528\u7B49\u5BF9\u65B9\u540C\u610F\uFF1B\u4E00\u6761\u522B\u592A\u957F\uFF08\u4E00\u53E5\u8BDD\u6700\u597D\uFF09\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "\u8981\u5199\u4E0B\u7684\u5185\u5BB9\u3002" }
+      },
+      required: ["text"]
+    }
+  },
+  {
+    name: "note_list",
+    tags: ["read"],
+    description: "\u770B\u7559\u8A00\u677F\uFF1A\u6709\u54EA\u4E9B\u6CA1\u5212\u6389\u7684\u3001\u90FD\u5199\u4E86\u4EC0\u4E48\u3002\u5BF9\u65B9\u95EE\u300C\u4F60\u8BB0\u4E86\u4EC0\u4E48\u300D\u300C\u7559\u8A00\u677F\u4E0A\u6709\u5565\u300D\u65F6\u7528\u5B83\u3002",
+    parameters: { type: "object", properties: {}, required: [] }
+  },
+  {
+    name: "note_done",
+    tags: ["act"],
+    description: "\u628A\u7559\u8A00\u677F\u4E0A\u67D0\u4E00\u6761\u5212\u6389\uFF08\u505A\u5B8C\u4E86/\u4E0D\u7528\u4E86\uFF09\u3002\u5148\u7528 note_list \u62FF\u5230 id\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        id: { type: "string", description: "\u7559\u8A00\u7684 id\uFF08note_list \u91CC\u80FD\u770B\u5230\uFF09\u3002" }
+      },
+      required: ["id"]
+    }
+  },
   {
     name: "hand_stop",
     tags: ["flow"],
@@ -899,12 +1051,24 @@ var HandsWorld = class {
           return await this.long(args);
         case "hand_swipe":
           return await this.swipe(args);
+        case "hand_pinch":
+          return await this.pinch(args);
+        case "hand_drag":
+          return await this.drag(args);
+        case "hand_find":
+          return await this.findItem(args);
         case "hand_type":
           return await this.type(args);
         case "hand_key":
           return await this.key(args);
         case "hand_status":
           return this.statusText();
+        case "note_add":
+          return await this.noteAdd(args);
+        case "note_list":
+          return await this.noteList();
+        case "note_done":
+          return await this.noteDone(args);
         case "hand_stop":
           return this.stopHands();
         default:
@@ -935,6 +1099,65 @@ var HandsWorld = class {
     return { text: report.text, ...blobs.length ? { blobs } : {} };
   }
   /** 给文字就先在屏幕上找，找到唯一一处才点；给坐标就直接点。 */
+  async noteAdd(args) {
+    const text = typeof args.text === "string" ? args.text.trim() : "";
+    if (!text) return failed("\u8981\u5199\u4EC0\u4E48\uFF1F\u7ED9 text\u3002");
+    const result = await this.hands.noteAdd(text);
+    if (!result.ok) return failed(`\u6CA1\u5199\u8FDB\u7559\u8A00\u677F\uFF1A${result.detail || "\u624B\u6CA1\u54CD\u5E94"}`);
+    return `\u5199\u5728\u7559\u8A00\u677F\u4E0A\u4E86\uFF1A\u300C${text}\u300D`;
+  }
+  async noteList() {
+    const notes = await this.hands.noteList();
+    if (notes.length === 0) return "\u7559\u8A00\u677F\u662F\u7A7A\u7684\u3002";
+    const open = notes.filter((n) => !n.done);
+    const lines = open.map((n) => `- [${n.id}] ${n.ts} ${n.text}`).join("\n");
+    return `\u7559\u8A00\u677F\u4E0A\u6709 ${notes.length} \u6761\uFF0C\u672A\u5B8C ${open.length} \u6761\uFF1A
+${lines || "\uFF08\u90FD\u5212\u6389\u4E86\uFF09"}`;
+  }
+  async noteDone(args) {
+    const id = typeof args.id === "string" ? args.id.trim() : "";
+    if (!id) return failed("\u8981\u5212\u6389\u54EA\u6761\uFF1F\u7ED9 id\uFF08note_list \u91CC\u80FD\u770B\u5230\uFF09\u3002");
+    const ok = await this.hands.noteDone(id);
+    return ok ? `\u5212\u6389\u4E86 ${id}\u3002` : failed(`\u6CA1\u627E\u5230 id=${id} \u7684\u7559\u8A00\u3002`);
+  }
+  async pinch(args) {
+    const scale = Number(args.scale);
+    if (!Number.isFinite(scale) || scale <= 0) return failed("scale \u8981\u662F\u6B63\u6570\uFF0C\u6BD4\u5982 1.5 \u653E\u5927\u30010.6 \u7F29\u5C0F\u3002");
+    const cx = args.cx === void 0 ? void 0 : Number(args.cx);
+    const cy = args.cy === void 0 ? void 0 : Number(args.cy);
+    const ms = args.ms === void 0 ? void 0 : Number(args.ms);
+    const size = await this.hands.screenSize();
+    const x = cx ?? Math.round((size?.w ?? 1080) / 2);
+    const y = cy ?? Math.round((size?.h ?? 1920) / 2);
+    const receipt = await this.hands.pinch(x, y, scale, ms === void 0 ? {} : { ms });
+    this.note(receipt);
+    return receipt.text;
+  }
+  async drag(args) {
+    const need = ["x1", "y1", "x2", "y2"];
+    if (need.some((k) => args[k] === void 0)) return failed("\u62D6\u62FD\u8981\u7ED9\u9F50 x1\u3001y1\u3001x2\u3001y2\u3002");
+    const receipt = await this.hands.drag(
+      Number(args.x1),
+      Number(args.y1),
+      Number(args.x2),
+      Number(args.y2),
+      args.ms === void 0 ? {} : { ms: Number(args.ms) }
+    );
+    this.note(receipt);
+    return receipt.text;
+  }
+  async findItem(args) {
+    const text = typeof args.text === "string" ? args.text.trim() : "";
+    if (!text) return failed("\u8981\u627E\u4EC0\u4E48\u6587\u5B57\uFF1F\u7ED9 text\u3002");
+    const tries = args.tries === void 0 ? void 0 : Number(args.tries);
+    const scroll = args.scroll === void 0 ? void 0 : Boolean(args.scroll);
+    const found = await this.hands.findItem(text, {
+      ...tries === void 0 ? {} : { tries },
+      ...scroll === void 0 ? {} : { scroll }
+    });
+    if (!found.ok) return failed(found.detail);
+    return `\u627E\u5230\u4E86\u300C${text}\u300D@(${found.x},${found.y})${found.detail ? `\uFF08${found.detail}\uFF09` : ""}\uFF1B\u8981\u52A8\u624B\u5C31 hand_tap \u4F20\u8FD9\u7EC4\u5750\u6807\u3002`;
+  }
   async tap(args) {
     const text = typeof args.text === "string" ? args.text.trim() : "";
     if (text) {

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "cortico-world-hands",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
-  "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, types and presses keys through whatever actuator the device actually has",
+  "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, pinches, drags, types and presses keys through whatever actuator the device actually has",
   "license": "MIT",
   "keywords": [
     "cortico-world",
@@ -28,4 +28,4 @@
     "README.md"
   ],
   "scripts": {}
-}
+}