cortico-world-hands 0.14.1 → 0.16.0

package/dist/index.js

@@ -1105,6 +1105,16 @@ var Hands = class {
     const res = await post(this.port(cfg.handPort), `/hand/qq/reply?${t}${toq}&text=${encodeURIComponent(text)}`, "", 2e4);
     return res.body.toString("utf8");
   }
+  /** 本地模板匹配：找图（可选直接点）。 */
+  async matchImage(name, opts = {}) {
+    const cfg = this.cfg();
+    const credential = this.readToken();
+    const t = credential ? `t=${encodeURIComponent(credential)}` : "";
+    const th = opts.threshold ? `&threshold=${opts.threshold}` : "";
+    const path = opts.tap ? "/hand/tap_image" : "/hand/match";
+    const res = await get(this.port(cfg.handPort), `${path}?name=${encodeURIComponent(name)}${th}&${t}`, 15e3);
+    return res.body.toString("utf8");
+  }
   /** QQ 兜底：打开 QQ → 找到对话 → 粘贴 → 发送 → 切回。 */
   async qqOpenReply(text, to) {
     const cfg = this.cfg();
@@ -1369,6 +1379,38 @@ ${lines.join("\n")}`);
 
 // src/tools.ts
 var HANDS_TOOL_DECLS = [
+  {
+    name: "hand_match",
+    tags: ["read"],
+    description: "\u5728\u5C4F\u5E55\u4E0A\u627E\u4E00\u5F20\u5C0F\u56FE\uFF08\u6A21\u677F\uFF09\uFF1A\u8FD4\u56DE\u5B83\u7684\u771F\u5B9E\u5750\u6807\u548C\u76F8\u4F3C\u5EA6\u3002\u9002\u5408\u6E38\u620F\u6309\u94AE\u3001\u56FE\u6807\u8FD9\u7C7B\u6CA1\u6709\u6587\u5B57\u7684\u76EE\u6807\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "\u6A21\u677F\u540D\uFF08\u653E\u8FDB App \u7684 tpl \u76EE\u5F55\u7684 png \u6587\u4EF6\u540D\uFF0C\u53EF\u4E0D\u5E26 .png\uFF09" },
+        threshold: { type: "number", description: "\u76F8\u4F3C\u5EA6\u95E8\u69DB 0-1\uFF0C\u9ED8\u8BA4 0.86\uFF1B\u627E\u4E0D\u5230\u5C31\u8C03\u4F4E\u4E00\u70B9" }
+      },
+      required: ["name"]
+    }
+  },
+  {
+    name: "hand_tap_image",
+    tags: ["act"],
+    description: "\u627E\u5230\u90A3\u5F20\u5C0F\u56FE\u5E76\u70B9\u5B83\u2014\u2014\u6BD4\u6309\u5750\u6807\u70B9\u51C6\uFF0C\u4E5F\u4E0D\u4F1A\u70B9\u6B6A\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "\u6A21\u677F\u540D" },
+        threshold: { type: "number", description: "\u76F8\u4F3C\u5EA6\u95E8\u69DB\uFF0C\u9ED8\u8BA4 0.86" }
+      },
+      required: ["name"]
+    }
+  },
+  {
+    name: "hand_see",
+    tags: ["read"],
+    description: "\u53EA\u770B\u6587\u5B57\uFF1A\u5F53\u524D\u524D\u53F0\u662F\u8C01\u3001\u5C4F\u5E55\u4E0A\u6709\u54EA\u4E9B\u80FD\u70B9\u7684\u5B57\u3001\u5404\u81EA\u7684\u771F\u5B9E\u5750\u6807\u3002\u6BD4 hand_look \u5FEB\uFF08\u4E0D\u622A\u56FE\uFF09\uFF0C\u60F3\u5FEB\u901F\u770B\u6E05\u5C4F\u5E55\u65F6\u7528\u5B83\u3002",
+    parameters: { type: "object", properties: {} }
+  },
   {
     name: "hand_look",
     tags: ["read", "snapshot"],
@@ -2034,6 +2076,12 @@ ${lines.join("\n")}
       await this.guard();
       ctx.signal?.throwIfAborted();
       switch (name) {
+        case "hand_match":
+          return await this.matchImage(args, false);
+        case "hand_tap_image":
+          return await this.matchImage(args, true);
+        case "hand_see":
+          return await this.see();
         case "hand_look":
           return await this.look(args);
         case "hand_tap":
@@ -2123,6 +2171,38 @@ ${lines.join("\n")}
     this.recent.unshift(`${stamp} ${receipt.text}`);
     if (this.recent.length > 6) this.recent.length = 6;
   }
+  /** 点完等半秒，看前台有没有变；变了说一声，没变提醒她可能点歪了。 */
+  async verifyChange(beforePkg) {
+    try {
+      await new Promise((r) => setTimeout(r, 450));
+      const ui = await this.hands.ui();
+      const now = ui.ok ? ui.pkg : "";
+      if (beforePkg && now && beforePkg !== now) return `\uFF0C\u754C\u9762\u5DF2\u7ECF\u5207\u5230 ${now}`;
+      return "\uFF08\u5C4F\u5E55\u770B\u8D77\u6765\u6CA1\u53D8\uFF1B\u5982\u679C\u8BE5\u6709\u53CD\u5E94\uFF0C\u5148 hand_see \u770B\u4E00\u773C\u518D\u70B9\uFF09";
+    } catch {
+      return "";
+    }
+  }
+  /** 模板匹配：找图 / 找到就点。 */
+  async matchImage(args, tap) {
+    const name = typeof args.name === "string" ? args.name.trim() : "";
+    if (!name) return failed("\u8981\u6A21\u677F\u540D\uFF08png \u6587\u4EF6\u540D\uFF09");
+    const threshold = args.threshold === void 0 ? void 0 : Number(args.threshold);
+    const raw = await this.hands.matchImage(name, { threshold, tap });
+    try {
+      const r = JSON.parse(raw);
+      if (r.ok && r.found) {
+        return `${tap ? "\u70B9\u5230\u4E86" : "\u627E\u5230\u4E86"}\u300C${name}\u300D@(${r.x},${r.y})\uFF0C\u76F8\u4F3C\u5EA6 ${r.score}\uFF0C\u7528\u4E86 ${r.ms}ms`;
+      }
+      return failed(r.error || `\u5C4F\u5E55\u4E0A\u6CA1\u627E\u5230\u300C${name}\u300D\uFF08\u53EF\u4EE5\u8C03\u4F4E threshold \u518D\u8BD5\uFF09`);
+    } catch {
+      return failed(raw.slice(0, 120));
+    }
+  }
+  /** 纯文字版看屏：快，不带图。 */
+  async see() {
+    return await this.hands.look("text").then((r) => r.text);
+  }
   async look(args) {
     const detail = args.detail === "screen" || args.detail === "text" ? args.detail : "both";
     const report = await this.hands.look(detail);
@@ -2456,9 +2536,12 @@ ${lines || "\uFF08\u90FD\u5212\u6389\u4E86\uFF09"}`;
 ${list}`);
         }
         const only = hits2[0];
+        const beforeUi = await this.hands.ui().catch(() => null);
+        const before = beforeUi && beforeUi.ok ? beforeUi.pkg : "";
         const receipt3 = await this.hands.tap(only.center.x, only.center.y);
         this.note(receipt3);
-        return `${receipt3.text}\uFF08\u70B9\u7684\u662F\u300C${only.text}\u300D\uFF09`;
+        const after = await this.verifyChange(before);
+        return `${receipt3.text}\uFF08\u70B9\u7684\u662F\u300C${only.text}\u300D\uFF09${after}`;
       }
       const hit = hits2[nth];
       if (!hit) return failed(`\u300C${text}\u300D\u53EA\u6709 ${hits2.length} \u5904\uFF0C\u6CA1\u6709\u7B2C ${nth} \u4E2A\u3002`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortico-world-hands",
-  "version": "0.14.1",
+  "version": "0.16.0",
   "type": "module",
   "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, pinches, drags, types and presses keys through whatever actuator the device actually has",
   "license": "MIT",