cortico-world-hands 0.20.3 → 0.20.5

package/dist/index.js

@@ -719,6 +719,12 @@ var MIN_GESTURE_MS = 20;
 var MAX_GESTURE_MS = 2e4;
 var HandError = class extends Error {
 };
+function parseRect(raw) {
+  if (!raw) return void 0;
+  const m = raw.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+  if (!m) return void 0;
+  return [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])];
+}
 function get(port, path, timeoutMs) {
   return new Promise((resolve2, reject) => {
     const req = httpGet({ host: "127.0.0.1", port, path, timeout: timeoutMs }, (res) => {
@@ -1016,6 +1022,20 @@ var EyesClient = class {
     }
     return { ok: body.ok !== false, pkg: flat(body.pkg, 120), items };
   }
+  /** 只截一块并放大：x,y,w,h 是真实像素。 */
+  async zoom(x, y, w, h, maxWidth = 1080, quality = 80) {
+    const cfg = this.cfg();
+    const path = `/screen.jpg?x=${Math.max(0, Math.round(x))}&y=${Math.max(0, Math.round(y))}&w=${Math.max(8, Math.round(w))}&h=${Math.max(8, Math.round(h))}&max=${Math.round(maxWidth)}&q=${Math.round(quality)}`;
+    const res = await get(this.handPort(), path, 12e3);
+    if (res.status !== 200 || res.body.length === 0) return null;
+    return {
+      bytes: new Uint8Array(res.body),
+      width: Number.parseInt(res.header("x-width"), 10) || 0,
+      height: Number.parseInt(res.header("x-height"), 10) || 0,
+      pkg: "",
+      ageMs: 0
+    };
+  }
   async frame(maxWidth, quality) {
     try {
       const w = Math.max(160, Math.min(2160, Math.round(maxWidth)));
@@ -1051,6 +1071,10 @@ var describeCommand = (command) => {
   }
 };
 var Hands = class {
+  /** 读屏幕上的字（转发给眼睛客户端）。 */
+  async uiSnapshot() {
+    return this.eyes.ui();
+  }
   eyes;
   rootActuator;
   shellActuator;
@@ -1096,7 +1120,7 @@ var Hands = class {
     for (const item of ui.items) {
       if (!item.center) continue;
       if (!item.text.toLowerCase().includes(query)) continue;
-      hits2.push({ text: item.text, center: item.center });
+      hits2.push({ text: item.text, center: item.center, rect: parseRect(item.bounds) });
       if (hits2.length >= 20) break;
     }
     return hits2;
@@ -2122,6 +2146,45 @@ var HANDS_TOOL_DECLS = [
       },
       required: ["content"]
     }
+  },
+  {
+    name: "hand_zoom",
+    tags: ["read"],
+    description: "\u53EA\u622A\u5C4F\u5E55\u7684\u4E00\u5C0F\u5757\u5E76\u653E\u5927\u2014\u2014\u5C0F\u5B57\u3001\u5C0F\u6309\u94AE\u770B\u4E0D\u6E05\u65F6\u7528\u5B83\u3002\u5148 hand_see \u62FF\u5230\u5750\u6807\uFF0C\u518D\u628A\u8FD9\u5757\u7684 x/y/\u5BBD/\u9AD8 \u4EA4\u7ED9\u5B83\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        x: { type: "integer", description: "\u5DE6\u4E0A\u89D2\u6A2A\u5750\u6807\uFF08\u771F\u5B9E\u50CF\u7D20\uFF09" },
+        y: { type: "integer", description: "\u5DE6\u4E0A\u89D2\u7EB5\u5750\u6807" },
+        w: { type: "integer", description: "\u5BBD" },
+        h: { type: "integer", description: "\u9AD8" }
+      },
+      required: ["x", "y", "w", "h"]
+    }
+  },
+  {
+    name: "hand_read_page",
+    tags: ["read"],
+    description: "\u628A\u4E00\u6574\u9875\u8BFB\u5B8C\uFF1A\u5F53\u524D\u5C4F\u7684\u6587\u5B57 + \u5F80\u4E0A\u6EDA\u51E0\u6B21\u628A\u540E\u9762\u7684\u4E5F\u8BFB\u8FDB\u6765\uFF0C\u53BB\u91CD\u540E\u62FC\u6210\u4E00\u6BB5\u3002\u957F\u5217\u8868\u3001\u957F\u6587\u7AE0\u7528\u5B83\uFF0C\u522B\u4E00\u5C4F\u4E00\u5C4F\u95EE\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        pages: { type: "integer", description: "\u6700\u591A\u7FFB\u51E0\u5C4F\uFF0C\u9ED8\u8BA4 3\uFF0C\u6700\u591A 8" }
+      }
+    }
+  },
+  {
+    name: "hand_read_text",
+    tags: ["read"],
+    description: "\u8BFB\u67D0\u4E00\u5904\u6587\u5B57\uFF1B\u5B57\u592A\u5C0F\uFF08\u9AD8\u5EA6\u4E0D\u8DB3 40 \u50CF\u7D20\uFF09\u4F1A\u81EA\u52A8\u628A\u90A3\u4E00\u5757\u653E\u5927\u518D\u7ED9\u4F60\u770B\uFF0C\u7701\u5F97\u4F60\u81EA\u5DF1\u8C03 hand_zoom\u3002",
+    parameters: {
+      type: "object",
+      properties: {
+        text: { type: "string", description: "\u8981\u8BFB\u7684\u90A3\u5904\u6587\u5B57\uFF08\u53EF\u4EE5\u53EA\u7ED9\u4E00\u90E8\u5206\uFF09" },
+        nth: { type: "integer", description: "\u540C\u540D\u591A\u5904\u65F6\u53D6\u7B2C\u51E0\u4E2A\uFF0C\u4ECE 0 \u5F00\u59CB" }
+      },
+      required: ["text"]
+    }
   }
 ];
 
@@ -2530,6 +2593,67 @@ ${lines.join("\n")}
           return await this.runSync(true);
         case "sync_status":
           return this.lastSync ? `\u4E0A\u6B21\u540C\u6B65 ${this.lastSync.at.slice(0, 19)}\uFF0C\u8BB0\u5FC6 ${this.lastSync.lines} \u6761\uFF0C\u4EBA\u683C ${this.lastSync.persona} \u5B57` : "\u8FD8\u6CA1\u540C\u6B65\u8FC7";
+        case "hand_read_page": {
+          const pages = Math.max(1, Math.min(8, Math.round(Number(args.pages) || 3)));
+          const seen = /* @__PURE__ */ new Set();
+          const out = [];
+          for (let i = 0; i < pages; i++) {
+            const ui = await this.hands.uiSnapshot();
+            if (!ui.ok) break;
+            for (const item of ui.items) {
+              const t = (item.text || "").trim();
+              if (!t || t.length < 2 || seen.has(t)) continue;
+              seen.add(t);
+              out.push(t);
+            }
+            if (i < pages - 1) {
+              await this.hands.swipe(1550, 1500, 1550, 700, 420);
+              await new Promise((r) => setTimeout(r, 550));
+            }
+          }
+          if (out.length === 0) return failed("\u8FD9\u4E00\u9875\u6CA1\u8BFB\u5230\u80FD\u5FF5\u7684\u5B57");
+          const text = out.join("\uFF5C");
+          return `\u8FD9\u4E00\u9875\u8BFB\u4E86 ${out.length} \u6BB5
+${text.slice(0, 2200)}`;
+        }
+        case "hand_read_text": {
+          const needle = typeof args.text === "string" ? args.text.trim() : "";
+          if (!needle) return failed("\u8981\u8BFB\u54EA\u4E00\u6BB5\uFF1F\u7ED9 text");
+          const hits2 = await this.hands.findText(needle);
+          if (hits2.length === 0) return failed(`\u5C4F\u5E55\u4E0A\u6CA1\u627E\u5230\u300C${needle}\u300D`);
+          const nth = args.nth === void 0 ? 0 : Math.max(0, Math.round(Number(args.nth)));
+          const hit = hits2[nth];
+          if (!hit) return failed(`\u300C${needle}\u300D\u53EA\u6709 ${hits2.length} \u5904`);
+          const b = hit.rect;
+          const h = b ? b[3] - b[1] : 0;
+          if (h > 0 && h < 40) {
+            const pad = 12;
+            const f = await this.hands.zoom(
+              Math.max(0, b[0] - pad),
+              Math.max(0, b[1] - pad),
+              Math.max(16, b[2] - b[0] + pad * 2),
+              Math.max(16, b[3] - b[1] + pad * 2)
+            );
+            if (f) {
+              return {
+                text: `\u300C${hit.text}\u300D\u5B57\u592A\u5C0F\uFF08\u9AD8 ${h}px\uFF09\uFF0C\u6211\u628A\u8FD9\u5757\u653E\u5927\u4E86`,
+                blobs: [{ bytes: f.bytes, mime: "image/jpeg", name: "zoom.jpg", fallbackText: "\uFF08\u8FD9\u4E00\u5757\u7684\u653E\u5927\u56FE\uFF09" }]
+              };
+            }
+          }
+          return `\u300C${hit.text}\u300D@(${hit.center.x},${hit.center.y})`;
+        }
+        case "hand_zoom": {
+          const n = (v) => Math.round(Number(v));
+          const x = n(args.x), y = n(args.y), w = n(args.w), h = n(args.h);
+          if (![x, y, w, h].every(Number.isFinite) || w <= 0 || h <= 0) return failed("\u8981\u7ED9 x/y/w/h\uFF08\u771F\u5B9E\u50CF\u7D20\uFF09");
+          const f = await this.hands.zoom(x, y, w, h);
+          if (!f) return failed("\u8FD9\u4E00\u5757\u6CA1\u622A\u5230\uFF08\u5750\u6807\u53EF\u80FD\u8D85\u51FA\u5C4F\u5E55\uFF09");
+          return {
+            text: `\u653E\u5927\u770B @(${x},${y}) ${w}\xD7${h}`,
+            blobs: [{ bytes: f.bytes, mime: "image/jpeg", name: "zoom.jpg", fallbackText: "\uFF08\u8FD9\u662F\u4ED6\u6307\u5B9A\u90A3\u4E00\u5757\u653E\u5927\u540E\u7684\u622A\u56FE\uFF09" }]
+          };
+        }
         case "hand_see":
           return await this.see();
         case "hand_look":
@@ -2689,7 +2813,7 @@ ${lines.join("\n")}
   async verifyChange(beforePkg) {
     try {
       await new Promise((r) => setTimeout(r, 450));
-      const ui = await this.hands.ui();
+      const ui = await this.hands.uiSnapshot();
       const now = ui.ok ? ui.pkg : "";
       if (beforePkg && now && beforePkg !== now) return `\uFF0C\u754C\u9762\u5DF2\u7ECF\u5207\u5230 ${now}`;
       return "\uFF08\u5C4F\u5E55\u770B\u8D77\u6765\u6CA1\u53D8\uFF1B\u5982\u679C\u8BE5\u6709\u53CD\u5E94\uFF0C\u5148 hand_see \u770B\u4E00\u773C\u518D\u70B9\uFF09";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortico-world-hands",
-  "version": "0.20.3",
+  "version": "0.20.5",
   "type": "module",
   "description": "Cortico World: hands for the desktop pet — looks at the phone screen, then taps, swipes, pinches, drags, types and presses keys through whatever actuator the device actually has",
   "license": "MIT",