@rebind.gg/mcp-server 0.1.2 → 0.1.4

package/README.md

@@ -60,9 +60,16 @@ Off unless set.
 cursor and sending relative HID deltas (which map 1:1 to pixels), correcting
 against the true cursor until it lands. This sidesteps the relay's own absolute
 move, whose delta is computed from a cached cursor position that goes stale and
-flings the pointer to a screen edge. A mouse scale probe still runs at startup;
-if pointer ballistics are detected (enhance pointer precision, DPI mismatch) the
-probe reports it, since acceleration would break the 1:1 assumption.
+flings the pointer to a screen edge. A mouse scale probe runs on the first
+click/move_mouse; if pointer ballistics are detected (enhance pointer precision,
+DPI mismatch) the probe reports it, since acceleration would break the 1:1
+assumption.
+
+Tool calls are serialized through a single queue: there is one physical cursor
+and keyboard, so parallel tool calls from the model (e.g. two `move_mouse` in
+one turn) run one after another instead of interleaving HID input. All results
+are reported in the coordinate space of the last screenshot, the same space the
+model emits.
 
 ## Develop
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rebind.gg/mcp-server",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "MCP server exposing Rebind as computer-use tools: screenshot, verified click, type, key, scroll, and window control over the Remote Access protocol.",
   "type": "module",
   "bin": {

package/src/coords.ts

@@ -28,6 +28,13 @@ export function mapToScreen(shot: ShotMapping, ix: number, iy: number): Point {
   };
 }
 
+export function mapToImage(shot: ShotMapping, sx: number, sy: number): Point {
+  return {
+    x: Math.round((sx - shot.originX) * shot.scale),
+    y: Math.round((sy - shot.originY) * shot.scale),
+  };
+}
+
 export function inImageBounds(
   shot: ShotMapping,
   ix: number,

package/src/index.ts

@@ -12,8 +12,9 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { RebindRemote } from "@rebind.gg/client-ts";
 import { z } from "zod";
+import { mapToImage } from "./coords.ts";
 import { renderScreenshot } from "./image.ts";
-import { Session } from "./session.ts";
+import { CLICK_EPS_PX, Session } from "./session.ts";
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
@@ -51,13 +52,8 @@ async function main() {
   await client.connect();
 
   const session = new Session(client);
-  try {
-    await session.calibrate();
-  } catch (e) {
-    session.calibration = { calibrated: false, reason: String(e) };
-  }
 
-  const server = new McpServer({ name: "rebind", version: "0.1.2" });
+  const server = new McpServer({ name: "rebind", version: "0.1.4" });
 
   // REBIND_AGENT_LOG=<path>: opt-in telemetry — one JSONL line per tool call
   // ({ts, tool, ok, latency_ms, note}). No screenshot bytes, no user content.
@@ -74,17 +70,28 @@ async function main() {
       // telemetry must never break a tool call
     }
   };
+  // one physical cursor and keyboard: tool calls must never overlap. Models
+  // routinely emit parallel calls in a single turn (two move_mouse at once)
+  // and the MCP SDK runs handlers concurrently; interleaved HID deltas and
+  // cursor readbacks make every overlapping move land nowhere. Chain all
+  // handlers through a single queue.
+  let chain: Promise<unknown> = Promise.resolve();
   const register = (name: string, config: object, handler: (args: any) => Promise<any>) => {
-    server.registerTool(name, config as any, async (args: any) => {
-      const t0 = performance.now();
-      try {
-        const result = await handler(args);
-        logCall(name, true, t0);
-        return result;
-      } catch (e) {
-        logCall(name, false, t0, String(e));
-        throw e;
-      }
+    server.registerTool(name, config as any, (args: any) => {
+      const run = async () => {
+        const t0 = performance.now();
+        try {
+          const result = await handler(args);
+          logCall(name, true, t0);
+          return result;
+        } catch (e) {
+          logCall(name, false, t0, String(e));
+          throw e;
+        }
+      };
+      const p = chain.then(run, run);
+      chain = p.catch(() => {});
+      return p;
     });
   };
 
@@ -292,10 +299,13 @@ async function main() {
     async ({ x, y, button, double }) => {
       const target = session.mapImagePoint(x, y);
       const r = await session.verifiedClick(target.x, target.y, button, double);
+      // report in the same image space the model emits — mixing in raw screen
+      // coordinates sends smaller models into ad-hoc rescaling math
+      const at = mapToImage(session.lastShot!, r.landed.x, r.landed.y);
       return text(
         r.ok
-          ? `clicked ${button}${double ? " (double)" : ""} at screen (${r.landed.x}, ${r.landed.y})${r.corrected ? " after one correction" : ""}`
-          : `WARNING: cursor landed at (${r.landed.x}, ${r.landed.y}) instead of (${target.x}, ${target.y}) — the click may have missed. Take a screenshot to verify.`,
+          ? `clicked ${button}${double ? " (double)" : ""} at image (${at.x}, ${at.y})${r.corrected ? " after one correction" : ""}`
+          : `WARNING: cursor landed at image (${at.x}, ${at.y}) instead of (${x}, ${y}) — the click may have missed. Take a screenshot to verify.`,
       );
     },
   );
@@ -313,7 +323,14 @@ async function main() {
     async ({ x, y }) => {
       const target = session.mapImagePoint(x, y);
       const landed = await session.moveTo(target.x, target.y);
-      return text(`cursor at screen (${landed.x}, ${landed.y})`);
+      const at = mapToImage(session.lastShot!, landed.x, landed.y);
+      const missed =
+        Math.hypot(landed.x - target.x, landed.y - target.y) > CLICK_EPS_PX;
+      return text(
+        missed
+          ? `WARNING: cursor stopped at image (${at.x}, ${at.y}) instead of (${x}, ${y}) — take a fresh screenshot and retry`
+          : `cursor at image (${at.x}, ${at.y})`,
+      );
     },
   );
 
@@ -413,7 +430,8 @@ async function main() {
     "calibrate",
     {
       description:
-        "Re-run the mouse scale probe (auto-runs at startup). Verifies relative mouse counts map 1:1 to pixels; reports pointer-ballistics misconfiguration.",
+        "Run the mouse scale probe. Verifies relative mouse counts map 1:1 to pixels; reports pointer-ballistics misconfiguration. " +
+        "Auto-runs on first click/move_mouse if not already calibrated; call this tool explicitly to re-run.",
     },
     async () => text(JSON.stringify(await session.calibrate())),
   );

package/src/session.ts

@@ -54,9 +54,20 @@ export interface CalibrationResult {
 export class Session {
   lastShot: ShotMapping | null = null;
   calibration: CalibrationResult | null = null;
+  private _calibrating = false;
 
   constructor(readonly client: RebindRemote) {}
 
+  async ensureCalibrated(): Promise<void> {
+    if (this.calibration !== null || this._calibrating) return;
+    this._calibrating = true;
+    try {
+      await this.calibrate();
+    } finally {
+      this._calibrating = false;
+    }
+  }
+
   /** map a point in the last screenshot's image space to screen coords. */
   mapImagePoint(ix: number, iy: number): Point {
     if (!this.lastShot) {
@@ -83,6 +94,7 @@ export class Session {
    * the true cursor. Converges to the exact pixel, usually in a single pass.
    */
   async moveTo(x: number, y: number): Promise<Point & { passes: number }> {
+    await this.ensureCalibrated();
     let at = await this.client.systemMouse();
     let passes = 0;
     while (passes < MOVE_MAX_PASSES) {