@rebind.gg/mcp-server 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rebind
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,52 @@
+# @rebind.gg/mcp-server
+
+Stdio MCP server that exposes Rebind as computer-use tools: `screenshot`,
+`screenshot_window`, `zoom`, `click` (verified), `move_mouse`, `type`, `key`,
+`scroll`, `run_lua`, `wait`, window control, and `calibrate`. Wraps
+`@rebind.gg/client-ts`; requires a remote-control
+server running in Rebind — `remote.luau` (recommended) or `remote_access.lua`
+(protocol >= 1.1.0). The `click` tool clicks via `hid.press`, which is
+coroutine-only; use `remote.luau` or a current `remote_access.lua` (older copies
+called it outside a coroutine, so clicks silently failed).
+
+## Use
+
+```json
+{
+  "mcpServers": {
+    "rebind": {
+      "command": "bunx",
+      "args": ["@rebind.gg/mcp-server"],
+      "env": { "REBIND_URL": "ws://127.0.0.1:19561" }
+    }
+  }
+}
+```
+
+`REBIND_TOKEN` is required when the server has an auth token set (the **Auth
+token** panel field on `remote.luau`, or `AUTH_TOKEN` on `remote_access.lua`).
+
+Check the relay is reachable before wiring it up:
+
+```sh
+bunx @rebind.gg/mcp-server --selftest
+```
+
+Set `REBIND_AGENT_LOG=<path>` to append one JSONL line per tool call
+(`{ts, tool, ok, latency_ms, note}`) — no screenshot bytes, no user content.
+Off unless set.
+
+`click` and `move_mouse` reach an absolute screen point by reading the real OS
+cursor and sending relative HID deltas (which map 1:1 to pixels), correcting
+against the true cursor until it lands. This sidesteps the relay's own absolute
+move, whose delta is computed from a cached cursor position that goes stale and
+flings the pointer to a screen edge. A mouse scale probe still runs at startup;
+if pointer ballistics are detected (enhance pointer precision, DPI mismatch) the
+probe reports it, since acceleration would break the 1:1 assumption.
+
+## Develop
+
+```sh
+bun test packages/mcp-server/tests/
+bun run --cwd packages/mcp-server typecheck
+```

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@rebind.gg/mcp-server",
+  "version": "0.1.0",
+  "description": "MCP server exposing Rebind as computer-use tools: screenshot, verified click, type, key, scroll, and window control over the Remote Access protocol.",
+  "type": "module",
+  "bin": {
+    "rebind-mcp-server": "./src/index.ts"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "license": "MIT",
+  "author": {
+    "name": "US Input Company",
+    "email": "inquiries@usinput.com",
+    "url": "https://usinput.com"
+  },
+  "homepage": "https://rebind.gg",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/usinput/rebind-client-ts",
+    "directory": "packages/mcp-server"
+  },
+  "bugs": {
+    "email": "support@rebind.gg"
+  },
+  "keywords": [
+    "rebind",
+    "mcp",
+    "model-context-protocol",
+    "computer-use",
+    "openclaw",
+    "hid",
+    "automation",
+    "keyboard",
+    "mouse"
+  ],
+  "engines": {
+    "bun": ">=1.1"
+  },
+  "scripts": {
+    "start": "bun run src/index.ts",
+    "selftest": "bun run src/index.ts --selftest",
+    "test": "bun test tests/",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "@rebind.gg/client-ts": "0.2.0",
+    "sharp": "^0.34.2",
+    "zod": "^3.25.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/coords.ts

@@ -0,0 +1,37 @@
+// pure image-space -> screen-space mapping for the last screenshot taken.
+// the model only ever sees and emits coordinates in the pixel space of the
+// screenshot in front of it; all mapping back to signed virtual-desktop
+// screen coordinates lives here.
+
+export interface ShotMapping {
+  /** 1-based index of the captured display, for region re-captures (zoom). */
+  displayIndex: number;
+  /** signed virtual-desktop origin of the captured display. */
+  originX: number;
+  originY: number;
+  /** image pixels per screen point (capture scale x resize factor). */
+  scale: number;
+  /** dims of the image the model saw, for bounds checking. */
+  imageWidth: number;
+  imageHeight: number;
+}
+
+export interface Point {
+  x: number;
+  y: number;
+}
+
+export function mapToScreen(shot: ShotMapping, ix: number, iy: number): Point {
+  return {
+    x: Math.round(shot.originX + ix / shot.scale),
+    y: Math.round(shot.originY + iy / shot.scale),
+  };
+}
+
+export function inImageBounds(
+  shot: ShotMapping,
+  ix: number,
+  iy: number,
+): boolean {
+  return ix >= 0 && ix < shot.imageWidth && iy >= 0 && iy < shot.imageHeight;
+}

package/src/image.ts

@@ -0,0 +1,85 @@
+// the model-facing image pipeline: decode the native-resolution PNG from
+// Screen.Capture, downscale to the vision sweet spot, draw the cursor
+// crosshair in image space, and encode webp for the MCP image block.
+
+import sharp from "sharp";
+
+/** long-edge target. XGA-ish is the vision-accuracy sweet spot and keeps the
+ * payload small; raise only when small text must be read (use zoom instead). */
+export const DEFAULT_MAX_EDGE = 1280;
+
+/** odd so the center lands on a pixel. */
+const CROSSHAIR_SIZE = 25;
+
+export interface RenderedShot {
+  /** webp bytes for the MCP image content block. */
+  data: Buffer;
+  width: number;
+  height: number;
+  /** image pixels per screen point of the captured area. */
+  scale: number;
+}
+
+function crosshairSvg(): Buffer {
+  const s = CROSSHAIR_SIZE;
+  const c = (s - 1) / 2;
+  return Buffer.from(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}">` +
+      `<line x1="${c}" y1="0" x2="${c}" y2="${s}" stroke="white" stroke-width="5"/>` +
+      `<line x1="0" y1="${c}" x2="${s}" y2="${c}" stroke="white" stroke-width="5"/>` +
+      `<line x1="${c}" y1="0" x2="${c}" y2="${s}" stroke="red" stroke-width="2"/>` +
+      `<line x1="0" y1="${c}" x2="${s}" y2="${c}" stroke="red" stroke-width="2"/>` +
+      `</svg>`,
+  );
+}
+
+const clamp = (v: number, lo: number, hi: number) =>
+  Math.min(Math.max(v, lo), Math.max(lo, hi));
+
+export async function renderScreenshot(opts: {
+  /** native-resolution lossless PNG from Screen.Capture. */
+  png: Buffer;
+  /** logical size of the captured area in screen points. */
+  logicalWidth: number;
+  logicalHeight: number;
+  /** cursor relative to the captured area, in screen points; omit to skip the marker. */
+  cursor?: { x: number; y: number };
+  maxEdge?: number;
+}): Promise<RenderedShot> {
+  const maxEdge = opts.maxEdge ?? DEFAULT_MAX_EDGE;
+  const meta = await sharp(opts.png).metadata();
+  const nativeW = meta.width;
+  const nativeH = meta.height;
+  if (!nativeW || !nativeH) throw new Error("could not decode capture PNG");
+
+  let outW = nativeW;
+  let outH = nativeH;
+  const long = Math.max(nativeW, nativeH);
+  let img = sharp(opts.png);
+  if (long > maxEdge) {
+    const f = maxEdge / long;
+    outW = Math.max(1, Math.round(nativeW * f));
+    outH = Math.max(1, Math.round(nativeH * f));
+    img = img.resize(outW, outH);
+  }
+  const scale = outW / opts.logicalWidth;
+
+  if (opts.cursor) {
+    const px = Math.round(opts.cursor.x * scale);
+    const py = Math.round(opts.cursor.y * scale);
+    // skip the marker when the cursor sits outside the captured area
+    if (px >= 0 && px < outW && py >= 0 && py < outH) {
+      const half = (CROSSHAIR_SIZE - 1) / 2;
+      img = img.composite([
+        {
+          input: crosshairSvg(),
+          left: clamp(px - half, 0, outW - CROSSHAIR_SIZE),
+          top: clamp(py - half, 0, outH - CROSSHAIR_SIZE),
+        },
+      ]);
+    }
+  }
+
+  const data = await img.webp({ quality: 80 }).toBuffer();
+  return { data, width: outW, height: outH, scale };
+}

package/src/index.ts

@@ -0,0 +1,456 @@
+#!/usr/bin/env bun
+
+// stdio MCP server wrapping @rebind.gg/client-ts. all model-facing features
+// live here: image pipeline, coordinate bookkeeping, verified click,
+// calibration. the Rust/Lua layers below ship primitives only.
+//
+// config: REBIND_URL (default ws://127.0.0.1:19561), REBIND_TOKEN (optional).
+
+import { appendFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { RebindRemote } from "@rebind.gg/client-ts";
+import { z } from "zod";
+import { renderScreenshot } from "./image.ts";
+import { Session } from "./session.ts";
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+function text(t: string) {
+  return { content: [{ type: "text" as const, text: t }] };
+}
+
+async function main() {
+  const url = process.env.REBIND_URL ?? "ws://127.0.0.1:19561";
+  const client = new RebindRemote(url, {
+    token: process.env.REBIND_TOKEN ?? "",
+    timeoutMs: 15000,
+  });
+
+  // --selftest: verify the relay is reachable and capturing, then exit. Used by
+  // the installer and by users to diagnose the #1 first-run failure (relay not
+  // loaded) without wiring up the MCP transport.
+  if (process.argv.includes("--selftest")) {
+    try {
+      await client.connect();
+      const cap = await client.screenCapture({});
+      console.error(
+        `selftest OK: relay ${url}, display ${cap.display.index} ${cap.display.width}x${cap.display.height}`,
+      );
+      process.exit(0);
+    } catch (e) {
+      console.error(
+        `selftest FAILED: Rebind relay not reachable on ${url}.\n` +
+          `Start Rebind and load the Remote Access script (remote_access.lua).\n(${e})`,
+      );
+      process.exit(1);
+    }
+  }
+
+  await client.connect();
+
+  const session = new Session(client);
+  try {
+    await session.calibrate();
+  } catch (e) {
+    session.calibration = { calibrated: false, reason: String(e) };
+  }
+
+  const server = new McpServer({ name: "rebind", version: "0.1.0" });
+
+  // REBIND_AGENT_LOG=<path>: opt-in telemetry — one JSONL line per tool call
+  // ({ts, tool, ok, latency_ms, note}). No screenshot bytes, no user content.
+  // No-op when unset.
+  const telemetryPath = process.env.REBIND_AGENT_LOG?.replace(/^~(?=\/)/, homedir());
+  const logCall = (tool: string, ok: boolean, t0: number, note?: string) => {
+    if (!telemetryPath) return;
+    try {
+      appendFileSync(
+        telemetryPath,
+        `${JSON.stringify({ ts: new Date().toISOString(), tool, ok, latency_ms: Math.round(performance.now() - t0), ...(note ? { note } : {}) })}\n`,
+      );
+    } catch {
+      // telemetry must never break a tool call
+    }
+  };
+  const register = (name: string, config: object, handler: (args: any) => Promise<any>) => {
+    server.registerTool(name, config as any, async (args: any) => {
+      const t0 = performance.now();
+      try {
+        const result = await handler(args);
+        logCall(name, true, t0);
+        return result;
+      } catch (e) {
+        logCall(name, false, t0, String(e));
+        throw e;
+      }
+    });
+  };
+
+  register(
+    "screenshot",
+    {
+      description:
+        "Capture a display and return it as an image. The cursor is marked with a red crosshair. " +
+        "All click/move coordinates you emit must be pixel coordinates in the MOST RECENT screenshot. " +
+        "After every action, take a new screenshot and verify the action had the intended effect before continuing; if not, correct and retry.",
+      inputSchema: {
+        display: z
+          .number()
+          .int()
+          .min(1)
+          .optional()
+          .describe(
+            "1-based display index from list_displays; defaults to the display under the cursor",
+          ),
+      },
+    },
+    async ({ display }) => {
+      const cap = await client.screenCapture({ display });
+      const shot = await renderScreenshot({
+        png: Buffer.from(cap.image, "base64"),
+        logicalWidth: cap.display.width,
+        logicalHeight: cap.display.height,
+        cursor: {
+          x: cap.cursor.x - cap.display.x,
+          y: cap.cursor.y - cap.display.y,
+        },
+      });
+      session.lastShot = {
+        displayIndex: cap.display.index,
+        originX: cap.display.x,
+        originY: cap.display.y,
+        scale: shot.scale,
+        imageWidth: shot.width,
+        imageHeight: shot.height,
+      };
+      const win = await client.systemWindow().catch(() => null);
+      const lines = [
+        `display ${cap.display.index}: ${cap.display.width}x${cap.display.height} at (${cap.display.x}, ${cap.display.y})${cap.display.primary ? " (primary)" : ""}`,
+        `image: ${shot.width}x${shot.height} — emit click/move coordinates in this image's pixel space`,
+        `cursor: image (${Math.round((cap.cursor.x - cap.display.x) * shot.scale)}, ${Math.round((cap.cursor.y - cap.display.y) * shot.scale)}), marked with a red crosshair`,
+        win ? `active window: "${win.title}" (${win.process})` : null,
+        session.calibration && !session.calibration.calibrated
+          ? `WARNING: mouse not calibrated — ${session.calibration.reason}. Clicks may land off-target; verify each one.`
+          : null,
+      ].filter(Boolean);
+      return {
+        content: [
+          { type: "text" as const, text: lines.join("\n") },
+          {
+            type: "image" as const,
+            data: shot.data.toString("base64"),
+            mimeType: "image/webp",
+          },
+        ],
+      };
+    },
+  );
+
+  register(
+    "screenshot_window",
+    {
+      description:
+        "Capture just ONE window (by title substring, or the active window if omitted) instead of a whole display — smaller, cheaper, and monitor-proof. " +
+        "Click/move coordinates you emit map into THIS capture exactly like screenshot. Prefer this over a full-screen screenshot to read or verify a specific app.",
+      inputSchema: {
+        title: z
+          .string()
+          .optional()
+          .describe("window title substring; omit to capture the active window"),
+      },
+    },
+    async ({ title }) => {
+      const win = title
+        ? (await client.windowList(title))[0]
+        : await client.systemWindow();
+      if (!win) throw new Error(`no window matches "${title}"`);
+      const cap = await client.screenCaptureWindow({ title: win.title });
+      // remote.luau captures the window rect clamped to its display; recover the
+      // capture's screen-space origin so image coords map back to the screen.
+      const originX = Math.max(cap.display.x, win.x);
+      const originY = Math.max(cap.display.y, win.y);
+      const rx = Math.max(0, win.x - cap.display.x);
+      const ry = Math.max(0, win.y - cap.display.y);
+      const logicalWidth = Math.min(win.width, cap.display.width - rx);
+      const logicalHeight = Math.min(win.height, cap.display.height - ry);
+      const curInside =
+        cap.cursor.x >= originX &&
+        cap.cursor.x < originX + logicalWidth &&
+        cap.cursor.y >= originY &&
+        cap.cursor.y < originY + logicalHeight;
+      const shot = await renderScreenshot({
+        png: Buffer.from(cap.image, "base64"),
+        logicalWidth,
+        logicalHeight,
+        cursor: curInside
+          ? { x: cap.cursor.x - originX, y: cap.cursor.y - originY }
+          : undefined,
+      });
+      session.lastShot = {
+        displayIndex: cap.display.index,
+        originX,
+        originY,
+        scale: shot.scale,
+        imageWidth: shot.width,
+        imageHeight: shot.height,
+      };
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `window "${win.title}" at screen (${win.x}, ${win.y}), ${logicalWidth}x${logicalHeight} — emit click/move coords in this image's pixel space (${shot.width}x${shot.height})`,
+          },
+          {
+            type: "image" as const,
+            data: shot.data.toString("base64"),
+            mimeType: "image/webp",
+          },
+        ],
+      };
+    },
+  );
+
+  register(
+    "zoom",
+    {
+      description:
+        "Capture a region of the last screenshot at full native resolution, for reading small text or inspecting small targets. " +
+        "READ-ONLY: click coordinates still refer to the last full screenshot, not the zoomed image.",
+      inputSchema: {
+        x: z
+          .number()
+          .describe("region top-left x, in last-screenshot image pixels"),
+        y: z
+          .number()
+          .describe("region top-left y, in last-screenshot image pixels"),
+        w: z
+          .number()
+          .min(1)
+          .describe("region width, in last-screenshot image pixels"),
+        h: z
+          .number()
+          .min(1)
+          .describe("region height, in last-screenshot image pixels"),
+      },
+    },
+    async ({ x, y, w, h }) => {
+      const shot = session.lastShot;
+      if (!shot)
+        throw new Error("no screenshot taken yet — call screenshot first");
+      const topLeft = session.mapImagePoint(x, y);
+      const cap = await client.screenCapture({
+        display: shot.displayIndex,
+        region: {
+          x: topLeft.x - shot.originX,
+          y: topLeft.y - shot.originY,
+          w: Math.ceil(w / shot.scale),
+          h: Math.ceil(h / shot.scale),
+        },
+      });
+      const rendered = await renderScreenshot({
+        png: Buffer.from(cap.image, "base64"),
+        logicalWidth: Math.ceil(w / shot.scale),
+        logicalHeight: Math.ceil(h / shot.scale),
+      });
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `zoomed region (${x}, ${y}) ${w}x${h} of the last screenshot. Coordinates for clicks still refer to the last full screenshot.`,
+          },
+          {
+            type: "image" as const,
+            data: rendered.data.toString("base64"),
+            mimeType: "image/webp",
+          },
+        ],
+      };
+    },
+  );
+
+  register(
+    "list_displays",
+    { description: "Enumerate displays with their virtual-desktop geometry." },
+    async () => text(JSON.stringify(await client.screenDisplays(), null, 2)),
+  );
+
+  register(
+    "click",
+    {
+      description:
+        "Move the mouse to a point in the last screenshot and click. The landing position is verified and corrected before pressing. " +
+        "Prefer keyboard shortcuts (the key tool) for fiddly widgets like address bars and dropdowns.",
+      inputSchema: {
+        x: z.number().describe("x in last-screenshot image pixels"),
+        y: z.number().describe("y in last-screenshot image pixels"),
+        button: z.enum(["left", "right", "middle"]).default("left"),
+        double: z.boolean().default(false),
+      },
+    },
+    async ({ x, y, button, double }) => {
+      const target = session.mapImagePoint(x, y);
+      const r = await session.verifiedClick(target.x, target.y, button, double);
+      return text(
+        r.ok
+          ? `clicked ${button}${double ? " (double)" : ""} at screen (${r.landed.x}, ${r.landed.y})${r.corrected ? " after one correction" : ""}`
+          : `WARNING: cursor landed at (${r.landed.x}, ${r.landed.y}) instead of (${target.x}, ${target.y}) — the click may have missed. Take a screenshot to verify.`,
+      );
+    },
+  );
+
+  register(
+    "move_mouse",
+    {
+      description:
+        "Move the mouse to a point in the last screenshot without clicking.",
+      inputSchema: {
+        x: z.number().describe("x in last-screenshot image pixels"),
+        y: z.number().describe("y in last-screenshot image pixels"),
+      },
+    },
+    async ({ x, y }) => {
+      const target = session.mapImagePoint(x, y);
+      const landed = await session.moveTo(target.x, target.y);
+      return text(`cursor at screen (${landed.x}, ${landed.y})`);
+    },
+  );
+
+  register(
+    "type",
+    {
+      description: "Type text into the focused element via the keyboard.",
+      inputSchema: { text: z.string() },
+    },
+    async ({ text: t }) => {
+      client.hidType(t);
+      // typing is paced by the relay; give it time proportional to length
+      await sleep(Math.min(50 + t.length * 15, 5000));
+      return text(`typed ${t.length} characters`);
+    },
+  );
+
+  register(
+    "key",
+    {
+      description:
+        'Press a key or combo. Names: "Enter", "Escape", "Tab", "Backspace", arrows ("Up", "Down", "Left", "Right"), letters/digits, modifiers "LCtrl", "LAlt", "LShift", "LMeta". ' +
+        'Combos join with "+", e.g. "LCtrl+L" (focus address bar), "LCtrl+C", "LAlt+Tab".',
+      inputSchema: { combo: z.string() },
+    },
+    async ({ combo }) => {
+      client.hidPress(combo, 30);
+      await sleep(80);
+      return text(`pressed ${combo}`);
+    },
+  );
+
+  register(
+    "scroll",
+    {
+      description:
+        "Scroll the mouse wheel at the current cursor position. Positive = up, negative = down.",
+      inputSchema: { clicks: z.number().int().describe("wheel clicks") },
+    },
+    async ({ clicks }) => {
+      client.hidScroll(clicks);
+      await sleep(80);
+      return text(`scrolled ${clicks} clicks`);
+    },
+  );
+
+  register(
+    "list_windows",
+    {
+      description: "List open windows (handle, title, process, geometry).",
+      inputSchema: {
+        filter: z.string().optional().describe("title substring filter"),
+      },
+    },
+    async ({ filter }) =>
+      text(JSON.stringify(await client.windowList(filter), null, 2)),
+  );
+
+  register(
+    "focus_window",
+    {
+      description:
+        "Bring a window to the foreground by handle (from list_windows).",
+      inputSchema: { handle: z.number() },
+    },
+    async ({ handle }) => {
+      await client.windowActivate(handle);
+      return text(`activated window ${handle}`);
+    },
+  );
+
+  register(
+    "wait",
+    {
+      description: "Pause before the next action (page loads, animations).",
+      inputSchema: { ms: z.number().int().min(1).max(10000) },
+    },
+    async ({ ms }) => {
+      await sleep(ms);
+      return text(`waited ${ms}ms`);
+    },
+  );
+
+  register(
+    "get_cursor",
+    { description: "Read the current cursor position in screen coordinates." },
+    async () => text(JSON.stringify(await client.systemMouse())),
+  );
+
+  register(
+    "active_window",
+    { description: "Read the currently focused window." },
+    async () => text(JSON.stringify(await client.systemWindow())),
+  );
+
+  register(
+    "calibrate",
+    {
+      description:
+        "Re-run the mouse scale probe (auto-runs at startup). Verifies relative mouse counts map 1:1 to pixels; reports pointer-ballistics misconfiguration.",
+    },
+    async () => text(JSON.stringify(await session.calibrate())),
+  );
+
+  register(
+    "run_lua",
+    {
+      description:
+        "Run a Luau script on the connected machine through Rebind's engine and return its `return` value as JSON. " +
+        "PREFER THIS over the screenshot→click loop for any DETERMINISTIC sequence — it is far faster, cheaper, and " +
+        "reliable, and it sidesteps multi-monitor guessing. Use it to: launch apps (`System.ExecDetached(\"cmd\", {\"/c\",\"start\",\"\",url})`), " +
+        "wait for and place windows on a known display (`local h=Window.Wait(title, ms); Window.Move(h, x, y, w, h)`), " +
+        "run shell commands (`System.Exec(cmd) -> {exit,stdout,stderr}`), find UI by color (`Screen.SearchForColor(region,hex)`), " +
+        "type, click, read pixels, manage processes. Namespaces: System, HID, Input, Screen, Window, Process, File, Clipboard, Net, Macro, Env, Log. " +
+        "Fall back to `screenshot` only for genuinely visual reasoning the script cannot resolve. End the script with `return <value>` to get data back.",
+      inputSchema: {
+        source: z
+          .string()
+          .describe("Luau source. End with `return <value>` to return data."),
+      },
+    },
+    async ({ source }) => {
+      const result = await client.luaExec(source);
+      return text(
+        typeof result === "string" ? result : JSON.stringify(result, null, 2),
+      );
+    },
+  );
+
+  await server.connect(new StdioServerTransport());
+  // stdout carries the MCP protocol — log to stderr only
+  console.error(
+    `rebind-mcp-server: connected to ${url} (calibrated=${session.calibration?.calibrated ?? false})`,
+  );
+}
+
+main().catch((e) => {
+  console.error(`rebind-mcp-server: fatal: ${e}`);
+  process.exit(1);
+});

package/src/session.ts

@@ -0,0 +1,188 @@
+// per-connection state and the two composed behaviors built from existing
+// protocol primitives: verified click (move -> read back -> correct once ->
+// press) and calibration (relative-move scale probe at session start).
+
+import type { RebindRemote } from "@rebind.gg/client-ts";
+import {
+  inImageBounds,
+  mapToScreen,
+  type Point,
+  type ShotMapping,
+} from "./coords.ts";
+
+/** acceptable landing error. relative counts map 1:1 to pixels, but readback
+ * (GetCursorPos) lags the just-sent USB move under load, so keep a small margin. */
+export const CLICK_EPS_PX = 5;
+
+/** OS cursor + SystemState refresh time after a move burst, before reading back.
+ * too short and the readback is stale, the loop mis-corrects, and it diverges. */
+const SETTLE_MS = 70;
+
+/** correction passes: read real cursor, step the residual, repeat. */
+const MOVE_MAX_PASSES = 6;
+/** target pixels per glide step — smaller is smoother (more USB frames). */
+const MOVE_STEP_PX = 40;
+/** cap on glide steps in one pass, so a cross-screen move stays bounded. */
+const MOVE_MAX_STEPS = 40;
+/** delay between glide steps within a pass. */
+const MOVE_STEP_MS = 8;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const dist = (a: Point, b: Point) => Math.hypot(a.x - b.x, a.y - b.y);
+
+export type MouseButton = "left" | "right" | "middle";
+
+const BUTTON_CODE: Record<MouseButton, string> = {
+  left: "Mouse1",
+  right: "Mouse2",
+  middle: "Mouse3",
+};
+
+export interface ClickResult {
+  ok: boolean;
+  landed: Point;
+  corrected: boolean;
+}
+
+export interface CalibrationResult {
+  calibrated: boolean;
+  /** input counts per screen pixel when the ratio is constant. */
+  k?: number;
+  reason?: string;
+}
+
+export class Session {
+  lastShot: ShotMapping | null = null;
+  calibration: CalibrationResult | null = null;
+
+  constructor(readonly client: RebindRemote) {}
+
+  /** map a point in the last screenshot's image space to screen coords. */
+  mapImagePoint(ix: number, iy: number): Point {
+    if (!this.lastShot) {
+      throw new Error(
+        "no screenshot taken yet — call the screenshot tool first",
+      );
+    }
+    if (!inImageBounds(this.lastShot, ix, iy)) {
+      throw new Error(
+        `(${ix}, ${iy}) is outside the last screenshot (${this.lastShot.imageWidth}x${this.lastShot.imageHeight}) — take a fresh screenshot`,
+      );
+    }
+    return mapToScreen(this.lastShot, ix, iy);
+  }
+
+  /**
+   * Move the OS cursor to absolute screen (x, y).
+   *
+   * The relay's hid.move_smooth / hid.move_to derive their delta from a cached
+   * cursor position that goes stale and flings the pointer to a screen edge
+   * (observed: target (800,500) landing at (0,0)). Relative counts map 1:1 to
+   * pixels, so we read the REAL cursor, glide the residual in small relative
+   * steps (human-like, not an instant jump), then read back and correct against
+   * the true cursor. Converges to the exact pixel, usually in a single pass.
+   */
+  async moveTo(x: number, y: number): Promise<Point & { passes: number }> {
+    let at = await this.client.systemMouse();
+    let passes = 0;
+    while (passes < MOVE_MAX_PASSES) {
+      const dx = x - at.x;
+      const dy = y - at.y;
+      const distance = Math.hypot(dx, dy);
+      if (distance <= CLICK_EPS_PX) break;
+      passes++;
+      const steps = Math.min(
+        MOVE_MAX_STEPS,
+        Math.max(1, Math.round(distance / MOVE_STEP_PX)),
+      );
+      let sentX = 0;
+      let sentY = 0;
+      for (let s = 1; s <= steps; s++) {
+        const nx = Math.round((dx * s) / steps) - sentX;
+        const ny = Math.round((dy * s) / steps) - sentY;
+        sentX += nx;
+        sentY += ny;
+        this.client.hidMove(nx, ny);
+        if (s < steps) await sleep(MOVE_STEP_MS);
+      }
+      await sleep(SETTLE_MS);
+      at = await this.client.systemMouse();
+    }
+    return { x: at.x, y: at.y, passes };
+  }
+
+  /** move (self-correcting), then press. */
+  async verifiedClick(
+    x: number,
+    y: number,
+    button: MouseButton = "left",
+    double = false,
+  ): Promise<ClickResult> {
+    const target = { x, y };
+    const at = await this.moveTo(x, y);
+    const code = BUTTON_CODE[button];
+    this.client.hidPress(code, 30);
+    if (double) {
+      await sleep(80);
+      this.client.hidPress(code, 30);
+    }
+    return {
+      ok: dist(at, target) <= CLICK_EPS_PX,
+      landed: { x: at.x, y: at.y },
+      corrected: at.passes > 1,
+    };
+  }
+
+  /**
+   * relative-move scale probe. moves to screen center (deterministic anchor
+   * away from edge clamp), nudges by increasing counts on each axis, and
+   * checks whether counts-per-pixel is constant. a magnitude-dependent ratio
+   * means ballistics in the chain (EPP on, DPI mismatch, software fallback) —
+   * recorded and surfaced to the agent instead of failing via missed clicks.
+   */
+  async calibrate(): Promise<CalibrationResult> {
+    const done = (r: CalibrationResult): CalibrationResult => {
+      this.calibration = r;
+      return r;
+    };
+    // 8 as the smallest probe: large enough to dodge integer quantization,
+    // small enough that ballistic acceleration curves still diverge by 256.
+    const probes = [8, 64, 256];
+    const res = await this.client.screenResolution();
+    const saved = await this.client.systemMouse();
+    const cx = Math.floor(res.width / 2);
+    const cy = Math.floor(res.height / 2);
+
+    const ratios: number[] = [];
+    try {
+      for (const axis of ["x", "y"] as const) {
+        for (const n of probes) {
+          const before = await this.moveTo(cx, cy);
+          this.client.hidMove(axis === "x" ? n : 0, axis === "y" ? n : 0);
+          await sleep(SETTLE_MS);
+          const after = await this.client.systemMouse();
+          const moved = axis === "x" ? after.x - before.x : after.y - before.y;
+          if (moved <= 0) {
+            return done({
+              calibrated: false,
+              reason: `${axis}-axis probe of ${n} counts produced no movement`,
+            });
+          }
+          ratios.push(n / moved);
+        }
+      }
+    } finally {
+      await this.moveTo(saved.x, saved.y);
+    }
+
+    const mean = ratios.reduce((a, b) => a + b, 0) / ratios.length;
+    const maxDev = Math.max(...ratios.map((r) => Math.abs(r - mean) / mean));
+    if (maxDev > 0.15) {
+      return done({
+        calibrated: false,
+        reason: `counts-per-pixel varies with magnitude (±${Math.round(maxDev * 100)}%) — pointer ballistics in the chain (enhance pointer precision, DPI mismatch, or software fallback)`,
+      });
+    }
+    return done({ calibrated: true, k: mean });
+  }
+}