@rebind.gg/mcp-server 0.2.4 → 0.3.0

package/README.md

@@ -5,7 +5,9 @@ Stdio MCP server that turns [Rebind](https://rebind.gg) into a computer-use tool
 **Requirements**
 
 - [Bun](https://bun.sh) >= 1.1 (the server runs via `bunx`)
-- The [Rebind app](https://rebind.gg/download) running the free [Remote Control](https://www.rebind.gg/marketplace/script_AAHteeDcYPGOT7GfXZ_OK) package (`rebind install @rebind/remote-control`)
+- The [Rebind app](https://rebind.gg/download) running the free [Remote Control](https://www.rebind.gg/marketplace/script_AAHteeDcYPGOT7GfXZ_OK) package (`rebind install @rebind/remote-control --allow`)
+
+The first time the server sends a command, Remote Control asks on that computer whether to allow **Rebind MCP server**. Allow it once; the name is remembered. Tool calls fail with `awaiting_approval` until you answer. Remote Control accepts connections from its own computer only unless **Local only** is turned off in its Options, which an agent on another computer or inside WSL needs. `run_lua`, `focus_window`, `position_window` and `read_page` also need **Allow custom Luau** turned on there; the server reads that setting at startup.
 
 ## Install
 
@@ -41,7 +43,7 @@ Any MCP client takes the stdio block directly:
 bunx @rebind.gg/mcp-server --selftest
 ```
 
-Connects, takes a capture, and reports the display — the fastest way to catch the #1 first-run failure (relay not loaded).
+Connects, waits up to a minute for the approval prompt on a first run, takes a capture, and reports the display. It is the fastest way to catch the #1 first-run failure (relay not loaded).
 
 ## Tools
 
@@ -55,7 +57,7 @@ Connects, takes a capture, and reports the display — the fastest way to catch
 | `type` | Type text into the focused element. |
 | `key` | Press a key or combo, e.g. `Enter`, `LCtrl+L`, `LAlt+Tab`. |
 | `scroll` | Scroll at the cursor. Positive scrolls down, negative up. `unit` is `wheel` (default, fine wheel clicks) or `page` (one PageDown or PageUp per unit, capped at 10). |
-| `read_page` | Extract the focused browser's current page as structured text (URL, forms, interactive elements, plain text) via an address-bar bookmarklet; the full dump, including cleaned HTML, lands in the clipboard. The result is read back through AppleScript (`URL of active tab of front window`), so the tool is registered only when the relay host is macOS, and the front browser must answer that query. Safari is detected but blocks the bookmarklet path. |
+| `read_page` | Extract the focused browser's current page as structured text (URL, forms, interactive elements, plain text) via an address-bar bookmarklet; the full dump, including cleaned HTML, lands in the clipboard. The result is read back through AppleScript (`URL of active tab of front window`), so the tool is registered only when the relay host is macOS and **Allow custom Luau** is on, and the front browser must answer that query. Safari is detected but blocks the bookmarklet path. |
 | `meditate` | Research a problem page until a reasoning model has the data to solve it: fetches the URL directly, pulls the script bundles behind a JS shell so its API endpoints are visible, fetches those endpoints and assets, and reads the live page with vision. The browser steps (rendering a JS shell, same-origin fetches through the page, and the vision pass) need a macOS relay; elsewhere it reads pages over HTTP only. Returns a markdown dossier plus JSON `{error, problem, requirements, plan, submittable, form, live_captcha_hint, confidence, sources, assets, pending_fetch, notes, cost_usd}`. `error` is non-null when no problem could be identified or the page could not be read; partial data is still returned. Gated behind `REBIND_ALLOW_DELEGATION=1` (spends OpenRouter credits). See the pipeline below. |
 | `run_lua` | Run a Luau script on the connected machine and return its `return` value as JSON — the fast path for any deterministic sequence (launch apps, place windows, shell commands, pixel search). |
 | `lua_docs` | Full Luau SDK type definitions for `run_lua`. |
@@ -84,7 +86,11 @@ Connects, takes a capture, and reports the display — the fastest way to catch
 
 **One coordinate space.** All results are reported in the pixel space of the last screenshot — the same space the model emits — so no rescaling math ever falls on the model.
 
-**Machine-aware orientation.** At startup the server probes the connected machine through the relay (OS, shortcut modifier, available shell tools, installed apps) and serves it as the MCP `instructions` block, along with a policy: prefer `run_lua` for deterministic work, keyboard over mouse, and screenshots last. The full Luau SDK type definitions are embedded inline so the model never has to guess `run_lua` APIs (including the coroutine-safe HID path — `HID.Down`/`Up`/`MoveTo`/`Combo` work in a one-shot chunk, `HID.Press` does not). The OS is read from the relay, not `process.platform`, so the guidance is correct even when the server and the relay run on different machines.
+**Machine-aware orientation.** At startup the server reads the OS from Remote Control's `hello` banner and, when custom Luau is allowed, probes the connected machine through the relay (available shell tools, installed apps) and serves it as the MCP `instructions` block, along with a policy: prefer `run_lua` for deterministic work, keyboard over mouse, and screenshots last. The full Luau SDK type definitions are embedded inline so the model never has to guess `run_lua` APIs (including the coroutine-safe HID path: `HID.Down`/`Up`/`MoveTo`/`Combo` work in a one-shot chunk, `HID.Press` does not). The OS is read from the relay, not `process.platform`, so the guidance is correct even when the server and the relay run on different machines.
+
+## Save what worked
+
+When the user wants a working sequence kept, the server's instructions tell the agent to save it as a Rebind script that runs with no agent: `rebind init <dir>`, the sequence in `main.luau` bound to a key, `[runtime] permissions` in `rebind.toml`, then `rebind check <dir>`, `rebind pack <dir>`, `rebind install <name>.rbp --allow` (`--force` to replace) and `rebind script start packages/<name>/main.luau`, on the computer running Rebind.
 
 ## Solve pipeline
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rebind.gg/mcp-server",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "description": "MCP server exposing Rebind as computer-use tools: screenshot, verified click, type, key, scroll, and window control over the Remote Control protocol.",
   "type": "module",
   "bin": {

package/src/index.ts

@@ -10,7 +10,7 @@ import { appendFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { RebindRemote } from "@rebind.gg/client-ts";
+import { RebindRemote, ServerError } from "@rebind.gg/client-ts";
 import { z } from "zod";
 import pkg from "../package.json";
 import { mapToImage } from "./coords.ts";
@@ -29,6 +29,7 @@ import {
   PROBE_CLIS_CMD,
   probeAppsCmd,
   RUN_LUA_DESC,
+  SAVE_WORK_DOCTRINE,
   toolPriority,
 } from "./prompts.ts";
 import { CLICK_EPS_PX, Session } from "./session.ts";
@@ -282,11 +283,16 @@ async function v1Sdk(): Promise<string> {
 // signal because this process may run on a different machine than the relay.
 // Probes are best-effort: a locked-down remote (no exec permission) just
 // yields a shorter card. Machine facts and general technique only.
-// The probed OS is returned alongside: tools that pick a shortcut modifier
+// The OS comes from the hello banner when the server reports it (Remote
+// Control 0.3.0+), so it is right even with lua.exec off; older servers fall
+// back to a uname probe. When the banner says lua.exec is off, the probes and
+// read_page (both built on lua.exec) are skipped.
+// The OS is returned alongside: tools that pick a shortcut modifier
 // (read_page) need it, and re-probing would be a second relay round-trip.
 async function buildInstructions(
   client: RebindRemote,
-): Promise<{ instructions: string; os: OS }> {
+  hello: Hello,
+): Promise<{ instructions: string; os: OS; hasLua: boolean }> {
   const sh = async (cmd: string): Promise<string> => {
     try {
       const r = await client.luaExec(
@@ -297,14 +303,20 @@ async function buildInstructions(
       return "";
     }
   };
-  const uname = await sh("uname");
-  const os: OS = /darwin/i.test(uname)
-    ? "macOS"
-    : /linux/i.test(uname)
-      ? "Linux"
-      : "Windows";
-  const clis = os === "Windows" ? "" : await sh(PROBE_CLIS_CMD);
-  const apps = os === "Windows" ? "" : await sh(probeAppsCmd(os));
+  const hasLua = hello.gates?.lua !== false;
+  const os: OS =
+    HELLO_OS[hello.platform ?? ""] ??
+    (await (async () => {
+      const uname = hasLua ? await sh("uname") : "";
+      return /darwin/i.test(uname)
+        ? "macOS"
+        : /linux/i.test(uname)
+          ? "Linux"
+          : "Windows";
+    })());
+  const probe = hasLua && os !== "Windows";
+  const clis = probe ? await sh(PROBE_CLIS_CMD) : "";
+  const apps = probe ? await sh(probeAppsCmd(os)) : "";
   // Embed the full Luau SDK inline so the model never has to guess run_lua APIs
   // or make a lua_docs round-trip. It carries the coroutine-safety notes that
   // matter for HID: HID.Press sleeps and is coroutine-only, but HID.Down/Up/
@@ -320,8 +332,9 @@ async function buildInstructions(
     apps
       ? `GUI applications installed (open via ${openAppHow(os)}): ${apps}`
       : "",
-    `Work in as few steps as possible. ${DETERMINISTIC_WORK_DOCTRINE}`,
-    toolPriority(true),
+    `Work in as few steps as possible.${hasLua ? ` ${DETERMINISTIC_WORK_DOCTRINE}` : ""}`,
+    SAVE_WORK_DOCTRINE,
+    toolPriority(hasLua),
     BATCHING_DOCTRINE,
     "When you do need to see the screen, prefer screenshot_window over a full-display screenshot — a window shot costs roughly a quarter of the image tokens. Every screenshot stays in your context for the rest of the session, so each one you take keeps costing you on every later turn: verify with values (clipboard, file contents, run_lua reads) where possible.",
     guiTips(os),
@@ -330,14 +343,34 @@ async function buildInstructions(
   ]
     .filter(Boolean)
     .join("\n");
-  return { instructions, os };
+  return { instructions, os, hasLua };
 }
 
+/** the fields of the Remote Control hello banner this server reads */
+type Hello = { platform?: string; gates?: { lua?: boolean } };
+
+const HELLO_OS: Record<string, OS> = {
+  macos: "macOS",
+  linux: "Linux",
+  windows: "Windows",
+};
+
+/** the name Remote Control shows in its approval prompt and remembers */
+const APP_NAME = "Rebind MCP server";
+
 async function main() {
   const url = process.env.REBIND_URL ?? "ws://127.0.0.1:19561";
-  const client = new RebindRemote(url, {
+  // name this client on every connect, reconnects included: Remote Control
+  // approves programs by the name they declare in hello. setState fires before
+  // connect() resolves, so the hello goes out ahead of any tool call.
+  let hello: Promise<Hello> = Promise.resolve({});
+  const client: RebindRemote = new RebindRemote(url, {
     token: process.env.REBIND_TOKEN ?? "",
     timeoutMs: 15000,
+    onStateChange: (s) => {
+      if (s !== "connected") return;
+      hello = client.call<Hello>("hello", { app: APP_NAME }).catch(() => ({}));
+    },
   });
 
   // --selftest: verify the relay is reachable and capturing, then exit. Used by
@@ -346,15 +379,41 @@ async function main() {
   if (process.argv.includes("--selftest")) {
     try {
       await client.connect();
-      const cap = await client.screenCapture({});
+      await hello;
+      // a first connection waits on the approval prompt in Remote Control:
+      // retry for up to a minute while the person decides.
+      const deadline = Date.now() + 60_000;
+      let cap: Awaited<ReturnType<RebindRemote["screenCapture"]>>;
+      for (;;) {
+        try {
+          cap = await client.screenCapture({});
+          break;
+        } catch (e) {
+          const waiting =
+            e instanceof ServerError && e.code === "awaiting_approval";
+          if (!waiting || Date.now() + 2000 > deadline) throw e;
+          console.error(
+            `selftest: allow "${APP_NAME}" in the Remote Control prompt on that computer...`,
+          );
+          await sleep(2000);
+        }
+      }
       console.error(
         `selftest OK: relay ${url}, display ${cap.display.index} ${cap.display.width}x${cap.display.height}`,
       );
       process.exit(0);
     } catch (e) {
+      if (e instanceof ServerError) {
+        console.error(
+          e.code === "awaiting_approval" || e.code === "not_approved"
+            ? `selftest FAILED: "${APP_NAME}" was not allowed in Remote Control's prompt on ${url}. (${e.message})`
+            : `selftest FAILED: Remote Control on ${url} refused the capture. (${e.message})`,
+        );
+        process.exit(1);
+      }
       console.error(
         `selftest FAILED: Rebind relay not reachable on ${url}.\n` +
-          `Start Rebind and the Remote Control package (rebind install @rebind/remote-control).\n(${e})`,
+          `Start Rebind and the Remote Control package (rebind install @rebind/remote-control --allow).\n(${e})`,
       );
       process.exit(1);
     }
@@ -364,7 +423,10 @@ async function main() {
 
   const session = new Session(client);
 
-  const { instructions, os } = await buildInstructions(client);
+  const { instructions, os, hasLua } = await buildInstructions(
+    client,
+    await hello,
+  );
   const server = new McpServer(
     { name: "rebind", version: pkg.version },
     { instructions },
@@ -1080,9 +1142,9 @@ return { ok = Window.IsActive(h), name = name }
   );
 
   // read_page finds the front app through App.Front and reads its result back
-  // through AppleScript (tabUrl): both exist only on a macOS relay. A failed
-  // OS probe reads as "Windows", so this never offers the tool where it fails.
-  if (os === "macOS") {
+  // through AppleScript (tabUrl), both over lua.exec: offered only on a macOS
+  // relay with lua.exec on, so never where it fails.
+  if (os === "macOS" && hasLua) {
     register(
       "read_page",
       {

package/src/prompts.ts

@@ -69,6 +69,10 @@ export function guiTips(os: OS): string {
 export const DETERMINISTIC_WORK_DOCTRINE =
   "run_lua is your PRIMARY tool — reach for it FIRST and use it aggressively. It exposes the ENTIRE machine SDK (every namespace below), so almost anything a task needs — file operations, text processing, arithmetic, clipboard, launching and positioning apps, shell commands, reading the screen — is ONE run_lua call, never a click loop. Do not operate calculators, browsers, or file managers through the GUI when a script can do the work. Fall back to screenshot/click ONLY for state that exists purely on screen and cannot be scripted.";
 
+/** MCP server only (the bench keeps its own prompt): keep what worked. */
+export const SAVE_WORK_DOCTRINE =
+  "When a sequence works and the user wants to keep it, save it as a Rebind script so it runs again with no agent and no tokens. On the computer running Rebind: `rebind init <dir>` scaffolds rebind.toml and main.luau; put the sequence in main.luau bound to a key (Bind or OnDown; hooks never fire inside run_lua, they do in an installed script); add `[runtime] permissions = [...]` to rebind.toml listing only what it uses; then run `rebind check <dir>`, `rebind pack <dir>` (writes <name>.rbp in the current directory), `rebind install <name>.rbp --allow` (it prints what the package asks for; add --force to replace an earlier install) and `rebind script start packages/<name>/main.luau`. Tell the user which key runs it, and that it runs until Rebind restarts and can be started again from Rebind's scripts list.";
+
 // --- tool descriptions (shared only where the underlying behavior is identical
 // across both surfaces: run_lua, lua_docs, and key all hit the same relay) ---
 

package/src/rebind.d.luau

@@ -490,6 +490,57 @@ export type RbBindOptions = {
   device: string?,
 }
 
+-- rb.ai and rb.phone are Rebind Pro (a 7-day trial starts on first use).
+-- Each rb.ai call waits for the model, so call it inside rb.run; its cost
+-- counts against the monthly AI allowance. A script that declares
+-- permissions needs ai, or ai:decide, ai:find or ai:read. A window option
+-- names a window by title (or app name on macOS); the call reads the screen
+-- pixels inside that window's rectangle, through the screen: permission.
+
+export type AiDecideOptions = {
+  choices: { string },
+  -- send the display under the cursor (the target window under screen:target)
+  screen: boolean?,
+  -- send this window instead
+  window: string?,
+}
+
+export type AiFindOptions = {
+  -- look in this window; default: the display under the cursor
+  window: string?,
+}
+
+export type AiReadOptions = {
+  window: string?,
+  -- read only around this point (screen coordinates, such as rb.ai.find's)
+  at: AiPoint?,
+}
+
+-- screen coordinates, for rb.hid.move_to
+export type AiPoint = {
+  x: number,
+  y: number,
+}
+
+export type RbAi = {
+  -- picks one of opts.choices; returns it and the model's confidence (0 to 1)
+  decide: (question: string, opts: AiDecideOptions) -> (string, number),
+  -- the centre of what is described, or nil when it is not visible
+  find: (what: string, opts: AiFindOptions?) -> AiPoint?,
+  -- the text described, as it appears on screen
+  read: (what: string, opts: AiReadOptions?) -> string,
+}
+
+export type PhoneResult = {
+  delivered: boolean,
+  reason: string?,
+}
+
+export type RbPhone = {
+  -- no phone app yet: returns { delivered = false, reason = "no phone connected" }
+  notify: (text: string, title: string?) -> PhoneResult,
+}
+
 -- events for rb.on (Rebind 3.9.0 and later), with the arguments each
 -- handler receives. For down, up, move and scroll, a handler that returns
 -- false swallows the input.
@@ -690,7 +741,7 @@ declare rb: {
     get_pos: (handle: number?) -> WindowPos,
     get_pid: (handle: number?) -> number,
     activate: (handle: number) -> (),
-    move: (handle: number, rect: RbWindowRect?) -> (),
+    move: (handle: number?, rect: RbWindowRect?) -> (),
     set_always_on_top: (handle: number, enabled: boolean) -> (),
     set_transparency: (handle: number, alpha: number) -> (),
     set_title: (handle: number, title: string) -> (),
@@ -839,6 +890,9 @@ declare rb: {
     delete_key: (key_path: string) -> (),
     create_key: (key_path: string) -> (),
   },
+  -- Rebind Pro; require("@rebind/pro") returns rb
+  ai: RbAi,
+  phone: RbPhone,
 }
 
 -- v1 names
@@ -1041,7 +1095,7 @@ declare Window: {
   GetPos: (handle: number?) -> WindowPos,
   GetPID: (handle: number?) -> number,
   Activate: (handle: number) -> (),
-  Move: (handle: number, x: number?, y: number?, width: number?, height: number?) -> (),
+  Move: (handle: number?, x: number?, y: number?, width: number?, height: number?) -> (),
   SetAlwaysOnTop: (handle: number, enabled: boolean) -> (),
   SetTransparency: (handle: number, alpha: number) -> (),
   SetTitle: (handle: number, title: string) -> (),