@rebind.gg/mcp-server 0.1.5 → 0.2.0

package/src/keys.test.ts

@@ -0,0 +1,15 @@
+import { expect, test } from "bun:test";
+import { normalizeCombo } from "./keys";
+
+test("normalizes every Command spelling models emit", () => {
+  for (const alias of ["LCmd", "Cmd", "Command", "⌘", "Super", "LWin", "Win"]) {
+    expect(normalizeCombo(`${alias}+L`)).toBe("LMeta+L");
+  }
+  expect(normalizeCombo("RCmd+Tab")).toBe("RMeta+Tab");
+});
+
+test("passes through already-valid combos and bare keys", () => {
+  expect(normalizeCombo("LCtrl+Shift+P")).toBe("LCtrl+Shift+P");
+  expect(normalizeCombo("Enter")).toBe("Enter");
+  expect(normalizeCombo(" LMeta + A ")).toBe("LMeta+A");
+});

package/src/keys.ts

@@ -0,0 +1,31 @@
+// Shared key-combo normalization for every surface that sends model-authored
+// chords to the relay (the MCP server's `key` tool and the bench executor).
+//
+// Models emit "LCmd"/"Cmd"/"Command"/"⌘" for the macOS Command key, but the SDK
+// key parser only knows LMeta/Meta/Cmd (not the L-prefixed "LCmd"). Without
+// normalization the modifier is silently dropped and the bare letter types.
+
+const META = new Set([
+  "LCMD",
+  "CMD",
+  "COMMAND",
+  "⌘",
+  "LMETA",
+  "META",
+  "SUPER",
+  "LWIN",
+  "WIN",
+]);
+const RMETA = new Set(["RCMD", "RCOMMAND", "RMETA", "RWIN"]);
+
+export function normalizeCombo(combo: string): string {
+  return combo
+    .split("+")
+    .map((t) => {
+      const u = t.trim().toUpperCase();
+      if (META.has(u)) return "LMeta";
+      if (RMETA.has(u)) return "RMeta";
+      return t.trim();
+    })
+    .join("+");
+}

package/src/llm.ts

@@ -0,0 +1,157 @@
+// Minimal OpenRouter tool-calling step. LEAF module: node fs/fetch only, ZERO
+// imports from the MCP SDK / image pipeline / session — the bench re-exports
+// llmStep from here and must never transitively pull those in (see the covenant
+// in ./prompts.ts). Both the bench loop and the server's delegate_task /
+// rebind_versus tools call it.
+//
+// Deliberately NOT @rebind/llm: that transport force-adds a web_search tool and
+// drops usage. Here we want a clean, controlled step — tools in, the raw
+// assistant message + parsed tool calls + exact usage out. Temperature 0.
+
+const URL =
+  process.env.OPENROUTER_URL || "https://openrouter.ai/api/v1/chat/completions";
+// hard ceiling on one model round-trip. fetch has no built-in timeout, so a
+// wedged connection would hang forever; abort so the caller fails and moves on.
+const LLM_TIMEOUT_MS = Number(process.env.BENCH_LLM_TIMEOUT || 60000);
+const MAX_RETRIES = Number(process.env.OPENROUTER_MAX_RETRIES || 3);
+
+// key resolution: env var first, then the repo root's decrypted .env.production
+// so `just bench-demo` (and a bare server) work from a fresh shell without
+// exporting anything. Resolved LAZILY — importing this module must have no fs
+// side effect (the production server imports it just to register two tools).
+let KEY: string | undefined;
+function key(): string {
+  if (KEY !== undefined) return KEY;
+  KEY = process.env.OPENROUTER_API_KEY || keyFromRootEnv();
+  return KEY;
+}
+function keyFromRootEnv(): string {
+  try {
+    const txt = require("node:fs").readFileSync(
+      new globalThis.URL("../../../.env.production", import.meta.url),
+      "utf8",
+    ) as string;
+    return /^OPENROUTER_API_KEY=(.*)$/m.exec(txt)?.[1]?.trim() ?? "";
+  } catch {
+    return "";
+  }
+}
+
+export interface Tool {
+  type: "function";
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+
+// OpenAI/OpenRouter message shape. `content` is a string or multimodal parts.
+export type Msg = Record<string, unknown>;
+
+export interface ParsedCall {
+  id: string;
+  name: string;
+  args: Record<string, unknown>;
+}
+
+export interface Step {
+  /** raw assistant message, pushed back verbatim so tool_call_ids line up */
+  message: Msg;
+  toolCalls: ParsedCall[];
+  /** OpenRouter-reported usage for THIS request; cost is exact USD credits. */
+  usage?: { prompt: number; completion: number; cost: number };
+}
+
+const safeParse = (s: string): Record<string, unknown> => {
+  try {
+    return JSON.parse(s || "{}");
+  } catch {
+    return {};
+  }
+};
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export async function llmStep(
+  messages: Msg[],
+  tools: Tool[],
+  model: string,
+): Promise<Step> {
+  if (!key()) throw new Error("set OPENROUTER_API_KEY (Bun auto-loads .env)");
+
+  // usage.include makes OpenRouter return exact token counts + USD cost per
+  // request — no client-side pricing table to drift. Omit `tools` entirely when
+  // empty: strict providers 400 on an empty tools array.
+  const body = JSON.stringify({
+    model,
+    messages,
+    ...(tools.length ? { tools } : {}),
+    max_tokens: 4096,
+    temperature: 0,
+    usage: { include: true },
+  });
+
+  let res: Response;
+  for (let attempt = 0; ; attempt++) {
+    try {
+      res = await fetch(URL, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${key()}`,
+          "Content-Type": "application/json",
+          "HTTP-Referer": "https://rebind.gg",
+          "X-Title": "Rebind Agent Bench",
+        },
+        body,
+        signal: AbortSignal.timeout(LLM_TIMEOUT_MS),
+      });
+    } catch (e) {
+      if (e instanceof Error && e.name === "TimeoutError") {
+        throw new Error(
+          `openrouter request timed out after ${LLM_TIMEOUT_MS}ms`,
+        );
+      }
+      throw e;
+    }
+    if (res.ok) break;
+    // 429 (rate limit) and 5xx are transient — parallel fan-out reliably trips
+    // 429 on the flash tiers. Back off and retry; surface anything else at once.
+    if ((res.status === 429 || res.status >= 500) && attempt < MAX_RETRIES) {
+      await sleep(400 * 2 ** attempt);
+      continue;
+    }
+    throw new Error(`openrouter ${res.status}: ${await res.text()}`);
+  }
+
+  const j = (await res.json()) as {
+    choices?: {
+      message?: {
+        content?: string | null;
+        tool_calls?: {
+          id: string;
+          function: { name: string; arguments: string };
+        }[];
+      };
+    }[];
+    usage?: {
+      prompt_tokens?: number;
+      completion_tokens?: number;
+      cost?: number;
+    };
+  };
+  const m = j.choices?.[0]?.message ?? { content: null };
+  const toolCalls: ParsedCall[] = (m.tool_calls ?? []).map((t) => ({
+    id: t.id,
+    name: t.function.name,
+    args: safeParse(t.function.arguments),
+  }));
+  const usage = j.usage
+    ? {
+        prompt: j.usage.prompt_tokens ?? 0,
+        completion: j.usage.completion_tokens ?? 0,
+        cost: j.usage.cost ?? 0,
+      }
+    : undefined;
+  return { message: m as Msg, toolCalls, usage };
+}

package/src/prompts.ts

@@ -0,0 +1,101 @@
+// Canonical agent-facing copy, shared by the Rebind MCP server (this package)
+// and the agent bench (packages/mcp-server/bench). ONE source of truth for the tool
+// descriptions, efficiency doctrine, and machine-probe commands that both
+// surfaces show the model, so they can never drift.
+//
+// PURE strings + string builders — ZERO runtime imports. That is deliberate:
+// the colocated bench (../bench) imports this by relative path and must NOT pull
+// in the MCP SDK, image pipeline, or calibration that live elsewhere in src/.
+//
+// What is NOT here, by design: the screenshot/click descriptions. The server's
+// click is verified + calibrated; the bench's is a primitive hidMoveTo+press.
+// A shared description would make one of them lie, so each keeps its own.
+
+export type OS = "macOS" | "Linux" | "Windows";
+
+// --- machine probes (run relay-side through System.Exec) ---
+
+/** CLIs worth surfacing to the model; prints the ones present, space-separated. */
+export const PROBE_CLIS_CMD = `for c in python3 osascript jq curl sed awk sort uniq grep wc open unzip tar; do command -v $c >/dev/null 2>&1 && printf "%s " $c; done`;
+
+/** List up to 40 installed GUI apps, comma-separated, for the given OS. */
+export function probeAppsCmd(os: OS): string {
+  return os === "macOS"
+    ? `ls /Applications /System/Applications 2>/dev/null | grep '\\.app$' | sed 's/\\.app$//' | sort -u | head -40 | tr '\\n' ','`
+    : `ls /usr/share/applications 2>/dev/null | sed 's/\\.desktop$//' | sort -u | head -40 | tr '\\n' ','`;
+}
+
+/** How to launch a GUI app by name via run_lua, per OS. */
+export function openAppHow(os: OS): string {
+  if (os === "macOS") return 'System.ExecDetached("open", {"-a", "<Name>"})';
+  if (os === "Linux") return 'System.ExecDetached("<name>", {})';
+  return 'System.ExecDetached("cmd", {"/c","start","","<name>"})';
+}
+
+/** The shortcut modifier per OS — the #1 fix for wrong-key shortcut failures.
+ * Returned as a fragment the caller wraps ("...shortcuts use <this>."). */
+export function modifierNote(os: OS): string {
+  return os === "macOS"
+    ? "LMeta (Command): LMeta+A select all, LMeta+C copy, LMeta+V paste, LMeta+L focus the browser address bar"
+    : "LCtrl: LCtrl+A select all, LCtrl+C copy, LCtrl+V paste, LCtrl+L focus the address bar";
+}
+
+// --- efficiency doctrine ---
+
+/** Tool-preference ladder. run_lua only appears when the surface/arm has it. */
+export function toolPriority(hasLua: boolean): string {
+  return `Tool priority, most to least preferred: ${hasLua ? "script (run_lua) > " : ""}keyboard > mouse > screenshot. If an application accepts keyboard input for the job — typed digits and operators, shortcuts, menu accelerators, type-to-select — use the keyboard: it is deterministic and needs no coordinates. Use the mouse only when no keyboard path exists, and screenshots only when you genuinely need to see state.`;
+}
+
+export const BATCHING_DOCTRINE =
+  "When you must drive the GUI: do not verify after every action. Batch predictable action sequences — keystrokes especially — and verify once at the end via values (file contents, clipboard, command output) rather than screenshots.";
+
+/** GUI traps that reliably strand an agent, per OS. Returned as one line or "". */
+export function guiTips(os: OS): string {
+  if (os === "macOS")
+    return (
+      "GUI traps to avoid on macOS: (1) In Finder, Return/Enter RENAMES the selected item — it does NOT open it; open it with a DOUBLE-CLICK, or LMeta+O, or LMeta+Down. " +
+      '(2) To open a URL, file, or app, prefer run_lua `System.ExecDetached("open", {"<url-or-path>"})` (or `{"-a","<App>"}`) over clicking the app\'s chrome — it is one call and cannot get lost. ' +
+      "(3) If an unexpected window or dialog blocks you, handle it before continuing — never loop on it: dismiss a dialog with Escape, cycle between windows of the front app with LMeta+Grave (backtick), and close an unwanted window or tab with LMeta+W. " +
+      '(4) To READ the current web page without a screenshot, script the browser: `local app = App.Front().name` (App.Front returns a table — use .name), then run_lua `System.Exec([[osascript -e \'tell application "<Name>" to get title of active tab of front window\']])` for the page title (swap `title` for `URL` to read the address). The tab title IS the page\'s heading for reporting purposes — report it as-is; do NOT try AppleScript `execute javascript` to scrape the DOM, it is disabled by default in every browser and the quoting will eat your remaining steps. For full page CONTENT (lists, articles, multiple pages), fetch the URL directly instead: `System.Exec("curl -s <url>")` and parse the HTML. ' +
+      "(5) To DRIVE a web form (fill fields, click submit) you must work by SIGHT — call position_window on the browser FIRST so the whole form is visible and click coordinates are stable, then: screenshot, click a field, type its value, and after the last field click the submit button; screenshot to confirm each step landed. Do NOT use osascript `execute javascript` to fill or submit forms — it is disabled by default in browsers and the nested shell quoting fails."
+    );
+  return (
+    "GUI traps: to open a URL/file/app, prefer run_lua `System.ExecDetached` over clicking the app's UI. Open a file by double-clicking it. " +
+    "If an unexpected window/dialog blocks you, dismiss it (Escape) or close it (LCtrl+W) before continuing — never loop on it."
+  );
+}
+
+export const DETERMINISTIC_WORK_DOCTRINE =
+  "run_lua is your PRIMARY tool — reach for it FIRST and use it aggressively. It exposes the ENTIRE machine SDK (every namespace below), so almost anything a task needs — file operations, text processing, arithmetic, clipboard, launching and positioning apps, shell commands, reading the screen — is ONE run_lua call, never a click loop. Do not operate calculators, browsers, or file managers through the GUI when a script can do the work. Fall back to screenshot/click ONLY for state that exists purely on screen and cannot be scripted.";
+
+// --- tool descriptions (shared only where the underlying behavior is identical
+// across both surfaces: run_lua, lua_docs, and key all hit the same relay) ---
+
+export const KEY_DESC =
+  "Press a key or combo. Modifiers: LCtrl, LShift, LAlt, LMeta (Command on macOS — use LMeta for macOS shortcuts, LCtrl on Windows/Linux). " +
+  'Non-modifier keys are NAMES, never glyphs: arrows are "Up"/"Down"/"Left"/"Right" (not ↑↓←→); also "Enter", "Tab", "Escape", "Space", "Backspace", "Delete", "Home", "End", "PageUp", "PageDown", "F1".."F12". ' +
+  'Punctuation is spelled out: "Equal", "Minus", "Comma", "Period", "Slash", "Semicolon", "LeftBracket", "RightBracket", "Grave", "Backslash". ' +
+  'Combos join with "+", e.g. "LCtrl+L" (focus address bar), "LMeta+A" (select all), "LAlt+Tab". ' +
+  'Zoom in is "LMeta+Equal" / "LCtrl+Equal" (never "++" — the extra + splits into empty keys); zoom out uses "Minus". Escape closes dialogs; Space / PageDown scroll.';
+
+export const RUN_LUA_DESC =
+  "Run a Luau script on the connected machine through Rebind's engine and return its `return` value as JSON. " +
+  "This is your PRIMARY tool and FIRST choice — it exposes the ENTIRE machine SDK (every namespace listed below; call lua_docs for the full function list) and is far faster, cheaper, and more reliable than the screenshot-and-click loop, which it sidesteps entirely. Default to it for ANYTHING that can be scripted. " +
+  "The chunk runs ONCE and must end with `return <value>`: script hooks (OnDown/OnTick) never fire here, and coroutine-only calls " +
+  '(Sleep, HID.Press, Dialog file pickers, Screen.Capture, Screen.FindImage, Screen.SearchForColor over 200k pixels) error — wait with System.Exec("sleep 0.5") instead, and take screenshots with the screenshot tool, not from a chunk. There is no Lua stdlib `io`/`os`/`require`. ' +
+  "Use it to launch apps (System.ExecDetached), place windows (Window.Wait / Window.Move — on macOS the window handle is advisory and Move targets the FRONTMOST window, so bring the app forward first), run shell commands " +
+  "(System.Exec(cmd) -> {exit,stdout,stderr}, absolute paths fine; inside a chunk it runs asynchronously on a background thread and does not block, so a slow command is safe), find UI by color in a small region (Screen.SearchForColor({x1,y1,x2,y2}, \"rrggbb\", tolerance), region under 200k pixels), type, click, read pixels (Screen.GetPixelColor), manage processes. " +
+  "File.* (Read/Write/Append/ReadJSON/WriteJSON/Exists/Delete/List/MkDir/RmDir/Copy/Move/GetSize/GetTime/GetScriptDir) is jailed to the script dir: paths RELATIVE, absolute rejected. " +
+  'A folder described as on the Desktop or in home is reachable at its BARE name from the jail root (e.g. File.List("myfolder")) — never prefix "Desktop/" or "~". ' +
+  "JSON is JSON.Parse(s) and JSON.Stringify(v) — there is no JSON.Decode/Encode; for JSON files File.ReadJSON/File.WriteJSON is one call. " +
+  "System.Exec commands run in an UNRELATED working directory — a relative shell path does NOT land in the File jail, so do file work with File.* (File.Write always works in the jail), and use absolute paths inside shell commands. " +
+  'System.Exec takes ONE shell-command STRING — System.Exec("ls /tmp") — and its optional second parameter is {timeout, cwd}, NOT an argument array (an args table there is silently ignored and the bare command runs); only System.ExecDetached takes an args array. ' +
+  'There is NO io.* or os.* — file work is File.*, shell work is System.Exec. App has Front/IsFront/IsRunning/Activate/Hide/Quit only — there is no App.List (installed apps are listed in your instructions). HID.Press takes ONE key; fire chords with HID.Combo("LMeta+C"). ' +
+  "All namespaces: System, HID, Input, Screen, Window, App, Process, File, Clipboard, Net, Env, Log, UI, Bind, Timer, Math, JSON, Pipe, Audio, Dialog, Script, Regex, Config, Hash, Codec, Macro, Registry. " +
+  'Example (move all .log files into d/logs and report the count): `File.MkDir("d/logs"); local n=0; for _,f in ipairs(File.List("d")) do if f:match("%.log$") then File.Move("d/"..f, "d/logs/"..f); n=n+1 end end; return n`. ' +
+  "Call `lua_docs` for exact signatures BEFORE writing nontrivial scripts — do not guess APIs.";
+
+export const LUA_DOCS_DESC =
+  "Return the full Luau SDK type definitions for run_lua: every namespace, function signature, and type (~5k tokens, one call is enough). " +
+  "Read this BEFORE writing any nontrivial run_lua script instead of guessing APIs.";

package/src/rebind.d.luau

@@ -183,51 +183,38 @@ export type MacroAction = MacroMoveAction | MacroExplicitAction
 
 -- ui widget option types
 
-export type ToggleOpts = {
+export type WidgetOpts = {
   label: string?,
+  tooltip: string?,
+  group: string?,
+  tab: string?,
+  showIf: string?,
 }
 
-export type SliderOpts = {
-  label: string?,
+export type ToggleOpts = WidgetOpts
+
+export type SliderOpts = WidgetOpts & {
   min: number?,
   max: number?,
   step: number?,
   suffix: string?,
-  tooltip: string?,
-  group: string?,
-  tab: string?,
-  showIf: string?,
 }
 
-export type KeybindOpts = {
-  label: string?,
-}
+export type KeybindOpts = WidgetOpts
 
-export type SelectOpts = {
-  label: string?,
-}
+export type SelectOpts = WidgetOpts
 
-export type TextOpts = {
-  label: string?,
+export type TextOpts = WidgetOpts & {
   placeholder: string?,
   maxLength: number?,
-  tooltip: string?,
-  group: string?,
-  tab: string?,
-  showIf: string?,
 }
 
-export type ColorOpts = {
-  label: string?,
-  tooltip: string?,
-  group: string?,
-  tab: string?,
-  showIf: string?,
-}
+export type ColorOpts = WidgetOpts
 
 export type RecordOpts = {
   ignore_mouse: boolean?,
   ignore_keyboard: boolean?,
+  -- default "normal"
   precision: string?,
 }
 
@@ -251,14 +238,20 @@ export type RebindInfo = {
   runtime_uuid: string,
   id: string?,
   instance: string,
+  -- host operating system: "windows" | "macos" | "linux"
+  platform: string,
 }
 
 -- global functions
 
 declare function Run(fn: () -> ()): TaskHandle
 -- RunAsync was removed. Use Run() directly.
+-- coroutine-only: Sleep must be called inside Run()/a scheduled coroutine.
+-- Calling it from a plain hook body or a one-shot exec chunk errors at runtime.
 declare function Sleep(ms: number): ()
 declare function After(ms: number, fn: () -> ()): TaskHandle
+-- wraps fn so it runs on the coroutine scheduler; when auto-spawned the call
+-- returns false instead of fn's results.
 declare function Async(fn: (...any) -> ...any): (...any) -> ...any
 declare function print(...: any): ()
 declare function require(module: string): any
@@ -270,7 +263,9 @@ declare _REBIND: RebindInfo
 -- system namespace
 
 export type ExecOptions = {
+  -- milliseconds; default 5000. The process is killed on expiry and Exec errors.
   timeout: number?,
+  -- working directory for the command; default: the host process's cwd.
   cwd: string?,
 }
 
@@ -289,8 +284,18 @@ declare System: {
   Mouse: () -> (number, number),
   Screen: () -> (number, number),
   Window: () -> WindowInfo,
+  -- runs cmd through the shell (sh -c / cmd.exe), output capped at 64KB per
+  -- stream. Requires the 'exec' permission. Return shape {exit,stdout,stderr}
+  -- unchanged. Inside Run()/a coroutine it runs async off the tick thread
+  -- (yields, resumes with the result on a later tick). From a plain hook body
+  -- or one-shot chunk it falls back to bounded blocking (deprecated; warns
+  -- once, blocks the tick thread — wrap it in Run()).
   Exec: (cmd: string, options: ExecOptions?) -> ExecResult,
+  -- fire-and-forget spawn (no shell, argv array); returns the pid.
   ExecDetached: (cmd: string, args: { string }?, options: ExecDetachedOptions?) -> number,
+  -- types text through the host OS (Unicode aware), never through the HID
+  -- transport. raises when the OS refuses the injection.
+  TypeText: (text: string) -> (),
 }
 
 -- hid namespace
@@ -298,6 +303,8 @@ declare System: {
 declare HID: {
   Down: (code: string) -> (),
   Up: (code: string) -> (),
+  -- coroutine-only (sleeps for holdMs): call inside Run(). From a plain
+  -- chunk use Down+Up or Combo instead.
   Press: (code: string, holdMs: number?) -> (),
   -- fire a chord in one synchronous call (press L→R, release R→L). no sleep,
   -- so callable directly from a Bind action / OnDown. modifier tokens accept
@@ -361,6 +368,8 @@ declare Timer: {
 -- macro namespace
 
 declare Macro: {
+  -- mode: "parallel" (default, plays alongside running macros) | "replace"
+  -- (stops all running macros first).
   Play: (macro: { MacroAction }, speed: number?, mode: string?) -> MacroHandle,
   StopAll: () -> (),
   Record: (options: RecordOpts?) -> (),
@@ -378,6 +387,7 @@ declare Math: {
   Spline: (macro: { MacroAction }, tension: number) -> { MacroAction },
   Resample: (macro: { MacroAction }, intervalMs: number) -> { MacroAction },
   TimeComp: (macro: { MacroAction }, targetMs: number) -> { MacroAction },
+  -- mode: "linear" | anything else = positional curve (default).
   Interpolate: (macro: { MacroAction }, intervalMs: number, mode: string?) -> { MacroAction },
 }
 
@@ -426,10 +436,52 @@ export type MonitorEntry = {
   primary: boolean,
 }
 
+export type CaptureOptions = {
+  -- 1-based display index; default: the display under the cursor.
+  display: number?,
+  -- display-local logical coords; default: the whole display.
+  region: { x: number, y: number, w: number, h: number }?,
+  -- downscale so the long edge is at most this; default and 0: native.
+  max_edge: number?,
+}
+
+export type CaptureResult = {
+  image: string, -- base64 PNG
+  width: number,
+  height: number,
+  display: MonitorEntry,
+  cursor: { x: number, y: number },
+}
+
+export type FindImageOptions = {
+  display: number?,
+  region: { x: number, y: number, w: number, h: number }?,
+  -- 0.0 exact .. 1.0, per-channel fraction of 255 (same scale as SearchForColor).
+  tolerance: number?,
+  -- ratios to resample the needle by before scanning, tried in order; default { 1 }.
+  scales: { number }?,
+}
+
+export type ImageMatch = {
+  x: number,
+  y: number,
+  width: number,
+  height: number,
+}
+
 declare Screen: {
   GetPixelColor: (x: number?, y: number?) -> string,
+  -- reads the published frame. regions up to 200k physical pixels answer on the
+  -- calling thread; larger ones yield inside Run() and are refused outside it.
   SearchForColor: (region: {number}, color: string, tolerance: number?) -> PixelMatch?,
   List: () -> { MonitorEntry },
+  -- yields inside Run(); refused outside a coroutine.
+  Capture: (options: CaptureOptions?) -> CaptureResult,
+  -- needle is a PNG path relative to the script directory. yields inside Run(); refused outside a coroutine.
+  FindImage: (needle: string, options: FindImageOptions?) -> ImageMatch?,
+  -- usable desktop of the display containing the point (cursor when omitted).
+  -- Windows only; raises elsewhere.
+  WorkArea: (x: number?, y: number?) -> WindowPos,
 }
 
 -- window namespace
@@ -455,6 +507,10 @@ declare Window: {
   Kill: (handle: number?) -> (),
   IsVisible: (handle: number) -> boolean,
   IsActive: (handle: number?) -> boolean,
+  GetForeground: () -> number?,
+  FromPoint: (x: number, y: number) -> number?,
+  -- "minimized" | "maximized" | "normal". Windows only; raises elsewhere.
+  GetState: (handle: number?) -> string,
   Wait: (title: string, timeout: number?) -> number?,
   WaitActive: (title: string, timeout: number?) -> boolean,
   WaitClose: (title: string, timeout: number?) -> boolean,
@@ -510,16 +566,19 @@ declare File: {
   Append: (path: string, content: string) -> (),
   Exists: (path: string) -> boolean,
   Delete: (path: string) -> boolean,
+  -- nil path lists the script's own directory. Paths are relative to the
+  -- script dir (the File.* jail); absolute paths are rejected.
   List: (path: string?) -> { string },
   MkDir: (path: string) -> boolean,
-  RmDir: (path: string) -> (),
+  -- recursive; returns true even when the directory did not exist.
+  RmDir: (path: string) -> boolean,
+  Copy: (src: string, dst: string) -> (),
+  Move: (src: string, dst: string) -> (),
   ReadJSON: (path: string) -> any,
   WriteJSON: (path: string, value: any) -> (),
   GetScriptDir: () -> string,
   GetSize: (path: string) -> number,
   GetTime: (path: string) -> number,
-  Copy: (src: string, dst: string) -> (),
-  Move: (src: string, dst: string) -> (),
   IsDir: (path: string) -> boolean,
   IsFile: (path: string) -> boolean,
 }
@@ -566,6 +625,8 @@ export type DialogFileOptions = {
   filters: { DialogFileFilter }?,
 }
 
+-- every Dialog call is coroutine-only: it yields until the dialog closes and
+-- errors outside Run(). the file and folder pickers return nil on cancel.
 declare Dialog: {
   Message: (text: string, options: DialogMessageOptions?) -> (),
   Confirm: (text: string, options: DialogMessageOptions?) -> boolean,
@@ -694,5 +755,9 @@ declare Registry: {
 -- declare function OnMove(dx: number, dy: number): boolean?  -- return value only used with mouse_block=true (hardware mode only)
 -- declare function OnScroll(delta: number): boolean?
 -- declare function OnScrollH(delta: number): boolean?
--- declare function OnError(): ()
+-- declare function OnInputDown(key: string): ()
+-- declare function OnInputUp(key: string, durationMs: number): ()
+-- declare function OnInputMove(dx: number, dy: number): ()
+-- declare function OnInputScroll(delta: number): ()
+-- declare function OnError(message: string): ()
 -- declare function OnReload(): ()

package/src/session.ts

@@ -77,7 +77,8 @@ export class Session {
     }
     if (!inImageBounds(this.lastShot, ix, iy)) {
       throw new Error(
-        `(${ix}, ${iy}) is outside the last screenshot (${this.lastShot.imageWidth}x${this.lastShot.imageHeight}) — take a fresh screenshot`,
+        `(${ix}, ${iy}) is outside the last screenshot, which is ${this.lastShot.imageWidth}x${this.lastShot.imageHeight} pixels, 0-indexed from the top-left. ` +
+          `Emit pixel coordinates within it — if you emitted 0-1000-normalized coordinates, multiply by the image dimensions instead — or take a fresh screenshot.`,
       );
     }
     return mapToScreen(this.lastShot, ix, iy);