@rebind.gg/mcp-server 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -55,8 +55,8 @@ Connects, takes a capture, and reports the display — the fastest way to catch
55
55
  | `type` | Type text into the focused element. |
56
56
  | `key` | Press a key or combo, e.g. `Enter`, `LCtrl+L`, `LAlt+Tab`. |
57
57
  | `scroll` | Scroll at the cursor. Positive scrolls down, negative up. `unit` is `wheel` (default, fine wheel clicks) or `page` (one PageDown or PageUp per unit, capped at 10). |
58
- | `read_page` | Extract the focused browser's current page as structured text (URL, forms, interactive elements, plain text) via an address-bar bookmarklet; the full dump, including cleaned HTML, lands in the clipboard. The result is read back through AppleScript (`URL of active tab of front window`), so the relay host must be macOS and the front browser must answer that query. Safari is detected but blocks the bookmarklet path. |
59
- | `meditate` | Research a problem page until a reasoning model has the data to solve it: fetches the URL directly, pulls the script bundles behind a JS shell so its API endpoints are visible, fetches those endpoints and assets, and reads the live page with vision. Returns a markdown dossier plus JSON `{error, problem, requirements, plan, submittable, form, live_captcha_hint, confidence, sources, assets, pending_fetch, notes, cost_usd}`. `error` is non-null when no problem could be identified or the page could not be read; partial data is still returned. Gated behind `REBIND_ALLOW_DELEGATION=1` (spends OpenRouter credits). See the pipeline below. |
58
+ | `read_page` | Extract the focused browser's current page as structured text (URL, forms, interactive elements, plain text) via an address-bar bookmarklet; the full dump, including cleaned HTML, lands in the clipboard. The result is read back through AppleScript (`URL of active tab of front window`), so the tool is registered only when the relay host is macOS, and the front browser must answer that query. Safari is detected but blocks the bookmarklet path. |
59
+ | `meditate` | Research a problem page until a reasoning model has the data to solve it: fetches the URL directly, pulls the script bundles behind a JS shell so its API endpoints are visible, fetches those endpoints and assets, and reads the live page with vision. The browser steps (rendering a JS shell, same-origin fetches through the page, and the vision pass) need a macOS relay; elsewhere it reads pages over HTTP only. Returns a markdown dossier plus JSON `{error, problem, requirements, plan, submittable, form, live_captcha_hint, confidence, sources, assets, pending_fetch, notes, cost_usd}`. `error` is non-null when no problem could be identified or the page could not be read; partial data is still returned. Gated behind `REBIND_ALLOW_DELEGATION=1` (spends OpenRouter credits). See the pipeline below. |
60
60
  | `run_lua` | Run a Luau script on the connected machine and return its `return` value as JSON — the fast path for any deterministic sequence (launch apps, place windows, shell commands, pixel search). |
61
61
  | `lua_docs` | Full Luau SDK type definitions for `run_lua`. |
62
62
  | `list_displays` / `list_windows` / `focus_window` / `position_window` / `active_window` | Display and window enumeration, focus, and placement. |
@@ -90,7 +90,7 @@ Connects, takes a capture, and reports the display — the fastest way to catch
90
90
 
91
91
  `meditate` gathers; it does not submit. The intended flow, all gated behind `REBIND_ALLOW_DELEGATION=1`:
92
92
 
93
- 1. **`meditate`** → a dossier. Evidence is gathered by **fetching, not by driving the GUI**: a direct GET returns the server's own bytes, and when that comes back a JS shell the script bundles are pulled in so the endpoints the page calls are visible to the model, which then requests them by id. Same-origin URLs are fetched **from inside the page** — the browser's own session, cookies and TLS answer them, so an endpoint that refuses a bare curl responds normally; cross-origin falls back to relay-side curl. The rendered DOM is read when a fetch cannot see the content. It then leaves the browser **open on the problem page** and returns `form` (per-field `{value, source, op}`, traced to evidence) and `submittable`. It never bakes in the captcha — that can rotate, so it is read live at submit time, and its absence does not make a dossier unsubmittable.
93
+ 1. **`meditate`** → a dossier. Evidence is gathered by **fetching, not by driving the GUI**: a direct GET returns the server's own bytes, and when that comes back a JS shell the script bundles are pulled in so the endpoints the page calls are visible to the model, which then requests them by id. On a macOS relay, same-origin URLs are fetched **from inside the page** — the browser's own session, cookies and TLS answer them, so an endpoint that refuses a bare curl responds normally; cross-origin falls back to relay-side curl. The rendered DOM is read when a fetch cannot see the content, also on a macOS relay only. It then leaves the browser **open on the problem page** and returns `form` (per-field `{value, source, op}`, traced to evidence) and `submittable`. It never bakes in the captcha — that can rotate, so it is read live at submit time, and its absence does not make a dossier unsubmittable.
94
94
  2. **`rebind_versus`** → feed it the dossier to attack the reconciled values before anything irreversible happens.
95
95
  3. **Solve (caller-driven, by sight)** on the still-open page: re-navigate to the anchor and confirm the URL; screenshot → click field → type each `form` value; tick attachment checkboxes; read the **live** captcha off the page; then **re-extract the filled form and diff it against the dossier** — on any mismatch or unresolved conflict, stop (do not submit). Only on a clean diff, click submit and verify. When `submittable` is false there is no form to fill; surface the answer instead.
96
96
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@rebind.gg/mcp-server",
3
- "version": "0.2.0",
3
+ "version": "0.2.2",
4
4
  "description": "MCP server exposing Rebind as computer-use tools: screenshot, verified click, type, key, scroll, and window control over the Remote Control protocol.",
5
5
  "type": "module",
6
6
  "bin": {
package/src/index.ts CHANGED
@@ -12,6 +12,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
12
12
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
13
13
  import { RebindRemote } from "@rebind.gg/client-ts";
14
14
  import { z } from "zod";
15
+ import pkg from "../package.json";
15
16
  import { mapToImage } from "./coords.ts";
16
17
  import { renderScreenshot } from "./image.ts";
17
18
  import { normalizeCombo } from "./keys.ts";
@@ -97,8 +98,6 @@ function versusMessages(
97
98
  // asked to re-emit the whole growing dossier blows the model's output cap and
98
99
  // truncates the JSON; splitting keeps every response small and bounds cost.
99
100
 
100
- /** stop the loop once the model reports at least this much confidence */
101
- const MEDITATE_CONFIDENT = 0.8;
102
101
  /** max distinct pages read (problem page + follows) before the loop must stop */
103
102
  const MEDITATE_MAX_PAGES = 10;
104
103
  // Budgets sized for a 1M-token context, not a 4k one. These were tight enough
@@ -358,7 +357,7 @@ async function main() {
358
357
 
359
358
  const { instructions, os } = await buildInstructions(client);
360
359
  const server = new McpServer(
361
- { name: "rebind", version: "0.1.6" },
360
+ { name: "rebind", version: pkg.version },
362
361
  { instructions },
363
362
  );
364
363
 
@@ -979,9 +978,10 @@ return { ok = true, proc = proc }
979
978
  },
980
979
  async ({ handle }) => {
981
980
  // handle-based activation can raise without focusing on macOS; fall back
982
- // to app-name activation (App.Activate, macOS-only, no-op elsewhere via
983
- // pcall). Decided relay-side: this process may run on a different
984
- // machine than the relay, so process.platform is the wrong signal.
981
+ // to app-name activation (App.Activate, pcall-guarded because it raises
982
+ // where the relay has no App backend). Decided relay-side: this process
983
+ // may run on a different machine than the relay, so process.platform is
984
+ // the wrong signal.
985
985
  const r = (await client.luaExec(`
986
986
  local h = ${handle}
987
987
  Window.Activate(h)
@@ -1070,53 +1070,58 @@ return { ok = Window.IsActive(h), name = name }
1070
1070
  async () => text(JSON.stringify(await session.calibrate())),
1071
1071
  );
1072
1072
 
1073
- register(
1074
- "read_page",
1075
- {
1076
- description:
1077
- "Extract the CURRENT web page from the focused browser window as structured text — page URL, forms and their fields (with scrollY positions), other interactive elements, and the page's full plain text — and leave the complete dump (including cleaned HTML with absolute links) in the system clipboard. " +
1078
- "Works in ANY browser: it focuses the address bar via the keyboard and runs a javascript: bookmarklet, so the page must already be loaded and the browser must be the frontmost window (use focus_window first). " +
1079
- "Far cheaper and more reliable than screenshots for reading page content or finding form fields. Overwrites the clipboard. Safari blocks javascript: URLs — use another browser there.",
1080
- inputSchema: {
1081
- html: z
1082
- .boolean()
1083
- .default(false)
1084
- .describe(
1085
- "include the cleaned-HTML section in the response (always present in the clipboard; often very large)",
1086
- ),
1073
+ // read_page finds the front app through App.Front and reads its result back
1074
+ // through AppleScript (tabUrl): both exist only on a macOS relay. A failed
1075
+ // OS probe reads as "Windows", so this never offers the tool where it fails.
1076
+ if (os === "macOS") {
1077
+ register(
1078
+ "read_page",
1079
+ {
1080
+ description:
1081
+ "Extract the CURRENT web page from the focused browser window as structured text — page URL, forms and their fields (with scrollY positions), other interactive elements, and the page's full plain text — and leave the complete dump (including cleaned HTML with absolute links) in the system clipboard. " +
1082
+ "It focuses the address bar via the keyboard and runs a javascript: bookmarklet, so the page must already be loaded and the browser must be the frontmost window (use focus_window first). The result is read back through AppleScript, so the front browser must answer the active-tab URL query. " +
1083
+ "Far cheaper and more reliable than screenshots for reading page content or finding form fields. Overwrites the clipboard. Safari blocks javascript: URLs — use another browser there.",
1084
+ inputSchema: {
1085
+ html: z
1086
+ .boolean()
1087
+ .default(false)
1088
+ .describe(
1089
+ "include the cleaned-HTML section in the response (always present in the clipboard; often very large)",
1090
+ ),
1091
+ },
1087
1092
  },
1088
- },
1089
- async ({ html }) => {
1090
- // Target whichever browser is actually in front, and remember the page we
1091
- // are on: a dropped paste sends Enter to search and navigates away, so the
1092
- // retry has to be able to put the user back where they were.
1093
- const front = await frontApp();
1094
- const app = BROWSER_RE.test(front) ? front : undefined;
1095
- const here = app ? await tabUrl(app) : "";
1096
- const clip = await extractPage(app, here || undefined);
1097
- if (!clip) {
1098
- return text(
1099
- "WARNING: page extraction did not complete — the parsed page never arrived. " +
1100
- "Likely causes: the focused window is not a browser, the page blocks javascript: URLs (strict CSP, browser start page, or Safari), or the page is still loading. " +
1101
- "Take a screenshot to check state, then retry.",
1102
- );
1103
- }
1104
- const cut = clip.indexOf("\n\n=== Cleaned HTML");
1105
- let body = html || cut === -1 ? clip : clip.slice(0, cut);
1106
- const notes: string[] = [];
1107
- if (!html && cut !== -1)
1108
- notes.push(
1109
- "[cleaned HTML omitted — the full dump is in the clipboard; pass html=true to include it]",
1110
- );
1111
- if (body.length > 60_000) {
1112
- body = body.slice(0, 60_000);
1113
- notes.push(
1114
- "[truncated at 60000 chars — the full dump is in the clipboard]",
1115
- );
1116
- }
1117
- return text(notes.length ? `${body}\n\n${notes.join("\n")}` : body);
1118
- },
1119
- );
1093
+ async ({ html }) => {
1094
+ // Target whichever browser is actually in front, and remember the page we
1095
+ // are on: a dropped paste sends Enter to search and navigates away, so the
1096
+ // retry has to be able to put the user back where they were.
1097
+ const front = await frontApp();
1098
+ const app = BROWSER_RE.test(front) ? front : undefined;
1099
+ const here = app ? await tabUrl(app) : "";
1100
+ const clip = await extractPage(app, here || undefined);
1101
+ if (!clip) {
1102
+ return text(
1103
+ "WARNING: page extraction did not complete — the parsed page never arrived. " +
1104
+ "Likely causes: the focused window is not a browser, the page blocks javascript: URLs (strict CSP, browser start page, or Safari), or the page is still loading. " +
1105
+ "Take a screenshot to check state, then retry.",
1106
+ );
1107
+ }
1108
+ const cut = clip.indexOf("\n\n=== Cleaned HTML");
1109
+ let body = html || cut === -1 ? clip : clip.slice(0, cut);
1110
+ const notes: string[] = [];
1111
+ if (!html && cut !== -1)
1112
+ notes.push(
1113
+ "[cleaned HTML omitted — the full dump is in the clipboard; pass html=true to include it]",
1114
+ );
1115
+ if (body.length > 60_000) {
1116
+ body = body.slice(0, 60_000);
1117
+ notes.push(
1118
+ "[truncated at 60000 chars — the full dump is in the clipboard]",
1119
+ );
1120
+ }
1121
+ return text(notes.length ? `${body}\n\n${notes.join("\n")}` : body);
1122
+ },
1123
+ );
1124
+ }
1120
1125
 
1121
1126
  register(
1122
1127
  "run_lua",
@@ -1283,9 +1288,9 @@ return { ok = Window.IsActive(h), name = name }
1283
1288
  "meditate",
1284
1289
  {
1285
1290
  description:
1286
- "Research a problem page until enough data is collected to solve it. Fetches `url` directly (relay-side curl) and, when the response is a JS shell, pulls its script bundles so the API endpoints behind the page are visible; falls back to extracting the rendered page in a browser when a fetch cannot see the content. A reasoning model then iterates: identify the problem, fetch the URLs and API endpoints that close information gaps, download required assets, until it reaches confidence >= 0.8, has nothing left to fetch, or hits max_rounds. A final pass renders everything into a solving dossier. " +
1291
+ "Research a problem page until enough data is collected to solve it. Fetches `url` directly (relay-side curl) and, when the response is a JS shell, pulls its script bundles so the API endpoints behind the page are visible; falls back to extracting the rendered page in a browser when a fetch cannot see the content. A reasoning model then iterates: identify the problem, fetch the URLs and API endpoints that close information gaps, download required assets, until it has nothing left to fetch or hits max_rounds. A final pass renders everything into a solving dossier. " +
1287
1292
  "Returns the dossier as markdown plus a JSON plan {error, problem, requirements, plan, confidence, sources, assets, pending_fetch, notes, cost_usd}. `error` is non-null (with the partial data still returned, never thrown) when no clear problem is identifiable or the browser could not be driven. Downloaded `assets` paths are on the RELAY host. " +
1288
- "Drives the real browser (takes over keyboard/clipboard for the duration) and spends OpenRouter credits. Refuses non-public URLs (loopback/private/link-local). Safari cannot be driven — pass `browser` to name another.",
1293
+ "On a macOS relay it also drives the real browser (takes over keyboard/clipboard for the duration) to render a JS shell, fetch same-origin URLs through the page, and read the live captcha; elsewhere it reads pages over HTTP only. Spends OpenRouter credits. Refuses non-public URLs (loopback/private/link-local). Safari cannot be driven — pass `browser` to name another.",
1289
1294
  inputSchema: {
1290
1295
  url: z.string().url().describe("page that explains the problem"),
1291
1296
  goal: z
@@ -1467,20 +1472,25 @@ return { ok = Window.IsActive(h), name = name }
1467
1472
  // evidence. The per-page budget keeps only the FIRST chars, so the
1468
1473
  // highest-signal content has to come first.
1469
1474
  let first = isShell ? null : raw;
1470
- // Always wait for the browser and place the window, whatever the fetch
1471
- // returned. The page is left open for the vision pass and for whoever
1472
- // solves it by sight afterwards, so a predictable window rect is not
1473
- // specific to the render path — it is the point of opening a browser
1474
- // at all. Positioning only when a fetch happened to come back thin
1475
- // left the solver working against an arbitrarily sized window.
1475
+ // Where the browser can be driven, always wait for it and place the
1476
+ // window, whatever the fetch returned. The page is left open for the
1477
+ // vision pass and for whoever solves it by sight afterwards, so a
1478
+ // predictable window rect is not specific to the render path — it is
1479
+ // the point of opening a browser at all. Positioning only when a
1480
+ // fetch happened to come back thin left the solver working against
1481
+ // an arbitrarily sized window. Driving the browser needs App.Front
1482
+ // and the AppleScript readback, which exist only on a macOS relay;
1483
+ // elsewhere the run stays on HTTP.
1484
+ const drive = os === "macOS";
1476
1485
  let front = "";
1477
- for (let i = 0; i < 12; i++) {
1486
+ for (let i = 0; drive && i < 12; i++) {
1478
1487
  await sleep(700);
1479
1488
  front = await frontApp();
1480
1489
  if (BROWSER_RE.test(front)) break;
1481
1490
  }
1482
1491
  const haveBrowser = BROWSER_RE.test(front);
1483
- if (!haveBrowser)
1492
+ if (!drive) note("browser steps skipped: they need a macOS relay");
1493
+ else if (!haveBrowser)
1484
1494
  note(
1485
1495
  browser
1486
1496
  ? `browser "${browser}" never reached the foreground (front app: ${front || "unknown"}) — is it installed under that exact name? try the default browser or a different name`
@@ -1509,7 +1519,9 @@ return { ok = Window.IsActive(h), name = name }
1509
1519
  null,
1510
1520
  haveBrowser
1511
1521
  ? "could not read the problem page — the fetch returned nothing and the rendered page could not be extracted"
1512
- : `could not read the problem page — it is a JS shell needing a browser to render, but none launched${browser ? ` (requested "${browser}")` : ""}. Start the browser, or pass a browser name that is installed`,
1522
+ : !drive
1523
+ ? "could not read the problem page: the fetch returned nothing, and rendering it needs a browser driven on a macOS relay"
1524
+ : `could not read the problem page — it is a JS shell needing a browser to render, but none launched${browser ? ` (requested "${browser}")` : ""}. Start the browser, or pass a browser name that is installed`,
1513
1525
  );
1514
1526
  pages.push({ url: pageUrl, dump: trimDump(first) });
1515
1527
  seen.add(normUrl(pageUrl));
@@ -1623,7 +1635,11 @@ return { ok = Window.IsActive(h), name = name }
1623
1635
  return false;
1624
1636
  }
1625
1637
  });
1626
- const fetched = await pageFetch(sameOrigin, browser, pageUrl);
1638
+ const fetched = await pageFetch(
1639
+ drive ? sameOrigin : [],
1640
+ browser,
1641
+ pageUrl,
1642
+ );
1627
1643
  for (const u of todo) {
1628
1644
  const viaPage = fetched[u];
1629
1645
  const body =
@@ -1641,37 +1657,40 @@ return { ok = Window.IsActive(h), name = name }
1641
1657
  // 4. revisit the anchor so the browser is left on the form page for
1642
1658
  // the solver, and run an ISOLATED vision pass to confirm the form and
1643
1659
  // read the live captcha. Its failure must never sink the dossier.
1644
- await gotoUrl(pageUrl);
1645
- try {
1646
- const shot = await captureScreen();
1647
- const vstep = await llmStep(
1648
- [
1649
- { role: "system", content: MEDITATE_VISION_SYS },
1650
- {
1651
- role: "user",
1652
- content: [
1653
- { type: "text", text: "Read this page." },
1654
- { type: "image_url", image_url: { url: shot.dataUrl } },
1655
- ],
1656
- },
1657
- ],
1658
- [],
1659
- VISION_MODEL || model_,
1660
- );
1661
- cost += vstep.usage?.cost ?? 0;
1662
- const vr = parseJsonObject<VisionRead>(
1663
- String(vstep.message.content ?? ""),
1664
- );
1665
- if (vr) {
1666
- liveCaptcha = vr.captcha_text ?? null;
1660
+ // Both need the live page, so they run only when a browser was driven.
1661
+ if (haveBrowser) {
1662
+ await gotoUrl(pageUrl);
1663
+ try {
1664
+ const shot = await captureScreen();
1665
+ const vstep = await llmStep(
1666
+ [
1667
+ { role: "system", content: MEDITATE_VISION_SYS },
1668
+ {
1669
+ role: "user",
1670
+ content: [
1671
+ { type: "text", text: "Read this page." },
1672
+ { type: "image_url", image_url: { url: shot.dataUrl } },
1673
+ ],
1674
+ },
1675
+ ],
1676
+ [],
1677
+ VISION_MODEL || model_,
1678
+ );
1679
+ cost += vstep.usage?.cost ?? 0;
1680
+ const vr = parseJsonObject<VisionRead>(
1681
+ String(vstep.message.content ?? ""),
1682
+ );
1683
+ if (vr) {
1684
+ liveCaptcha = vr.captcha_text ?? null;
1685
+ note(
1686
+ `vision: has_form=${!!vr.has_form} captcha=${!!vr.captcha_present}${vr.notes ? ` — ${vr.notes}` : ""}`,
1687
+ );
1688
+ }
1689
+ } catch (e) {
1667
1690
  note(
1668
- `vision: has_form=${!!vr.has_form} captcha=${!!vr.captcha_present}${vr.notes ? ` — ${vr.notes}` : ""}`,
1691
+ `vision pass skipped (model may lack image input): ${e instanceof Error ? e.message : e}`,
1669
1692
  );
1670
1693
  }
1671
- } catch (e) {
1672
- note(
1673
- `vision pass skipped (model may lack image input): ${e instanceof Error ? e.message : e}`,
1674
- );
1675
1694
  }
1676
1695
 
1677
1696
  // 5. one final text call turns everything gathered into the dossier
package/src/prompts.ts CHANGED
@@ -85,7 +85,7 @@ export const RUN_LUA_DESC =
85
85
  "The chunk runs ONCE and must end with `return <value>`: script hooks (OnDown/OnTick) never fire here, and coroutine-only calls " +
86
86
  '(Sleep, HID.Press, Dialog file pickers, Screen.Capture, Screen.FindImage, Screen.SearchForColor over 200k pixels) error — wait with System.Exec("sleep 0.5") instead, and take screenshots with the screenshot tool, not from a chunk. There is no Lua stdlib `io`/`os`/`require`. ' +
87
87
  "Use it to launch apps (System.ExecDetached), place windows (Window.Wait / Window.Move — on macOS the window handle is advisory and Move targets the FRONTMOST window, so bring the app forward first), run shell commands " +
88
- "(System.Exec(cmd) -> {exit,stdout,stderr}, absolute paths fine; inside a chunk it runs asynchronously on a background thread and does not block, so a slow command is safe), find UI by color in a small region (Screen.SearchForColor({x1,y1,x2,y2}, \"rrggbb\", tolerance), region under 200k pixels), type, click, read pixels (Screen.GetPixelColor), manage processes. " +
88
+ '(System.Exec(cmd) -> {exit,stdout,stderr}, absolute paths fine; inside a chunk it runs asynchronously on a background thread and does not block, so a slow command is safe), find UI by color in a small region (Screen.SearchForColor({x1,y1,x2,y2}, "rrggbb", tolerance), region under 200k pixels), type, click, read pixels (Screen.GetPixelColor), manage processes. ' +
89
89
  "File.* (Read/Write/Append/ReadJSON/WriteJSON/Exists/Delete/List/MkDir/RmDir/Copy/Move/GetSize/GetTime/GetScriptDir) is jailed to the script dir: paths RELATIVE, absolute rejected. " +
90
90
  'A folder described as on the Desktop or in home is reachable at its BARE name from the jail root (e.g. File.List("myfolder")) — never prefix "Desktop/" or "~". ' +
91
91
  "JSON is JSON.Parse(s) and JSON.Stringify(v) — there is no JSON.Decode/Encode; for JSON files File.ReadJSON/File.WriteJSON is one call. " +
package/src/rebind.d.luau CHANGED
@@ -35,6 +35,14 @@ export type ServerHandle = {
35
35
  Stop: (self: ServerHandle) -> (),
36
36
  }
37
37
 
38
+ -- UDP: socket handle returned by Net.UDPListen. SendTo sends from the bound
39
+ -- address and returns false when the send fails or the socket is stopped.
40
+ export type UdpHandle = {
41
+ port: number,
42
+ SendTo: (self: UdpHandle, ip: string, port: number, bytes: string) -> boolean,
43
+ Stop: (self: UdpHandle) -> (),
44
+ }
45
+
38
46
  -- WebSocket: server-side per-connection handle passed to OnConnect/OnMessage/OnClose.
39
47
  -- `id` is a monotonic identifier unique to this connection.
40
48
  export type WsClient = {
@@ -112,6 +120,19 @@ export type WindowPos = {
112
120
  height: number,
113
121
  }
114
122
 
123
+ -- the input device a key, button, move or scroll came from. hooks get nil
124
+ -- where the source cannot tell devices apart (Windows and macOS software mode)
125
+ export type InputDevice = {
126
+ id: string,
127
+ name: string,
128
+ nickname: string?,
129
+ kind: "keyboard" | "mouse" | "other",
130
+ vid: string,
131
+ pid: string,
132
+ source: "vm" | "host",
133
+ exact: boolean,
134
+ }
135
+
115
136
  export type MousePos = {
116
137
  x: number,
117
138
  y: number,
@@ -224,9 +245,10 @@ export type UiProxy = { [string]: any }
224
245
 
225
246
  -- bind config for table-style binding
226
247
  export type BindConfig = {
227
- when: (() -> boolean)?,
228
- action: (() -> boolean?)?,
229
- release: (() -> ())?,
248
+ device: string?, -- nickname or id: the bind answers only to that device
249
+ when: ((device: InputDevice?) -> boolean)?,
250
+ action: ((device: InputDevice?) -> boolean?)?,
251
+ release: ((device: InputDevice?) -> ())?,
230
252
  }
231
253
 
232
254
  -- runtime identity table
@@ -300,6 +322,20 @@ declare System: {
300
322
 
301
323
  -- hid namespace
302
324
 
325
+ -- a joypad state; every field is optional and defaults to neutral. buttons
326
+ -- lists pressed buttons 1-32, hat is 0 (up) to 7 clockwise (nil = centered),
327
+ -- x/y/z/rz are -1..1 (0 = center), slider and dial are 0..1.
328
+ export type JoypadState = {
329
+ buttons: { number }?,
330
+ hat: number?,
331
+ x: number?,
332
+ y: number?,
333
+ z: number?,
334
+ rz: number?,
335
+ slider: number?,
336
+ dial: number?,
337
+ }
338
+
303
339
  declare HID: {
304
340
  Down: (code: string) -> (),
305
341
  Up: (code: string) -> (),
@@ -319,6 +355,15 @@ declare HID: {
319
355
  Scroll: (amount: number) -> (),
320
356
  SetMouseMode: (mode: string) -> (),
321
357
  GetMouseMode: () -> string,
358
+ -- keyboard HID usage for a key name, or nil (media keys return nil). Down,
359
+ -- Up, Press and Combo also accept "usage:N" (decimal or 0x hex, 0x04-0xE7).
360
+ Usage: (name: string) -> number?,
361
+ -- set the whole joypad state (last write wins across scripts). Needs a
362
+ -- Rebind Link whose USB profile has a joypad; raises otherwise. Keyboard
363
+ -- and mouse calls raise the same way when the profile leaves them out.
364
+ Joypad: (state: JoypadState?) -> (),
365
+ -- back to neutral; the relay also does this when a script stops or errors
366
+ JoypadReset: () -> (),
322
367
  }
323
368
 
324
369
  -- input namespace
@@ -354,7 +399,7 @@ declare Bind: typeof(setmetatable({} :: {
354
399
  Toggle: (key: string, uiId: string) -> BindHandle,
355
400
  Remap: (from: string, to: string) -> BindHandle,
356
401
  }, {} :: {
357
- __call: (self: any, key: string, config: (() -> boolean?) | BindConfig) -> BindHandle,
402
+ __call: (self: any, key: string, config: ((device: InputDevice?) -> boolean?) | BindConfig) -> BindHandle,
358
403
  }))
359
404
 
360
405
  -- timer namespace
@@ -374,6 +419,9 @@ declare Macro: {
374
419
  StopAll: () -> (),
375
420
  Record: (options: RecordOpts?) -> (),
376
421
  Finish: () -> { MacroAction },
422
+ -- the saved-macro format: a `macro=true` modeline, then `return` of the
423
+ -- actions. require() of a saved macro returns the actions.
424
+ ToScript: (macro: { MacroAction }, name: string) -> string,
377
425
  Stream: (macro: { MacroAction }) -> (),
378
426
  Abort: () -> (),
379
427
  }
@@ -517,9 +565,12 @@ declare Window: {
517
565
  }
518
566
 
519
567
  -- app namespace: application-level control (Keyboard Maestro's Application
520
- -- Control). names match case-insensitively by app display name. Front/IsRunning/
521
- -- Activate are permission-free; Hide/Quit/IsHidden drive an AppleEvent and
522
- -- prompt for macOS Automation permission on first use.
568
+ -- Control). macOS: names match case-insensitively by app display name;
569
+ -- Front/IsRunning/Activate are permission-free; Hide/Quit/IsHidden drive an
570
+ -- AppleEvent and prompt for macOS Automation permission on first use.
571
+ -- Windows: names are executable names (case-insensitive, ".exe" optional);
572
+ -- Activate focuses a running app and does not launch one; Quit closes its
573
+ -- windows; Hide/IsHidden raise. Linux: every call raises.
523
574
 
524
575
  export type AppInfo = {
525
576
  name: string,
@@ -549,6 +600,10 @@ declare Net: {
549
600
  Listen: (port: number, handler: (request: HttpRequest) -> HttpResponseSpec | string) -> ServerHandle,
550
601
  WSListen: (port: number, handlers: WsListenHandlers) -> WsServerHandle,
551
602
  WSConnect: (url: string, handlers: WsConnectHandlers) -> WsConnectHandle,
603
+ -- bind a UDP socket; handler receives each datagram's raw bytes and sender.
604
+ UDPListen: (bindIp: string, port: number, handler: (data: string, ip: string, port: number) -> ()) -> UdpHandle,
605
+ -- this computer's IPv4 address on its default route, or nil when there is none.
606
+ LocalIP: () -> string?,
552
607
  }
553
608
 
554
609
  -- pipe namespace
@@ -750,14 +805,14 @@ declare Registry: {
750
805
  -- declare function OnFocus(window: WindowInfo): ()
751
806
  -- declare function OnBlur(window: WindowInfo): ()
752
807
  -- declare function OnTick(dtMs: number): () -- ms elapsed since the previous OnTick (0 on the first)
753
- -- declare function OnDown(key: string): boolean?
754
- -- declare function OnUp(key: string, durationMs: number): boolean?
755
- -- declare function OnMove(dx: number, dy: number): boolean? -- return value only used with mouse_block=true (hardware mode only)
756
- -- declare function OnScroll(delta: number): boolean?
808
+ -- declare function OnDown(key: string, device: InputDevice?): boolean?
809
+ -- declare function OnUp(key: string, durationMs: number, device: InputDevice?): boolean?
810
+ -- declare function OnMove(dx: number, dy: number, device: InputDevice?): boolean? -- return value only used with mouse_block=true (hardware mode only)
811
+ -- declare function OnScroll(delta: number, device: InputDevice?): boolean?
757
812
  -- declare function OnScrollH(delta: number): boolean?
758
- -- declare function OnInputDown(key: string): ()
759
- -- declare function OnInputUp(key: string, durationMs: number): ()
760
- -- declare function OnInputMove(dx: number, dy: number): ()
761
- -- declare function OnInputScroll(delta: number): ()
813
+ -- declare function OnInputDown(key: string, device: InputDevice?): ()
814
+ -- declare function OnInputUp(key: string, durationMs: number, device: InputDevice?): ()
815
+ -- declare function OnInputMove(dx: number, dy: number, device: InputDevice?): ()
816
+ -- declare function OnInputScroll(delta: number, device: InputDevice?): ()
762
817
  -- declare function OnError(message: string): ()
763
818
  -- declare function OnReload(): ()