@remcp/runtime 0.2.77 → 0.2.79

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+## 0.2.79 — 2026-09-21
+
+- Explicit Wayland keyboard and key-by-key text input now focuses and verifies the requested target window before dispatch, with bounded direct window cycling and verified fallbacks.
+- Native Wayland cursor telemetry fails closed instead of trusting stale XWayland `xdotool` coordinates; `computer_snapshot` keeps that capture non-fatal and structured.
+- Regression coverage is headless-safe while still enforcing the snapshot normalization/error contract.
+- Lockstep patch release with the ReMCP client/server at `0.2.79`.
+
 ## 0.2.77 — 2026-09-21
 
 - Fixed `browser_action` typing so the documented `text` argument reaches CDP `Input.insertText` with Unicode intact; `value` and `text_value` remain compatible aliases.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.77",
+  "version": "0.2.79",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",

package/src/extended/catalog.mjs

@@ -455,7 +455,7 @@ const inputBackendSelector = {
   backend: e(['auto','x11','portal'], 'Input backend preference. auto uses proven local backends and already-authorized portal input; portal explicitly requests XDG RemoteDesktop consent on Wayland; x11 forces XWayland/xdotool only for operations where read-back testing shows it is reliable. GNOME Wayland keyboard/text deliberately reject x11.'),
 };
 const inputWindowSelector = {
-  window_id: s('Optional target window id returned by list_windows. Useful for deterministic XWayland keyboard input on Wayland.'),
+  window_id: s('Optional target window id returned by list_windows. On Wayland the target is focused and verified before keyboard/text input is dispatched.'),
   pid: n('Optional target process id.'),
   app: s('Optional target application/process name substring.'),
   title: s('Optional target window title substring.'),
@@ -538,8 +538,8 @@ export const extendedToolDefinitions = [
   define('ui_snapshot', 'UI accessibility snapshot', 'Read the active desktop accessibility/UI Automation tree with compact labels, semantic hierarchy, bounds, state and suggested actions. Default scope is the active application/window; scope=desktop requests the full desktop tree. Explicit pid/app/window_title scopes override the active default. browser_dom asks the OS accessibility provider for the web-document subtree when available.', o({ scope:e(['active','desktop'],'Default active. Use desktop only when cross-application discovery is required.'), pid:n('Optional owning process id scope.'), app:s('Optional application/process name substring scope.'), window_title:s('Optional top-level accessibility window/frame title scope.'), browser_dom:b('Prefer the native browser document accessibility subtree instead of browser chrome when supported.'), semantic_tree:b('Include compact semantic hierarchy text; default true.'), max_nodes:n('Maximum nodes; default 500, maximum 5000.'), max_depth:n('Maximum traversal depth; default 8, maximum 32.') }), readOnlyLive, desktopHandlers.ui_snapshot, ['ui']),
   define('ui_find', 'Find UI elements', 'Find native accessible desktop elements by label/id/name/role/AutomationId. Reuses the latest five-second semantic snapshot for fast repeated targeting; refresh=true forces a new OS accessibility capture.', o({ ...uiSelector, refresh:b('Force a fresh accessibility capture instead of the short-lived snapshot cache.'), limit:n('Maximum matches; default 20.'), max_nodes:n(), max_depth:n() }), readOnlyLive, desktopHandlers.ui_find, ['ui']),
   define('ui_action', 'Act on UI element', 'Act on a native accessibility/UI Automation element by id, label or semantic selector. Supports invoke/focus/value/toggle/selection, expand-collapse, range values and scroll-into-view before falling back to coordinates.', o({ action:e(['click','invoke','focus','set_value','select','toggle','expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection']), ...uiSelector, value:{description:'New text or numeric range value.'} }, ['action']), openMutating, desktopHandlers.ui_action, ['ui']),
-  define('type_text', 'Type text', 'Enter Unicode text into a native semantic accessibility target or focused native control. Auto prefers verified accessibility editing; explicit semantic targets fail safely when the platform rejects the write instead of claiming an unverified paste. Use browser_action for Chromium page DOM. Clipboard/keys fallbacks remain available for focused/window-level input when appropriate.', o({ text:s('Text to enter.'), method:e(['auto','accessibility','clipboard','keys']), clear:b('Clear existing text before input when keyboard/paste fallback is used.'), caret_position:e(['start','idle','end']), press_enter:b('Press Enter after typing.'), delay_ms:n('Per-key delay for method=keys.'), ...uiSelector, window_id:s('Optional XWayland target window id for clipboard/keys methods.'), title:s('Optional target window title substring for clipboard/keys methods.'), ...inputBackendSelector }, ['text']), openMutating, desktopHandlers.type_text, ['input']),
-  define('keyboard', 'Keyboard shortcut', 'Send a keyboard shortcut or key press to the active desktop application or an explicitly selected window. On Wayland deterministic XWayland input should use window_id/pid/app/title; backend=portal explicitly requests the consent-backed XDG RemoteDesktop path.', o({ shortcut:s(), key:s(), keys:{ type:'array', items:{type:'string'} }, ...inputWindowSelector, ...inputBackendSelector }), openMutating, desktopHandlers.keyboard, ['input']),
+  define('type_text', 'Type text', 'Enter Unicode text into a native semantic accessibility target or focused native control. Auto prefers verified accessibility editing; explicit semantic targets fail safely when the platform rejects the write instead of claiming an unverified paste. Use browser_action for Chromium page DOM. Clipboard/keys fallbacks remain available for focused/window-level input when appropriate, and explicit Wayland window targets are focused and verified before dispatch.', o({ text:s('Text to enter.'), method:e(['auto','accessibility','clipboard','keys'], 'Default auto prefers verified accessibility editing and falls back to clipboard paste. keys sends physical key events and therefore follows the active keyboard layout.'), clear:b('Clear existing text before input when keyboard/paste fallback is used.'), caret_position:e(['start','idle','end']), press_enter:b('Press Enter after typing.'), delay_ms:n('Per-key delay for method=keys.'), ...uiSelector, window_id:s('Optional target window id; the window is focused before auto/accessibility/clipboard/keys input and Wayland verifies focus before dispatch.'), title:s('Optional target window title substring; focused before input when no semantic element target is supplied.'), ...inputBackendSelector }, ['text']), openMutating, desktopHandlers.type_text, ['input']),
+  define('keyboard', 'Keyboard shortcut', 'Send a keyboard shortcut or key press to the active desktop application or an explicitly selected window. On Wayland, window_id/pid/app/title targets are focused and verified before input is dispatched; backend=portal explicitly requests the consent-backed XDG RemoteDesktop path.', o({ shortcut:s(), key:s(), keys:{ type:'array', items:{type:'string'} }, ...inputWindowSelector, ...inputBackendSelector }), openMutating, desktopHandlers.keyboard, ['input']),
   define('pointer', 'Pointer action', 'Move the pointer or perform click, double-click, right-click, mouse-down, or mouse-up at desktop screen coordinates. Wayland supports XWayland fallback and an explicitly consented XDG RemoteDesktop backend.', o({ action:e(['move','click','double_click','right_click','down','up']), x:n(), y:n(), button:e(['left','right','middle']), ...inputBackendSelector }, ['action']), openMutating, desktopHandlers.pointer, ['pointer']),
   define('drag_drop', 'Drag and drop', 'Drag between desktop coordinates or semantic UI element IDs returned by ui_snapshot/ui_find. Element IDs are resolved to current accessibility bounds before input. GNOME Wayland requires the consent-backed Remote Desktop portal for real drag delivery; unreliable XTEST/xdotool drag is rejected.', o({ from_x:n(), from_y:n(), to_x:n(), to_y:n(), from_id:s('Source UI element id; use instead of from_x/from_y.'), to_id:s('Destination UI element id; use instead of to_x/to_y.'), button:e(['left','right','middle']), hold_ms:n(), duration_ms:n(), ...inputBackendSelector }), openMutating, desktopHandlers.drag_drop, ['drag']),
   define('scroll', 'Scroll', 'Scroll vertically or horizontally using native wheel input. Target a semantic UI element/label or x/y first so the wheel is delivered to the intended region; direction+wheel_times is a convenience over normalized deltas.', o({ delta_x:n('Horizontal delta; positive means right.'), delta_y:n('Vertical delta; positive means down.'), delta:n('Alias for delta_y.'), direction:e(['up','down','left','right']), wheel_times:n('Number of 120-unit wheel steps when direction is used; 1..50.'), x:n(), y:n(), ...uiSelector, ...inputBackendSelector }), mutatingNonDestructive, desktopHandlers.scroll, ['pointer']),

package/src/extended/desktop-linux.mjs

@@ -252,6 +252,36 @@ async function focusInputWindow(windowId) {
   if (result.code === 0) await new Promise(resolve => setTimeout(resolve, 40));
 }
 
+function explicitInputWindowSelector(args = {}) {
+  const dedicatedWindowId = optionalString(args.window_id || args.windowId);
+  const dedicatedTitle = optionalString(args.title);
+  const hasSemanticElementTarget = Boolean(
+    optionalString(args.id)
+    || optionalString(args.name)
+    || optionalString(args.role)
+    || optionalString(args.automation_id || args.automationId)
+  );
+  if (!dedicatedWindowId && !dedicatedTitle && hasSemanticElementTarget) return null;
+  const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
+  const app = optionalString(args.app);
+  const scopedTitle = optionalString(args.window_title || args.windowTitle);
+  if (!dedicatedWindowId && !dedicatedTitle && pid == null && !app && !scopedTitle) return null;
+  return {
+    ...(dedicatedWindowId ? { id:dedicatedWindowId } : {}),
+    ...(pid != null ? { pid } : {}),
+    ...(app ? { app } : {}),
+    ...((dedicatedTitle || scopedTitle) ? { title:dedicatedTitle || scopedTitle } : {}),
+  };
+}
+
+async function focusExplicitWaylandInputTarget(args = {}) {
+  const selector = explicitInputWindowSelector(args);
+  if (!selector) return null;
+  const row = await windowMatch(selector);
+  await windowAction({ action:'focus', id:row.id, backend:inputBackend(args) });
+  return row;
+}
+
 async function nativeWaylandWindowAction(row, action, args) {
   const backend = inputBackend(args);
   if (backend === 'x11') unavailable('Native Wayland window action', 'use backend=portal for compositor-managed windows');
@@ -350,23 +380,31 @@ export async function windowAction(args) {
       if (focusState.exact) return text('Window action focus already matched the target window.');
 
       const rows = JSON.parse((await listWindows()).content?.[0]?.text || '[]');
-      const appCycles = Math.min(30, Math.max(4, new Set(rows.map(item => Number(item.pid) || String(item.app || ''))).size + 2));
+      const windowCycles = Math.min(60, Math.max(8, rows.length + 4));
+      for (let index = 0; index < windowCycles; index += 1) {
+        await portalShortcut('ALT+ESC', portalOptions);
+        await new Promise(resolve => setTimeout(resolve, 140));
+        focusState = await targetAccessibilityFocus(row);
+        if (focusState.exact) return text('Window action focus completed via verified Wayland Alt+Esc.');
+      }
+
+      const appCycles = Math.min(12, Math.max(3, new Set(rows.map(item => Number(item.pid) || String(item.app || ''))).size + 1));
       for (let index = 0; index < appCycles; index += 1) {
         await portalShortcut('ALT+TAB', portalOptions);
-        await new Promise(resolve => setTimeout(resolve, 120));
+        await new Promise(resolve => setTimeout(resolve, 160));
         focusState = await targetAccessibilityFocus(row);
-        if (focusState.exact) return text('Window action focus completed via verified Wayland Alt+Tab.');
+        if (focusState.exact) return text('Window action focus completed via verified Wayland Alt+Tab fallback.');
         if (focusState.active) {
           const sameAppWindows = rows.filter(item => Number(item.pid) === Number(row.pid)).length;
-          for (let windowIndex = 0; windowIndex < Math.min(12, Math.max(2, sameAppWindows + 2)); windowIndex += 1) {
+          for (let windowIndex = 0; windowIndex < Math.min(8, Math.max(2, sameAppWindows + 1)); windowIndex += 1) {
             await portalShortcut('ALT+`', portalOptions);
-            await new Promise(resolve => setTimeout(resolve, 100));
+            await new Promise(resolve => setTimeout(resolve, 120));
             focusState = await targetAccessibilityFocus(row);
-            if (focusState.exact) return text('Window action focus completed via verified Wayland window cycling.');
+            if (focusState.exact) return text('Window action focus completed via verified Wayland same-app fallback.');
           }
         }
       }
-      throw new Error(`Could not verify focus for ${row.title || row.app || row.id} after cycling Wayland applications`);
+      throw new Error(`Could not verify focus for ${row.title || row.app || row.id} after bounded Wayland window cycling`);
     }
     if (!commandExists('wmctrl')) unavailable('Window focus', 'install wmctrl');
     await runFile('wmctrl', ['-ia', id], { label: 'window focus' });
@@ -805,6 +843,7 @@ export async function keyboard(args = {}) {
     return argv;
   };
   if (isWaylandSession()) {
+    await focusExplicitWaylandInputTarget(args);
     if (backend !== 'portal' && commandExists('wtype')) {
       const result = await runFile('wtype', wtypeArgs(), { label: 'keyboard', allowFailure: true, timeout: 5000 });
       if (result.code === 0) return text(`Sent ${shortcut} via wtype.`);
@@ -842,6 +881,7 @@ export async function typeTextKeys(value, args = {}) {
   const delay = clamp(args.delay_ms, 1, 0, 1000);
   const backend = inputBackend(args);
   if (isWaylandSession()) {
+    await focusExplicitWaylandInputTarget(args);
     if (backend !== 'portal' && commandExists('wtype')) {
       const result = await runFile('wtype', ['-d', String(delay), '--', String(value)], { label: 'type text', timeout: 30_000, allowFailure: true });
       if (result.code === 0) return { ...result, backend: 'wtype' };
@@ -1063,6 +1103,9 @@ export async function scroll(args = {}) {
 }
 
 export async function cursorPosition() {
+  if (isWaylandSession()) {
+    unavailable('Cursor position', 'native Wayland does not expose an authoritative global cursor position; XWayland xdotool coordinates may be stale');
+  }
   if (!commandExists('xdotool')) unavailable('Cursor position', 'install xdotool');
   const result = await runFile('xdotool', ['getmouselocation','--shell'], { label:'cursor position', allowFailure:true, timeout:2000 });
   const x = Number(result.stdout.match(/^X=(-?\d+)$/m)?.[1]);

package/src/extended/desktop.mjs

@@ -618,6 +618,23 @@ export async function waitForUi(args = {}) {
   return jsonResult({ state:'timeout', wanted:state, matched:nodes.length, attempts, elapsed_ms:Date.now() - startedAt, nodes, ...(state === 'changed' ? { before:baseline } : {}) });
 }
 
+function explicitTypeTextWindowTarget(args = {}, hasElementSelector = false) {
+  const windowId = optionalString(args.window_id || args.windowId);
+  const title = optionalString(args.title);
+  if (!windowId && !title && hasElementSelector) return null;
+  const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
+  const app = optionalString(args.app);
+  const scopedTitle = optionalString(args.window_title || args.windowTitle);
+  if (!windowId && !title && pid == null && !app && !scopedTitle) return null;
+  return {
+    ...(windowId ? { id:windowId } : {}),
+    ...(pid != null ? { pid } : {}),
+    ...(app ? { app } : {}),
+    ...((title || scopedTitle) ? { title:title || scopedTitle } : {}),
+    ...(args.backend ? { backend:args.backend } : {}),
+  };
+}
+
 export async function typeText(args = {}) {
   const value = String(args.text ?? '');
   const method = requireEnum(args.method || 'auto', 'method', ['auto','accessibility','clipboard','keys']);
@@ -626,6 +643,8 @@ export async function typeText(args = {}) {
     optionalString(resolvedArgs.id) || optionalString(resolvedArgs.name) || optionalString(resolvedArgs.role) ||
     optionalString(resolvedArgs.automation_id || resolvedArgs.automationId)
   );
+  const explicitWindowTarget = explicitTypeTextWindowTarget(resolvedArgs, hasElementSelector);
+  if (explicitWindowTarget) await windowAction({ action:'focus', ...explicitWindowTarget });
   const clear = args.clear === true;
   const pressEnter = args.press_enter === true;
   const caret = requireEnum(args.caret_position || 'idle', 'caret_position', ['start','idle','end']);