@remcp/runtime 0.2.110 → 0.2.112

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.2.112 — 2026-09-25
+
+- Linux EIS absolute pointer motion converts desktop-wide coordinates into the selected libei region before dispatch and keeps the current pointer device when it remains valid, eliminating accepted-but-undelivered absolute moves.
+- Wayland portal drag-and-drop uses one region-safe target transition while pressed, then reasserts the final target before release so `pointerdown → pointermove → pointerup` remains intact.
+- The live computer-use smoke is serialized per user to prevent concurrent desktop-input races; the final ASUS GNOME Wayland run passed 82/82 checks. Runtime tool names and schemas are unchanged.
+
+## 0.2.111 — 2026-09-25
+
+- `read_document` reads `.xlsx` files again while `allowedRoots` confinement is enabled: the reader no longer re-enters its own temp staging path through the allowlist.
+- Linux window inventory refreshes X11 rows after the AT-SPI walk, window selectors resolve X11/XWayland targets without a full semantic rebuild, and `window_action focus` verifies direct XWayland activation before falling back to portal cycling. Runtime tool names and schemas are unchanged.
+
 ## 0.2.110 — 2026-09-25
 
 - Bulk handlers (`read_multiple_files`, `read_files`, `replace_in_files`, `write_files`, `copy_paths`, `move_paths`, `delete_paths`, `create_archive`) and the directory walk now honour client cancellation between items, so an aborted call stops at the next item boundary instead of finishing on the device; single-file and external-tool (`tar`/`unzip`) operations remain atomic and are not interrupted mid-step.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.110",
+  "version": "0.2.112",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",

package/src/extended/desktop-linux.mjs

@@ -55,7 +55,7 @@ async function x11PixelBounds(id) {
 }
 
 export async function listWindows() {
-  const wmRows = commandExists('wmctrl')
+  let wmRows = commandExists('wmctrl')
     ? parseWindows((await runFile('wmctrl', ['-lpGx'], { label: 'window inventory', allowFailure:true, timeout:3000 })).stdout)
     : [];
   if (!isWaylandSession()) {
@@ -85,6 +85,15 @@ export async function listWindows() {
     });
   } catch {}
 
+  // AT-SPI enumeration can take several seconds on a busy desktop. Refresh the cheap
+  // X11 inventory afterwards so an XWayland window that appeared while AT-SPI was
+  // walking the tree is merged with its semantic window instead of being misclassified
+  // as native Wayland for the rest of this call.
+  if (commandExists('wmctrl')) {
+    const refreshedWmRows = parseWindows((await runFile('wmctrl', ['-lpGx'], { label:'window inventory refresh', allowFailure:true, timeout:3000 })).stdout);
+    if (refreshedWmRows.length) wmRows = refreshedWmRows;
+  }
+
   const pixelBounds = new Map();
   if (commandExists('xwininfo')) {
     // wmctrl geometry on Mutter/XWayland is decoration-shifted and cannot be used for pixel crops.
@@ -151,18 +160,50 @@ export async function listWindows() {
   return jsonResult(rows);
 }
 
+function windowSelector(args = {}) {
+  return {
+    id: optionalString(args.id || args.window_id || args.windowId),
+    pid: Number.isInteger(Number(args.pid)) ? Number(args.pid) : null,
+    app: optionalString(args.app),
+    title: optionalString(args.title || args.window_title || args.windowTitle),
+  };
+}
+
+function windowMatchesSelector(item, selector) {
+  return (!selector.id || item.id === selector.id || item.wm_id === selector.id || item.ui_id === selector.id)
+    && (selector.pid == null || Number(item.pid) === selector.pid)
+    && (!selector.app || String(item.app || '').toLowerCase().includes(selector.app.toLowerCase()))
+    && (!selector.title || String(item.title || '').toLowerCase().includes(selector.title.toLowerCase()));
+}
+
+async function fastX11WindowMatch(args) {
+  if (!commandExists('wmctrl')) return null;
+  const selector = windowSelector(args);
+  const rows = parseWindows((await runFile('wmctrl', ['-lpGx'], {
+    label:'window match',
+    allowFailure:true,
+    timeout:2000,
+  })).stdout);
+  const row = rows.find(item => windowMatchesSelector({ ...item, wm_id:item.id, ui_id:null }, selector));
+  if (!row) return null;
+  return {
+    ...row,
+    wm_id:row.id,
+    ui_id:null,
+    backend:isWaylandSession() ? 'xwayland' : 'x11',
+  };
+}
+
 async function windowMatch(args) {
+  // Most desktop actions target an X11/XWayland window. Resolve that cheap native id
+  // first instead of rebuilding the entire AT-SPI inventory before every action.
+  // Native Wayland windows have no wmctrl row and fall through to the semantic inventory.
+  const fast = await fastX11WindowMatch(args);
+  if (fast) return fast;
+
   const rows = JSON.parse((await listWindows()).content[0].text);
-  const id = optionalString(args.id || args.window_id || args.windowId);
-  const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
-  const app = optionalString(args.app);
-  const title = optionalString(args.title || args.window_title || args.windowTitle);
-  const row = rows.find(item =>
-    (!id || item.id === id || item.wm_id === id || item.ui_id === id)
-    && (pid == null || item.pid === pid)
-    && (!app || String(item.app || '').toLowerCase().includes(app.toLowerCase()))
-    && (!title || String(item.title || '').toLowerCase().includes(title.toLowerCase()))
-  );
+  const selector = windowSelector(args);
+  const row = rows.find(item => windowMatchesSelector(item, selector));
   if (!row) throw new Error('No matching window found');
   return row;
 }
@@ -502,7 +543,24 @@ export async function windowAction(args) {
           if (focusState.exact) return text('Window action focus completed via verified AT-SPI focus.');
         } catch {}
       }
-      if (backend === 'x11') unavailable('Wayland window focus', 'AT-SPI focus did not take effect and X11 focus cannot target this Wayland window; use backend=portal');
+      // XWayland windows have a real X11 wm_id even inside a Wayland session. Prefer
+      // direct activation for those windows, verify the result through accessibility,
+      // and only fall back to compositor-level cycling if direct activation did not take.
+      // This avoids dozens of Alt+Esc/AT-SPI round trips on busy desktops.
+      if (row.wm_id && backend !== 'portal' && commandExists('wmctrl')) {
+        const activated = await runFile('wmctrl', ['-ia', row.wm_id], { label:'XWayland window focus', allowFailure:true, timeout:2000 });
+        if (activated.code === 0) {
+          if (commandExists('xdotool')) {
+            await runFile('xdotool', ['windowactivate','--sync',row.wm_id], { label:'XWayland window focus sync', allowFailure:true, timeout:3000 });
+          }
+          await new Promise(resolve => setTimeout(resolve, 80));
+          const focusState = await targetAccessibilityFocus(row);
+          if (focusState.exact) return text('Window action focus completed via verified XWayland activation.');
+        }
+      }
+      if (backend === 'x11') unavailable('Wayland window focus', row.wm_id
+        ? 'direct XWayland activation did not produce verifiable focus'
+        : 'the target is native Wayland and has no X11 window id; use backend=portal');
       if (!waylandPortalCandidate()) unavailable('Wayland window focus', 'no XDG RemoteDesktop portal is available');
       if (backend === 'auto' && !hasWaylandRemoteDesktopGrant()) portalPermissionHint('Wayland window focus');
       const portalOptions = { timeoutMs: backend === 'portal' ? 120_000 : 2500 };
@@ -1212,22 +1270,16 @@ export async function dragDrop(args = {}) {
     await portalPointerButton(button, true, portalOptions);
     await new Promise(resolve => setTimeout(resolve, clamp(args.hold_ms, 120, 60, 5000)));
 
-    // A single absolute jump while the button is held can be coalesced by Mutter/
-    // XWayland and never become a target-visible pointermove. Send a short,
-    // human-like path so drag-aware toolkits observe movement before button-up.
-    const distance = Math.hypot(toX - fromX, toY - fromY);
-    const steps = Math.max(4, Math.min(20, Math.ceil(distance / 24)));
-    const durationMs = clamp(args.duration_ms, 160, 80, 5000);
-    const stepDelay = Math.max(8, Math.round(durationMs / steps));
-    for (let step = 1; step <= steps; step += 1) {
-      const ratio = step / steps;
-      await portalMoveTo(
-        fromX + (toX - fromX) * ratio,
-        fromY + (toY - fromY) * ratio,
-        portalOptions,
-      );
-      await new Promise(resolve => setTimeout(resolve, stepDelay));
-    }
+    // Keep the pressed pointer on a single absolute-motion transition. Repeated
+    // absolute moves can cross EIS region/device contexts and lose the logical
+    // pressed state. The EIS helper converts the desktop target into region-local
+    // coordinates, so one target move is sufficient and preserves buttons=1.
+    await portalMoveTo(toX, toY, portalOptions);
+    await new Promise(resolve => setTimeout(resolve, clamp(args.duration_ms, 160, 80, 5000)));
+    // Reassert the final target immediately before release. On Mutter/XWayland this
+    // mirrors a real pointer-up gesture and ensures the release is delivered as a
+    // pointerup/mouseup at the dragged target instead of only clearing button state.
+    await portalMoveTo(toX, toY, portalOptions);
     await new Promise(resolve => setTimeout(resolve, 40));
     await portalPointerButton(button, false, portalOptions);
     await new Promise(resolve => setTimeout(resolve, 80));

package/src/extended/documents.mjs

@@ -334,10 +334,12 @@ function readWorksheetCells(xml, sharedStrings, maxCells) {
 }
 
 async function readXlsx(filePath, args) {
-  const source = await snapshotDocumentFile(filePath);
   const dir = await tempDir('remcp-xlsx-read-');
   try {
-    await extractZip(source.path, dir);
+    // extractZip already snapshots the user-approved source into a private temp file
+    // before invoking unzip. A second snapshot here used to re-open ReMCP's own temp
+    // path through allowedRoots, so read_document failed whenever confinement was on.
+    await extractZip(filePath, dir);
     const workbook = await readFile(path.join(dir,'xl','workbook.xml'),'utf8');
     const rels = await readFile(path.join(dir,'xl','_rels','workbook.xml.rels'),'utf8');
     const sheet = workbookSheetPath(workbook, rels, optionalString(args.sheet));
@@ -347,7 +349,6 @@ async function readXlsx(filePath, args) {
     const cells = readWorksheetCells(worksheet, shared, clamp(args.max_cells,5000,1,50_000));
     return { kind:'xlsx', sheet:sheet.name, count:cells.length, cells };
   } finally {
-    await removeTemp(source.dir);
     await removeTemp(dir);
   }
 }

package/src/helpers/wayland-eis-helper.py

@@ -216,6 +216,8 @@ class EiSender:
         self.lib.ei_device_has_capability.restype = ctypes.c_bool
         self.lib.ei_device_get_region_at.argtypes = [c_void_p, ctypes.c_double, ctypes.c_double]
         self.lib.ei_device_get_region_at.restype = c_void_p
+        self.lib.ei_region_convert_point.argtypes = [c_void_p, ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double)]
+        self.lib.ei_region_convert_point.restype = ctypes.c_bool
         self.lib.ei_device_start_emulating.argtypes = [c_void_p, ctypes.c_uint32]
         self.lib.ei_device_stop_emulating.argtypes = [c_void_p]
         self.lib.ei_device_keyboard_key.argtypes = [c_void_p, ctypes.c_uint32, ctypes.c_bool]
@@ -367,17 +369,46 @@ class EiSender:
                 return state["ptr"]
         raise RuntimeError(f"no resumed EIS device with capability {cap}")
 
+    def _absolute_motion_for_device(self, device, x, y):
+        if not self.lib.ei_device_has_capability(device, EI_CAP_POINTER_ABSOLUTE):
+            return None
+        region = self.lib.ei_device_get_region_at(device, float(x), float(y))
+        if not region:
+            return None
+        motion_x = ctypes.c_double(float(x))
+        motion_y = ctypes.c_double(float(y))
+        if not self.lib.ei_region_convert_point(region, ctypes.byref(motion_x), ctypes.byref(motion_y)):
+            raise RuntimeError(f"failed to convert desktop coordinate ({x}, {y}) into the selected EIS region")
+        return motion_x.value, motion_y.value
+
     def device_for_absolute(self, x, y):
         candidates = []
+
+        # Preserve pointer identity across absolute motion whenever the current
+        # device still covers the destination. This is required for pointer
+        # capture during drags: switching EIS devices mid-gesture can drop the
+        # captured pointer before its button release is observed.
+        preferred = self.devices.get(self.pointer_device_key)
+        if preferred and preferred["resumed"] and preferred["emulating"]:
+            device = preferred["ptr"]
+            if self.lib.ei_device_has_capability(device, EI_CAP_POINTER_ABSOLUTE):
+                candidates.append(device)
+                converted = self._absolute_motion_for_device(device, x, y)
+                if converted is not None:
+                    return device, converted[0], converted[1]
+
         for state in self.devices.values():
             if not state["resumed"] or not state["emulating"]:
                 continue
             device = state["ptr"]
+            if preferred and self._device_key(device) == self.pointer_device_key:
+                continue
             if not self.lib.ei_device_has_capability(device, EI_CAP_POINTER_ABSOLUTE):
                 continue
             candidates.append(device)
-            if self.lib.ei_device_get_region_at(device, float(x), float(y)):
-                return device
+            converted = self._absolute_motion_for_device(device, x, y)
+            if converted is not None:
+                return device, converted[0], converted[1]
         if candidates:
             raise RuntimeError(f"no absolute-pointer EIS region contains desktop coordinate ({x}, {y})")
         raise RuntimeError("no resumed EIS device with absolute-pointer capability")
@@ -424,8 +455,8 @@ class EiSender:
         elif op == "motion_absolute":
             x = float(payload["x"])
             y = float(payload["y"])
-            device = self.device_for_absolute(x, y)
-            self.lib.ei_device_pointer_motion_absolute(device, x, y)
+            device, motion_x, motion_y = self.device_for_absolute(x, y)
+            self.lib.ei_device_pointer_motion_absolute(device, motion_x, motion_y)
             self.remember_pointer(device, (x, y))
             self.frame(device)
         elif op == "button":