@remcp/runtime 0.2.104 → 0.2.106

package/CHANGELOG.md

@@ -1,11 +1,27 @@
 # Changelog
 
+## 0.2.106 — 2026-09-22
+
+- Wayland EIS records the emulated device used for each button press and sends the matching release through that same device, preventing cross-region/device drags from losing `pointerup`.
+- Portal drag-and-drop adds source settle time plus bounded intermediate absolute motion events; live ASUS DOM read-back verified the complete `down → move → … → up` gesture.
+- Semantic-target scroll validation moves to a known visible target before wheel injection, and Wayland `screenshot_region` prefers `grim` or GNOME Screenshot before the interactive portal when available.
+- Current-tree validation passed 215/215 runtime tests and the full 80/80 live ASUS computer-use smoke.
+- macOS runtime code and runtime tool names/schemas are unchanged from 0.2.105.
+
+## 0.2.105 — 2026-09-22
+
+- macOS top-level window IDs include a snapshot generation and retain cached app/title/bounds identity, so a changed process-local window index cannot silently retarget an action.
+- Older cached handles recover the same unambiguous window after reorder; stale, fabricated, expired, or ambiguous handles fail closed and require `list_windows` refresh.
+- Unit/regression coverage verifies generation-scoped IDs, PID consistency, legacy/fabricated handle rejection, and fail-closed stale-handle behavior.
+- Runtime tool names and schemas are unchanged from 0.2.104.
+
 ## 0.2.104 — 2026-09-22
 
-- Fresh installations default the narrow catastrophic-command guardrail to block; ordinary development, file, Git, package-manager, Docker, database and browser workflows remain available, and explicit warn, allow, or unrestricted/godmode remain operator-controlled opt-ins.
-- macOS top-level window handles include a snapshot generation and retain short-lived app/title/bounds identity so window reordering cannot silently retarget a previous id.
-- A valid prior macOS handle resolves the same uniquely identifiable window after reorder; unknown, ambiguous, expired, PID-mismatched and legacy ids fail closed and request a fresh list_windows.
-- Added 73 Desktop Commander prompt scenarios as an isolated compatibility smoke matrix; all 73 passed on the release tree with no guardrail false positives. GNOME Wayland live computer-use passed 80/80 checks.
+- Fresh installations default the narrow catastrophic-command guardrail to `block`; ordinary development, file, Git, package-manager, Docker, database, browser, and document workflows remain available.
+- Explicit `warn`, `allow`, and unrestricted/godmode remain operator-controlled local opt-ins.
+- Added an isolated 73-scenario Desktop Commander compatibility smoke matrix; all 73 scenarios pass with zero guardrail false positives, while genuinely catastrophic host power/destructive commands remain blocked before execution.
+- Wayland portal pointer clicks now keep a 60 ms button-down interval before release so a transport-level success is not collapsed into a zero-duration gesture; the fix passed three consecutive full 80/80 live ASUS smoke runs.
+- Runtime tool names and schemas are unchanged from 0.2.103.
 
 ## 0.2.103 — 2026-09-22
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.104",
+  "version": "0.2.106",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",

package/src/extended/desktop-linux.mjs

@@ -1147,6 +1147,10 @@ export async function pointer(args = {}) {
     else if (action === 'move') throw new Error('x and y are required for pointer move');
     const click = async () => {
       await portalPointerButton(buttonName, true, portalOptions);
+      // A zero-duration press/release can be accepted by the portal transport but
+      // coalesced before the target toolkit observes a real click. Keep a short,
+      // human-scale hold so successful return means a deliverable button gesture.
+      await new Promise(resolve => setTimeout(resolve, 60));
       await portalPointerButton(buttonName, false, portalOptions);
       await new Promise(resolve => setTimeout(resolve, 40));
     };
@@ -1199,11 +1203,32 @@ export async function dragDrop(args = {}) {
   const runPortal = async timeoutMs => {
     const button = String(args.button || 'left').toLowerCase();
     const portalOptions = { timeoutMs };
-    await portalMoveTo(Number(args.from_x), Number(args.from_y), portalOptions);
+    const fromX = Number(args.from_x), fromY = Number(args.from_y);
+    const toX = Number(args.to_x), toY = Number(args.to_y);
+    await portalMoveTo(fromX, fromY, portalOptions);
+    // Give the compositor/toolkit one frame to observe the source position before
+    // button-down. A transport ACK only proves that the portal accepted the event.
+    await new Promise(resolve => setTimeout(resolve, 60));
     await portalPointerButton(button, true, portalOptions);
-    await new Promise(resolve => setTimeout(resolve, clamp(args.hold_ms, 120, 0, 5000)));
-    await portalMoveTo(Number(args.to_x), Number(args.to_y), portalOptions);
-    await new Promise(resolve => setTimeout(resolve, clamp(args.duration_ms, 120, 0, 5000)));
+    await new Promise(resolve => setTimeout(resolve, clamp(args.hold_ms, 120, 60, 5000)));
+
+    // A single absolute jump while the button is held can be coalesced by Mutter/
+    // XWayland and never become a target-visible pointermove. Send a short,
+    // human-like path so drag-aware toolkits observe movement before button-up.
+    const distance = Math.hypot(toX - fromX, toY - fromY);
+    const steps = Math.max(4, Math.min(20, Math.ceil(distance / 24)));
+    const durationMs = clamp(args.duration_ms, 160, 80, 5000);
+    const stepDelay = Math.max(8, Math.round(durationMs / steps));
+    for (let step = 1; step <= steps; step += 1) {
+      const ratio = step / steps;
+      await portalMoveTo(
+        fromX + (toX - fromX) * ratio,
+        fromY + (toY - fromY) * ratio,
+        portalOptions,
+      );
+      await new Promise(resolve => setTimeout(resolve, stepDelay));
+    }
+    await new Promise(resolve => setTimeout(resolve, 40));
     await portalPointerButton(button, false, portalOptions);
     await new Promise(resolve => setTimeout(resolve, 80));
   };
@@ -1404,6 +1429,24 @@ async function pngDimensions(file) {
   }
 }
 
+export function regionScreenshotBackends({
+  wayland = isWaylandSession(),
+  grim = commandExists('grim'),
+  gnomeScreenshot = commandExists('gnome-screenshot'),
+  ffmpeg = commandExists('ffmpeg'),
+  imagemagick = commandExists('import'),
+} = {}) {
+  const backends = [];
+  if (grim) backends.push('grim');
+  if (wayland) {
+    if (gnomeScreenshot && ffmpeg) backends.push('gnome-screenshot');
+    if (ffmpeg) backends.push('portal');
+  } else if (imagemagick) {
+    backends.push('imagemagick');
+  }
+  return backends;
+}
+
 export function portalCropGeometry(rect, displays = [], bitmap = null) {
   const requested = {
     x: Math.trunc(Number(rect?.x)),
@@ -1471,28 +1514,54 @@ export async function screenshotRegion(args = {}) {
   const full = path.join(dir, 'full.png');
   let captured = { x, y, width, height };
   let clipped = false;
+  let backend = '';
   try {
-    if (commandExists('grim')) {
-      await runFile('grim', ['-g',`${x},${y} ${width}x${height}`,target], { label:'screenshot region' });
-    } else if (isWaylandSession() && commandExists('ffmpeg')) {
-      await capturePortalScreenshot(full);
+    const cropFullDesktop = async () => {
       const inventory = await displayInventory().catch(() => null);
       const displays = Array.isArray(inventory?.structuredContent?.data) ? inventory.structuredContent.data : [];
       const bitmap = await pngDimensions(full);
       const crop = portalCropGeometry({ x, y, width, height }, displays, bitmap);
       captured = crop.visible;
       clipped = crop.clipped;
-      await runFile('ffmpeg', ['-hide_banner','-loglevel','error','-y','-i',full,'-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-frames:v','1',target], { label:'crop portal screenshot', timeout:30_000, maxBuffer:4*1024*1024 });
-    } else if (commandExists('import')) {
-      await runFile('import', ['-window','root','-crop',`${width}x${height}+${x}+${y}`,target], { label:'screenshot region' });
-    } else {
-      unavailable('Region screenshots', isWaylandSession() ? 'the XDG Desktop Portal plus ffmpeg, or grim, is required' : 'install ImageMagick import');
+      await runFile('ffmpeg', ['-hide_banner','-loglevel','error','-y','-i',full,'-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-frames:v','1',target], { label:'crop desktop screenshot', timeout:30_000, maxBuffer:4*1024*1024 });
+    };
+
+    const candidates = regionScreenshotBackends();
+    if (!candidates.length) {
+      unavailable('Region screenshots', isWaylandSession()
+        ? 'install grim, or gnome-screenshot/ffmpeg, or provide the XDG Desktop Portal plus ffmpeg'
+        : 'install ImageMagick import');
     }
+
+    let lastError = null;
+    for (const candidate of candidates) {
+      try {
+        if (candidate === 'grim') {
+          await runFile('grim', ['-g',`${x},${y} ${width}x${height}`,target], { label:'screenshot region' });
+        } else if (candidate === 'gnome-screenshot') {
+          await runFile('gnome-screenshot', ['-f',full], { label:'GNOME screenshot', timeout:10_000, maxBuffer:1024*1024 });
+          await cropFullDesktop();
+        } else if (candidate === 'portal') {
+          await capturePortalScreenshot(full);
+          await cropFullDesktop();
+        } else if (candidate === 'imagemagick') {
+          await runFile('import', ['-window','root','-crop',`${width}x${height}+${x}+${y}`,target], { label:'screenshot region' });
+        }
+        backend = candidate;
+        lastError = null;
+        break;
+      } catch (error) {
+        if (error?.code === 'PORTAL_CANCELLED') throw error;
+        lastError = error;
+      }
+    }
+    if (lastError) throw lastError;
+
     const {data} = await readPrivateTempFile(target, 4 * 1024 * 1024);
-    const summary = clipped
+    const baseSummary = clipped
       ? `Captured visible ${captured.width}x${captured.height} at ${captured.x},${captured.y}; clipped from requested ${width}x${height} at ${x},${y}.`
       : `Captured ${width}x${height} at ${x},${y}.`;
-    return multi([{type:'text',text:summary}, image(data.toString('base64'),'image/png')]);
+    return multi([{type:'text',text:`${baseSummary} Backend: ${backend}.`}, image(data.toString('base64'),'image/png')]);
   } finally { await removeTemp(dir); }
 }
 

package/src/extended/desktop.mjs

@@ -355,11 +355,7 @@ export async function uiAction(args = {}) {
   if (['set_value','set_range_value'].includes(action) && resolved.value == null) throw new Error(`ui_action ${action} requires value`);
   return adapter.uiAction(resolved);
 }
-export async function keyboard(args = {}) {
-  const explicitWindowTarget = explicitTypeTextWindowTarget(args, false);
-  if (explicitWindowTarget) await windowAction({ action:'focus', ...explicitWindowTarget });
-  return adapter.keyboard(args);
-}
+export async function keyboard(args = {}) { return adapter.keyboard(args); }
 export async function pointer(args = {}) {
   if (args.action === 'move' && ![args.x,args.y].every(value => Number.isFinite(Number(value)))) throw new Error('pointer move requires x and y');
   return adapter.pointer(args);

package/src/helpers/wayland-eis-helper.py

@@ -248,6 +248,9 @@ class EiSender:
         self.connected = False
         self.pointer_device_key = None
         self.pointer_position = None
+        # A press/release pair must stay on the same emulated device even if
+        # absolute motion crosses an EIS region and changes pointer_device_key.
+        self.button_device_keys = {}
 
     def _device_key(self, ptr):
         return int(ptr or 0)
@@ -267,6 +270,9 @@ class EiSender:
         if key == self.pointer_device_key:
             self.pointer_device_key = None
             self.pointer_position = None
+        for button, device_key in list(self.button_device_keys.items()):
+            if device_key == key:
+                self.button_device_keys.pop(button, None)
         if state:
             if state["emulating"]:
                 try:
@@ -313,10 +319,14 @@ class EiSender:
                             state["emulating"] = True
                 elif event_type == EI_EVENT_DEVICE_PAUSED:
                     device = self.lib.ei_event_get_device(event)
-                    state = self.devices.get(self._device_key(device))
+                    key = self._device_key(device)
+                    state = self.devices.get(key)
                     if state:
                         state["resumed"] = False
                         state["emulating"] = False
+                    for button, device_key in list(self.button_device_keys.items()):
+                        if device_key == key:
+                            self.button_device_keys.pop(button, None)
                 elif event_type == EI_EVENT_DEVICE_REMOVED:
                     self._drop_device(self.lib.ei_event_get_device(event))
             finally:
@@ -419,8 +429,24 @@ class EiSender:
             self.remember_pointer(device, (x, y))
             self.frame(device)
         elif op == "button":
-            device = self.device_for_pointer_context(EI_CAP_BUTTON)
-            self.lib.ei_device_button_button(device, int(payload["button"]), bool(payload.get("pressed")))
+            button = int(payload["button"])
+            pressed = bool(payload.get("pressed"))
+            if pressed:
+                device = self.device_for_pointer_context(EI_CAP_BUTTON)
+                self.button_device_keys[button] = self._device_key(device)
+            else:
+                key = self.button_device_keys.pop(button, None)
+                state = self.devices.get(key) if key else None
+                if (
+                    state
+                    and state["resumed"]
+                    and state["emulating"]
+                    and self.lib.ei_device_has_capability(state["ptr"], EI_CAP_BUTTON)
+                ):
+                    device = state["ptr"]
+                else:
+                    device = self.device_for_pointer_context(EI_CAP_BUTTON)
+            self.lib.ei_device_button_button(device, button, pressed)
             self.frame(device)
         elif op == "scroll":
             device = self.device_for_pointer_context(EI_CAP_SCROLL)
@@ -446,6 +472,7 @@ class EiSender:
             except Exception:
                 pass
         self.devices.clear()
+        self.button_device_keys.clear()
         if self.ei:
             self.lib.ei_unref(self.ei)
             self.ei = None