@remcp/runtime 0.2.79 → 0.2.81

package/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+## 0.2.81 — 2026-09-21
+
+- `record_screen` is advertised only when the current OS/session has a recorder backend the runtime can actually invoke.
+- macOS/Windows require `ffmpeg`; Linux Wayland requires `wf-recorder + timeout`; Linux X11 requires `ffmpeg`.
+- Availability errors are platform-specific, and regression coverage locks the capability matrix.
+- Lockstep patch release with the ReMCP client/server at `0.2.81`.
+
+## 0.2.80 — 2026-09-21
+
+- Wayland `type_text` can replace text in an inactive explicitly targeted native window without compositor focus when exactly one editable accessibility control is present.
+- Ambiguous or missing editables are never guessed; key/clipboard/Enter fallbacks retain verified window-focus requirements.
+- Tool descriptions now explain the semantic window-target path so agents do not request unnecessary focus before accessibility replacement.
+- Lockstep patch release with the ReMCP client/server at `0.2.80`.
+
 ## 0.2.79 — 2026-09-21
 
 - Explicit Wayland keyboard and key-by-key text input now focuses and verifies the requested target window before dispatch, with bounded direct window cycling and verified fallbacks.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.79",
+  "version": "0.2.81",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",
@@ -32,11 +32,11 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/antonbaider/remcp.git",
+    "url": "git+https://github.com/getremcp/remcp.git",
     "directory": "packages/runtime"
   },
   "bugs": {
-    "url": "https://github.com/antonbaider/remcp/issues"
+    "url": "https://github.com/getremcp/remcp/issues"
   },
   "publishConfig": {
     "access": "public"

package/src/extended/catalog.mjs

@@ -9,7 +9,7 @@ import { isWaylandSession } from '../screenshot-portal.mjs';
 import { waylandPortalCandidate } from '../wayland-remote-desktop.mjs';
 import { commandExists, runFile } from './common.mjs';
 import { desktopHandlers } from './desktop.mjs';
-import { diagnosticHandlers } from './diagnostics.mjs';
+import { diagnosticHandlers, recordScreenAvailable } from './diagnostics.mjs';
 import { documentHandlers } from './documents.mjs';
 
 const readOnly = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false };
@@ -538,7 +538,7 @@ export const extendedToolDefinitions = [
   define('ui_snapshot', 'UI accessibility snapshot', 'Read the active desktop accessibility/UI Automation tree with compact labels, semantic hierarchy, bounds, state and suggested actions. Default scope is the active application/window; scope=desktop requests the full desktop tree. Explicit pid/app/window_title scopes override the active default. browser_dom asks the OS accessibility provider for the web-document subtree when available.', o({ scope:e(['active','desktop'],'Default active. Use desktop only when cross-application discovery is required.'), pid:n('Optional owning process id scope.'), app:s('Optional application/process name substring scope.'), window_title:s('Optional top-level accessibility window/frame title scope.'), browser_dom:b('Prefer the native browser document accessibility subtree instead of browser chrome when supported.'), semantic_tree:b('Include compact semantic hierarchy text; default true.'), max_nodes:n('Maximum nodes; default 500, maximum 5000.'), max_depth:n('Maximum traversal depth; default 8, maximum 32.') }), readOnlyLive, desktopHandlers.ui_snapshot, ['ui']),
   define('ui_find', 'Find UI elements', 'Find native accessible desktop elements by label/id/name/role/AutomationId. Reuses the latest five-second semantic snapshot for fast repeated targeting; refresh=true forces a new OS accessibility capture.', o({ ...uiSelector, refresh:b('Force a fresh accessibility capture instead of the short-lived snapshot cache.'), limit:n('Maximum matches; default 20.'), max_nodes:n(), max_depth:n() }), readOnlyLive, desktopHandlers.ui_find, ['ui']),
   define('ui_action', 'Act on UI element', 'Act on a native accessibility/UI Automation element by id, label or semantic selector. Supports invoke/focus/value/toggle/selection, expand-collapse, range values and scroll-into-view before falling back to coordinates.', o({ action:e(['click','invoke','focus','set_value','select','toggle','expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection']), ...uiSelector, value:{description:'New text or numeric range value.'} }, ['action']), openMutating, desktopHandlers.ui_action, ['ui']),
-  define('type_text', 'Type text', 'Enter Unicode text into a native semantic accessibility target or focused native control. Auto prefers verified accessibility editing; explicit semantic targets fail safely when the platform rejects the write instead of claiming an unverified paste. Use browser_action for Chromium page DOM. Clipboard/keys fallbacks remain available for focused/window-level input when appropriate, and explicit Wayland window targets are focused and verified before dispatch.', o({ text:s('Text to enter.'), method:e(['auto','accessibility','clipboard','keys'], 'Default auto prefers verified accessibility editing and falls back to clipboard paste. keys sends physical key events and therefore follows the active keyboard layout.'), clear:b('Clear existing text before input when keyboard/paste fallback is used.'), caret_position:e(['start','idle','end']), press_enter:b('Press Enter after typing.'), delay_ms:n('Per-key delay for method=keys.'), ...uiSelector, window_id:s('Optional target window id; the window is focused before auto/accessibility/clipboard/keys input and Wayland verifies focus before dispatch.'), title:s('Optional target window title substring; focused before input when no semantic element target is supplied.'), ...inputBackendSelector }, ['text']), openMutating, desktopHandlers.type_text, ['input']),
+  define('type_text', 'Type text', 'Enter Unicode text into a native semantic accessibility target or focused native control. Auto prefers verified accessibility editing; with clear=true and an explicit window target it can replace the value of the single editable accessibility control in that window without compositor focus. Explicit semantic targets fail safely when the platform rejects the write instead of claiming an unverified paste. Use browser_action for Chromium page DOM. Clipboard/keys/Enter fallbacks still require verified window focus on Wayland.', o({ text:s('Text to enter.'), method:e(['auto','accessibility','clipboard','keys'], 'Default auto prefers verified accessibility editing and falls back to clipboard paste. keys sends physical key events and therefore follows the active keyboard layout.'), clear:b('Replace existing text for semantic/window-scoped accessibility input, or clear it before keyboard/paste fallback.'), caret_position:e(['start','idle','end']), press_enter:b('Press Enter after typing; physical Enter delivery requires verified window focus.'), delay_ms:n('Per-key delay for method=keys.'), ...uiSelector, window_id:s('Optional target window id. With auto/accessibility + clear=true, a unique editable control may be updated semantically without focus; keyboard/clipboard fallbacks verify focus before dispatch.'), title:s('Optional target window title substring. With auto/accessibility + clear=true, a unique editable control may be updated semantically without compositor focus.'), ...inputBackendSelector }, ['text']), openMutating, desktopHandlers.type_text, ['input']),
   define('keyboard', 'Keyboard shortcut', 'Send a keyboard shortcut or key press to the active desktop application or an explicitly selected window. On Wayland, window_id/pid/app/title targets are focused and verified before input is dispatched; backend=portal explicitly requests the consent-backed XDG RemoteDesktop path.', o({ shortcut:s(), key:s(), keys:{ type:'array', items:{type:'string'} }, ...inputWindowSelector, ...inputBackendSelector }), openMutating, desktopHandlers.keyboard, ['input']),
   define('pointer', 'Pointer action', 'Move the pointer or perform click, double-click, right-click, mouse-down, or mouse-up at desktop screen coordinates. Wayland supports XWayland fallback and an explicitly consented XDG RemoteDesktop backend.', o({ action:e(['move','click','double_click','right_click','down','up']), x:n(), y:n(), button:e(['left','right','middle']), ...inputBackendSelector }, ['action']), openMutating, desktopHandlers.pointer, ['pointer']),
   define('drag_drop', 'Drag and drop', 'Drag between desktop coordinates or semantic UI element IDs returned by ui_snapshot/ui_find. Element IDs are resolved to current accessibility bounds before input. GNOME Wayland requires the consent-backed Remote Desktop portal for real drag delivery; unreliable XTEST/xdotool drag is rejected.', o({ from_x:n(), from_y:n(), to_x:n(), to_y:n(), from_id:s('Source UI element id; use instead of from_x/from_y.'), to_id:s('Destination UI element id; use instead of to_x/to_y.'), button:e(['left','right','middle']), hold_ms:n(), duration_ms:n(), ...inputBackendSelector }), openMutating, desktopHandlers.drag_drop, ['drag']),
@@ -623,7 +623,7 @@ export async function capabilitySnapshot() {
     apps: win || mac || commandExists('dpkg-query') || commandExists('rpm'),
     audio: win || mac || commandExists('wpctl') || commandExists('pactl') || commandExists('amixer'),
     power: win || mac || commandExists('systemctl') || commandExists('loginctl'),
-    record: win || mac || commandExists('wf-recorder') || commandExists('ffmpeg'),
+    record: recordScreenAvailable({ platform:process.platform, wayland }),
     ooxml: win || (commandExists('zip') && commandExists('unzip')),
   };
 }

package/src/extended/desktop-linux.mjs

@@ -757,6 +757,37 @@ export async function clipboard(args = {}) {
   unavailable('Clipboard read', `${failures.join('; ') || 'install wl-clipboard or xclip'}`);
 }
 
+export async function typeTextWindowTarget(value, args = {}) {
+  const row = await windowMatch(args);
+  if (!row?.pid) return null;
+  const snapshot = await uiSnapshot({
+    pid:row.pid,
+    window_title:row.title || undefined,
+    max_nodes:2200,
+    max_depth:24,
+  });
+  let payload = null;
+  try { payload = JSON.parse(snapshot.content?.[0]?.text || '{}'); } catch {}
+  const nodes = Array.isArray(payload) ? payload : payload?.nodes || [];
+  const editableRoles = /(^|\b)(text|entry|textbox|edit|text field|search box|password text)(\b|$)/i;
+  const candidates = nodes.filter(node => node?.id && editableRoles.test(String(node.role || '')));
+  if (candidates.length !== 1) return null;
+
+  const target = candidates[0];
+  const result = await uiAction({ action:'set_value', id:target.id, value:String(value) });
+  let action = result.content?.[0]?.text || '';
+  try { action = JSON.parse(action); } catch {}
+  return {
+    id:target.id,
+    pid:row.pid,
+    app:row.app || target.app || '',
+    window:row.title || target.window || '',
+    role:target.role || '',
+    name:target.name || '',
+    action,
+  };
+}
+
 export async function typeTextFocused(value) {
   if (!commandExists('python3')) return null;
   const script = String.raw`import gi,json,sys

package/src/extended/desktop.mjs

@@ -644,11 +644,24 @@ export async function typeText(args = {}) {
     optionalString(resolvedArgs.automation_id || resolvedArgs.automationId)
   );
   const explicitWindowTarget = explicitTypeTextWindowTarget(resolvedArgs, hasElementSelector);
-  if (explicitWindowTarget) await windowAction({ action:'focus', ...explicitWindowTarget });
   const clear = args.clear === true;
   const pressEnter = args.press_enter === true;
   const caret = requireEnum(args.caret_position || 'idle', 'caret_position', ['start','idle','end']);
 
+  if (
+    explicitWindowTarget && !hasElementSelector && clear && !pressEnter
+    && (method === 'auto' || method === 'accessibility')
+    && typeof adapter.typeTextWindowTarget === 'function'
+  ) {
+    const semantic = await adapter.typeTextWindowTarget(value, resolvedArgs);
+    if (semantic) {
+      return jsonResult({ length:value.length, method:'accessibility', target:semantic, clear:true, press_enter:false });
+    }
+    if (method === 'accessibility') {
+      throw new Error('Accessibility-only text input needs exactly one editable control in the targeted window when no element selector is provided');
+    }
+  }
+
   if ((method === 'auto' || method === 'accessibility') && hasElementSelector) {
     try {
       const result = await uiAction({ ...resolvedArgs, action:'set_value', value });
@@ -671,7 +684,10 @@ export async function typeText(args = {}) {
       throw new Error(`Semantic text target rejected accessibility input. Refresh the UI target or use browser_action for browser page content. ${detail}`);
     }
   }
-  if ((method === 'auto' || method === 'accessibility') && !hasElementSelector && typeof adapter.typeTextFocused === 'function' && !clear && caret === 'idle') {
+  if (
+    !explicitWindowTarget && (method === 'auto' || method === 'accessibility') && !hasElementSelector
+    && typeof adapter.typeTextFocused === 'function' && !clear && caret === 'idle'
+  ) {
     const semantic = await adapter.typeTextFocused(value, resolvedArgs);
     if (semantic) {
       if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
@@ -680,9 +696,10 @@ export async function typeText(args = {}) {
     if (method === 'accessibility') throw new Error('No focused editable accessibility element is available');
   }
   if (method === 'accessibility') {
-    throw new Error('Accessibility-only text input requires a semantic element target for replacement, or a focused editable control with clear=false and caret_position=idle');
+    throw new Error('Accessibility-only text input requires a semantic element target, one editable control in the targeted window with clear=true, or a focused editable control with clear=false and caret_position=idle');
   }
 
+  if (explicitWindowTarget) await windowAction({ action:'focus', ...explicitWindowTarget });
   if (hasElementSelector) await uiAction({ ...resolvedArgs, action:'focus' });
   if (clear) {
     await keyboard({ ...resolvedArgs, shortcut:process.platform === 'darwin' ? 'CMD+A' : 'CTRL+A' });

package/src/extended/diagnostics.mjs

@@ -5,6 +5,7 @@ import path from 'node:path';
 import process from 'node:process';
 import { stat } from 'node:fs/promises';
 import { assertAllowedCommand } from '../policy.mjs';
+import { isWaylandSession } from '../screenshot-portal.mjs';
 import { resolveSafePath, text } from '../util.mjs';
 import {
   clamp,
@@ -243,6 +244,19 @@ export async function powerAction(args) {
   return text(`${policy.note ? `${policy.note}\n` : ''}Power action ${action} requested.`);
 }
 
+export function recordScreenAvailable({
+  platform = process.platform,
+  wayland = platform === 'linux' ? isWaylandSession() : false,
+  commandExistsFn = commandExists,
+} = {}) {
+  if (platform === 'linux') {
+    if (wayland) return commandExistsFn('wf-recorder') && commandExistsFn('timeout');
+    return commandExistsFn('ffmpeg');
+  }
+  if (platform === 'darwin' || platform === 'win32') return commandExistsFn('ffmpeg');
+  return false;
+}
+
 export async function recordScreen(args) {
   const seconds = clamp(args.duration_seconds, 5, 1, 120);
   const fps = clamp(args.fps, 15, 1, 60);
@@ -251,7 +265,14 @@ export async function recordScreen(args) {
     const result = await runFile('timeout', ['--signal=INT', `${seconds}s`, 'wf-recorder', '-f',destination,'-r',String(fps),'-c','libx264'], { label:'screen recording', timeout:(seconds+10)*1000, allowFailure:true });
     if (![0, 124, 130].includes(Number(result.code))) throw new Error(result.stderr.trim() || `wf-recorder exited ${result.code}`);
   } else {
-    if (!commandExists('ffmpeg')) unavailable('Screen recording', 'install wf-recorder on Wayland or ffmpeg');
+    if (process.platform === 'linux' && isWaylandSession()) {
+      unavailable('Screen recording', 'wf-recorder and timeout are required on Wayland');
+    }
+    if (!commandExists('ffmpeg')) {
+      if (process.platform === 'darwin') unavailable('Screen recording', 'ffmpeg is required on macOS');
+      if (process.platform === 'win32') unavailable('Screen recording', 'ffmpeg is required on Windows');
+      unavailable('Screen recording', 'ffmpeg is required on X11');
+    }
     let argv;
     if (process.platform === 'win32') argv=['-y','-f','gdigrab','-framerate',String(fps),'-i','desktop','-t',String(seconds),'-pix_fmt','yuv420p',destination];
     else if (process.platform === 'darwin') argv=['-y','-f','avfoundation','-framerate',String(fps),'-i','1:none','-t',String(seconds),'-pix_fmt','yuv420p',destination];