@remcp/runtime 0.2.162 → 0.2.163

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.162",
+  "version": "0.2.163",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",

package/src/extended/browser-readiness.mjs

@@ -0,0 +1,68 @@
+// Optional bounded local CDP readiness probe. This reports availability only;
+// it never exposes tab titles/URLs, starts Chrome, changes CDP flags or grants
+// permission to access targets. Actual actions always re-check browser policy.
+import { browserRemoteEnabled, endpointUrl, requestJson, validatedTargetWebSocketUrl } from './browser-endpoint.mjs';
+
+const records = new Map();
+const CACHE_MS = 3000;
+const MAX_ENTRIES = 8;
+const transportError = message =>
+  /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|abort|timed out|network error/i.test(message);
+
+export function classifyBrowserReadiness(version, targets, endpoint) {
+  if (!version || typeof version !== 'object' || !(version.Browser || version.webSocketDebuggerUrl)
+    || !Array.isArray(targets)) return { status:'invalid_response', ready:false, candidate_pages:0 };
+  let pages = 0;
+  for (const target of targets.slice(0, 1000)) {
+    if (target?.type !== 'page' || !target?.id || !target?.webSocketDebuggerUrl) continue;
+    try {
+      validatedTargetWebSocketUrl(target, endpointUrl(endpoint));
+      pages += 1;
+    } catch {
+      // The CDP server's untrusted page/socket metadata is not permission.
+    }
+  }
+  return { status:pages ? 'ready' : 'no_tabs', ready:pages > 0, candidate_pages:pages };
+}
+
+export async function probeBrowserReadiness({
+  endpoint,
+  timeoutMs = 350,
+  request = requestJson,
+  enabled = browserRemoteEnabled(),
+  now = Date.now(),
+  cache = request === requestJson,
+} = {}) {
+  if (!enabled) return {
+    status:'disabled', ready:false, candidate_pages:0, next_step:'use_native_accessibility',
+  };
+  let key;
+  try { key = endpointUrl(endpoint).origin; }
+  catch { return { status:'blocked_endpoint', ready:false, candidate_pages:0, next_step:'check_local_cdp_config' }; }
+  if (cache) {
+    const previous = records.get(key);
+    if (previous && previous.expires > now) return { ...previous.value, cache_hit:true };
+  }
+  const timeout = Math.max(100, Math.min(800, Number(timeoutMs) || 350));
+  let result;
+  try {
+    const version = await request('/json/version', endpoint, { timeout_ms:timeout });
+    // CDP version alone is not evidence a useful debuggable tab exists.
+    if (!version || typeof version !== 'object' || !(version.Browser || version.webSocketDebuggerUrl)) {
+      result = { status:'invalid_response', ready:false, candidate_pages:0 };
+    } else {
+      const targets = await request('/json/list', endpoint, { timeout_ms:timeout });
+      result = classifyBrowserReadiness(version, targets, endpoint);
+    }
+  } catch (error) {
+    result = { status:transportError(String(error?.message || '')) ? 'unavailable' : 'invalid_response',
+      ready:false, candidate_pages:0 };
+  }
+  const next_step = result.ready ? 'use_browser_tabs' : 'use_native_accessibility';
+  const output = { ...result, next_step };
+  if (cache) {
+    records.set(key, { expires:now + CACHE_MS, value:output });
+    while (records.size > MAX_ENTRIES) records.delete(records.keys().next().value);
+  }
+  return output;
+}

package/src/extended/catalog-schemas.mjs

@@ -164,6 +164,7 @@ export const OUTPUT_FIELDS = Object.freeze({
     cursor:outputObject('Pointer position when available.'),
     clipboard:outputObject('Clipboard metadata only unless clipboard content was explicitly requested.'),
     browser:outputObject('Optional browser DOM/CDP snapshot.'),
+    browser_readiness:outputObject('Optional bounded local CDP readiness status and fallback hint; no tab titles or URLs.'),
     ocr:outputObject('Optional OCR words/metadata.'),
     fallback_chain:outputObject('Which semantic/DOM/OCR/coordinate layers were requested and available.'),
     errors:outputArray(outputObject(), 'Non-fatal sub-capture errors.'),

package/src/extended/catalog.mjs

@@ -63,7 +63,7 @@ const browserTarget = {
 };
 
 export const extendedToolDefinitions = [
-  define('computer_snapshot', 'Computer snapshot', 'Capture one compact computer-use state bundle: displays, visible windows, labeled semantic accessibility state, clipboard metadata, optional native browser accessibility/CDP/OCR, and real screenshot pixels. Use include_ui=false for a fast screenshot-first state; screenshot_window_id/monitor/region scope pixels without adding another tool.', o({ max_ui_nodes:n('Maximum native accessibility nodes; default 800.'), max_ui_depth:n('Maximum native accessibility depth; default 12.'), include_ui:b('Include native accessibility tree; default true. False is the fast screenshot-only path.'), ui_scope:e(['active','desktop'],'Accessibility scope for computer_snapshot. Default active keeps the compact snapshot fast; desktop requests all accessible applications/windows.'), semantic_tree:b('Include compact semantic hierarchy text alongside nodes; default true.'), ui_browser_dom:b('Prefer the native browser accessibility document subtree when the OS exposes one; on Windows this also enables the Firefox MSAA/IAccessible fallback.'), include_screenshot:b('Include real rendered desktop pixels; default true.'), screenshot_region:{type:'array',minItems:4,maxItems:4,items:{type:'number'},description:'Virtual-desktop [left,top,right,bottom] region; takes precedence over full desktop.'}, screenshot_window_id:s('Capture the window returned by list_windows instead of the whole desktop.'), screenshot_monitor:s('Monitor name substring or "primary".'), screenshot_monitor_index:n('Zero-based monitor index.'), screenshot_padding:n('Pixels of context around a targeted screenshot; 0..200.'), grid_columns:n('Optional reference-grid column count returned as visual metadata; 0..20.'), grid_rows:n('Optional reference-grid row count returned as visual metadata; 0..20.'), include_browser:b('Include CDP browser accessibility when available. Default auto: only when the active app looks Chromium-based.'), browser_max_nodes:n('Maximum browser accessibility nodes; default 500, maximum 2000.'), include_ocr:b('Run optional local OCR. Default auto: only when native accessibility and browser DOM are unavailable. Requires tesseract on PATH.'), ocr_language:s('Tesseract language code, default eng; multiple languages may use eng+deu.'), ocr_max_words:n('Maximum OCR word boxes returned; default 2000, maximum 10000.'), ocr_psm:n('Tesseract page segmentation mode, 3..13; default 11.'), ocr_timeout_ms:n('OCR timeout, default 30000 ms, maximum 120000 ms.') }), readOnlyLive, desktopHandlers.computer_snapshot),
+  define('computer_snapshot', 'Computer snapshot', 'Capture one compact computer-use state bundle: displays, visible windows, labeled semantic accessibility state, clipboard metadata, optional native browser accessibility/CDP/OCR, and real screenshot pixels. For local browser availability without tab content or launching Chrome, request browser_readiness=true; otherwise no extra CDP probe is made. Use include_ui=false for a fast screenshot-first state; screenshot_window_id/monitor/region scope pixels without adding another tool.', o({ max_ui_nodes:n('Maximum native accessibility nodes; default 800.'), max_ui_depth:n('Maximum native accessibility depth; default 12.'), include_ui:b('Include native accessibility tree; default true. False is the fast screenshot-only path.'), ui_scope:e(['active','desktop'],'Accessibility scope for computer_snapshot. Default active keeps the compact snapshot fast; desktop requests all accessible applications/windows.'), semantic_tree:b('Include compact semantic hierarchy text alongside nodes; default true.'), ui_browser_dom:b('Prefer the native browser accessibility document subtree when the OS exposes one; on Windows this also enables the Firefox MSAA/IAccessible fallback.'), include_screenshot:b('Include real rendered desktop pixels; default true.'), screenshot_region:{type:'array',minItems:4,maxItems:4,items:{type:'number'},description:'Virtual-desktop [left,top,right,bottom] region; takes precedence over full desktop.'}, screenshot_window_id:s('Capture the window returned by list_windows instead of the whole desktop.'), screenshot_monitor:s('Monitor name substring or "primary".'), screenshot_monitor_index:n('Zero-based monitor index.'), screenshot_padding:n('Pixels of context around a targeted screenshot; 0..200.'), grid_columns:n('Optional reference-grid column count returned as visual metadata; 0..20.'), grid_rows:n('Optional reference-grid row count returned as visual metadata; 0..20.'), include_browser:b('Include CDP browser accessibility when available. Default auto: only when the active app looks Chromium-based.'), browser_readiness:b('Opt-in bounded CDP readiness status: disabled, blocked endpoint, unavailable, invalid response, no tabs, or ready. No tab URLs or titles; never launches Chrome. Default false.'), browser_max_nodes:n('Maximum browser accessibility nodes; default 500, maximum 2000.'), include_ocr:b('Run optional local OCR. Default auto: only when native accessibility and browser DOM are unavailable. Requires tesseract on PATH.'), ocr_language:s('Tesseract language code, default eng; multiple languages may use eng+deu.'), ocr_max_words:n('Maximum OCR word boxes returned; default 2000, maximum 10000.'), ocr_psm:n('Tesseract page segmentation mode, 3..13; default 11.'), ocr_timeout_ms:n('OCR timeout, default 30000 ms, maximum 120000 ms.') }), readOnlyLive, desktopHandlers.computer_snapshot),
   define(
     'computer_action',
     'Computer action',

package/src/extended/desktop-computer-actions.mjs

@@ -72,6 +72,17 @@ export function computerActionWindowTextArgs(args = {}) {
   };
 }
 
+// A UI element ID is scoped to the native accessibility tree. If it becomes
+// stale, never reinterpret a remaining name, selector, OCR hint or x/y as a
+// different target: refind that element in the same window first.
+export function hasPinnedUiElement(args = {}) {
+  return Boolean(args.id || args.automation_id || args.automationId);
+}
+
+function stalePinnedUiElement() {
+  return new Error('Pinned UI element could not be resolved; use ui_snapshot and ui_find to refresh its ID. No browser, OCR, window or coordinate fallback was attempted.');
+}
+
 export async function computerAction(args = {}) {
   const legacy = ['ui','pointer','window','type','keyboard','scroll','drag','clipboard','launch_app','open_path','reveal_path','notification'];
   const universal = [
@@ -89,6 +100,7 @@ export async function computerAction(args = {}) {
   const hasOcr = Boolean(ocrNeedle);
   const hasCoordinates = Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y));
   const automatic = target === 'auto';
+  const pinnedUi = automatic && hasPinnedUiElement(args);
 
   if (action === 'batch') {
     const steps = Array.isArray(args.steps) ? args.steps : [];
@@ -170,7 +182,10 @@ export async function computerAction(args = {}) {
     const failures = [];
     if (hasUi) {
       try { return await typeText({ ...args, method:'accessibility' }); }
-      catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
+      catch (error) {
+        if (pinnedUi) throw stalePinnedUiElement();
+        failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`);
+      }
     }
     if (hasBrowser) {
       if (await browserCapabilityAvailable(args.endpoint, 300)) {
@@ -203,7 +218,10 @@ export async function computerAction(args = {}) {
       let uiError = null;
       if (hasUi) {
         try { return await scroll(args); }
-        catch (error) { uiError = error; }
+        catch (error) {
+          if (pinnedUi) throw stalePinnedUiElement();
+          uiError = error;
+        }
       }
       if (hasBrowser) {
         if (await browserCapabilityAvailable(args.endpoint, 300)) return browserComputerAction(args, 'scroll');
@@ -244,7 +262,10 @@ export async function computerAction(args = {}) {
     let uiError = null;
     if (hasUi) {
       try { return await uiAction({ ...args, action:uiActionName }); }
-      catch (error) { uiError = error; }
+      catch (error) {
+        if (pinnedUi) throw stalePinnedUiElement();
+        uiError = error;
+      }
     }
     if (hasBrowser) return browserComputerAction(args, uiActionName);
     if (uiError) throw uiError;
@@ -288,7 +309,10 @@ export async function computerAction(args = {}) {
       const failures = [];
       if (hasUi) {
         try { return await uiAction({ ...args, action:'click' }); }
-        catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
+        catch (error) {
+        if (pinnedUi) throw stalePinnedUiElement();
+        failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`);
+      }
       }
       if (hasBrowser) {
         if (await browserCapabilityAvailable(args.endpoint, 300)) {

package/src/extended/desktop-computer-snapshot.mjs

@@ -9,6 +9,7 @@ import * as linux from './desktop-linux.mjs';
 import * as macos from './desktop-macos.mjs';
 import * as windows from './desktop-windows.mjs';
 import { browserAction, browserCapabilityAvailable, browserSnapshot } from './browser.mjs';
+import { probeBrowserReadiness } from './browser-readiness.mjs';
 import { findOcrText, ocrImage } from './ocr.mjs';
 
 
@@ -20,7 +21,7 @@ export async function computerSnapshot(args = {}) {
   const screenshotRequested = args.include_screenshot !== false || args.include_ocr === true;
   const captureScreenshot = desktopAvailable && screenshotRequested;
   const includeUi = args.include_ui !== false;
-  const [windowsResult, displaysResult, uiResult, clipboardResult, cursorResult, shotResult] = await Promise.allSettled([
+  const [windowsResult, displaysResult, uiResult, clipboardResult, cursorResult, shotResult, readinessResult] = await Promise.allSettled([
     desktopAvailable ? listWindows() : Promise.resolve(null),
     desktopAvailable ? displayInventory() : Promise.resolve(null),
     includeUi && desktopAvailable
@@ -35,6 +36,7 @@ export async function computerSnapshot(args = {}) {
     desktopAvailable ? clipboard({ action:'read' }) : Promise.resolve(null),
     desktopAvailable ? cursorPosition() : Promise.resolve(null),
     captureScreenshot ? computerScreenshotTask(args) : Promise.resolve(null),
+    args.browser_readiness === true ? probeBrowserReadiness() : Promise.resolve(null),
   ]);
   const parse = (result, fallback = null) => {
     if (result.status !== 'fulfilled' || !result.value || result.value.isError === true) return fallback;
@@ -112,9 +114,11 @@ export async function computerSnapshot(args = {}) {
   const activeLabel = [activeApp?.app, activeApp?.window, activeWindow?.app, activeWindow?.title].filter(Boolean).join(' ');
   const browserRequested = args.include_browser === true
     || (args.include_browser !== false && /chrome|chromium|edge|brave/i.test(activeLabel));
+  const readiness = args.browser_readiness === true && readinessResult.status === 'fulfilled'
+    ? objectRecord(readinessResult.value) : null;
   let browser = null;
   let browserError = null;
-  if (browserRequested && await browserCapabilityAvailable(undefined, 400)) {
+  if (browserRequested && (readiness ? readiness.ready : await browserCapabilityAvailable(undefined, 400))) {
     const hint = browserTitleHint(activeWindow);
     try {
       const browserResult = await browserSnapshot({
@@ -184,6 +188,7 @@ export async function computerSnapshot(args = {}) {
     cursor,
     ui,
     ...(browser ? { browser } : {}),
+    ...(args.browser_readiness === true ? { browser_readiness:readiness || { status:'invalid_response', ready:false, candidate_pages:0, next_step:'use_native_accessibility' } } : {}),
     ...(ocr ? { ocr } : {}),
     fallback_chain: {
       accessibility: { requested:includeUi, available:includeUi && desktopAvailable && uiResult.status === 'fulfilled' && Boolean(uiResult.value), nodes:uiNodes.length },