@remcp/runtime 0.2.157 → 0.2.159

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remcp/runtime",
-  "version": "0.2.157",
+  "version": "0.2.159",
   "description": "First-party ReMCP local device runtime: filesystem, terminal, computer-use, browser, diagnostics and document tools over MCP.",
   "type": "module",
   "license": "MIT",

package/src/extended/browser-actions.mjs

@@ -12,6 +12,7 @@ import { clamp, commandExists, jsonResult, optionalString, requireEnum, spawnDet
 import { assertBrowserRemoteEnabled, assertSafeEvaluationExpression } from './browser-endpoint.mjs';
 import { callPageFunction, evaluate, withTarget } from './browser-session.mjs';
 import { browserNavigate, browserSnapshot, browserTabs, findExpression } from './browser-read.mjs';
+import { executeBrowserWorkflow, validateBrowserWorkflow } from './browser-workflow.mjs';
 
 export async function browserFind(args) {
   if (!(args.selector || args.text || args.role)) throw new Error('browser_find requires selector, text, or role; use browser_snapshot to enumerate page structure');
@@ -89,11 +90,26 @@ export const PAGE_ELEMENT_ACTION = `function(selector, needle, action, value) {
     el.dispatchEvent(new Event('input', {bubbles:true}));
     el.dispatchEvent(new Event('change', {bubbles:true}));
   }
-  const b = el.getBoundingClientRect();
-  if (action === 'click' && (!b.width || !b.height)) throw new Error('Element has no clickable bounds');
+  let b = el.getBoundingClientRect();
+  if (action === 'click') {
+    // An inline element can span several text fragments. Its bounding box center
+    // can point to a completely unrelated input (or even a destructive button).
+    // Only return a point whose actual hit target belongs to this element.
+    const rectangles = Array.from(el.getClientRects());
+    const safe = rectangles.find(rect => {
+      if (!rect.width || !rect.height) return false;
+      const x = rect.x + rect.width / 2, y = rect.y + rect.height / 2;
+      const target = document.elementFromPoint(x, y);
+      return target && (target === el || el.contains(target));
+    });
+    if (!safe) throw new Error('Element has no unobstructed clickable fragment');
+    b = safe;
+  }
   return {
     tag: el.tagName.toLowerCase(),
-    value: 'value' in el ? String(el.value) : '',
+    // Never expose password or credential field contents in tool responses.
+    value: 'value' in el && !el.matches('input[type="password"],input[autocomplete="current-password"],input[autocomplete="new-password"],input[autocomplete="one-time-code"]') ?
+      String(el.value) : '',
     x: b.x, y: b.y, width: b.width, height: b.height,
   };
 }`;
@@ -188,9 +204,17 @@ export function browserScrollDeltas(args = {}) {
 }
 
 export async function browserAction(args) {
-  const action = requireEnum(args.action, 'action', ['click', 'focus', 'type', 'set_value', 'select', 'scroll', 'scroll_into_view', 'upload', 'press', 'set_viewport', 'fill_form']);
+  const action = requireEnum(args.action, 'action', ['click', 'focus', 'type', 'set_value', 'select', 'scroll', 'scroll_into_view', 'upload', 'press', 'set_viewport', 'fill_form', 'workflow']);
   const formFields = action === 'fill_form' ? validateFormFields(args.fields) : null;
+  const workflowSteps = action === 'workflow' ?
+    validateBrowserWorkflow(args.steps, {validateFormFields}) : null;
   return withTarget(args, async (session, target) => {
+    if (action === 'workflow') {
+      const result = await executeBrowserWorkflow(session, workflowSteps, {
+        pageElementAction:PAGE_ELEMENT_ACTION, pageFillForm:PAGE_FILL_FORM,
+      });
+      return jsonResult({target_id:target.id, action, result});
+    }
     if (action === 'fill_form') {
       const result = await callPageFunction(session, PAGE_FILL_FORM, [formFields]);
       return jsonResult({target_id:target.id, action, result});

package/src/extended/browser-workflow.mjs

@@ -0,0 +1,111 @@
+// Execute deterministic bounded UI steps on one guarded CDP target/session.
+// A workflow is explicitly requested by the caller after inspecting the page.
+// No arbitrary JavaScript, cross-target access, file uploads or implicit submits.
+import { assertResolvedPageUrl } from './browser-endpoint.mjs';
+import { callPageFunction, evaluate } from './browser-session.mjs';
+
+const ACTIONS = new Set(['fill_form','set_value','select','focus','click','wait_for']);
+const CONDITIONS = new Set(['selector','text','url_contains','load']);
+
+export function validateBrowserWorkflow(steps, { validateFormFields }) {
+  if (!Array.isArray(steps) || steps.length < 1 || steps.length > 12) {
+    throw new Error('browser workflow requires 1 to 12 steps');
+  }
+  return steps.map((raw, index) => {
+    const action = typeof raw?.action === 'string' ? raw.action : '';
+    if (!ACTIONS.has(action)) throw new Error('Unsupported browser workflow action at step ' + (index + 1));
+    if (action === 'fill_form') {
+      return { action, fields:validateFormFields(raw.fields) };
+    }
+    if (action === 'wait_for') {
+      const condition = String(raw.condition || 'selector');
+      if (!CONDITIONS.has(condition)) throw new Error('Unsupported browser wait condition at step ' + (index + 1));
+      const criterion = condition === 'selector' ? raw.selector : raw.text;
+      if (condition !== 'load' && (typeof criterion !== 'string' || !criterion.trim() || criterion.length > 512)) {
+        throw new Error('Browser workflow wait requires a bounded criterion at step ' + (index + 1));
+      }
+      const timeout_ms = raw.timeout_ms == null ? 1500 : Number(raw.timeout_ms);
+      if (!Number.isInteger(timeout_ms) || timeout_ms < 100 || timeout_ms > 4000) {
+        throw new Error('Browser workflow wait timeout must be 100-4000 ms');
+      }
+      return {action, condition, criterion:String(criterion || ''), timeout_ms};
+    }
+    if (typeof raw.selector !== 'string' || !raw.selector.trim() || raw.selector.length > 512) {
+      throw new Error('Browser workflow action requires a CSS selector at step ' + (index + 1));
+    }
+    if (['set_value','select'].includes(action) &&
+        (typeof raw.value !== 'string' && typeof raw.value !== 'number' && typeof raw.value !== 'boolean')) {
+      throw new Error('Browser workflow input requires a string/number/boolean value at step ' + (index + 1));
+    }
+    if (['set_value','select'].includes(action) && String(raw.value).length > 16384) {
+      throw new Error('Browser workflow input exceeds 16384 characters');
+    }
+    return {action, selector:raw.selector.trim(),
+      ...(['set_value','select'].includes(action) ? {value:String(raw.value)} : {})};
+  });
+}
+
+async function waitForStep(session, step, started) {
+  const deadline = Math.min(started + 12000, Date.now() + step.timeout_ms);
+  const expression = step.condition === 'selector' ?
+    'Boolean(document.querySelector('+JSON.stringify(step.criterion)+'))'
+    : step.condition === 'text' ?
+      '(document.body?.innerText||"").includes('+JSON.stringify(step.criterion)+')'
+      : step.condition === 'url_contains' ?
+        'location.href.includes('+JSON.stringify(step.criterion)+')'
+        : 'document.readyState === "complete"';
+  while (Date.now() < deadline) {
+    let state;
+    try {
+      // One CDP roundtrip validates the navigated page URL and the condition.
+      state = await evaluate(session, '({url:location.href,matched:Boolean(' + expression + ')})');
+    } catch (error) {
+      // Clicking a link can invalidate the JS execution context briefly.
+      if (!/execution context (was destroyed|not found)|cannot find context/i.test(String(error?.message || ''))) {
+        throw error;
+      }
+      await new Promise(resolve=>setTimeout(resolve, 75));
+      continue;
+    }
+    // Never accept a match on a private or unsafe page.
+    await assertResolvedPageUrl(state?.url);
+    if (state?.matched) return;
+    await new Promise(resolve=>setTimeout(resolve, 75));
+  }
+  throw new Error('Workflow wait timed out');
+}
+
+export async function executeBrowserWorkflow(session, steps, {
+  pageElementAction, pageFillForm,
+}) {
+  const started = Date.now();
+  const completed = [];
+  for (let index = 0; index < steps.length; index++) {
+    const step = steps[index];
+    if (Date.now() - started > 12000) throw new Error('Workflow exceeded 12 second budget after '+completed.length+' steps');
+    try {
+      const currentUrl = await evaluate(session, 'location.href');
+      await assertResolvedPageUrl(currentUrl);
+      if (step.action === 'fill_form') {
+        await callPageFunction(session,pageFillForm,[step.fields]);
+      } else if (step.action === 'wait_for') {
+        await waitForStep(session,step,started);
+      } else if (step.action === 'click') {
+        const bounds=await callPageFunction(session,pageElementAction,[step.selector,null,'click','']);
+        const x=Number(bounds.x)+Number(bounds.width)/2;
+        const y=Number(bounds.y)+Number(bounds.height)/2;
+        await session.send('Input.dispatchMouseEvent',{type:'mouseMoved',x,y,button:'none'});
+        await session.send('Input.dispatchMouseEvent',{type:'mousePressed',x,y,button:'left',clickCount:1});
+        await session.send('Input.dispatchMouseEvent',{type:'mouseReleased',x,y,button:'left',clickCount:1});
+      } else {
+        await callPageFunction(session,pageElementAction,[step.selector,null,step.action,step.value??'']);
+      }
+      completed.push({index:index+1,action:step.action,ok:true});
+    } catch (error) {
+      // Don't echo form values or page text in an error.
+      throw new Error('Browser workflow stopped at step '+(index+1)+' ('+step.action+'); '+
+        completed.length+' steps completed. '+(step.action==='wait_for'?'Condition not satisfied.':'Inspect the page before retrying.'));
+    }
+  }
+  return {completed:completed.length,steps:completed,elapsed_ms:Date.now()-started};
+}

package/src/extended/catalog.mjs

@@ -165,7 +165,7 @@ export const extendedToolDefinitions = [
   define('browser_navigate', 'Browser navigate', 'Open a new debuggable tab or navigate a selected browser page by URL, history back/forward, or reload through Chrome DevTools Protocol.', o({ ...browserTarget, action:e(['url','new_tab','back','forward','reload'], 'Default url when url is provided, otherwise reload. new_tab creates the first page target when needed.'), url:s('Destination URL for action=url or action=new_tab.'), wait:b('Wait for the page load event; default true.'), ignore_cache:b('Reload without cache when action=reload.') }), openMutatingNonDestructive, browserHandlers.browser_navigate, ['browser_cdp']),
   define('browser_snapshot', 'Browser snapshot', 'Return the selected browser page accessibility tree through Chrome DevTools Protocol and, when requested, attach a real rendered viewport PNG for visual verification. A selector is temporarily recentered for capture, its exact viewport bounds are returned, and the prior scroll position is restored before return.', o({ ...browserTarget, max_nodes:n('Maximum AX nodes; default 1500.'), include_screenshot:b('Attach a real CDP-rendered viewport PNG screenshot; default false.'), selector:s('When include_screenshot=true, center this CSS-selected element before capture and report its viewport bounds.') }), readOnlyOpen, browserHandlers.browser_snapshot, ['browser_cdp']),
   define('browser_find', 'Find browser element', 'Find visible DOM elements by CSS selector, text, or ARIA role and return reusable selectors plus text, value, and bounding boxes.', o({ ...browserTarget, selector:s('CSS selector.'), text:s('Visible text substring.'), role:s('ARIA role.'), limit:n() }), readOnlyOpen, browserHandlers.browser_find, ['browser_cdp']),
-  define('browser_action', 'Browser element action', 'Click, focus, type/set a value, fill up to 20 form fields in one guarded call, select, scroll a page or scrollable element, scroll an element into view, upload files, press a key, or set an exact responsive-test viewport in a debuggable browser page using DOM/CDP semantics.', o({ ...browserTarget, action:e(['click','focus','type','set_value','select','scroll','scroll_into_view','upload','press','set_viewport','fill_form']), selector:s('CSS selector for the target element; prefer the reusable selector returned by browser_find.'), fields:{type:'array',minItems:1,maxItems:20,items:{type:'object',additionalProperties:false,properties:{selector:s('CSS selector.'),value:{type:['string','boolean','number'],description:'New field value; not returned in results.'}},required:['selector','value']},description:'fill_form: 1–20 CSS form fields in one CDP call. Does not submit.'}, browser_text:s('Visible text used only to locate a DOM element; use this instead of overloading text for action=type.'), text:s('For action=type, Unicode text to insert. For non-type element actions without selector/browser_text, visible text may identify the target.'), clear:b('For action=type, clear the current value before inserting text.'), value:s('Value for action=set_value; also accepted as a legacy type payload when text is used as the locator.'), text_value:s('Compatibility alias for the type/set_value text value.'), option:s(), delta_x:n('Horizontal scroll delta; positive means right.'), delta_y:n('Vertical scroll delta; positive means down.'), delta:n('Alias for delta_y.'), direction:e(['up','down','left','right']), wheel_times:n('Number of 120-unit wheel steps for direction; 1..50.'), path:s(), paths:{type:'array',items:{type:'string'}}, key:s(), width:n('CSS viewport width for set_viewport.'), height:n('CSS viewport height for set_viewport.'), device_scale_factor:n('Device scale factor for set_viewport; default 1.'), mobile:b('Enable mobile emulation for set_viewport.') }, ['action']), openMutating, browserHandlers.browser_action, ['browser_cdp']),
+  define('browser_action', 'Browser element action', 'Click, focus, type/set a value, fill up to 20 form fields in one guarded call, select, scroll a page or scrollable element, scroll an element into view, upload files, press a key, or set an exact responsive-test viewport in a debuggable browser page using DOM/CDP semantics.', o({ ...browserTarget, action:e(['click','focus','type','set_value','select','scroll','scroll_into_view','upload','press','set_viewport','fill_form','workflow']), selector:s('CSS selector for the target element; prefer the reusable selector returned by browser_find.'), steps:{type:'array',minItems:1,maxItems:12,description:'Workflow on one page: explicit CSS actions and bounded waits, max 12 seconds. Never silently submits; click must be explicit.',items:{type:'object',additionalProperties:false,properties:{action:e(['fill_form','set_value','select','focus','click','wait_for']),selector:s('CSS selector for action or wait_for.'),value:{type:['string','number','boolean']},fields:{type:'array',description:'CSS-selected field edits within the workflow step; no implicit form submission.',minItems:1,maxItems:20,items:{type:'object',additionalProperties:false,properties:{selector:s(),value:{type:['string','number','boolean']}},required:['selector','value']}},condition:e(['selector','text','url_contains','load']),text:s('Text or URL fragment for wait condition.'),timeout_ms:n('Wait timeout 100..4000 ms; default 1500.')},required:['action']}}, fields:{type:'array',minItems:1,maxItems:20,items:{type:'object',additionalProperties:false,properties:{selector:s('CSS selector.'),value:{type:['string','boolean','number'],description:'New field value; not returned in results.'}},required:['selector','value']},description:'fill_form: 1–20 CSS form fields in one CDP call. Does not submit.'}, browser_text:s('Visible text used only to locate a DOM element; use this instead of overloading text for action=type.'), text:s('For action=type, Unicode text to insert. For non-type element actions without selector/browser_text, visible text may identify the target.'), clear:b('For action=type, clear the current value before inserting text.'), value:s('Value for action=set_value; also accepted as a legacy type payload when text is used as the locator.'), text_value:s('Compatibility alias for the type/set_value text value.'), option:s(), delta_x:n('Horizontal scroll delta; positive means right.'), delta_y:n('Vertical scroll delta; positive means down.'), delta:n('Alias for delta_y.'), direction:e(['up','down','left','right']), wheel_times:n('Number of 120-unit wheel steps for direction; 1..50.'), path:s(), paths:{type:'array',items:{type:'string'}}, key:s(), width:n('CSS viewport width for set_viewport.'), height:n('CSS viewport height for set_viewport.'), device_scale_factor:n('Device scale factor for set_viewport; default 1.'), mobile:b('Enable mobile emulation for set_viewport.') }, ['action']), openMutating, browserHandlers.browser_action, ['browser_cdp']),
   define('browser_wait', 'Wait for browser', 'Wait for DOM state, page text, URL, completed loading, a navigation away from the current URL, or a short network-idle period in a debuggable browser page. Use browser_evaluate separately for JavaScript predicates or page-specific inspection.', o({ ...browserTarget, condition:e(['selector','text','url_contains','load','navigation','network_idle']), selector:s(), text:s(), value:s(), poll_ms:n(), idle_ms:n('Required zero-in-flight network quiet window for network_idle; default 500 ms.') }), readOnlyOpen, browserHandlers.browser_wait, ['browser_cdp']),
   define('browser_evaluate', 'Evaluate browser JavaScript', 'Evaluate JavaScript in a selected debuggable browser page and return its serializable value. This is a powerful escape hatch for page-specific automation.', o({ ...browserTarget, expression:s('JavaScript expression.'), await_promise:b('Await a returned Promise; default true.') }, ['expression']), openMutating, browserHandlers.browser_evaluate, ['browser_evaluate']),