@remcp/runtime 0.2.153 → 0.2.155

package/src/extended/desktop-computer-actions.mjs

@@ -0,0 +1,346 @@
+import os from 'node:os';
+import process from 'node:process';
+import { fileToolHandlers } from '../tools/files.mjs';
+import { liveConfig, runtimeConfig } from '../config.mjs';
+import { assertAllowedCommand } from '../policy.mjs';
+import { multi, resolveSafePath, text } from '../util.mjs';
+import { clamp, desktopSessionAvailable, jsonResult, optionalString, requireEnum } from './common.mjs';
+import * as linux from './desktop-linux.mjs';
+import * as macos from './desktop-macos.mjs';
+import * as windows from './desktop-windows.mjs';
+import { browserAction, browserCapabilityAvailable, browserSnapshot } from './browser.mjs';
+import { findOcrText, ocrImage } from './ocr.mjs';
+
+
+import { clipboard, displayInventory, keyboard, listWindows, notification, pointer, resultValue, screenshotRegion, scroll, sleep, uiAction, uiSnapshot, windowAction } from './desktop-core.mjs';
+import { dragDrop, launchApp, openPath, revealPath, typeText, uiFind, waitForUi } from './desktop-ui-actions.mjs';
+import { clickOcrText } from './desktop-vision.mjs';
+import { computerSnapshot } from './desktop-computer-snapshot.mjs';
+
+export function browserComputerAction(args, action) {
+  const mapped = {
+    click:'click',
+    invoke:'click',
+    toggle:'click',
+    focus:'focus',
+    set_value:'set_value',
+    select:'select',
+    type:'type',
+    scroll:'scroll',
+  }[action];
+  if (!mapped) throw new Error(`Browser target does not support computer_action action: ${action}`);
+  const visibleText = optionalString(args.browser_text)
+    || optionalString(args.name)
+    || ((['click','invoke','toggle','focus'].includes(action) && optionalString(args.text)) ? optionalString(args.text) : null);
+  return browserAction({
+    endpoint: args.endpoint,
+    target_id: args.target_id,
+    title: args.title,
+    url_contains: args.url_contains,
+    timeout_ms: args.timeout_ms,
+    action: mapped,
+    selector: args.selector,
+    ...(visibleText ? { browser_text: visibleText } : {}),
+    ...(args.value != null ? { value: args.value } : {}),
+    ...(args.option != null ? { option: args.option } : {}),
+    ...(action === 'type' && args.text != null ? { text: args.text, clear:args.clear === true } : {}),
+    ...(action === 'scroll' ? {
+      delta_x:args.delta_x,
+      delta_y:args.delta_y,
+      delta:args.delta,
+      direction:args.direction,
+      wheel_times:args.wheel_times,
+    } : {}),
+  });
+}
+
+export function computerActionWindowTextArgs(args = {}) {
+  const {
+    id: _id,
+    label: _label,
+    name: _name,
+    role: _role,
+    automation_id: _automationId,
+    automationId: _automationIdAlias,
+    selector: _selector,
+    browser_text: _browserText,
+    ...windowArgs
+  } = args;
+  return {
+    ...windowArgs,
+    ...(args.window_title && !args.title ? { title:args.window_title } : {}),
+  };
+}
+
+export async function computerAction(args = {}) {
+  const legacy = ['ui','pointer','window','type','keyboard','scroll','drag','clipboard','launch_app','open_path','reveal_path','notification'];
+  const universal = [
+    'click','double_click','right_click','move','down','up','invoke','focus','set_value','select','toggle',
+    'expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection',
+    'minimize','maximize','restore','move_resize','resize','close','wait','batch','multi_select','multi_edit',
+  ];
+  const action = requireEnum(args.action, 'action', [...legacy, ...universal]);
+  const explicitTarget = optionalString(args.target || args.target_type);
+  const target = explicitTarget ? requireEnum(explicitTarget, 'target', ['auto','ui','ui_element','browser','ocr','coordinates','window']) : 'auto';
+  const nested = { ...args, action: args.operation || args.subaction };
+  const hasUi = Boolean(args.label != null || args.id || args.name || args.role || args.automation_id || args.automationId);
+  const hasBrowser = Boolean(args.selector || args.browser_text || args.name);
+  const ocrNeedle = optionalString(args.ocr_text || args.name || args.browser_text);
+  const hasOcr = Boolean(ocrNeedle);
+  const hasCoordinates = Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y));
+  const automatic = target === 'auto';
+
+  if (action === 'batch') {
+    const steps = Array.isArray(args.steps) ? args.steps : [];
+    if (!steps.length) throw new Error('computer_action batch requires a non-empty steps array');
+    if (steps.length > 50) throw new Error('computer_action batch supports at most 50 steps');
+    const stopOnError = args.stop_on_error !== false;
+    const results = [];
+    for (let index = 0; index < steps.length; index += 1) {
+      const step = steps[index];
+      if (!step || typeof step !== 'object' || Array.isArray(step)) throw new Error(`batch step ${index} must be an object`);
+      if (step.action === 'batch') throw new Error('nested computer_action batches are not allowed');
+      try {
+        const result = await computerAction(step);
+        results.push({ index, ok:true, action:step.action, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
+      } catch (error) {
+        const failure = { index, ok:false, action:step.action || null, error:String(error?.message || error).slice(0, 1000) };
+        results.push(failure);
+        if (stopOnError) return jsonResult({ action:'batch', ok:false, stopped_at:index, completed:index, results });
+      }
+    }
+    return jsonResult({ action:'batch', ok:results.every(row => row.ok), completed:results.length, results });
+  }
+
+  if (action === 'multi_select') {
+    const targets = Array.isArray(args.targets) ? args.targets : [];
+    if (!targets.length) throw new Error('multi_select requires a non-empty targets array');
+    if (targets.length > 100) throw new Error('multi_select supports at most 100 targets');
+    const results = [];
+    for (let index = 0; index < targets.length; index += 1) {
+      const targetSpec = targets[index];
+      const operation = index === 0 && args.replace_selection !== false ? 'select' : 'add_to_selection';
+      const result = await uiAction({ ...targetSpec, action:operation });
+      results.push({ index, action:operation, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
+    }
+    return jsonResult({ action:'multi_select', count:results.length, results });
+  }
+
+  if (action === 'multi_edit') {
+    const edits = Array.isArray(args.edits) ? args.edits : [];
+    if (!edits.length) throw new Error('multi_edit requires a non-empty edits array');
+    if (edits.length > 100) throw new Error('multi_edit supports at most 100 edits');
+    const results = [];
+    for (let index = 0; index < edits.length; index += 1) {
+      const edit = edits[index];
+      if (!edit || typeof edit !== 'object' || edit.text == null) throw new Error(`multi_edit item ${index} requires target fields plus text`);
+      const result = await typeText({ ...edit, method:edit.method || 'auto', clear:edit.clear !== false });
+      results.push({ index, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
+    }
+    return jsonResult({ action:'multi_edit', count:results.length, results });
+  }
+
+  if (action === 'wait') {
+    const requested = args.wait_ms != null ? Number(args.wait_ms) : Number(args.seconds) * 1000;
+    if (!Number.isFinite(requested) || requested < 0) throw new Error('computer_action wait requires wait_ms or seconds');
+    const waitMs = Math.min(120_000, Math.round(requested));
+    await sleep(waitMs);
+    return jsonResult({ action:'wait', waited_ms:waitMs, capped:requested > 120_000 });
+  }
+
+  if (action === 'ui') return uiAction(nested);
+  if (action === 'pointer') return pointer(nested);
+  if (action === 'window') return windowAction(nested);
+
+  const windowTarget = () => ({
+    ...args,
+    ...(args.window_id ? { id:args.window_id } : {}),
+    ...(args.window_title && !args.title ? { title:args.window_title } : {}),
+  });
+  const windowTextTarget = () => computerActionWindowTextArgs(args);
+
+  if (action === 'type') {
+    if (target === 'browser') return browserComputerAction(args, 'type');
+    if (target === 'ocr' || target === 'coordinates') {
+      throw new Error('computer_action type requires a native UI, browser, or window target');
+    }
+    if (target === 'window') return typeText(windowTextTarget());
+    if (!automatic) return typeText({ ...args, method:args.method || 'accessibility' });
+
+    const failures = [];
+    if (hasUi) {
+      try { return await typeText({ ...args, method:'accessibility' }); }
+      catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
+    }
+    if (hasBrowser) {
+      if (await browserCapabilityAvailable(args.endpoint, 300)) {
+        try { return await browserComputerAction(args, 'type'); }
+        catch (error) { failures.push(`browser DOM: ${String(error?.message || error).slice(0, 300)}`); }
+      } else {
+        failures.push('browser DOM: no local CDP endpoint is available');
+      }
+    }
+    if (args.window_id || args.pid != null || args.app || args.window_title || args.title) {
+      try { return await typeText(windowTextTarget()); }
+      catch (error) { failures.push(`window: ${String(error?.message || error).slice(0, 300)}`); }
+    }
+    if (failures.length) throw new Error(`computer_action type targeting failed: ${failures.join('; ')}`);
+    throw new Error('computer_action type requires a native UI, browser, or window target');
+  }
+
+  if (action === 'keyboard') {
+    if (target === 'browser' && args.key && !args.shortcut && !Array.isArray(args.keys)) {
+      return browserAction({ ...args, action:'press' });
+    }
+    return keyboard(args);
+  }
+
+  if (action === 'scroll') {
+    if (target === 'browser') return browserComputerAction(args, 'scroll');
+    if (target === 'ocr') throw new Error('computer_action scroll does not support OCR targets; use browser/UI/window/coordinates');
+
+    if (automatic) {
+      let uiError = null;
+      if (hasUi) {
+        try { return await scroll(args); }
+        catch (error) { uiError = error; }
+      }
+      if (hasBrowser) {
+        if (await browserCapabilityAvailable(args.endpoint, 300)) return browserComputerAction(args, 'scroll');
+        if (uiError) throw new Error(`computer_action scroll targeting failed: accessibility: ${String(uiError?.message || uiError).slice(0, 300)}; browser DOM: no local CDP endpoint is available`);
+        throw new Error('computer_action scroll browser target is unavailable because no local CDP endpoint is available');
+      }
+      if (args.window_id || args.pid != null || args.app || args.window_title || args.title) {
+        await windowAction({ ...windowTarget(), action:'focus' });
+      }
+      return scroll(args);
+    }
+
+    if (target === 'window') {
+      await windowAction({ ...windowTarget(), action:'focus' });
+      return scroll(args);
+    }
+    return scroll(args);
+  }
+
+  if (action === 'drag') {
+    if (target === 'browser' || target === 'ocr') {
+      throw new Error('computer_action drag currently requires native UI element ids or desktop coordinates; browser/OCR drag is not advertised');
+    }
+    if (target === 'window') await windowAction({ ...windowTarget(), action:'focus' });
+    return dragDrop(args);
+  }
+  if (action === 'clipboard') return clipboard(nested);
+  if (action === 'launch_app') return launchApp(args);
+  if (action === 'open_path') return openPath(args);
+  if (action === 'reveal_path') return revealPath(args);
+  if (action === 'notification') return notification(args);
+
+  const inferredTarget = automatic
+    ? (hasUi ? 'ui_element' : hasBrowser ? 'browser' : args.ocr_text ? 'ocr' : hasCoordinates ? 'coordinates' : 'window')
+    : target;
+
+  const tryUiThenBrowser = async uiActionName => {
+    let uiError = null;
+    if (hasUi) {
+      try { return await uiAction({ ...args, action:uiActionName }); }
+      catch (error) { uiError = error; }
+    }
+    if (hasBrowser) return browserComputerAction(args, uiActionName);
+    if (uiError) throw uiError;
+    return null;
+  };
+
+  if (['invoke','set_value','select','toggle','expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection'].includes(action)) {
+    const browserCompatible = ['invoke','set_value','select','toggle'].includes(action);
+    if (inferredTarget === 'browser') {
+      if (!browserCompatible) throw new Error(`Browser target does not support ${action}; use browser_action or native accessibility`);
+      return browserComputerAction(args, action);
+    }
+    if (inferredTarget === 'ui' || inferredTarget === 'ui_element') {
+      if (automatic && browserCompatible) {
+        const result = await tryUiThenBrowser(action);
+        if (result) return result;
+      }
+      return uiAction({ ...args, action });
+    }
+    throw new Error(`${action} requires a UI${browserCompatible ? ' or browser' : ''} target`);
+  }
+
+  if (action === 'focus') {
+    if (inferredTarget === 'browser') return browserComputerAction(args, 'focus');
+    if (inferredTarget === 'ui' || inferredTarget === 'ui_element') {
+      if (automatic) {
+        const result = await tryUiThenBrowser('focus');
+        if (result) return result;
+      }
+      return uiAction({ ...args, action:'focus' });
+    }
+    return windowAction({ ...windowTarget(), action:'focus' });
+  }
+
+  if (['minimize','maximize','restore','move_resize','resize','close'].includes(action)) {
+    return windowAction({ ...args, action });
+  }
+
+  if (['click','double_click','right_click','move','down','up'].includes(action)) {
+    if (action === 'click' && automatic) {
+      const failures = [];
+      if (hasUi) {
+        try { return await uiAction({ ...args, action:'click' }); }
+        catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
+      }
+      if (hasBrowser) {
+        if (await browserCapabilityAvailable(args.endpoint, 300)) {
+          try { return await browserComputerAction(args, 'click'); }
+          catch (error) { failures.push(`browser DOM: ${String(error?.message || error).slice(0, 300)}`); }
+        } else {
+          failures.push('browser DOM: no local CDP endpoint is available');
+        }
+      }
+      if (hasOcr) {
+        try { return await clickOcrText({ ...args, ocr_text:ocrNeedle }); }
+        catch (error) { failures.push(`OCR: ${String(error?.message || error).slice(0, 300)}`); }
+      }
+      if (hasCoordinates) return pointer({ ...args, action:'click' });
+      if (failures.length) throw new Error(`computer_action click targeting failed: ${failures.join('; ')}`);
+      throw new Error('computer_action click requires a UI/browser/OCR target or x/y coordinates');
+    }
+    if (inferredTarget === 'browser') {
+      if (action !== 'click') throw new Error(`Browser target does not support ${action}; use browser_action or coordinates`);
+      return browserComputerAction(args, 'click');
+    }
+    if (inferredTarget === 'ocr') {
+      if (action !== 'click') throw new Error(`OCR target only supports click; ${action} requires a semantic or coordinate target`);
+      return clickOcrText({ ...args, ocr_text:ocrNeedle });
+    }
+    if ((inferredTarget === 'ui' || inferredTarget === 'ui_element') && action === 'click') {
+      return uiAction({ ...args, action:'click' });
+    }
+    return pointer({ ...args, action });
+  }
+  throw new Error(`Unsupported computer_action action: ${action}`);
+}
+
+export const desktopHandlers = {
+  computer_snapshot: computerSnapshot,
+  computer_action: computerAction,
+  list_windows: listWindows,
+  window_action: windowAction,
+  launch_app: launchApp,
+  ui_snapshot: uiSnapshot,
+  ui_find: uiFind,
+  ui_action: uiAction,
+  type_text: typeText,
+  keyboard,
+  pointer,
+  drag_drop: dragDrop,
+  scroll,
+  wait_for_ui: waitForUi,
+  clipboard,
+  display_inventory: displayInventory,
+  screenshot_region: screenshotRegion,
+  open_path: openPath,
+  reveal_path: revealPath,
+  notification,
+};

package/src/extended/desktop-computer-snapshot.mjs

@@ -0,0 +1,245 @@
+import os from 'node:os';
+import process from 'node:process';
+import { fileToolHandlers } from '../tools/files.mjs';
+import { liveConfig, runtimeConfig } from '../config.mjs';
+import { assertAllowedCommand } from '../policy.mjs';
+import { multi, resolveSafePath, text } from '../util.mjs';
+import { clamp, desktopSessionAvailable, jsonResult, optionalString, requireEnum } from './common.mjs';
+import * as linux from './desktop-linux.mjs';
+import * as macos from './desktop-macos.mjs';
+import * as windows from './desktop-windows.mjs';
+import { browserAction, browserCapabilityAvailable, browserSnapshot } from './browser.mjs';
+import { findOcrText, ocrImage } from './ocr.mjs';
+
+
+import { boundedComputerSnapshot, clipboard, cursorPosition, displayInventory, listWindows, objectRecord, resultValue, screenshotRegion, snapshotError, suggestedUiAction, toolResultMessage, uiSnapshot } from './desktop-core.mjs';
+import { activeWindowCaptureBounds, browserTitleHint, computerScreenshotTask, hasExplicitComputerScreenshotTarget, ocrScreenshotResult } from './desktop-vision.mjs';
+
+export async function computerSnapshot(args = {}) {
+  const desktopAvailable = desktopSessionAvailable();
+  const screenshotRequested = args.include_screenshot !== false || args.include_ocr === true;
+  const captureScreenshot = desktopAvailable && screenshotRequested;
+  const includeUi = args.include_ui !== false;
+  const [windowsResult, displaysResult, uiResult, clipboardResult, cursorResult, shotResult] = await Promise.allSettled([
+    desktopAvailable ? listWindows() : Promise.resolve(null),
+    desktopAvailable ? displayInventory() : Promise.resolve(null),
+    includeUi && desktopAvailable
+      ? uiSnapshot({
+          max_nodes:args.max_ui_nodes || 800,
+          max_depth:args.max_ui_depth || 12,
+          semantic_tree:args.semantic_tree !== false,
+          active_only:args.ui_scope === 'desktop' ? false : true,
+          ...(args.ui_browser_dom === true ? { browser_dom:true } : {}),
+        })
+      : Promise.resolve(null),
+    desktopAvailable ? clipboard({ action:'read' }) : Promise.resolve(null),
+    desktopAvailable ? cursorPosition() : Promise.resolve(null),
+    captureScreenshot ? computerScreenshotTask(args) : Promise.resolve(null),
+  ]);
+  const parse = (result, fallback = null) => {
+    if (result.status !== 'fulfilled' || !result.value || result.value.isError === true) return fallback;
+    return resultValue(result.value) ?? fallback;
+  };
+  const shapeErrors = [];
+  const parsedWindows = parse(windowsResult, []);
+  const windows = Array.isArray(parsedWindows) ? parsedWindows : [];
+  if (parsedWindows !== windows && parsedWindows != null) {
+    shapeErrors.push({ source:'windows', message:'window inventory returned an invalid result shape' });
+  }
+  const parsedDisplays = parse(displaysResult, []);
+  const displays = Array.isArray(parsedDisplays) ? parsedDisplays : [];
+  if (parsedDisplays !== displays && parsedDisplays != null) {
+    shapeErrors.push({ source:'displays', message:'display inventory returned an invalid result shape' });
+  }
+  const parsedCursor = parse(cursorResult, { x:null, y:null });
+  const cursor = objectRecord(parsedCursor) || { x:null, y:null };
+  if (parsedCursor !== cursor && parsedCursor != null) {
+    shapeErrors.push({ source:'cursor', message:'cursor position returned an invalid result shape' });
+  }
+  const parsedUi = includeUi ? parse(uiResult, null) : null;
+  const ui = objectRecord(parsedUi) || { platform:process.platform, count:0, nodes:[], ...(includeUi ? { unavailable:true } : { skipped:true }) };
+  if (includeUi && parsedUi !== ui && parsedUi != null) {
+    shapeErrors.push({ source:'ui', message:'accessibility snapshot returned an invalid result shape' });
+  }
+  const clipboardOk = desktopAvailable && clipboardResult.status === 'fulfilled' && clipboardResult.value?.isError !== true && Boolean(clipboardResult.value);
+  const clipText = clipboardOk ? clipboardResult.value.content?.[0]?.text || '' : '';
+  const uiNodes = Array.isArray(ui?.nodes) ? ui.nodes : Array.isArray(ui) ? ui : [];
+  const focusedNodes = uiNodes.filter(node => node.focused === true);
+  const focusScore = node => {
+    const app = String(node.app || '').toLowerCase();
+    const windowName = String(node.window || '').trim();
+    const role = String(node.role || '').toLowerCase();
+    let score = 0;
+    if (windowName) score += 100;
+    if (app && app !== 'gnome-shell') score += 40;
+    if (node.name) score += 15;
+    if (!['window','filler','panel'].includes(role)) score += 15;
+    if (Number(node.width) >= 100 && Number(node.height) >= 100) score += 10;
+    if (/desktop icons/i.test(windowName)) score -= 80;
+    if (app === 'gnome-shell' && !windowName) score -= 120;
+    return score;
+  };
+  const focusedNode = focusedNodes.sort((a, b) => focusScore(b) - focusScore(a))[0] || null;
+  const activeTopLevel = uiNodes.find(node =>
+    node.active === true
+    && /frame|window|dialog/i.test(String(node.role || ''))
+    && String(node.window || node.name || '').trim()
+    && !/desktop icons/i.test(String(node.window || node.name || ''))
+  ) || null;
+  const activeNode = activeTopLevel
+    || focusedNode
+    || uiNodes.find(node => /frame|window|dialog/i.test(String(node.role || '')) && String(node.window || node.name || '').trim())
+    || uiNodes[0]
+    || null;
+  const windowRows = Array.isArray(windows) ? windows : [];
+  const activeWindow = activeNode ? windowRows.find(row => {
+    const activePid = Number(activeNode.pid);
+    const rowPid = Number(row.pid);
+    const wantedTitle = String(activeNode.window || activeNode.name || '').trim().toLowerCase();
+    const rowTitle = String(row.title || '').trim().toLowerCase();
+    const pidMatch = activePid > 0 && rowPid > 0 && rowPid === activePid;
+    // XWayland/WM enumeration can report pid=0 while AT-SPI still exposes the real application PID.
+    // A strong top-level title match is the safe fallback for joining those two views.
+    const titleMatch = Boolean(wantedTitle && rowTitle && (rowTitle === wantedTitle || rowTitle.includes(wantedTitle) || wantedTitle.includes(rowTitle)));
+    return (pidMatch || titleMatch) && (!wantedTitle || titleMatch || pidMatch);
+  }) || null : null;
+  const activeApp = activeNode ? {
+    pid: Number(activeNode.pid) || activeWindow?.pid || null,
+    app: activeNode.app || activeWindow?.app || null,
+    window: activeNode.window || activeWindow?.title || null,
+  } : null;
+
+  const activeLabel = [activeApp?.app, activeApp?.window, activeWindow?.app, activeWindow?.title].filter(Boolean).join(' ');
+  const browserRequested = args.include_browser === true
+    || (args.include_browser !== false && /chrome|chromium|edge|brave/i.test(activeLabel));
+  let browser = null;
+  let browserError = null;
+  if (browserRequested && await browserCapabilityAvailable(undefined, 400)) {
+    const hint = browserTitleHint(activeWindow);
+    try {
+      const browserResult = await browserSnapshot({
+        ...(hint ? { title:hint } : {}),
+        max_nodes:clamp(args.browser_max_nodes, 500, 1, 2000),
+        timeout_ms:3000,
+      });
+      if (browserResult?.isError === true) throw new Error(toolResultMessage(browserResult));
+      browser = objectRecord(resultValue(browserResult));
+      if (!browser) throw new Error('browser snapshot returned an invalid result shape');
+    } catch (error) {
+      if (hint) {
+        try {
+          const fallbackResult = await browserSnapshot({
+            max_nodes:clamp(args.browser_max_nodes, 500, 1, 2000),
+            timeout_ms:3000,
+          });
+          if (fallbackResult?.isError === true) throw new Error(toolResultMessage(fallbackResult));
+          browser = objectRecord(resultValue(fallbackResult));
+          if (!browser) throw new Error('browser snapshot returned an invalid result shape');
+        } catch (fallbackError) {
+          browserError = String(fallbackError?.message || fallbackError);
+        }
+      } else {
+        browserError = String(error?.message || error);
+      }
+    }
+  }
+
+  const ocrRequested = args.include_ocr === true
+    || (desktopAvailable && args.include_ocr !== false && screenshotRequested && uiNodes.length === 0 && !browser);
+  const shouldOcr = desktopAvailable && ocrRequested;
+  let ocr = null;
+  let ocrCaptureBounds = null;
+  let ocrShotResult = shotResult;
+  // A whole-desktop OCR pass is noisy and slow when the active application is already known.
+  // Keep the normal screenshot contract unchanged, but use a second tightly scoped capture for OCR.
+  if (shouldOcr && !hasExplicitComputerScreenshotTarget(args)) {
+    const bounds = activeWindowCaptureBounds(activeWindow);
+    if (bounds) {
+      try {
+        const targeted = await screenshotRegion({ ...bounds, padding:0 });
+        ocrShotResult = { status:'fulfilled', value:targeted };
+        ocrCaptureBounds = bounds;
+      } catch {
+        // Fall back to the already captured desktop screenshot below.
+      }
+    }
+  }
+  if (shouldOcr) {
+    ocr = ocrShotResult.status === 'fulfilled' && ocrShotResult.value
+      ? await ocrScreenshotResult(ocrShotResult.value, args)
+      : { available:false, reason:'screenshot capture failed before OCR could run' };
+    if (ocr?.available && ocrCaptureBounds) {
+      ocr = { ...ocr, coordinate_space:'capture_pixels', capture_bounds:ocrCaptureBounds };
+    }
+  }
+
+  const payload = {
+    device: { hostname: os.hostname(), platform: process.platform, arch: process.arch },
+    platform: process.platform,
+    hostname: os.hostname(),
+    active_app: activeApp,
+    active_window: activeWindow,
+    windows,
+    displays,
+    cursor,
+    ui,
+    ...(browser ? { browser } : {}),
+    ...(ocr ? { ocr } : {}),
+    fallback_chain: {
+      accessibility: { requested:includeUi, available:includeUi && desktopAvailable && uiResult.status === 'fulfilled' && Boolean(uiResult.value), nodes:uiNodes.length },
+      native_browser_dom: {
+        requested:includeUi && args.ui_browser_dom === true,
+        available:Boolean(includeUi && ui?.browser_dom?.available),
+        provider:ui?.browser_dom?.provider || null,
+      },
+      browser_dom: { requested:browserRequested, available:Boolean(browser) },
+      ocr: { requested:ocrRequested, available:Boolean(ocr?.available), backend:ocr?.backend || null },
+      vision_screenshot: { available:shotResult.status === 'fulfilled' && Boolean(shotResult.value?.content?.some(part => part.type === 'image')) },
+    },
+    visual: {
+      coordinate_space:'virtual_desktop',
+      labeled_targets:uiNodes.filter(node => Number.isInteger(node.label)).slice(0, 500).map(node => ({
+        label:node.label,
+        id:node.id || null,
+        role:node.role || null,
+        name:String(node.name || '').slice(0, 240),
+        x:Number(node.x) || 0,
+        y:Number(node.y) || 0,
+        width:Number(node.width) || 0,
+        height:Number(node.height) || 0,
+        action:node.suggested_action || suggestedUiAction(node),
+      })),
+      ...(Number(args.grid_columns) > 0 || Number(args.grid_rows) > 0 ? {
+        reference_grid:{
+          columns:clamp(args.grid_columns, 0, 0, 20),
+          rows:clamp(args.grid_rows, 0, 0, 20),
+        },
+      } : {}),
+      screenshot_scope:Array.isArray(args.screenshot_region) ? 'region'
+        : args.screenshot_window_id ? 'window'
+        : (args.screenshot_monitor != null || args.screenshot_monitor_index != null) ? 'monitor'
+        : 'desktop',
+    },
+    clipboard: { available:clipboardOk, length:clipText.length },
+    errors: [
+      ...[
+        ['windows', windowsResult],
+        ['displays', displaysResult],
+        ...(includeUi ? [['ui', uiResult]] : []),
+        ['clipboard', clipboardResult],
+        ['cursor', cursorResult],
+        ...(captureScreenshot ? [['screenshot', shotResult]] : []),
+      ].map(([source, result]) => snapshotError(source, result)).filter(Boolean),
+      ...shapeErrors,
+      ...(browserError ? [{ source:'browser', message:browserError.slice(0, 2000) }] : []),
+    ],
+  };
+  const boundedPayload = boundedComputerSnapshot(payload);
+  const parts = [{ type: 'text', text: JSON.stringify(boundedPayload, null, 2) }];
+  if (args.include_screenshot !== false && shotResult.status === 'fulfilled' && shotResult.value?.content) {
+    for (const part of shotResult.value.content) if (part.type === 'image') parts.push(part);
+  }
+  const result = multi(parts);
+  result.structuredContent = boundedPayload;
+  return result;
+}