@remcp/runtime 0.2.154 → 0.2.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1583 +1,6 @@
1
- import os from 'node:os';
2
- import process from 'node:process';
3
- import { fileToolHandlers } from '../tools/files.mjs';
4
- import { liveConfig, runtimeConfig } from '../config.mjs';
5
- import { assertAllowedCommand } from '../policy.mjs';
6
- import { multi, resolveSafePath, text } from '../util.mjs';
7
- import { clamp, desktopSessionAvailable, jsonResult, optionalString, requireEnum } from './common.mjs';
8
- import * as linux from './desktop-linux.mjs';
9
- import * as macos from './desktop-macos.mjs';
10
- import * as windows from './desktop-windows.mjs';
11
- import { browserAction, browserCapabilityAvailable, browserSnapshot } from './browser.mjs';
12
- import { findOcrText, ocrImage } from './ocr.mjs';
13
-
14
- const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
15
- const adapter = process.platform === 'win32' ? windows : process.platform === 'darwin' ? macos : linux;
16
-
17
- const UI_CACHE_TTL_MS = 5_000;
18
- let latestUiSnapshot = null;
19
-
20
- function suggestedUiAction(node = {}) {
21
- const role = String(node.role || '').toLowerCase();
22
- const patterns = Array.isArray(node.patterns) ? node.patterns.map(value => String(value).toLowerCase()) : [];
23
- if (patterns.includes('toggle') || /check|switch/.test(role)) return 'toggle';
24
- if (patterns.includes('range_value') || /slider|spin/.test(role)) return 'set_range_value';
25
- if (patterns.includes('selection_item') || /radio|listitem|treeitem|tabitem|option/.test(role)) return 'select';
26
- if (patterns.includes('expand_collapse') || /combo|treeitem|menuitem/.test(role)) return 'expand';
27
- if (patterns.includes('value') || /edit|textbox|text field|entry|search/.test(role)) return 'set_value';
28
- if (patterns.includes('invoke') || /button|link|menuitem/.test(role)) return 'invoke';
29
- if (node.scrollable === true || patterns.includes('scroll') || patterns.includes('scroll_item')) return 'scroll_into_view';
30
- return 'click';
31
- }
32
-
33
- function semanticTreeText(nodes = [], maxChars = 48_000) {
34
- const lines = [];
35
- let chars = 0;
36
- for (const node of nodes) {
37
- const depth = Math.max(0, Math.min(32, Number(node.depth) || 0));
38
- const label = Number.isInteger(node.label) ? `#${node.label} ` : '';
39
- const role = String(node.role || 'element');
40
- const name = String(node.name || node.value || '').replace(/\s+/g, ' ').trim();
41
- const states = [
42
- node.focused === true ? 'focused' : '',
43
- node.enabled === false ? 'disabled' : '',
44
- node.offscreen === true ? 'offscreen' : '',
45
- node.password === true ? 'password' : '',
46
- node.toggle_state != null ? `toggle:${node.toggle_state}` : '',
47
- node.expand_collapse_state ? `state:${node.expand_collapse_state}` : '',
48
- node.range_value != null ? `value:${node.range_value}` : '',
49
- ].filter(Boolean);
50
- const line = `${' '.repeat(depth)}${label}${role}${name ? ` "${name.slice(0, 240)}"` : ''} [${node.suggested_action || suggestedUiAction(node)}]${states.length ? ` ${states.map(v => `[${v}]`).join(' ')}` : ''}`;
51
- if (chars + line.length + (lines.length ? 1 : 0) > maxChars) {
52
- lines.push('… semantic tree truncated …');
53
- break;
54
- }
55
- lines.push(line);
56
- chars += line.length + (lines.length > 1 ? 1 : 0);
57
- }
58
- return lines.join('\n');
59
- }
60
-
61
- function enrichUiPayload(value, args = {}) {
62
- const base = Array.isArray(value) ? { platform:process.platform, count:value.length, nodes:value } : { ...(value || {}) };
63
- const source = Array.isArray(base.nodes) ? base.nodes : [];
64
- let label = 0;
65
- const nodes = source.map(node => {
66
- const x = Number(node.x), y = Number(node.y), width = Number(node.width), height = Number(node.height);
67
- const usableBounds = [x, y, width, height].every(Number.isFinite) && width > 0 && height > 0;
68
- const actionable = usableBounds && node.offscreen !== true && (
69
- node.enabled !== false || node.focused === true || String(node.name || '').trim() || String(node.role || '').trim()
70
- );
71
- const enriched = {
72
- ...node,
73
- ...(usableBounds ? { center_x:Math.round(x + width / 2), center_y:Math.round(y + height / 2) } : {}),
74
- suggested_action:suggestedUiAction(node),
75
- };
76
- if (actionable) enriched.label = label++;
77
- return enriched;
78
- });
79
- const payload = {
80
- ...base,
81
- count:Number(base.count) || nodes.length,
82
- nodes,
83
- label_count:label,
84
- semantic_tree:args.semantic_tree === false ? undefined : semanticTreeText(nodes),
85
- captured_at:new Date().toISOString(),
86
- };
87
- latestUiSnapshot = { at:Date.now(), payload, args:{ ...args } };
88
- return payload;
89
- }
90
-
91
- function cachedUiNodes(args = {}) {
92
- if (!latestUiSnapshot || Date.now() - latestUiSnapshot.at > UI_CACHE_TTL_MS) return null;
93
- const requestedPid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
94
- const cachedNodes = latestUiSnapshot.payload?.nodes;
95
- if (!Array.isArray(cachedNodes)) return null;
96
- if (requestedPid != null && !cachedNodes.some(node => Number(node.pid) === requestedPid)) return null;
97
- if (args.app && !cachedNodes.some(node => String(node.app || '').toLowerCase().includes(String(args.app).toLowerCase()))) return null;
98
- if ((args.window_title || args.windowTitle) && !cachedNodes.some(node => String(node.window || node.window_title || '').toLowerCase().includes(String(args.window_title || args.windowTitle).toLowerCase()))) return null;
99
- return cachedNodes;
100
- }
101
-
102
- function resolveCachedLabel(value, field = 'label') {
103
- if (value == null || value === '') return null;
104
- const label = Number(value);
105
- if (!Number.isInteger(label) || label < 0) throw new Error(`${field} must be a non-negative integer label returned by ui_snapshot/ui_find`);
106
- if (!latestUiSnapshot || Date.now() - latestUiSnapshot.at > UI_CACHE_TTL_MS) {
107
- throw new Error(`${field} refers to an expired UI snapshot; refresh ui_snapshot/ui_find before acting`);
108
- }
109
- const nodes = latestUiSnapshot.payload?.nodes;
110
- if (!Array.isArray(nodes)) throw new Error(`${field} requires a prior ui_snapshot/ui_find on the current desktop state`);
111
- const node = nodes.find(item => Number(item.label) === label);
112
- if (!node) throw new Error(`${field} ${label} is not present in the latest UI snapshot; refresh ui_snapshot`);
113
- return node;
114
- }
115
-
116
- function withResolvedUiLabel(args = {}) {
117
- const node = resolveCachedLabel(args.label);
118
- if (!node) return args;
119
- return {
120
- ...args,
121
- id:args.id || node.id,
122
- pid:args.pid ?? node.pid,
123
- app:args.app || node.app,
124
- window_title:args.window_title || args.windowTitle || node.window || node.window_title,
125
- name:args.name || (!node.id ? node.name : undefined),
126
- role:args.role || (!node.id ? node.role : undefined),
127
- };
128
- }
129
-
130
- function resultValue(result) {
131
- if (result?.structuredContent && !(Object.keys(result.structuredContent).length === 1 && typeof result.structuredContent.text === 'string')) {
132
- if (Object.keys(result.structuredContent).length === 1 && Object.hasOwn(result.structuredContent, 'data')) return result.structuredContent.data;
133
- return result.structuredContent;
134
- }
135
- const raw = result?.content?.find(part => part.type === 'text')?.text;
136
- if (typeof raw !== 'string') return null;
137
- try { return JSON.parse(raw); } catch { return raw; }
138
- }
139
-
140
- function objectRecord(value) {
141
- return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
142
- }
143
-
144
- function toolResultMessage(result) {
145
- const text = result?.content?.find(part => part.type === 'text')?.text;
146
- return String(text || 'tool returned an error result').slice(0, 2000);
147
- }
148
-
149
- function snapshotError(source, settled) {
150
- if (settled?.status === 'rejected') {
151
- return { source, message:String(settled.reason?.message || settled.reason || 'capture rejected').slice(0, 2000) };
152
- }
153
- if (settled?.status === 'fulfilled' && settled.value?.isError === true) {
154
- return { source, message:toolResultMessage(settled.value) };
155
- }
156
- return null;
157
- }
158
-
159
- function normalizeSnapshotErrors(errors) {
160
- if (!Array.isArray(errors)) return [];
161
- return errors.slice(0, 50).map((error, index) => {
162
- if (objectRecord(error)) {
163
- return {
164
- ...error,
165
- source:String(error.source || 'snapshot').slice(0, 120),
166
- message:String(error.message || error.error || 'snapshot capture error').slice(0, 2000),
167
- };
168
- }
169
- return { source:'snapshot', message:String(error ?? `snapshot error ${index + 1}`).slice(0, 2000) };
170
- });
171
- }
172
-
173
- function jsonBytes(value) {
174
- return Buffer.byteLength(JSON.stringify(value), 'utf8');
175
- }
176
-
177
- export function boundedComputerSnapshot(payload) {
178
- const normalized = {
179
- ...payload,
180
- active_window:objectRecord(payload?.active_window),
181
- windows:Array.isArray(payload?.windows) ? payload.windows : [],
182
- displays:Array.isArray(payload?.displays) ? payload.displays : [],
183
- cursor:objectRecord(payload?.cursor) || { x:null, y:null },
184
- ui:objectRecord(payload?.ui) || { platform:process.platform, count:0, nodes:[] },
185
- clipboard:objectRecord(payload?.clipboard) || { available:false, length:0 },
186
- errors:normalizeSnapshotErrors(payload?.errors),
187
- };
188
- const budget = Math.min(768 * 1024, Math.max(16 * 1024, Math.floor(Number(liveConfig('maxOutputBytes') || 0) * 0.35)));
189
- const originalBytes = jsonBytes(normalized);
190
- if (originalBytes <= budget) {
191
- return { ...normalized, snapshot_bytes:originalBytes, snapshot_budget_bytes:budget, snapshot_truncated:false };
192
- }
193
-
194
- const bounded = {
195
- ...normalized,
196
- windows:[...normalized.windows],
197
- errors:normalizeSnapshotErrors(normalized.errors),
198
- ui:{
199
- ...normalized.ui,
200
- ...(Array.isArray(normalized.ui.nodes) ? { nodes:[...normalized.ui.nodes] } : {}),
201
- },
202
- browser:objectRecord(normalized.browser) ? {
203
- ...normalized.browser,
204
- ...(Array.isArray(normalized.browser.nodes) ? { nodes:[...normalized.browser.nodes] } : {}),
205
- } : undefined,
206
- ocr:objectRecord(normalized.ocr) ? {
207
- ...normalized.ocr,
208
- text:String(normalized.ocr.text || '').slice(0, 50_000),
209
- ...(Array.isArray(normalized.ocr.words) ? { words:[...normalized.ocr.words] } : {}),
210
- ...(Array.isArray(normalized.ocr.lines) ? { lines:[...normalized.ocr.lines] } : {}),
211
- } : undefined,
212
- snapshot_original_bytes:originalBytes,
213
- snapshot_budget_bytes:budget,
214
- snapshot_truncated:true,
215
- };
216
-
217
- const lists = [
218
- [bounded.ocr, 'words', 50],
219
- [bounded.ui, 'nodes', 100],
220
- [bounded.browser, 'nodes', 100],
221
- [bounded.visual, 'labeled_targets', 25],
222
- [bounded.ocr, 'lines', 25],
223
- [bounded, 'windows', 20],
224
- ];
225
- let guard = 0;
226
- while (jsonBytes(bounded) > budget && guard < 32) {
227
- guard += 1;
228
- let changed = false;
229
- for (const [container, key, floor] of lists) {
230
- const values = container?.[key];
231
- if (!Array.isArray(values) || values.length <= floor) continue;
232
- const nextLength = Math.max(floor, Math.floor(values.length * 0.6));
233
- container[key] = values.slice(0, nextLength);
234
- container.truncated = true;
235
- container[`returned_${key}`] = nextLength;
236
- changed = true;
237
- if (jsonBytes(bounded) <= budget) break;
238
- }
239
- if (!changed) break;
240
- }
241
-
242
- if (jsonBytes(bounded) > budget && bounded.ui && typeof bounded.ui.semantic_tree === 'string') {
243
- bounded.ui.semantic_tree = bounded.ui.semantic_tree.slice(0, 8_000);
244
- bounded.ui.semantic_tree_truncated = true;
245
- }
246
-
247
- if (jsonBytes(bounded) > budget) {
248
- if (bounded.ui && typeof bounded.ui === 'object') {
249
- bounded.ui = {
250
- platform:bounded.ui.platform || null,
251
- count:Number(bounded.ui.count) || (Array.isArray(bounded.ui.nodes) ? bounded.ui.nodes.length : 0),
252
- nodes:[],
253
- truncated:true,
254
- reason:'snapshot payload budget',
255
- };
256
- }
257
- if (bounded.browser && typeof bounded.browser === 'object') {
258
- bounded.browser = {
259
- target_id:bounded.browser.target_id || null,
260
- title:String(bounded.browser.title || '').slice(0, 500),
261
- url:String(bounded.browser.url || '').slice(0, 2000),
262
- count:Number(bounded.browser.count) || (Array.isArray(bounded.browser.nodes) ? bounded.browser.nodes.length : 0),
263
- nodes:[],
264
- truncated:true,
265
- reason:'snapshot payload budget',
266
- };
267
- }
268
- if (bounded.ocr && typeof bounded.ocr === 'object') {
269
- bounded.ocr = {
270
- available:Boolean(bounded.ocr.available),
271
- backend:bounded.ocr.backend || null,
272
- language:bounded.ocr.language || null,
273
- text:String(bounded.ocr.text || '').slice(0, 10_000),
274
- count:Number(bounded.ocr.count) || 0,
275
- total_words:Number(bounded.ocr.total_words) || 0,
276
- words:[],
277
- lines:[],
278
- truncated:true,
279
- reason:'snapshot payload budget',
280
- };
281
- }
282
- if (Array.isArray(bounded.windows)) bounded.windows = bounded.windows.slice(0, 10);
283
- }
284
-
285
- bounded.snapshot_bytes = jsonBytes(bounded);
286
- return bounded;
287
- }
288
-
289
- function bestMonitor(window, displays) {
290
- if (!Array.isArray(displays)) return null;
291
- const wx1 = Number(window.x), wy1 = Number(window.y), ww = Number(window.width), wh = Number(window.height);
292
- if (![wx1, wy1, ww, wh].every(Number.isFinite) || ww <= 0 || wh <= 0) return null;
293
- const wx2 = wx1 + ww, wy2 = wy1 + wh;
294
- let best = null;
295
- let bestArea = 0;
296
- for (let index = 0; index < displays.length; index += 1) {
297
- const display = displays[index];
298
- const dx1 = Number(display.x), dy1 = Number(display.y), dw = Number(display.width), dh = Number(display.height);
299
- if (![dx1, dy1, dw, dh].every(Number.isFinite) || dw <= 0 || dh <= 0) continue;
300
- const area = Math.max(0, Math.min(wx2, dx1 + dw) - Math.max(wx1, dx1)) * Math.max(0, Math.min(wy2, dy1 + dh) - Math.max(wy1, dy1));
301
- if (area > bestArea) { bestArea = area; best = { index, ...display }; }
302
- }
303
- return best;
304
- }
305
-
306
- export async function listWindows(args = {}) {
307
- const raw = await adapter.listWindows(args);
308
- if (args.include_monitor === false) return raw;
309
- const windows = resultValue(raw);
310
- if (!Array.isArray(windows)) return raw;
311
- let displays = null;
312
- try { displays = resultValue(await adapter.displayInventory()); } catch {}
313
- if (!Array.isArray(displays)) return jsonResult(windows.map(window => ({ ...window, monitor: window.monitor ?? null })));
314
- return jsonResult(windows.map(window => {
315
- const monitor = bestMonitor(window, displays);
316
- return { ...window, monitor: monitor?.name ?? null, monitor_index: monitor?.index ?? null };
317
- }));
318
- }
319
- export async function windowAction(args = {}) {
320
- const hasTarget = Boolean(args.id || Number.isInteger(Number(args.pid)) || args.app || args.title);
321
- if (!hasTarget) throw new Error('window_action requires id, pid, app, or title');
322
- const action = optionalString(args.action);
323
- if (action === 'move' && ![args.x,args.y].every(value => Number.isFinite(Number(value)))) throw new Error('window_action move requires x and y');
324
- if (action === 'resize' && ![args.width,args.height].every(value => Number.isFinite(Number(value)))) throw new Error('window_action resize requires width and height');
325
- if (action === 'move_resize' && ![args.x,args.y,args.width,args.height].every(value => Number.isFinite(Number(value)))) throw new Error('window_action move_resize requires x, y, width and height');
326
- return adapter.windowAction(args);
327
- }
328
- export async function uiSnapshot(args = {}) {
329
- const explicitlyScoped = args.pid != null || args.app || args.window_title || args.windowTitle;
330
- const activeOnly = args.active_only != null
331
- ? args.active_only === true
332
- : (!explicitlyScoped && args.scope !== 'desktop');
333
- let effective = { ...args, active_only:activeOnly };
334
- let raw = await adapter.uiSnapshot(effective);
335
- let value = resultValue(raw);
336
- const requestedDepth = Number(effective.max_depth);
337
- const isEmptySnapshot = payload => Array.isArray(payload)
338
- ? payload.length === 0
339
- : Array.isArray(payload?.nodes) && payload.nodes.length === 0;
340
- if (process.platform === 'linux' && isEmptySnapshot(value)) {
341
- // GNOME/AT-SPI can transiently report an empty tree. Retry once at the requested
342
- // depth before reducing traversal depth, so valid deep trees are not discarded.
343
- await sleep(40);
344
- raw = await adapter.uiSnapshot(effective);
345
- value = resultValue(raw);
346
- if (isEmptySnapshot(value) && Number.isFinite(requestedDepth) && requestedDepth > 12) {
347
- effective = { ...effective, max_depth:12 };
348
- raw = await adapter.uiSnapshot(effective);
349
- value = resultValue(raw);
350
- }
351
- }
352
- return jsonResult(enrichUiPayload(value, effective));
353
- }
354
- export async function uiAction(args = {}) {
355
- const resolved = withResolvedUiLabel(args);
356
- const hasTarget = Boolean(resolved.id || resolved.label != null || resolved.name || resolved.role || resolved.automation_id || resolved.automationId);
357
- if (!hasTarget) throw new Error('ui_action requires id, label, name, role, or automation_id');
358
- const action = optionalString(resolved.action);
359
- if (['set_value','set_range_value'].includes(action) && resolved.value == null) throw new Error(`ui_action ${action} requires value`);
360
- return adapter.uiAction(resolved);
361
- }
362
- export async function keyboard(args = {}) { return adapter.keyboard(args); }
363
- export async function pointer(args = {}) {
364
- if (args.action === 'move' && ![args.x,args.y].every(value => Number.isFinite(Number(value)))) throw new Error('pointer move requires x and y');
365
- return adapter.pointer(args);
366
- }
367
- export async function clipboard(args = {}) { return adapter.clipboard(args); }
368
- export async function displayInventory(args = {}) { return adapter.displayInventory(args); }
369
- export async function cursorPosition(args = {}) { return typeof adapter.cursorPosition === 'function' ? adapter.cursorPosition(args) : jsonResult({ x:null, y:null }); }
370
-
371
- export async function screenshotRegion(args = {}) {
372
- let x = Number(args.x), y = Number(args.y), width = Number(args.width), height = Number(args.height);
373
- const hasRect = [x, y, width, height].every(Number.isFinite) && width > 0 && height > 0;
374
- if (!hasRect) {
375
- const windowId = optionalString(args.window_id || args.id);
376
- const app = optionalString(args.app);
377
- const title = optionalString(args.title);
378
- const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
379
- if (windowId || app || title || pid != null) {
380
- let row = null;
381
- if (windowId && typeof adapter.resolveWindowTarget === 'function') {
382
- row = resultValue(await adapter.resolveWindowTarget({
383
- id:windowId,
384
- ...(pid != null ? { pid } : {}),
385
- ...(app ? { app } : {}),
386
- ...(title ? { title } : {}),
387
- }));
388
- } else {
389
- const rows = resultValue(await listWindows({ include_monitor:false }));
390
- row = Array.isArray(rows) ? rows.find(item =>
391
- (!windowId || item.id === windowId)
392
- && (pid == null || Number(item.pid) === pid)
393
- && (!app || String(item.app || '').toLowerCase().includes(app.toLowerCase()))
394
- && (!title || String(item.title || '').toLowerCase().includes(title.toLowerCase()))
395
- ) : null;
396
- }
397
- if (!row) throw new Error('No matching window found for screenshot_region');
398
- x = Number(row.x); y = Number(row.y); width = Number(row.width); height = Number(row.height);
399
- } else if (args.monitor != null || args.monitor_index != null) {
400
- const displays = resultValue(await displayInventory());
401
- if (!Array.isArray(displays)) throw new Error('This platform does not expose normalized monitor geometry for screenshot_region');
402
- const index = Number.isInteger(Number(args.monitor_index)) ? Number(args.monitor_index) : null;
403
- const monitor = optionalString(args.monitor);
404
- const row = displays.find((item, itemIndex) =>
405
- (index == null || itemIndex === index)
406
- && (!monitor || (monitor.toLowerCase() === 'primary' ? Boolean(item.primary) : String(item.name || item.display_name || '').toLowerCase().includes(monitor.toLowerCase())))
407
- );
408
- if (!row) throw new Error('No matching monitor found for screenshot_region');
409
- x = Number(row.x); y = Number(row.y); width = Number(row.width); height = Number(row.height);
410
- }
411
- }
412
- if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) {
413
- throw new Error('Provide x/y/width/height, a window selector, or a monitor selector');
414
- }
415
- const padding = clamp(args.padding, 0, 0, 200);
416
- if (padding) { x -= padding; y -= padding; width += padding * 2; height += padding * 2; }
417
- return adapter.screenshotRegion({ ...args, x, y, width, height });
418
- }
419
- export async function notification(args = {}) { return adapter.notification(args); }
420
- export async function scroll(args = {}) {
421
- const direction = optionalString(args.direction);
422
- const hasDelta = [args.delta_x,args.delta_y,args.delta].some(value => value != null && Number.isFinite(Number(value)));
423
- if (!direction && !hasDelta) throw new Error('scroll requires direction or a delta');
424
- const times = clamp(args.wheel_times, 1, 1, 50);
425
- let deltaX = Number(args.delta_x || 0);
426
- let deltaY = Number(args.delta_y ?? args.delta ?? 0);
427
- if (direction) {
428
- const normalized = requireEnum(direction, 'direction', ['up','down','left','right']);
429
- if (normalized === 'up') deltaY = -120 * times;
430
- if (normalized === 'down') deltaY = 120 * times;
431
- if (normalized === 'left') deltaX = -120 * times;
432
- if (normalized === 'right') deltaX = 120 * times;
433
- }
434
- const semanticTarget = args.label != null || args.id || args.name || args.role || args.automation_id || args.automationId;
435
- if (semanticTarget) {
436
- const nodes = await uiMatches({ ...args, limit:1, refresh:true });
437
- const node = nodes[0];
438
- if (!node) throw new Error('Scroll target UI element was not found');
439
- const x = Number(node.center_x ?? (Number(node.x) + Number(node.width) / 2));
440
- const y = Number(node.center_y ?? (Number(node.y) + Number(node.height) / 2));
441
- if (Number.isFinite(x) && Number.isFinite(y)) await pointer({ action:'move', x:Math.round(x), y:Math.round(y), backend:args.backend });
442
- } else if (Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y))) {
443
- await pointer({ action:'move', x:Number(args.x), y:Number(args.y), backend:args.backend });
444
- }
445
- return adapter.scroll({ ...args, delta_x:deltaX, delta_y:deltaY });
446
- }
447
-
448
- function matches(node, args) {
449
- const id = optionalString(args.id);
450
- const label = args.label == null || args.label === '' ? null : Number(args.label);
451
- const name = optionalString(args.name);
452
- const role = optionalString(args.role);
453
- const value = optionalString(args.value);
454
- const automationId = optionalString(args.automation_id || args.automationId);
455
- const app = optionalString(args.app);
456
- const windowTitle = optionalString(args.window_title || args.windowTitle);
457
- const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
458
- if (label != null && (!Number.isInteger(label) || Number(node.label) !== label)) return false;
459
- if (id && node.id !== id) return false;
460
- if (pid != null && Number(node.pid) !== pid) return false;
461
- if (app && !String(node.app || '').toLowerCase().includes(app.toLowerCase())) return false;
462
- if (windowTitle && !String(node.window || node.window_title || '').toLowerCase().includes(windowTitle.toLowerCase())) return false;
463
- if (name && !String(node.name || '').toLowerCase().includes(name.toLowerCase())) return false;
464
- if (role && !String(node.role || '').toLowerCase().includes(role.toLowerCase())) return false;
465
- if (value && !String(node.value || '').toLowerCase().includes(value.toLowerCase())) return false;
466
- if (automationId && String(node.automationId || '') !== automationId) return false;
467
- return true;
468
- }
469
-
470
- async function uiMatches(args = {}) {
471
- const resolved = args.label != null ? withResolvedUiLabel(args) : args;
472
- const cached = resolved.refresh === true ? null : cachedUiNodes(resolved);
473
- if (cached) {
474
- const hits = cached.filter(node => matches(node, resolved)).slice(0, clamp(resolved.limit, 20, 1, 200));
475
- if (hits.length || resolved.cache_only === true || resolved.id || resolved.label != null) return hits;
476
- }
477
- const stableLinuxPid = optionalString(resolved.id)?.match(/^linux:(\d+):/)?.[1];
478
- const scopedPid = resolved.pid != null ? resolved.pid : stableLinuxPid ? Number(stableLinuxPid) : null;
479
- const result = await uiSnapshot({
480
- max_nodes: resolved.max_nodes || 2000,
481
- max_depth: resolved.max_depth || (process.platform === 'linux' ? 12 : 16),
482
- ...(scopedPid != null ? { pid: scopedPid } : {}),
483
- ...(resolved.app ? { app: resolved.app } : {}),
484
- ...((resolved.window_title || resolved.windowTitle) ? { window_title: resolved.window_title || resolved.windowTitle } : {}),
485
- ...(resolved.browser_dom === true ? { browser_dom:true } : {}),
486
- });
487
- const payload = resultValue(result) || {};
488
- const nodes = Array.isArray(payload) ? payload : payload.nodes || [];
489
- return nodes.filter(node => matches(node, resolved)).slice(0, clamp(resolved.limit, 20, 1, 200));
490
- }
491
-
492
- export async function uiFind(args = {}) {
493
- const hasSelector = Boolean(args.id || args.label != null || args.name || args.role || args.automation_id || args.automationId);
494
- if (!hasSelector) throw new Error('ui_find requires id, label, name, role, or automation_id; use ui_snapshot to enumerate UI');
495
- const nodes = await uiMatches(args);
496
- return jsonResult({ count: nodes.length, nodes });
497
- }
498
-
499
- function uiFingerprint(nodes) {
500
- return JSON.stringify((nodes || []).map(node => ({
501
- id: node.id ?? null,
502
- role: node.role ?? null,
503
- name: node.name ?? null,
504
- value: node.value ?? null,
505
- x: Number(node.x || 0), y: Number(node.y || 0),
506
- width: Number(node.width || 0), height: Number(node.height || 0),
507
- })));
508
- }
509
-
510
- function uiTextContains(node, needle) {
511
- if (!needle) return true;
512
- const normalized = String(needle).toLowerCase();
513
- return [
514
- node.name, node.value, node.description, node.help_text, node.helpText,
515
- node.role, node.window, node.window_title, node.automationId,
516
- ].some(value => String(value || '').toLowerCase().includes(normalized));
517
- }
518
-
519
- async function uiConditionMatches(args, condition) {
520
- if (condition === 'active_window') {
521
- const expected = optionalString(args.window_title || args.windowTitle || args.text || args.name);
522
- const nodes = await uiMatches({ ...args, name:undefined, role:undefined, value:undefined, id:undefined, label:undefined, limit:200 });
523
- const active = nodes.filter(node => node.active === true || (
524
- node.focused === true && /window|frame|dialog/i.test(String(node.role || ''))
525
- ));
526
- const matched = expected ? active.filter(node => uiTextContains(node, expected)) : active;
527
- return { matched:matched.length > 0, nodes:matched, detail:matched[0]?.window || matched[0]?.name || null };
528
- }
529
- if (condition === 'text_exists') {
530
- const needle = optionalString(args.text || args.name || args.value);
531
- if (!needle) throw new Error('text is required for wait_for_ui condition=text_exists');
532
- const cached = cachedUiNodes(args);
533
- let nodes = cached || [];
534
- if (!cached) {
535
- const snapshot = resultValue(await uiSnapshot({
536
- max_nodes:args.max_nodes || 2000,
537
- max_depth:args.max_depth || (process.platform === 'linux' ? 12 : 16),
538
- ...(args.pid != null ? { pid:args.pid } : {}),
539
- ...(args.app ? { app:args.app } : {}),
540
- ...((args.window_title || args.windowTitle) ? { window_title:args.window_title || args.windowTitle } : {}),
541
- ...(args.browser_dom === true ? { browser_dom:true } : {}),
542
- })) || {};
543
- nodes = Array.isArray(snapshot) ? snapshot : snapshot.nodes || [];
544
- }
545
- const matched = nodes.filter(node => uiTextContains(node, needle)).slice(0, clamp(args.limit, 20, 1, 200));
546
- return { matched:matched.length > 0, nodes:matched, detail:needle };
547
- }
548
-
549
- const nodes = await uiMatches(args);
550
- if (condition === 'element_enabled') {
551
- const enabled = nodes.filter(node => node.enabled === true || node.disabled === false);
552
- return { matched:enabled.length > 0, nodes:enabled };
553
- }
554
- if (condition === 'focused_element') {
555
- const focused = nodes.filter(node => node.focused === true || node.has_focused === true);
556
- return { matched:focused.length > 0, nodes:focused };
557
- }
558
- return { matched:nodes.length > 0, nodes };
559
- }
560
-
561
- export async function waitForUi(args = {}) {
562
- const timeoutMs = clamp(args.timeout_ms, 10_000, 100, 120_000);
563
- const pollMs = clamp(args.poll_ms, 250, 50, 5000);
564
- const rawCondition = optionalString(args.condition);
565
- const rawState = optionalString(args.state);
566
- if (!rawCondition && !rawState) throw new Error('wait_for_ui requires state or condition');
567
- const aliases = { text:'text_exists', window:'active_window', element:'element_exists', enabled:'element_enabled', focused:'focused_element' };
568
- const condition = rawCondition
569
- ? requireEnum(aliases[rawCondition] || rawCondition, 'condition', ['text_exists','active_window','element_exists','element_enabled','focused_element'])
570
- : null;
571
- const state = condition ? null : requireEnum(rawState, 'state', ['present', 'absent', 'changed']);
572
- const hasSelector = Boolean(args.id || args.label != null || args.name || args.role || args.automation_id || args.automationId);
573
- if (['element_exists','element_enabled'].includes(condition) && !hasSelector) throw new Error(`wait_for_ui condition=${condition} requires a semantic target`);
574
- if (condition === 'active_window' && !(args.text || args.name || args.window_title || args.windowTitle)) throw new Error('wait_for_ui condition=active_window requires text, name, or window_title');
575
- if (['present','absent'].includes(state) && !hasSelector) throw new Error(`wait_for_ui state=${state} requires a semantic target`);
576
- const deadline = Date.now() + timeoutMs;
577
- const startedAt = Date.now();
578
- let attempts = 0;
579
-
580
- if (condition) {
581
- let evaluation = { matched:false, nodes:[] };
582
- do {
583
- attempts += 1;
584
- // A wait must observe fresh state, not repeatedly query the same 5-second cache.
585
- latestUiSnapshot = null;
586
- evaluation = await uiConditionMatches(args, condition);
587
- if (evaluation.matched) {
588
- return jsonResult({
589
- condition,
590
- matched:true,
591
- attempts,
592
- elapsed_ms:Date.now() - startedAt,
593
- count:evaluation.nodes.length,
594
- nodes:evaluation.nodes,
595
- ...(evaluation.detail ? { detail:evaluation.detail } : {}),
596
- });
597
- }
598
- await sleep(pollMs);
599
- } while (Date.now() < deadline);
600
- return jsonResult({
601
- condition,
602
- matched:false,
603
- state:'timeout',
604
- attempts,
605
- elapsed_ms:Date.now() - startedAt,
606
- count:evaluation.nodes?.length || 0,
607
- nodes:evaluation.nodes || [],
608
- });
609
- }
610
-
611
- latestUiSnapshot = null;
612
- const baseline = state === 'changed' ? await uiMatches(args) : [];
613
- const baselineFingerprint = state === 'changed' ? uiFingerprint(baseline) : null;
614
- let nodes = baseline;
615
- do {
616
- attempts += 1;
617
- latestUiSnapshot = null;
618
- nodes = await uiMatches(args);
619
- const changed = state === 'changed' && uiFingerprint(nodes) !== baselineFingerprint;
620
- if ((state === 'present' && nodes.length) || (state === 'absent' && !nodes.length) || changed) {
621
- return jsonResult({
622
- state,
623
- matched: nodes.length,
624
- attempts,
625
- elapsed_ms:Date.now() - startedAt,
626
- nodes,
627
- ...(state === 'changed' ? { before: baseline } : {}),
628
- });
629
- }
630
- await sleep(pollMs);
631
- } while (Date.now() < deadline);
632
- return jsonResult({ state:'timeout', wanted:state, matched:nodes.length, attempts, elapsed_ms:Date.now() - startedAt, nodes, ...(state === 'changed' ? { before:baseline } : {}) });
633
- }
634
-
635
- function explicitTypeTextWindowTarget(args = {}, hasElementSelector = false) {
636
- const windowId = optionalString(args.window_id || args.windowId);
637
- const title = optionalString(args.title);
638
- if (!windowId && !title && hasElementSelector) return null;
639
- const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
640
- const app = optionalString(args.app);
641
- const scopedTitle = optionalString(args.window_title || args.windowTitle);
642
- if (!windowId && !title && pid == null && !app && !scopedTitle) return null;
643
- return {
644
- ...(windowId ? { id:windowId } : {}),
645
- ...(pid != null ? { pid } : {}),
646
- ...(app ? { app } : {}),
647
- ...((title || scopedTitle) ? { title:title || scopedTitle } : {}),
648
- ...(args.backend ? { backend:args.backend } : {}),
649
- };
650
- }
651
-
652
- export async function typeText(args = {}) {
653
- const value = String(args.text ?? '');
654
- const method = requireEnum(args.method || 'auto', 'method', ['auto','accessibility','clipboard','keys']);
655
- const resolvedArgs = args.label != null ? withResolvedUiLabel(args) : args;
656
- const hasElementSelector = Boolean(
657
- optionalString(resolvedArgs.id) || optionalString(resolvedArgs.name) || optionalString(resolvedArgs.role) ||
658
- optionalString(resolvedArgs.automation_id || resolvedArgs.automationId)
659
- );
660
- const explicitWindowTarget = explicitTypeTextWindowTarget(resolvedArgs, hasElementSelector);
661
- const clear = args.clear === true;
662
- const pressEnter = args.press_enter === true;
663
- const caret = requireEnum(args.caret_position || 'idle', 'caret_position', ['start','idle','end']);
664
-
665
- if (
666
- explicitWindowTarget && !hasElementSelector && clear && !pressEnter
667
- && (method === 'auto' || method === 'accessibility')
668
- && typeof adapter.typeTextWindowTarget === 'function'
669
- ) {
670
- const semantic = await adapter.typeTextWindowTarget(value, resolvedArgs);
671
- if (semantic) {
672
- return jsonResult({ length:value.length, method:'accessibility', target:semantic, clear:true, press_enter:false });
673
- }
674
- if (method === 'accessibility') {
675
- throw new Error('Accessibility-only text input needs exactly one editable control in the targeted window when no element selector is provided');
676
- }
677
- }
678
-
679
- if ((method === 'auto' || method === 'accessibility') && hasElementSelector) {
680
- try {
681
- const result = await uiAction({ ...resolvedArgs, action:'set_value', value });
682
- const raw = result.content?.[0]?.text || '';
683
- let target = raw;
684
- try { target = JSON.parse(raw); } catch {}
685
- if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
686
- return jsonResult({ length:value.length, method:'accessibility', target, clear:true, press_enter:pressEnter });
687
- } catch (error) {
688
- if (method === 'accessibility') throw error;
689
- if (!clear && caret === 'idle' && typeof adapter.typeTextFocused === 'function') {
690
- await uiAction({ ...resolvedArgs, action:'focus' });
691
- const semantic = await adapter.typeTextFocused(value, resolvedArgs);
692
- if (semantic) {
693
- if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
694
- return jsonResult({ length:value.length, method:'accessibility', target:semantic, press_enter:pressEnter });
695
- }
696
- }
697
- const detail = error instanceof Error ? error.message : String(error);
698
- throw new Error(`Semantic text target rejected accessibility input. Refresh the UI target or use browser_action for browser page content. ${detail}`);
699
- }
700
- }
701
- if (
702
- !explicitWindowTarget && (method === 'auto' || method === 'accessibility') && !hasElementSelector
703
- && typeof adapter.typeTextFocused === 'function' && !clear && caret === 'idle'
704
- ) {
705
- const semantic = await adapter.typeTextFocused(value, resolvedArgs);
706
- if (semantic) {
707
- if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
708
- return jsonResult({ length:value.length, method:'accessibility', target:semantic, press_enter:pressEnter });
709
- }
710
- if (method === 'accessibility') throw new Error('No focused editable accessibility element is available');
711
- }
712
- if (method === 'accessibility') {
713
- throw new Error('Accessibility-only text input requires a semantic element target, one editable control in the targeted window with clear=true, or a focused editable control with clear=false and caret_position=idle');
714
- }
715
-
716
- if (explicitWindowTarget) await windowAction({ action:'focus', ...explicitWindowTarget });
717
- if (hasElementSelector) await uiAction({ ...resolvedArgs, action:'focus' });
718
- if (clear) {
719
- await keyboard({ ...resolvedArgs, shortcut:process.platform === 'darwin' ? 'CMD+A' : 'CTRL+A' });
720
- await keyboard({ ...resolvedArgs, key:'BACKSPACE' });
721
- }
722
- if (caret === 'start') await keyboard({ ...resolvedArgs, key:'HOME' });
723
- if (caret === 'end') await keyboard({ ...resolvedArgs, key:'END' });
724
-
725
- if (method === 'keys' && typeof adapter.typeTextKeys === 'function') {
726
- const result = await adapter.typeTextKeys(value, resolvedArgs);
727
- if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
728
- return jsonResult({ length:value.length, method:'keys', press_enter:pressEnter, ...(result?.backend ? { backend:result.backend } : {}) });
729
- }
730
-
731
- let prior = null;
732
- let priorReadable = false;
733
- try {
734
- const previous = await clipboard({ action:'read' });
735
- prior = previous.content?.[0]?.text ?? '';
736
- priorReadable = true;
737
- } catch {}
738
- try {
739
- await clipboard({ action:'write', text:value });
740
- await sleep(50);
741
- await keyboard({ ...resolvedArgs, shortcut:process.platform === 'darwin' ? 'CMD+V' : 'CTRL+V' });
742
- await sleep(75);
743
- if (pressEnter) await keyboard({ ...resolvedArgs, key:'ENTER' });
744
- } finally {
745
- if (priorReadable) {
746
- try { await clipboard({ action:'write', text:prior }); } catch {}
747
- }
748
- }
749
- return jsonResult({ length:value.length, method:'clipboard_paste', clipboard_restored:priorReadable, press_enter:pressEnter });
750
- }
751
-
752
- async function uiElementCenter(id, field) {
753
- const matches = await uiMatches({ id, limit: 1, max_nodes: 5000, max_depth: 32, refresh:true });
754
- const node = matches[0];
755
- if (!node) throw new Error(`${field} UI element was not found: ${id}`);
756
- const x = Number(node.x), y = Number(node.y), width = Number(node.width), height = Number(node.height);
757
- if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) {
758
- throw new Error(`${field} UI element has no usable screen bounds: ${id}`);
759
- }
760
- return { x: Math.round(x + width / 2), y: Math.round(y + height / 2), node };
761
- }
762
-
763
- export async function dragDrop(args = {}) {
764
- let fromX = Number(args.from_x), fromY = Number(args.from_y);
765
- let toX = Number(args.to_x), toY = Number(args.to_y);
766
- let fromNode = null;
767
- let toNode = null;
768
- if (!Number.isFinite(fromX) || !Number.isFinite(fromY)) {
769
- const id = optionalString(args.from_id);
770
- if (!id) throw new Error('Provide from_x/from_y or from_id');
771
- const resolved = await uiElementCenter(id, 'from_id');
772
- fromX = resolved.x; fromY = resolved.y; fromNode = resolved.node;
773
- }
774
- if (!Number.isFinite(toX) || !Number.isFinite(toY)) {
775
- const id = optionalString(args.to_id);
776
- if (!id) throw new Error('Provide to_x/to_y or to_id');
777
- const resolved = await uiElementCenter(id, 'to_id');
778
- toX = resolved.x; toY = resolved.y; toNode = resolved.node;
779
- }
780
- if (typeof adapter.dragDrop === 'function') {
781
- const result = await adapter.dragDrop({ ...args, from_x: fromX, from_y: fromY, to_x: toX, to_y: toY });
782
- if (!fromNode && !toNode) return result;
783
- return jsonResult({ from: [fromX, fromY], to: [toX, toY], from_element: fromNode?.id || null, to_element: toNode?.id || null });
784
- }
785
- await pointer({ action: 'move', x: fromX, y: fromY, button: args.button });
786
- await pointer({ action: 'down', x: fromX, y: fromY, button: args.button });
787
- await sleep(clamp(args.hold_ms, 120, 0, 5000));
788
- await pointer({ action: 'move', x: toX, y: toY, button: args.button });
789
- await sleep(clamp(args.duration_ms, 120, 0, 5000));
790
- await pointer({ action: 'up', x: toX, y: toY, button: args.button });
791
- return jsonResult({ from: [fromX, fromY], to: [toX, toY], from_element: fromNode?.id || null, to_element: toNode?.id || null });
792
- }
793
-
794
- export async function launchApp(args = {}) {
795
- const app = optionalString(args.app || args.path);
796
- if (!app) throw new Error('app is required');
797
- const argv = Array.isArray(args.args) ? args.args.map(String) : [];
798
- const cwd = args.cwd != null ? await resolveSafePath(args.cwd, 'cwd') : null;
799
- const policy = assertAllowedCommand([app, ...argv].join(' '));
800
- const result = await adapter.launchApp(app, argv, { ...args, cwd });
801
- const body = result?.content?.[0]?.text || (typeof result === 'string' ? result : `Launched ${app}.`);
802
-
803
- const waitHint = optionalString(args.wait_for_window);
804
- if (waitHint) {
805
- const timeoutMs = clamp(args.wait_timeout_ms, 10_000, 100, 120_000);
806
- const deadline = Date.now() + timeoutMs;
807
- let match = null;
808
- do {
809
- const windows = resultValue(await listWindows({ include_monitor:false }));
810
- match = Array.isArray(windows) ? windows.find(row =>
811
- String(row.title || '').toLowerCase().includes(waitHint.toLowerCase())
812
- || String(row.app || '').toLowerCase().includes(waitHint.toLowerCase())
813
- ) : null;
814
- if (match) break;
815
- await sleep(100);
816
- } while (Date.now() < deadline);
817
- if (!match) throw new Error(`Application launched but no window matching "${waitHint}" appeared within ${timeoutMs} ms`);
818
- return text(`${policy.note ? `${policy.note}\n` : ''}${body}\nWindow ready: ${match.title || match.app || match.id}`);
819
- }
820
- return text(`${policy.note ? `${policy.note}\n` : ''}${body}`);
821
- }
822
-
823
- export async function openPath(args = {}) {
824
- const target = await resolveSafePath(args.path, 'path');
825
- await adapter.openPath(target);
826
- return text(`Opened ${target}.`);
827
- }
828
-
829
- export async function revealPath(args = {}) {
830
- const target = await resolveSafePath(args.path, 'path');
831
- await adapter.revealPath(target);
832
- return text(`Revealed ${target}.`);
833
- }
834
-
835
- function screenshotImagePart(screenshotResult) {
836
- return screenshotResult?.content?.find(part => part.type === 'image' && typeof part.data === 'string') || null;
837
- }
838
-
839
- async function ocrScreenshotResult(screenshotResult, args = {}) {
840
- const imagePart = screenshotImagePart(screenshotResult);
841
- if (!imagePart) return { available:false, reason:'no screenshot image is available for OCR' };
842
- return ocrImage(Buffer.from(imagePart.data, 'base64'), {
843
- language:args.ocr_language,
844
- max_words:args.ocr_max_words,
845
- psm:args.ocr_psm,
846
- timeout_ms:args.ocr_timeout_ms,
847
- });
848
- }
849
-
850
- function pngDimensions(imagePart) {
851
- if (!imagePart || imagePart.mimeType !== 'image/png') return null;
852
- let buffer;
853
- try { buffer = Buffer.from(imagePart.data, 'base64'); } catch { return null; }
854
- if (buffer.length < 24 || buffer.toString('ascii', 1, 4) !== 'PNG') return null;
855
- const width = buffer.readUInt32BE(16);
856
- const height = buffer.readUInt32BE(20);
857
- return width > 0 && height > 0 ? { width, height } : null;
858
- }
859
-
860
- function virtualDesktopBounds(displays) {
861
- if (!Array.isArray(displays) || !displays.length) return null;
862
- const valid = displays.filter(display =>
863
- [display.x, display.y, display.width, display.height].every(value => Number.isFinite(Number(value)))
864
- && Number(display.width) > 0
865
- && Number(display.height) > 0
866
- );
867
- if (!valid.length) return null;
868
- const x = Math.min(...valid.map(display => Number(display.x)));
869
- const y = Math.min(...valid.map(display => Number(display.y)));
870
- const right = Math.max(...valid.map(display => Number(display.x) + Number(display.width)));
871
- const bottom = Math.max(...valid.map(display => Number(display.y) + Number(display.height)));
872
- return { x, y, width:right - x, height:bottom - y };
873
- }
874
-
875
- export function mapOcrBoxToDesktop(box, imagePart, displays, captureBounds = null) {
876
- const image = pngDimensions(imagePart);
877
- const desktop = captureBounds || virtualDesktopBounds(displays);
878
- const centerX = Number(box?.x) + Number(box?.width) / 2;
879
- const centerY = Number(box?.y) + Number(box?.height) / 2;
880
- if (!Number.isFinite(centerX) || !Number.isFinite(centerY)) throw new Error('OCR match has invalid bounds');
881
- if (!image || !desktop || desktop.width <= 0 || desktop.height <= 0) {
882
- return { x:Math.round(centerX), y:Math.round(centerY), scale_x:1, scale_y:1, desktop_origin:[0,0] };
883
- }
884
- const scaleX = image.width / desktop.width;
885
- const scaleY = image.height / desktop.height;
886
- if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX <= 0 || scaleY <= 0) {
887
- throw new Error('Could not map OCR pixels to desktop coordinates');
888
- }
889
- return {
890
- x:Math.round(desktop.x + centerX / scaleX),
891
- y:Math.round(desktop.y + centerY / scaleY),
892
- scale_x:scaleX,
893
- scale_y:scaleY,
894
- desktop_origin:[desktop.x, desktop.y],
895
- screenshot_size:[image.width, image.height],
896
- desktop_size:[desktop.width, desktop.height],
897
- };
898
- }
899
-
900
- async function clickOcrText(args = {}) {
901
- const needle = optionalString(args.ocr_text || args.name || args.browser_text);
902
- if (!needle) throw new Error('ocr_text is required for OCR targeting');
903
-
904
- let screenshot = null;
905
- let captureBounds = null;
906
- const explicitRegion = Array.isArray(args.ocr_region) && args.ocr_region.length === 4
907
- ? args.ocr_region.map(Number)
908
- : null;
909
- if (explicitRegion?.every(Number.isFinite)) {
910
- const [left, top, right, bottom] = explicitRegion;
911
- if (right <= left || bottom <= top) throw new Error('ocr_region must be [left,top,right,bottom] with positive area');
912
- captureBounds = { x:Math.round(left), y:Math.round(top), width:Math.round(right-left), height:Math.round(bottom-top) };
913
- screenshot = await screenshotRegion({ ...captureBounds, padding:0 });
914
- }
915
-
916
- // Without an explicit OCR region, discover the current semantic foreground first. OCR on a tightly
917
- // scoped active window is much more accurate than OCR on a dense whole-desktop screenshot and still
918
- // falls back safely below.
919
- if (!screenshot) try {
920
- const state = resultValue(await computerSnapshot({
921
- include_screenshot:false,
922
- include_browser:false,
923
- include_ocr:false,
924
- max_ui_nodes:300,
925
- max_ui_depth:8,
926
- }));
927
- captureBounds = activeWindowCaptureBounds(state?.active_window);
928
- if (captureBounds) screenshot = await screenshotRegion({ ...captureBounds, padding:0 });
929
- } catch {}
930
-
931
- if (!screenshot) {
932
- screenshot = await fileToolHandlers.take_screenshot({
933
- keep:false,
934
- ...(runtimeConfig.allowedRoots[0] ? { directory:runtimeConfig.allowedRoots[0] } : {}),
935
- });
936
- captureBounds = null;
937
- }
938
- const deadline = Date.now() + clamp(args.timeout_ms, 1500, 300, 10_000);
939
- let imagePart = null;
940
- let ocr = null;
941
- let match = null;
942
- let attempts = 0;
943
- for (let attempt = 0; attempt < 3; attempt += 1) {
944
- attempts = attempt + 1;
945
- imagePart = screenshotImagePart(screenshot);
946
- ocr = await ocrScreenshotResult(screenshot, args);
947
- if (!ocr.available) throw new Error(`OCR is unavailable: ${ocr.reason || 'unknown OCR error'}`);
948
- match = findOcrText(ocr, needle);
949
- if (match) break;
950
- if (attempt >= 2 || Date.now() >= deadline) break;
951
- await sleep(Math.min(150, Math.max(1, deadline - Date.now())));
952
- if (captureBounds) {
953
- screenshot = await screenshotRegion({ ...captureBounds, padding:0 });
954
- } else {
955
- screenshot = await fileToolHandlers.take_screenshot({
956
- keep:false,
957
- ...(runtimeConfig.allowedRoots[0] ? { directory:runtimeConfig.allowedRoots[0] } : {}),
958
- });
959
- }
960
- }
961
- if (!match) throw new Error(`OCR text not found after ${attempts} capture${attempts === 1 ? '' : 's'}: ${needle}`);
962
- let displays = null;
963
- try { displays = resultValue(await displayInventory()); } catch {}
964
- const point = mapOcrBoxToDesktop(match, imagePart, displays, captureBounds);
965
- const pointerResult = await pointer({ ...args, action:'click', x:point.x, y:point.y });
966
- return jsonResult({
967
- target:'ocr',
968
- action:'click',
969
- query:needle,
970
- attempts,
971
- match,
972
- point:{ x:point.x, y:point.y },
973
- mapping:{
974
- scale_x:point.scale_x,
975
- scale_y:point.scale_y,
976
- desktop_origin:point.desktop_origin,
977
- screenshot_size:point.screenshot_size || null,
978
- desktop_size:point.desktop_size || null,
979
- capture_bounds:captureBounds,
980
- },
981
- pointer:resultValue(pointerResult) || pointerResult?.content?.[0]?.text || null,
982
- });
983
- }
984
-
985
- function browserTitleHint(activeWindow) {
986
- const title = optionalString(activeWindow?.title);
987
- if (!title) return null;
988
- const stripped = title.replace(/\s+[-—]\s+(Google Chrome|Chromium|Microsoft Edge|Brave)(?:\s+Browser)?$/i, '').trim();
989
- return stripped || null;
990
- }
991
-
992
- function hasExplicitComputerScreenshotTarget(args = {}) {
993
- return (Array.isArray(args.screenshot_region) && args.screenshot_region.length === 4)
994
- || Boolean(args.screenshot_window_id)
995
- || args.screenshot_monitor != null
996
- || args.screenshot_monitor_index != null;
997
- }
998
-
999
- function activeWindowCaptureBounds(window) {
1000
- if (!window) return null;
1001
- const x = Number(window.x), y = Number(window.y), width = Number(window.width), height = Number(window.height);
1002
- if (![x,y,width,height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
1003
- return { x:Math.round(x), y:Math.round(y), width:Math.round(width), height:Math.round(height) };
1004
- }
1005
-
1006
- function computerScreenshotTask(args = {}) {
1007
- const region = Array.isArray(args.screenshot_region) ? args.screenshot_region.map(Number) : null;
1008
- const targeted = region
1009
- || args.screenshot_window_id
1010
- || args.screenshot_monitor != null
1011
- || args.screenshot_monitor_index != null;
1012
- if (targeted) {
1013
- const targetArgs = {
1014
- padding:args.screenshot_padding,
1015
- ...(region?.length === 4 ? { x:region[0], y:region[1], width:region[2] - region[0], height:region[3] - region[1] } : {}),
1016
- ...(args.screenshot_window_id ? { window_id:args.screenshot_window_id } : {}),
1017
- ...(args.screenshot_monitor != null ? { monitor:args.screenshot_monitor } : {}),
1018
- ...(args.screenshot_monitor_index != null ? { monitor_index:args.screenshot_monitor_index } : {}),
1019
- };
1020
- return screenshotRegion(targetArgs);
1021
- }
1022
- return fileToolHandlers.take_screenshot({
1023
- keep:false,
1024
- ...(runtimeConfig.allowedRoots[0] ? { directory:runtimeConfig.allowedRoots[0] } : {}),
1025
- });
1026
- }
1027
-
1028
- export async function computerSnapshot(args = {}) {
1029
- const desktopAvailable = desktopSessionAvailable();
1030
- const screenshotRequested = args.include_screenshot !== false || args.include_ocr === true;
1031
- const captureScreenshot = desktopAvailable && screenshotRequested;
1032
- const includeUi = args.include_ui !== false;
1033
- const [windowsResult, displaysResult, uiResult, clipboardResult, cursorResult, shotResult] = await Promise.allSettled([
1034
- desktopAvailable ? listWindows() : Promise.resolve(null),
1035
- desktopAvailable ? displayInventory() : Promise.resolve(null),
1036
- includeUi && desktopAvailable
1037
- ? uiSnapshot({
1038
- max_nodes:args.max_ui_nodes || 800,
1039
- max_depth:args.max_ui_depth || 12,
1040
- semantic_tree:args.semantic_tree !== false,
1041
- active_only:args.ui_scope === 'desktop' ? false : true,
1042
- ...(args.ui_browser_dom === true ? { browser_dom:true } : {}),
1043
- })
1044
- : Promise.resolve(null),
1045
- desktopAvailable ? clipboard({ action:'read' }) : Promise.resolve(null),
1046
- desktopAvailable ? cursorPosition() : Promise.resolve(null),
1047
- captureScreenshot ? computerScreenshotTask(args) : Promise.resolve(null),
1048
- ]);
1049
- const parse = (result, fallback = null) => {
1050
- if (result.status !== 'fulfilled' || !result.value || result.value.isError === true) return fallback;
1051
- return resultValue(result.value) ?? fallback;
1052
- };
1053
- const shapeErrors = [];
1054
- const parsedWindows = parse(windowsResult, []);
1055
- const windows = Array.isArray(parsedWindows) ? parsedWindows : [];
1056
- if (parsedWindows !== windows && parsedWindows != null) {
1057
- shapeErrors.push({ source:'windows', message:'window inventory returned an invalid result shape' });
1058
- }
1059
- const parsedDisplays = parse(displaysResult, []);
1060
- const displays = Array.isArray(parsedDisplays) ? parsedDisplays : [];
1061
- if (parsedDisplays !== displays && parsedDisplays != null) {
1062
- shapeErrors.push({ source:'displays', message:'display inventory returned an invalid result shape' });
1063
- }
1064
- const parsedCursor = parse(cursorResult, { x:null, y:null });
1065
- const cursor = objectRecord(parsedCursor) || { x:null, y:null };
1066
- if (parsedCursor !== cursor && parsedCursor != null) {
1067
- shapeErrors.push({ source:'cursor', message:'cursor position returned an invalid result shape' });
1068
- }
1069
- const parsedUi = includeUi ? parse(uiResult, null) : null;
1070
- const ui = objectRecord(parsedUi) || { platform:process.platform, count:0, nodes:[], ...(includeUi ? { unavailable:true } : { skipped:true }) };
1071
- if (includeUi && parsedUi !== ui && parsedUi != null) {
1072
- shapeErrors.push({ source:'ui', message:'accessibility snapshot returned an invalid result shape' });
1073
- }
1074
- const clipboardOk = desktopAvailable && clipboardResult.status === 'fulfilled' && clipboardResult.value?.isError !== true && Boolean(clipboardResult.value);
1075
- const clipText = clipboardOk ? clipboardResult.value.content?.[0]?.text || '' : '';
1076
- const uiNodes = Array.isArray(ui?.nodes) ? ui.nodes : Array.isArray(ui) ? ui : [];
1077
- const focusedNodes = uiNodes.filter(node => node.focused === true);
1078
- const focusScore = node => {
1079
- const app = String(node.app || '').toLowerCase();
1080
- const windowName = String(node.window || '').trim();
1081
- const role = String(node.role || '').toLowerCase();
1082
- let score = 0;
1083
- if (windowName) score += 100;
1084
- if (app && app !== 'gnome-shell') score += 40;
1085
- if (node.name) score += 15;
1086
- if (!['window','filler','panel'].includes(role)) score += 15;
1087
- if (Number(node.width) >= 100 && Number(node.height) >= 100) score += 10;
1088
- if (/desktop icons/i.test(windowName)) score -= 80;
1089
- if (app === 'gnome-shell' && !windowName) score -= 120;
1090
- return score;
1091
- };
1092
- const focusedNode = focusedNodes.sort((a, b) => focusScore(b) - focusScore(a))[0] || null;
1093
- const activeTopLevel = uiNodes.find(node =>
1094
- node.active === true
1095
- && /frame|window|dialog/i.test(String(node.role || ''))
1096
- && String(node.window || node.name || '').trim()
1097
- && !/desktop icons/i.test(String(node.window || node.name || ''))
1098
- ) || null;
1099
- const activeNode = activeTopLevel
1100
- || focusedNode
1101
- || uiNodes.find(node => /frame|window|dialog/i.test(String(node.role || '')) && String(node.window || node.name || '').trim())
1102
- || uiNodes[0]
1103
- || null;
1104
- const windowRows = Array.isArray(windows) ? windows : [];
1105
- const activeWindow = activeNode ? windowRows.find(row => {
1106
- const activePid = Number(activeNode.pid);
1107
- const rowPid = Number(row.pid);
1108
- const wantedTitle = String(activeNode.window || activeNode.name || '').trim().toLowerCase();
1109
- const rowTitle = String(row.title || '').trim().toLowerCase();
1110
- const pidMatch = activePid > 0 && rowPid > 0 && rowPid === activePid;
1111
- // XWayland/WM enumeration can report pid=0 while AT-SPI still exposes the real application PID.
1112
- // A strong top-level title match is the safe fallback for joining those two views.
1113
- const titleMatch = Boolean(wantedTitle && rowTitle && (rowTitle === wantedTitle || rowTitle.includes(wantedTitle) || wantedTitle.includes(rowTitle)));
1114
- return (pidMatch || titleMatch) && (!wantedTitle || titleMatch || pidMatch);
1115
- }) || null : null;
1116
- const activeApp = activeNode ? {
1117
- pid: Number(activeNode.pid) || activeWindow?.pid || null,
1118
- app: activeNode.app || activeWindow?.app || null,
1119
- window: activeNode.window || activeWindow?.title || null,
1120
- } : null;
1121
-
1122
- const activeLabel = [activeApp?.app, activeApp?.window, activeWindow?.app, activeWindow?.title].filter(Boolean).join(' ');
1123
- const browserRequested = args.include_browser === true
1124
- || (args.include_browser !== false && /chrome|chromium|edge|brave/i.test(activeLabel));
1125
- let browser = null;
1126
- let browserError = null;
1127
- if (browserRequested && await browserCapabilityAvailable(undefined, 400)) {
1128
- const hint = browserTitleHint(activeWindow);
1129
- try {
1130
- const browserResult = await browserSnapshot({
1131
- ...(hint ? { title:hint } : {}),
1132
- max_nodes:clamp(args.browser_max_nodes, 500, 1, 2000),
1133
- timeout_ms:3000,
1134
- });
1135
- if (browserResult?.isError === true) throw new Error(toolResultMessage(browserResult));
1136
- browser = objectRecord(resultValue(browserResult));
1137
- if (!browser) throw new Error('browser snapshot returned an invalid result shape');
1138
- } catch (error) {
1139
- if (hint) {
1140
- try {
1141
- const fallbackResult = await browserSnapshot({
1142
- max_nodes:clamp(args.browser_max_nodes, 500, 1, 2000),
1143
- timeout_ms:3000,
1144
- });
1145
- if (fallbackResult?.isError === true) throw new Error(toolResultMessage(fallbackResult));
1146
- browser = objectRecord(resultValue(fallbackResult));
1147
- if (!browser) throw new Error('browser snapshot returned an invalid result shape');
1148
- } catch (fallbackError) {
1149
- browserError = String(fallbackError?.message || fallbackError);
1150
- }
1151
- } else {
1152
- browserError = String(error?.message || error);
1153
- }
1154
- }
1155
- }
1156
-
1157
- const ocrRequested = args.include_ocr === true
1158
- || (desktopAvailable && args.include_ocr !== false && screenshotRequested && uiNodes.length === 0 && !browser);
1159
- const shouldOcr = desktopAvailable && ocrRequested;
1160
- let ocr = null;
1161
- let ocrCaptureBounds = null;
1162
- let ocrShotResult = shotResult;
1163
- // A whole-desktop OCR pass is noisy and slow when the active application is already known.
1164
- // Keep the normal screenshot contract unchanged, but use a second tightly scoped capture for OCR.
1165
- if (shouldOcr && !hasExplicitComputerScreenshotTarget(args)) {
1166
- const bounds = activeWindowCaptureBounds(activeWindow);
1167
- if (bounds) {
1168
- try {
1169
- const targeted = await screenshotRegion({ ...bounds, padding:0 });
1170
- ocrShotResult = { status:'fulfilled', value:targeted };
1171
- ocrCaptureBounds = bounds;
1172
- } catch {
1173
- // Fall back to the already captured desktop screenshot below.
1174
- }
1175
- }
1176
- }
1177
- if (shouldOcr) {
1178
- ocr = ocrShotResult.status === 'fulfilled' && ocrShotResult.value
1179
- ? await ocrScreenshotResult(ocrShotResult.value, args)
1180
- : { available:false, reason:'screenshot capture failed before OCR could run' };
1181
- if (ocr?.available && ocrCaptureBounds) {
1182
- ocr = { ...ocr, coordinate_space:'capture_pixels', capture_bounds:ocrCaptureBounds };
1183
- }
1184
- }
1185
-
1186
- const payload = {
1187
- device: { hostname: os.hostname(), platform: process.platform, arch: process.arch },
1188
- platform: process.platform,
1189
- hostname: os.hostname(),
1190
- active_app: activeApp,
1191
- active_window: activeWindow,
1192
- windows,
1193
- displays,
1194
- cursor,
1195
- ui,
1196
- ...(browser ? { browser } : {}),
1197
- ...(ocr ? { ocr } : {}),
1198
- fallback_chain: {
1199
- accessibility: { requested:includeUi, available:includeUi && desktopAvailable && uiResult.status === 'fulfilled' && Boolean(uiResult.value), nodes:uiNodes.length },
1200
- native_browser_dom: {
1201
- requested:includeUi && args.ui_browser_dom === true,
1202
- available:Boolean(includeUi && ui?.browser_dom?.available),
1203
- provider:ui?.browser_dom?.provider || null,
1204
- },
1205
- browser_dom: { requested:browserRequested, available:Boolean(browser) },
1206
- ocr: { requested:ocrRequested, available:Boolean(ocr?.available), backend:ocr?.backend || null },
1207
- vision_screenshot: { available:shotResult.status === 'fulfilled' && Boolean(shotResult.value?.content?.some(part => part.type === 'image')) },
1208
- },
1209
- visual: {
1210
- coordinate_space:'virtual_desktop',
1211
- labeled_targets:uiNodes.filter(node => Number.isInteger(node.label)).slice(0, 500).map(node => ({
1212
- label:node.label,
1213
- id:node.id || null,
1214
- role:node.role || null,
1215
- name:String(node.name || '').slice(0, 240),
1216
- x:Number(node.x) || 0,
1217
- y:Number(node.y) || 0,
1218
- width:Number(node.width) || 0,
1219
- height:Number(node.height) || 0,
1220
- action:node.suggested_action || suggestedUiAction(node),
1221
- })),
1222
- ...(Number(args.grid_columns) > 0 || Number(args.grid_rows) > 0 ? {
1223
- reference_grid:{
1224
- columns:clamp(args.grid_columns, 0, 0, 20),
1225
- rows:clamp(args.grid_rows, 0, 0, 20),
1226
- },
1227
- } : {}),
1228
- screenshot_scope:Array.isArray(args.screenshot_region) ? 'region'
1229
- : args.screenshot_window_id ? 'window'
1230
- : (args.screenshot_monitor != null || args.screenshot_monitor_index != null) ? 'monitor'
1231
- : 'desktop',
1232
- },
1233
- clipboard: { available:clipboardOk, length:clipText.length },
1234
- errors: [
1235
- ...[
1236
- ['windows', windowsResult],
1237
- ['displays', displaysResult],
1238
- ...(includeUi ? [['ui', uiResult]] : []),
1239
- ['clipboard', clipboardResult],
1240
- ['cursor', cursorResult],
1241
- ...(captureScreenshot ? [['screenshot', shotResult]] : []),
1242
- ].map(([source, result]) => snapshotError(source, result)).filter(Boolean),
1243
- ...shapeErrors,
1244
- ...(browserError ? [{ source:'browser', message:browserError.slice(0, 2000) }] : []),
1245
- ],
1246
- };
1247
- const boundedPayload = boundedComputerSnapshot(payload);
1248
- const parts = [{ type: 'text', text: JSON.stringify(boundedPayload, null, 2) }];
1249
- if (args.include_screenshot !== false && shotResult.status === 'fulfilled' && shotResult.value?.content) {
1250
- for (const part of shotResult.value.content) if (part.type === 'image') parts.push(part);
1251
- }
1252
- const result = multi(parts);
1253
- result.structuredContent = boundedPayload;
1254
- return result;
1255
- }
1256
-
1257
- function browserComputerAction(args, action) {
1258
- const mapped = {
1259
- click:'click',
1260
- invoke:'click',
1261
- toggle:'click',
1262
- focus:'focus',
1263
- set_value:'set_value',
1264
- select:'select',
1265
- type:'type',
1266
- scroll:'scroll',
1267
- }[action];
1268
- if (!mapped) throw new Error(`Browser target does not support computer_action action: ${action}`);
1269
- const visibleText = optionalString(args.browser_text)
1270
- || optionalString(args.name)
1271
- || ((['click','invoke','toggle','focus'].includes(action) && optionalString(args.text)) ? optionalString(args.text) : null);
1272
- return browserAction({
1273
- endpoint: args.endpoint,
1274
- target_id: args.target_id,
1275
- title: args.title,
1276
- url_contains: args.url_contains,
1277
- timeout_ms: args.timeout_ms,
1278
- action: mapped,
1279
- selector: args.selector,
1280
- ...(visibleText ? { browser_text: visibleText } : {}),
1281
- ...(args.value != null ? { value: args.value } : {}),
1282
- ...(args.option != null ? { option: args.option } : {}),
1283
- ...(action === 'type' && args.text != null ? { text: args.text, clear:args.clear === true } : {}),
1284
- ...(action === 'scroll' ? {
1285
- delta_x:args.delta_x,
1286
- delta_y:args.delta_y,
1287
- delta:args.delta,
1288
- direction:args.direction,
1289
- wheel_times:args.wheel_times,
1290
- } : {}),
1291
- });
1292
- }
1293
-
1294
- export function computerActionWindowTextArgs(args = {}) {
1295
- const {
1296
- id: _id,
1297
- label: _label,
1298
- name: _name,
1299
- role: _role,
1300
- automation_id: _automationId,
1301
- automationId: _automationIdAlias,
1302
- selector: _selector,
1303
- browser_text: _browserText,
1304
- ...windowArgs
1305
- } = args;
1306
- return {
1307
- ...windowArgs,
1308
- ...(args.window_title && !args.title ? { title:args.window_title } : {}),
1309
- };
1310
- }
1311
-
1312
- export async function computerAction(args = {}) {
1313
- const legacy = ['ui','pointer','window','type','keyboard','scroll','drag','clipboard','launch_app','open_path','reveal_path','notification'];
1314
- const universal = [
1315
- 'click','double_click','right_click','move','down','up','invoke','focus','set_value','select','toggle',
1316
- 'expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection',
1317
- 'minimize','maximize','restore','move_resize','resize','close','wait','batch','multi_select','multi_edit',
1318
- ];
1319
- const action = requireEnum(args.action, 'action', [...legacy, ...universal]);
1320
- const explicitTarget = optionalString(args.target || args.target_type);
1321
- const target = explicitTarget ? requireEnum(explicitTarget, 'target', ['auto','ui','ui_element','browser','ocr','coordinates','window']) : 'auto';
1322
- const nested = { ...args, action: args.operation || args.subaction };
1323
- const hasUi = Boolean(args.label != null || args.id || args.name || args.role || args.automation_id || args.automationId);
1324
- const hasBrowser = Boolean(args.selector || args.browser_text || args.name);
1325
- const ocrNeedle = optionalString(args.ocr_text || args.name || args.browser_text);
1326
- const hasOcr = Boolean(ocrNeedle);
1327
- const hasCoordinates = Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y));
1328
- const automatic = target === 'auto';
1329
-
1330
- if (action === 'batch') {
1331
- const steps = Array.isArray(args.steps) ? args.steps : [];
1332
- if (!steps.length) throw new Error('computer_action batch requires a non-empty steps array');
1333
- if (steps.length > 50) throw new Error('computer_action batch supports at most 50 steps');
1334
- const stopOnError = args.stop_on_error !== false;
1335
- const results = [];
1336
- for (let index = 0; index < steps.length; index += 1) {
1337
- const step = steps[index];
1338
- if (!step || typeof step !== 'object' || Array.isArray(step)) throw new Error(`batch step ${index} must be an object`);
1339
- if (step.action === 'batch') throw new Error('nested computer_action batches are not allowed');
1340
- try {
1341
- const result = await computerAction(step);
1342
- results.push({ index, ok:true, action:step.action, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
1343
- } catch (error) {
1344
- const failure = { index, ok:false, action:step.action || null, error:String(error?.message || error).slice(0, 1000) };
1345
- results.push(failure);
1346
- if (stopOnError) return jsonResult({ action:'batch', ok:false, stopped_at:index, completed:index, results });
1347
- }
1348
- }
1349
- return jsonResult({ action:'batch', ok:results.every(row => row.ok), completed:results.length, results });
1350
- }
1351
-
1352
- if (action === 'multi_select') {
1353
- const targets = Array.isArray(args.targets) ? args.targets : [];
1354
- if (!targets.length) throw new Error('multi_select requires a non-empty targets array');
1355
- if (targets.length > 100) throw new Error('multi_select supports at most 100 targets');
1356
- const results = [];
1357
- for (let index = 0; index < targets.length; index += 1) {
1358
- const targetSpec = targets[index];
1359
- const operation = index === 0 && args.replace_selection !== false ? 'select' : 'add_to_selection';
1360
- const result = await uiAction({ ...targetSpec, action:operation });
1361
- results.push({ index, action:operation, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
1362
- }
1363
- return jsonResult({ action:'multi_select', count:results.length, results });
1364
- }
1365
-
1366
- if (action === 'multi_edit') {
1367
- const edits = Array.isArray(args.edits) ? args.edits : [];
1368
- if (!edits.length) throw new Error('multi_edit requires a non-empty edits array');
1369
- if (edits.length > 100) throw new Error('multi_edit supports at most 100 edits');
1370
- const results = [];
1371
- for (let index = 0; index < edits.length; index += 1) {
1372
- const edit = edits[index];
1373
- if (!edit || typeof edit !== 'object' || edit.text == null) throw new Error(`multi_edit item ${index} requires target fields plus text`);
1374
- const result = await typeText({ ...edit, method:edit.method || 'auto', clear:edit.clear !== false });
1375
- results.push({ index, result:resultValue(result) ?? result?.content?.[0]?.text ?? null });
1376
- }
1377
- return jsonResult({ action:'multi_edit', count:results.length, results });
1378
- }
1379
-
1380
- if (action === 'wait') {
1381
- const requested = args.wait_ms != null ? Number(args.wait_ms) : Number(args.seconds) * 1000;
1382
- if (!Number.isFinite(requested) || requested < 0) throw new Error('computer_action wait requires wait_ms or seconds');
1383
- const waitMs = Math.min(120_000, Math.round(requested));
1384
- await sleep(waitMs);
1385
- return jsonResult({ action:'wait', waited_ms:waitMs, capped:requested > 120_000 });
1386
- }
1387
-
1388
- if (action === 'ui') return uiAction(nested);
1389
- if (action === 'pointer') return pointer(nested);
1390
- if (action === 'window') return windowAction(nested);
1391
-
1392
- const windowTarget = () => ({
1393
- ...args,
1394
- ...(args.window_id ? { id:args.window_id } : {}),
1395
- ...(args.window_title && !args.title ? { title:args.window_title } : {}),
1396
- });
1397
- const windowTextTarget = () => computerActionWindowTextArgs(args);
1398
-
1399
- if (action === 'type') {
1400
- if (target === 'browser') return browserComputerAction(args, 'type');
1401
- if (target === 'ocr' || target === 'coordinates') {
1402
- throw new Error('computer_action type requires a native UI, browser, or window target');
1403
- }
1404
- if (target === 'window') return typeText(windowTextTarget());
1405
- if (!automatic) return typeText({ ...args, method:args.method || 'accessibility' });
1406
-
1407
- const failures = [];
1408
- if (hasUi) {
1409
- try { return await typeText({ ...args, method:'accessibility' }); }
1410
- catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
1411
- }
1412
- if (hasBrowser) {
1413
- if (await browserCapabilityAvailable(args.endpoint, 300)) {
1414
- try { return await browserComputerAction(args, 'type'); }
1415
- catch (error) { failures.push(`browser DOM: ${String(error?.message || error).slice(0, 300)}`); }
1416
- } else {
1417
- failures.push('browser DOM: no local CDP endpoint is available');
1418
- }
1419
- }
1420
- if (args.window_id || args.pid != null || args.app || args.window_title || args.title) {
1421
- try { return await typeText(windowTextTarget()); }
1422
- catch (error) { failures.push(`window: ${String(error?.message || error).slice(0, 300)}`); }
1423
- }
1424
- if (failures.length) throw new Error(`computer_action type targeting failed: ${failures.join('; ')}`);
1425
- throw new Error('computer_action type requires a native UI, browser, or window target');
1426
- }
1427
-
1428
- if (action === 'keyboard') {
1429
- if (target === 'browser' && args.key && !args.shortcut && !Array.isArray(args.keys)) {
1430
- return browserAction({ ...args, action:'press' });
1431
- }
1432
- return keyboard(args);
1433
- }
1434
-
1435
- if (action === 'scroll') {
1436
- if (target === 'browser') return browserComputerAction(args, 'scroll');
1437
- if (target === 'ocr') throw new Error('computer_action scroll does not support OCR targets; use browser/UI/window/coordinates');
1438
-
1439
- if (automatic) {
1440
- let uiError = null;
1441
- if (hasUi) {
1442
- try { return await scroll(args); }
1443
- catch (error) { uiError = error; }
1444
- }
1445
- if (hasBrowser) {
1446
- if (await browserCapabilityAvailable(args.endpoint, 300)) return browserComputerAction(args, 'scroll');
1447
- if (uiError) throw new Error(`computer_action scroll targeting failed: accessibility: ${String(uiError?.message || uiError).slice(0, 300)}; browser DOM: no local CDP endpoint is available`);
1448
- throw new Error('computer_action scroll browser target is unavailable because no local CDP endpoint is available');
1449
- }
1450
- if (args.window_id || args.pid != null || args.app || args.window_title || args.title) {
1451
- await windowAction({ ...windowTarget(), action:'focus' });
1452
- }
1453
- return scroll(args);
1454
- }
1455
-
1456
- if (target === 'window') {
1457
- await windowAction({ ...windowTarget(), action:'focus' });
1458
- return scroll(args);
1459
- }
1460
- return scroll(args);
1461
- }
1462
-
1463
- if (action === 'drag') {
1464
- if (target === 'browser' || target === 'ocr') {
1465
- throw new Error('computer_action drag currently requires native UI element ids or desktop coordinates; browser/OCR drag is not advertised');
1466
- }
1467
- if (target === 'window') await windowAction({ ...windowTarget(), action:'focus' });
1468
- return dragDrop(args);
1469
- }
1470
- if (action === 'clipboard') return clipboard(nested);
1471
- if (action === 'launch_app') return launchApp(args);
1472
- if (action === 'open_path') return openPath(args);
1473
- if (action === 'reveal_path') return revealPath(args);
1474
- if (action === 'notification') return notification(args);
1475
-
1476
- const inferredTarget = automatic
1477
- ? (hasUi ? 'ui_element' : hasBrowser ? 'browser' : args.ocr_text ? 'ocr' : hasCoordinates ? 'coordinates' : 'window')
1478
- : target;
1479
-
1480
- const tryUiThenBrowser = async uiActionName => {
1481
- let uiError = null;
1482
- if (hasUi) {
1483
- try { return await uiAction({ ...args, action:uiActionName }); }
1484
- catch (error) { uiError = error; }
1485
- }
1486
- if (hasBrowser) return browserComputerAction(args, uiActionName);
1487
- if (uiError) throw uiError;
1488
- return null;
1489
- };
1490
-
1491
- if (['invoke','set_value','select','toggle','expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection'].includes(action)) {
1492
- const browserCompatible = ['invoke','set_value','select','toggle'].includes(action);
1493
- if (inferredTarget === 'browser') {
1494
- if (!browserCompatible) throw new Error(`Browser target does not support ${action}; use browser_action or native accessibility`);
1495
- return browserComputerAction(args, action);
1496
- }
1497
- if (inferredTarget === 'ui' || inferredTarget === 'ui_element') {
1498
- if (automatic && browserCompatible) {
1499
- const result = await tryUiThenBrowser(action);
1500
- if (result) return result;
1501
- }
1502
- return uiAction({ ...args, action });
1503
- }
1504
- throw new Error(`${action} requires a UI${browserCompatible ? ' or browser' : ''} target`);
1505
- }
1506
-
1507
- if (action === 'focus') {
1508
- if (inferredTarget === 'browser') return browserComputerAction(args, 'focus');
1509
- if (inferredTarget === 'ui' || inferredTarget === 'ui_element') {
1510
- if (automatic) {
1511
- const result = await tryUiThenBrowser('focus');
1512
- if (result) return result;
1513
- }
1514
- return uiAction({ ...args, action:'focus' });
1515
- }
1516
- return windowAction({ ...windowTarget(), action:'focus' });
1517
- }
1518
-
1519
- if (['minimize','maximize','restore','move_resize','resize','close'].includes(action)) {
1520
- return windowAction({ ...args, action });
1521
- }
1522
-
1523
- if (['click','double_click','right_click','move','down','up'].includes(action)) {
1524
- if (action === 'click' && automatic) {
1525
- const failures = [];
1526
- if (hasUi) {
1527
- try { return await uiAction({ ...args, action:'click' }); }
1528
- catch (error) { failures.push(`accessibility: ${String(error?.message || error).slice(0, 300)}`); }
1529
- }
1530
- if (hasBrowser) {
1531
- if (await browserCapabilityAvailable(args.endpoint, 300)) {
1532
- try { return await browserComputerAction(args, 'click'); }
1533
- catch (error) { failures.push(`browser DOM: ${String(error?.message || error).slice(0, 300)}`); }
1534
- } else {
1535
- failures.push('browser DOM: no local CDP endpoint is available');
1536
- }
1537
- }
1538
- if (hasOcr) {
1539
- try { return await clickOcrText({ ...args, ocr_text:ocrNeedle }); }
1540
- catch (error) { failures.push(`OCR: ${String(error?.message || error).slice(0, 300)}`); }
1541
- }
1542
- if (hasCoordinates) return pointer({ ...args, action:'click' });
1543
- if (failures.length) throw new Error(`computer_action click targeting failed: ${failures.join('; ')}`);
1544
- throw new Error('computer_action click requires a UI/browser/OCR target or x/y coordinates');
1545
- }
1546
- if (inferredTarget === 'browser') {
1547
- if (action !== 'click') throw new Error(`Browser target does not support ${action}; use browser_action or coordinates`);
1548
- return browserComputerAction(args, 'click');
1549
- }
1550
- if (inferredTarget === 'ocr') {
1551
- if (action !== 'click') throw new Error(`OCR target only supports click; ${action} requires a semantic or coordinate target`);
1552
- return clickOcrText({ ...args, ocr_text:ocrNeedle });
1553
- }
1554
- if ((inferredTarget === 'ui' || inferredTarget === 'ui_element') && action === 'click') {
1555
- return uiAction({ ...args, action:'click' });
1556
- }
1557
- return pointer({ ...args, action });
1558
- }
1559
- throw new Error(`Unsupported computer_action action: ${action}`);
1560
- }
1561
-
1562
- export const desktopHandlers = {
1563
- computer_snapshot: computerSnapshot,
1564
- computer_action: computerAction,
1565
- list_windows: listWindows,
1566
- window_action: windowAction,
1567
- launch_app: launchApp,
1568
- ui_snapshot: uiSnapshot,
1569
- ui_find: uiFind,
1570
- ui_action: uiAction,
1571
- type_text: typeText,
1572
- keyboard,
1573
- pointer,
1574
- drag_drop: dragDrop,
1575
- scroll,
1576
- wait_for_ui: waitForUi,
1577
- clipboard,
1578
- display_inventory: displayInventory,
1579
- screenshot_region: screenshotRegion,
1580
- open_path: openPath,
1581
- reveal_path: revealPath,
1582
- notification,
1583
- };
1
+ // ReMCP cross-platform Desktop: stable public API, capability-specific modules.
2
+ export { boundedComputerSnapshot, listWindows, windowAction, uiSnapshot, uiAction, keyboard, pointer, clipboard, displayInventory, cursorPosition, screenshotRegion, notification, scroll } from './desktop-core.mjs';
3
+ export { uiFind, waitForUi, typeText, dragDrop, launchApp, openPath, revealPath } from './desktop-ui-actions.mjs';
4
+ export { mapOcrBoxToDesktop } from './desktop-vision.mjs';
5
+ export { computerSnapshot } from './desktop-computer-snapshot.mjs';
6
+ export { computerActionWindowTextArgs, computerAction, desktopHandlers } from './desktop-computer-actions.mjs';