@hybridlabor-api/aos 4.0.2 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/.claude/agents/database-reviewer.md +109 -0
  2. package/.claude/agents/go-build-resolver.md +112 -0
  3. package/.claude/agents/opensource-forker.md +216 -0
  4. package/.claude/agents/opensource-sanitizer.md +206 -0
  5. package/.claude/agents/security-reviewer.md +126 -0
  6. package/.claude/agents/silent-failure-hunter.md +68 -0
  7. package/CLAUDE.md +15 -0
  8. package/README.md +0 -13
  9. package/THIRD_PARTY_NOTICES.md +133 -0
  10. package/package.json +4 -2
  11. package/skills/basic/bdbmediastorm/SKILL.md +7 -1
  12. package/skills/basic/startcycle/SKILL.md +2 -0
  13. package/skills/bdbrainstorm/SKILL.md +1 -0
  14. package/skills/global_config/plan-canvas/SKILL.md +233 -0
  15. package/skills/global_config/plan-canvas/scripts/lib/loopback-guard.js +59 -0
  16. package/skills/global_config/plan-canvas/scripts/lib/plan-canvas/markdown.js +301 -0
  17. package/skills/global_config/plan-canvas/scripts/lib/plan-canvas/sdk.js +239 -0
  18. package/skills/global_config/plan-canvas/scripts/lib/plan-canvas/server.js +636 -0
  19. package/skills/global_config/plan-canvas/scripts/lib/plan-canvas/sessions.js +271 -0
  20. package/skills/global_config/plan-canvas/scripts/lib/plan-canvas/ui.js +630 -0
  21. package/skills/global_config/plan-canvas/scripts/plan-canvas.js +419 -0
  22. package/.agents/skills/firecrawl/SKILL.md +0 -149
  23. package/.agents/skills/firecrawl/rules/install.md +0 -82
  24. package/.agents/skills/firecrawl/rules/security.md +0 -26
  25. package/.agents/skills/firecrawl-agent/SKILL.md +0 -58
  26. package/.agents/skills/firecrawl-build/SKILL.md +0 -39
  27. package/.agents/skills/firecrawl-build-interact/SKILL.md +0 -68
  28. package/.agents/skills/firecrawl-build-onboarding/SKILL.md +0 -103
  29. package/.agents/skills/firecrawl-build-onboarding/references/auth-flow.md +0 -39
  30. package/.agents/skills/firecrawl-build-onboarding/references/project-setup.md +0 -20
  31. package/.agents/skills/firecrawl-build-onboarding/references/sdk-installation.md +0 -17
  32. package/.agents/skills/firecrawl-build-scrape/SKILL.md +0 -69
  33. package/.agents/skills/firecrawl-build-search/SKILL.md +0 -69
  34. package/.agents/skills/firecrawl-crawl/SKILL.md +0 -59
  35. package/.agents/skills/firecrawl-download/SKILL.md +0 -70
  36. package/.agents/skills/firecrawl-interact/SKILL.md +0 -84
  37. package/.agents/skills/firecrawl-map/SKILL.md +0 -51
  38. package/.agents/skills/firecrawl-scrape/SKILL.md +0 -69
  39. package/.agents/skills/firecrawl-search/SKILL.md +0 -60
  40. package/docs/sessions/AUDIT-HANDOVER-2026-08-28.md +0 -169
  41. package/docs/sessions/BDB_REMOTEOS_MCP_HANDOVER.md +0 -130
  42. package/docs/sessions/SESSION-HANDOVER-v3.13.md +0 -249
  43. package/mcps/RhinoMCP/cc-plugin/.claude/settings.json +0 -10
  44. package/mcps/after-effects-mcp/build/index.js +0 -840
  45. package/mcps/after-effects-mcp/build/scripts/applyEffect.jsx +0 -153
  46. package/mcps/after-effects-mcp/build/scripts/applyEffectTemplate.jsx +0 -218
  47. package/mcps/after-effects-mcp/build/scripts/createComposition.jsx +0 -71
  48. package/mcps/after-effects-mcp/build/scripts/createShapeLayer.jsx +0 -147
  49. package/mcps/after-effects-mcp/build/scripts/createSolidLayer.jsx +0 -114
  50. package/mcps/after-effects-mcp/build/scripts/createTextLayer.jsx +0 -115
  51. package/mcps/after-effects-mcp/build/scripts/getLayerInfo.jsx +0 -192
  52. package/mcps/after-effects-mcp/build/scripts/getProjectInfo.jsx +0 -90
  53. package/mcps/after-effects-mcp/build/scripts/listCompositions.jsx +0 -50
  54. package/mcps/after-effects-mcp/build/scripts/mcp-bridge-auto.jsx +0 -1773
  55. package/mcps/after-effects-mcp/build/scripts/setLayerProperties.jsx +0 -160
  56. package/mcps/bdb-remoteos-mcp/queue.db +0 -0
  57. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/__init__.cpython-312.pyc +0 -0
  58. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/incus_client.cpython-312.pyc +0 -0
  59. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/main.cpython-312.pyc +0 -0
  60. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/queue.cpython-312.pyc +0 -0
  61. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/schemas.cpython-312.pyc +0 -0
  62. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/server.cpython-312.pyc +0 -0
  63. package/mcps/bdb-remoteos-mcp/src/bdb_remoteos_mcp/__pycache__/webhook.cpython-312.pyc +0 -0
  64. package/mcps/bdb-remoteos-mcp/tests/__pycache__/__init__.cpython-312.pyc +0 -0
  65. package/mcps/bdb-remoteos-mcp/tests/__pycache__/mock_incus.cpython-312.pyc +0 -0
  66. package/mcps/bdb-remoteos-mcp/tests/__pycache__/test_mcp_server.cpython-312-pytest-9.1.1.pyc +0 -0
  67. package/mcps/bdb-remoteos-mcp/tests/__pycache__/test_security_redteam.cpython-312-pytest-9.1.1.pyc +0 -0
  68. package/mcps/bdb-remoteos-mcp/tests/__pycache__/test_webhook.cpython-312-pytest-9.1.1.pyc +0 -0
  69. package/mcps/computer-use-mcp/dist/client.d.ts +0 -150
  70. package/mcps/computer-use-mcp/dist/client.js +0 -136
  71. package/mcps/computer-use-mcp/dist/entrypoint.d.ts +0 -16
  72. package/mcps/computer-use-mcp/dist/entrypoint.js +0 -26
  73. package/mcps/computer-use-mcp/dist/native.d.ts +0 -212
  74. package/mcps/computer-use-mcp/dist/native.js +0 -50
  75. package/mcps/computer-use-mcp/dist/server.d.ts +0 -32
  76. package/mcps/computer-use-mcp/dist/server.js +0 -342
  77. package/mcps/computer-use-mcp/dist/session.d.ts +0 -101
  78. package/mcps/computer-use-mcp/dist/session.js +0 -2372
  79. package/skills/bdbsaastraining/scripts/__pycache__/build_profile.cpython-314.pyc +0 -0
@@ -1,2372 +0,0 @@
1
- /**
2
- * Session — resilient computer use session with in-process focus management.
3
- *
4
- * v4: Window-aware targeting with TargetState, focus strategies, and structured diagnostics.
5
- * Every mutating action: (1) resolve target, (2) ensure focus per strategy, (3) act, (4) update state.
6
- * Observation tools never mutate TargetState.
7
- * All runs in-process via NAPI — no child processes, no focus stealing.
8
- */
9
- import { loadNative } from './native.js';
10
- import { execFile, execFileSync } from 'child_process';
11
- const sleep = (ms) => new Promise(r => setTimeout(r, ms));
12
- /** Spawn a process with a hard timeout. Kills the child on overrun. */
13
- const defaultSpawnBounded = (cmd, args, timeoutMs) => new Promise(resolve => {
14
- const child = execFile(cmd, args, { timeout: 0, maxBuffer: 8 * 1024 * 1024 });
15
- let stdout = '';
16
- let stderr = '';
17
- child.stdout?.on('data', chunk => { stdout += chunk.toString(); });
18
- child.stderr?.on('data', chunk => { stderr += chunk.toString(); });
19
- let timedOut = false;
20
- const killer = setTimeout(() => {
21
- timedOut = true;
22
- try {
23
- child.kill('SIGKILL');
24
- }
25
- catch { /* ignore */ }
26
- }, Math.max(timeoutMs, 100));
27
- child.on('error', err => {
28
- clearTimeout(killer);
29
- resolve({ stdout, stderr: stderr || String(err), code: -1, timedOut });
30
- });
31
- child.on('close', code => {
32
- clearTimeout(killer);
33
- resolve({ stdout, stderr, code: code ?? -1, timedOut });
34
- });
35
- });
36
- /** Minimal `.sdef` parser — extracts suite/command/class names from the XML. */
37
- function parseSdef(xml, bundleId) {
38
- const suites = [];
39
- const suiteRe = /<suite\b[^>]*\bname="([^"]+)"[^>]*>([\s\S]*?)<\/suite>/g;
40
- let m;
41
- while ((m = suiteRe.exec(xml)) !== null) {
42
- const suiteName = m[1];
43
- const body = m[2];
44
- const commands = [];
45
- const cmdRe = /<command\b[^>]*\bname="([^"]+)"[^>]*(?:\/>|>([\s\S]*?)<\/command>)/g;
46
- let cm;
47
- while ((cm = cmdRe.exec(body)) !== null) {
48
- const desc = cm[0].match(/\bdescription="([^"]*)"/)?.[1];
49
- commands.push(desc ? { name: cm[1], description: desc } : { name: cm[1] });
50
- }
51
- const classes = [];
52
- const classRe = /<class\b[^>]*\bname="([^"]+)"[^>]*(?:\/>|>([\s\S]*?)<\/class>)/g;
53
- let classMatch;
54
- while ((classMatch = classRe.exec(body)) !== null) {
55
- const clsName = classMatch[1];
56
- const clsBody = classMatch[2] ?? '';
57
- const propNames = [];
58
- const propRe = /<property\b[^>]*\bname="([^"]+)"/g;
59
- let pm;
60
- while ((pm = propRe.exec(clsBody)) !== null) {
61
- propNames.push(pm[1]);
62
- }
63
- classes.push(propNames.length ? { name: clsName, properties: propNames } : { name: clsName });
64
- }
65
- suites.push({ name: suiteName, commands, classes });
66
- }
67
- return { bundleId, suites };
68
- }
69
- // ── Provider-aware screenshot defaults ────────────────────────────────────────
70
- const PROVIDER_WIDTH = {
71
- anthropic: 1024,
72
- openai: 1024,
73
- 'openai-low': 512,
74
- gemini: 768,
75
- llama: 1120,
76
- grok: 1024,
77
- mistral: 1024,
78
- qwen: 896,
79
- nova: 1024,
80
- 'deepseek-vl': 896,
81
- phi: 896,
82
- auto: 1024,
83
- };
84
- const PROVIDER_QUALITY = {
85
- anthropic: 80,
86
- openai: 80,
87
- gemini: 75,
88
- default: 80,
89
- };
90
- const TOOL_GUIDE_TABLE = [
91
- // ── Windows-specific entries (checked first on Windows) ─────────────────
92
- ...(process.platform === 'win32' ? [
93
- {
94
- pattern: /\b(file|folder|directory|rename|move|copy)\b.*\b(file|folder|directory|desktop)\b|\b(desktop)\b.*\b(file|folder|save|copy)\b/i,
95
- approach: 'scripting',
96
- toolSequence: ['filesystem', 'run_script'],
97
- explanation: 'Use the filesystem tool for file operations, or PowerShell via run_script for complex tasks. Faster than GUI clicks.',
98
- },
99
- {
100
- pattern: /\b(registry|regedit|hkey|hkcu|hklm)\b/i,
101
- approach: 'scripting',
102
- toolSequence: ['registry'],
103
- explanation: 'Use the registry tool for Windows Registry operations. Accepts PowerShell-format paths.',
104
- },
105
- {
106
- pattern: /\b(send|compose|reply|new|write).*(email|mail|message)\b/i,
107
- approach: 'scripting',
108
- toolSequence: ['run_script'],
109
- explanation: 'Use PowerShell via run_script to automate email. For Outlook: `$ol = New-Object -ComObject Outlook.Application; $mail = $ol.CreateItem(0)`.',
110
- },
111
- {
112
- pattern: /\b(open|visit|navigate).*(url|website|https?:|web\s*page|tab)\b/i,
113
- approach: 'scripting',
114
- toolSequence: ['run_script'],
115
- explanation: 'Use PowerShell: `Start-Process "https://example.com"` to open URLs in the default browser.',
116
- },
117
- {
118
- pattern: /\b(powershell|cmd|terminal|command|shell|script)\b/i,
119
- approach: 'scripting',
120
- toolSequence: ['run_script'],
121
- explanation: 'Use run_script with language "powershell" for system automation, CLI tools, and scripting.',
122
- },
123
- {
124
- pattern: /\b(process|task|kill|terminate|stop)\b/i,
125
- approach: 'scripting',
126
- toolSequence: ['process_kill'],
127
- explanation: 'Use process_kill to list or terminate processes by name or PID.',
128
- },
129
- {
130
- pattern: /\b(notify|notification|alert|toast)\b/i,
131
- approach: 'scripting',
132
- toolSequence: ['notification'],
133
- explanation: 'Use the notification tool to send Windows toast notifications.',
134
- },
135
- ] : []),
136
- // ── macOS-specific entries ──────────────────────────────────────────────
137
- ...(process.platform === 'darwin' ? [
138
- {
139
- pattern: /\b(send|compose|reply|new|write).*(email|mail|message)\b/i,
140
- bundleIdHints: ['com.apple.mail'],
141
- approach: 'scripting',
142
- toolSequence: ['get_app_capabilities', 'run_script'],
143
- explanation: 'Mail is scriptable. Use AppleScript `make new outgoing message` or `send` — one call replaces the whole compose flow.',
144
- },
145
- {
146
- pattern: /\b(open|visit|navigate).*(url|website|https?:|web\s*page|tab)\b/i,
147
- bundleIdHints: ['com.apple.Safari', 'com.google.Chrome'],
148
- approach: 'scripting',
149
- toolSequence: ['run_script'],
150
- explanation: 'Safari and Chrome are scriptable. `tell application "Safari" to open location "<url>"` beats screenshot-and-click.',
151
- },
152
- {
153
- pattern: /\b(spreadsheet|cell|row|column|numbers|sheet)\b/i,
154
- bundleIdHints: ['com.apple.iWork.Numbers', 'com.microsoft.Excel'],
155
- approach: 'scripting',
156
- toolSequence: ['get_app_dictionary', 'run_script'],
157
- explanation: 'Numbers is deeply scriptable — read/write cells via AppleScript. Fall back to fill_form if scripting is unavailable.',
158
- },
159
- {
160
- pattern: /\b(file|folder|directory|finder|rename|move|copy|desktop)\b/i,
161
- bundleIdHints: ['com.apple.finder'],
162
- approach: 'scripting',
163
- toolSequence: ['run_script'],
164
- explanation: 'Finder is scriptable (and shell is often even better). Prefer `osascript` or direct filesystem calls over GUI clicks.',
165
- },
166
- {
167
- pattern: /\b(calendar|event|reminder|note|todo|task)\b/i,
168
- bundleIdHints: ['com.apple.iCal', 'com.apple.reminders', 'com.apple.Notes'],
169
- approach: 'scripting',
170
- toolSequence: ['get_app_capabilities', 'run_script'],
171
- explanation: 'Calendar, Reminders, and Notes are all scriptable. One `make new <event/reminder/note>` call does the work.',
172
- },
173
- {
174
- pattern: /\b(play|pause|track|playlist|song|music)\b/i,
175
- bundleIdHints: ['com.apple.Music'],
176
- approach: 'scripting',
177
- toolSequence: ['run_script'],
178
- explanation: 'Music.app is scriptable: `tell application "Music" to play` / `pause` / `next track`.',
179
- },
180
- {
181
- pattern: /\b(imessage|send\s+message|chat|sms)\b/i,
182
- bundleIdHints: ['com.apple.iChat'],
183
- approach: 'scripting',
184
- toolSequence: ['run_script'],
185
- explanation: 'Messages is scriptable. Use AppleScript to send messages to a buddy without UI.',
186
- },
187
- ] : []),
188
- // ── Cross-platform entries ─────────────────────────────────────────────
189
- {
190
- pattern: /\b(fill|enter|type)\b.*\b(form|field|input)\b/i,
191
- approach: 'accessibility',
192
- toolSequence: ['get_ui_tree', 'fill_form'],
193
- explanation: 'Batch-set form fields by accessibility label — one fill_form call instead of click+type per field.',
194
- },
195
- {
196
- pattern: /\b(menu|menubar|file\s*(menu)?|edit\s*menu)\b/i,
197
- approach: 'accessibility',
198
- toolSequence: ['select_menu_item'],
199
- explanation: 'Use select_menu_item — walks the menu bar programmatically, faster and more reliable than visual navigation.',
200
- },
201
- {
202
- pattern: /\b(click|press|tap)\b.*\b(button|link)\b/i,
203
- approach: 'accessibility',
204
- toolSequence: ['find_element', 'press_button'],
205
- explanation: 'press_button finds buttons by label — avoids pixel-coordinate drift across window moves and resolution changes.',
206
- },
207
- {
208
- pattern: /\b(read|inspect|verify|check|examine|zoom|detail|small\s*text|tiny|pixel|magnif|enlarge|close.?up)\b/i,
209
- approach: 'coordinate',
210
- toolSequence: ['zoom'],
211
- explanation: 'Use the zoom tool to inspect a specific screen region at full native resolution. Pass region: [x1, y1, x2, y2] to crop without downscaling. Best for reading small text, verifying values, or checking pixel-level details. Default output is lossless PNG. Tip: take a screenshot first to identify the region coordinates, then zoom into the area of interest.',
212
- },
213
- {
214
- pattern: /\b(text|value|number|label|title|heading|content|status|what\s*does\s*it\s*say|read\s*the|what\s*is\s*written|what\s*does.*say)\b/i,
215
- approach: 'accessibility',
216
- toolSequence: ['get_ui_tree', 'zoom'],
217
- explanation: 'To read text on screen: first try get_ui_tree which returns element labels and values as structured data (fastest, no image needed). If the text is in an image or non-accessible element, use zoom with a tight region around the text for a full-resolution lossless PNG crop.',
218
- },
219
- {
220
- pattern: /\b(screenshot|see|show|look|observe|capture|screen)\b/i,
221
- approach: 'accessibility',
222
- toolSequence: ['screenshot', 'zoom'],
223
- explanation: 'Use screenshot for a full-screen overview (resized for efficiency). If you need to read specific text or inspect details, follow up with zoom on the region of interest — it returns full native resolution without downscaling. For structured UI data without an image, use get_ui_tree instead.',
224
- },
225
- {
226
- pattern: /.*/,
227
- approach: 'accessibility',
228
- toolSequence: ['get_app_capabilities', 'get_ui_tree', 'find_element', 'click_element'],
229
- explanation: 'No specific pattern matched. Probe capabilities, then prefer accessibility — fall back to coordinate input only as a last resort.',
230
- },
231
- ];
232
- function lookupToolGuide(taskDescription) {
233
- for (const entry of TOOL_GUIDE_TABLE) {
234
- if (entry.pattern.test(taskDescription)) {
235
- return {
236
- approach: entry.approach,
237
- toolSequence: entry.toolSequence,
238
- explanation: entry.explanation,
239
- ...(entry.bundleIdHints ? { bundleIdHints: entry.bundleIdHints } : {}),
240
- };
241
- }
242
- }
243
- // Unreachable — the final `.*` entry matches everything.
244
- return {
245
- approach: 'accessibility',
246
- toolSequence: ['get_ui_tree'],
247
- explanation: 'Default fallback.',
248
- };
249
- }
250
- // ── Errors ────────────────────────────────────────────────────────────────────
251
- class FocusError extends Error {
252
- details;
253
- constructor(details) {
254
- super(`Failed to focus ${details.requestedBundleId}`);
255
- this.details = details;
256
- this.name = 'FocusError';
257
- }
258
- }
259
- class WindowNotFoundError extends Error {
260
- windowId;
261
- constructor(windowId) {
262
- super(`Window not found: ${windowId}`);
263
- this.windowId = windowId;
264
- this.name = 'WindowNotFoundError';
265
- }
266
- }
267
- // ── v5.2: Session lock + runloop pump ─────────────────────────────────────────
268
- //
269
- // One computer-use session per Mac. Prevents two MCP processes from fighting
270
- // over the cursor. While the lock is held, we run a 1 ms CFRunLoop pump so
271
- // NSWorkspace KVO updates and @MainActor continuations make progress under
272
- // libuv — matches Claude Code's `drainRunLoop.ts` pattern.
273
- //
274
- // Both lock and pump are refcounted within a single process so nested
275
- // mutating calls (e.g. fill_form → set_value) don't tear down and rebuild.
276
- import * as fs from 'fs';
277
- import * as os from 'os';
278
- import * as path from 'path';
279
- const IS_WINDOWS = process.platform === 'win32';
280
- const IS_MACOS = process.platform === 'darwin';
281
- const IS_LINUX = process.platform === 'linux';
282
- const DEFAULT_LOCK_PATH = IS_WINDOWS
283
- ? path.join(os.tmpdir(), '.computer-use-mcp.lock')
284
- : '/tmp/.computer-use-mcp.lock';
285
- export class LockError extends Error {
286
- lockingPid;
287
- constructor(lockingPid) {
288
- super(lockingPid != null
289
- ? `computer-use session locked by PID ${lockingPid}`
290
- : 'computer-use session locked');
291
- this.name = 'LockError';
292
- this.lockingPid = lockingPid;
293
- }
294
- }
295
- function pidIsAlive(pid) {
296
- try {
297
- // Signal 0 is the probe signal — succeeds iff the process exists and we
298
- // have permission to signal it. Throws ESRCH when the process is gone.
299
- process.kill(pid, 0);
300
- return true;
301
- }
302
- catch (err) {
303
- // EPERM means the process exists but we can't signal — treat as alive.
304
- return err?.code === 'EPERM';
305
- }
306
- }
307
- /**
308
- * Tries to acquire the cross-process session lock.
309
- *
310
- * Returns a handle on success. Throws `LockError` when the lock is held by
311
- * another live process. Reclaims the lock when the holding PID is dead.
312
- */
313
- function acquireLockOnce(lockPath) {
314
- try {
315
- const fd = fs.openSync(lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_RDWR, 0o600);
316
- fs.writeSync(fd, String(process.pid));
317
- return {
318
- release: () => {
319
- try {
320
- fs.closeSync(fd);
321
- }
322
- catch { /* already closed */ }
323
- try {
324
- fs.unlinkSync(lockPath);
325
- }
326
- catch { /* already removed */ }
327
- },
328
- };
329
- }
330
- catch (err) {
331
- if (err?.code !== 'EEXIST')
332
- throw err;
333
- // Lock exists — inspect the holder.
334
- let holder = null;
335
- try {
336
- const raw = fs.readFileSync(lockPath, 'utf8').trim();
337
- const parsed = parseInt(raw, 10);
338
- if (Number.isFinite(parsed) && parsed > 0)
339
- holder = parsed;
340
- }
341
- catch { /* file vanished between openSync and readFileSync — retry */ }
342
- if (holder == null) {
343
- // Unreadable / empty lock: treat as dead.
344
- try {
345
- fs.unlinkSync(lockPath);
346
- }
347
- catch { /* race — someone else cleaned up */ }
348
- return acquireLockOnce(lockPath);
349
- }
350
- if (holder === process.pid) {
351
- // Our own process holds the lock but we dropped the FD ref —
352
- // reclaim by unlinking + re-creating.
353
- try {
354
- fs.unlinkSync(lockPath);
355
- }
356
- catch { /* ok */ }
357
- return acquireLockOnce(lockPath);
358
- }
359
- if (!pidIsAlive(holder)) {
360
- try {
361
- fs.unlinkSync(lockPath);
362
- }
363
- catch { /* race — ok */ }
364
- return acquireLockOnce(lockPath);
365
- }
366
- throw new LockError(holder);
367
- }
368
- }
369
- function makeLockPumpController(n, lockPath, disableLock) {
370
- let refcount = 0;
371
- let handle = null;
372
- let pumpInterval = null;
373
- function startPump() {
374
- if (IS_WINDOWS || IS_LINUX)
375
- return; // No CFRunLoop on Windows/Linux
376
- if (pumpInterval != null)
377
- return;
378
- pumpInterval = setInterval(() => {
379
- try {
380
- n.drainRunloop();
381
- }
382
- catch { /* never let the pump throw */ }
383
- }, 1);
384
- // Don't keep Node alive just for the pump.
385
- pumpInterval.unref?.();
386
- }
387
- function stopPump() {
388
- if (pumpInterval == null)
389
- return;
390
- clearInterval(pumpInterval);
391
- pumpInterval = null;
392
- }
393
- return {
394
- acquire() {
395
- if (refcount === 0 && !disableLock) {
396
- handle = acquireLockOnce(lockPath);
397
- }
398
- refcount++;
399
- if (refcount === 1)
400
- startPump();
401
- },
402
- release() {
403
- if (refcount === 0)
404
- return;
405
- refcount--;
406
- if (refcount === 0) {
407
- stopPump();
408
- if (handle) {
409
- const h = handle;
410
- handle = null;
411
- h.release();
412
- }
413
- }
414
- },
415
- get _refcount() { return refcount; },
416
- };
417
- }
418
- // ── Tools that mutate system state (take the session lock + pump) ─────────────
419
- //
420
- // Observation tools (screenshot, list_*, get_*) do not take the lock — they
421
- // must be callable concurrently (e.g. screenshots during a session held by
422
- // another process for diagnostics).
423
- const MUTATING_TOOLS = new Set([
424
- // Pointer + keyboard CGEvent
425
- 'left_click', 'right_click', 'middle_click', 'double_click', 'triple_click',
426
- 'mouse_move', 'left_click_drag', 'left_mouse_down', 'left_mouse_up', 'scroll',
427
- 'type', 'key', 'hold_key', 'write_clipboard', 'multi_select', 'multi_edit',
428
- // App / window activation
429
- 'activate_app', 'activate_window', 'open_application', 'hide_app', 'unhide_app',
430
- // Semantic AX mutations
431
- 'click_element', 'set_value', 'press_button', 'select_menu_item', 'fill_form',
432
- // Space/desktop mutation
433
- 'create_agent_space', 'move_window_to_space', 'remove_window_from_space', 'destroy_space',
434
- // Files, processes, registry, and notifications can all mutate host state
435
- 'filesystem', 'process_kill', 'registry', 'notification',
436
- // Scripting bridge (can mutate via AppleScript — treat conservatively)
437
- 'run_script',
438
- ]);
439
- // ── Session factory ───────────────────────────────────────────────────────────
440
- export function createSession(opts = {}) {
441
- const n = opts.native ?? loadNative();
442
- const spawnBounded = opts.spawnBounded ?? defaultSpawnBounded;
443
- let targetState;
444
- const visionEnabled = opts.vision !== false;
445
- // v5.2: cross-process lock + main-runloop pump. Mutating tool dispatch
446
- // acquires before running and releases in `finally`; observation tools
447
- // skip both to stay cheap and concurrent.
448
- //
449
- // Default: when a caller injects a mock native (opts.native set), we infer
450
- // this is a test harness and disable the cross-process lock by default.
451
- // Tests that *want* to exercise the lock can pass `disableSessionLock: false`
452
- // explicitly. Production (no opts.native → real NAPI module) defaults to
453
- // enabled. Callers can always override explicitly.
454
- const lockDisabledByDefault = opts.native != null;
455
- const lockPump = makeLockPumpController(n, opts.lockPath ?? DEFAULT_LOCK_PATH, opts.disableSessionLock ?? lockDisabledByDefault);
456
- // Best-effort cleanup if Node exits while a lock is held (SIGINT, SIGTERM,
457
- // uncaughtException). We force-release to drop the lockfile; a new session
458
- // started afterwards will see EEXIST and fall into the stale-PID recovery
459
- // path anyway, so this is belt + suspenders.
460
- //
461
- // Skipped when the cross-process lock is disabled (tests, in-process
462
- // multiple-sessions). Property-based tests create many sessions, which
463
- // would otherwise blow past the default MaxListeners on `process`.
464
- if (!(opts.disableSessionLock ?? lockDisabledByDefault)) {
465
- const forceRelease = () => {
466
- try {
467
- while (lockPump._refcount > 0)
468
- lockPump.release();
469
- }
470
- catch { /* ignore */ }
471
- };
472
- process.once('exit', forceRelease);
473
- process.once('SIGINT', () => { forceRelease(); process.exit(130); });
474
- process.once('SIGTERM', () => { forceRelease(); process.exit(143); });
475
- }
476
- const defaultProvider = opts.provider ?? process.env.COMPUTER_USE_PROVIDER ?? 'auto';
477
- // Screenshot dedup cache — keyed on the last encoded image hash.
478
- let _lastHash;
479
- let _lastResult;
480
- // Scripting dictionary cache — keyed by bundleId, invalidated on PID change.
481
- const dictionaryCache = new Map();
482
- // Cached space_id from the most recent successful create_agent_space.
483
- let cachedAgentSpaceId;
484
- // ── Target resolution ───────────────────────────────────────────────────
485
- function resolveTarget(args) {
486
- // 1. Explicit window_id (v5 semantic tools) or target_window_id (v4 input tools).
487
- const wid = typeof args.window_id === 'number' ? args.window_id
488
- : typeof args.target_window_id === 'number' ? args.target_window_id
489
- : undefined;
490
- if (wid !== undefined) {
491
- const win = n.getWindow(wid);
492
- if (!win)
493
- throw new WindowNotFoundError(wid);
494
- return { bundleId: win.bundleId ?? undefined, windowId: win.windowId };
495
- }
496
- // 2. Explicit target_app
497
- if (typeof args.target_app === 'string' && args.target_app.length > 0) {
498
- return { bundleId: args.target_app };
499
- }
500
- // 3. Current TargetState
501
- return { bundleId: targetState?.bundleId, windowId: targetState?.windowId };
502
- }
503
- // ── Focus strategy ──────────────────────────────────────────────────────
504
- function defaultStrategy(tool) {
505
- // Writes default to strict (corrupt-text risk); clicks default to best_effort.
506
- const strictTools = ['type', 'key', 'hold_key', 'set_value', 'fill_form'];
507
- if (strictTools.includes(tool))
508
- return 'strict';
509
- return 'best_effort';
510
- }
511
- function getStrategy(tool, args) {
512
- if (typeof args.focus_strategy === 'string') {
513
- const s = args.focus_strategy;
514
- if (s === 'strict' || s === 'best_effort' || s === 'none' || s === 'prepare_display')
515
- return s;
516
- }
517
- return defaultStrategy(tool);
518
- }
519
- function buildFocusFailure(requestedBundleId, requestedWindowId, frontmostBefore, frontmostAfter, activationAttempted) {
520
- const runningApp = n.listRunningApps().find(entry => entry.bundleId === requestedBundleId);
521
- let targetWindowVisible = null;
522
- if (requestedWindowId != null) {
523
- const win = n.getWindow(requestedWindowId);
524
- targetWindowVisible = win?.isOnScreen ?? false;
525
- }
526
- let suggestedRecovery;
527
- if (requestedWindowId != null && targetWindowVisible) {
528
- suggestedRecovery = 'activate_window';
529
- }
530
- else if (runningApp?.isHidden) {
531
- suggestedRecovery = 'unhide_app';
532
- }
533
- else {
534
- suggestedRecovery = 'open_application';
535
- }
536
- return {
537
- error: 'focus_failed',
538
- requestedBundleId,
539
- requestedWindowId,
540
- frontmostBefore,
541
- frontmostAfter,
542
- targetRunning: Boolean(runningApp),
543
- targetHidden: runningApp?.isHidden ?? false,
544
- targetWindowVisible,
545
- activationAttempted,
546
- suggestedRecovery,
547
- };
548
- }
549
- /**
550
- * Resolve the terminal host's bundle ID so prepare_display doesn't hide it.
551
- *
552
- * Order: `COMPUTER_USE_PREPARE_KEEP_VISIBLE` env (comma-separated, user
553
- * override) → `__CFBundleIdentifier` (set by the OS when launching a macOS
554
- * app) → `TERM_PROGRAM_BUNDLE_ID` → fallback `com.apple.Terminal`.
555
- */
556
- function resolveKeepVisibleBundles() {
557
- const envList = process.env.COMPUTER_USE_PREPARE_KEEP_VISIBLE;
558
- if (envList) {
559
- return envList.split(',').map(s => s.trim()).filter(Boolean);
560
- }
561
- if (IS_WINDOWS) {
562
- // On Windows, keep the terminal/IDE process visible
563
- return ['explorer.exe'];
564
- }
565
- if (IS_LINUX) {
566
- return ['gnome-shell', 'gnome-terminal', 'xterm'];
567
- }
568
- const terminal = process.env.__CFBundleIdentifier
569
- || process.env.TERM_PROGRAM_BUNDLE_ID
570
- || 'com.apple.Terminal';
571
- return [terminal];
572
- }
573
- // v5.2: per-dispatch slot capturing the list of bundles `prepare_display`
574
- // hid. Set inside `ensureFocusV4`, consumed by the outer dispatch wrapper
575
- // to decorate the response JSON. Reset to undefined at dispatch entry.
576
- let pendingHiddenBundleIds;
577
- async function ensureFocusV4(target, strategy) {
578
- if (strategy === 'none')
579
- return {};
580
- if (!target.bundleId)
581
- return {};
582
- // v5.2: prepare_display hides everything else before we even try to
583
- // activate. This is the hammer for focus-stealing apps (screenshot
584
- // watchers, NC banners). After this, activation is basically
585
- // deterministic because nothing else visible can steal focus.
586
- let hiddenBundleIds;
587
- if (strategy === 'prepare_display') {
588
- const keep = resolveKeepVisibleBundles();
589
- const result = n.prepareDisplay(target.bundleId, keep);
590
- hiddenBundleIds = result.hiddenBundleIds;
591
- pendingHiddenBundleIds = hiddenBundleIds;
592
- await sleep(50); // let the OS process the batched hide calls
593
- // Fall through to the normal activate+poll loop — after prepareDisplay,
594
- // the target should activate essentially instantly.
595
- }
596
- const front = n.getFrontmostApp();
597
- const frontBundleId = front?.bundleId ?? null;
598
- if (front?.bundleId === target.bundleId) {
599
- // App is frontmost — check window if strict + windowId
600
- if (strategy === 'strict' && target.windowId != null) {
601
- const win = n.getWindow(target.windowId);
602
- if (!win?.isOnScreen) {
603
- throw new FocusError(buildFocusFailure(target.bundleId, target.windowId, frontBundleId, frontBundleId, false));
604
- }
605
- }
606
- return { hiddenBundleIds };
607
- }
608
- // App not frontmost — attempt recovery
609
- const runningApp = n.listRunningApps().find(a => a.bundleId === target.bundleId);
610
- if (runningApp?.isHidden) {
611
- n.unhideApp(target.bundleId);
612
- await sleep(100);
613
- }
614
- n.activateApp(target.bundleId, 2000);
615
- await sleep(80);
616
- // Raise specific window if requested
617
- if (target.windowId != null) {
618
- try {
619
- n.activateWindow(target.windowId);
620
- }
621
- catch { /* best effort */ }
622
- await sleep(80);
623
- }
624
- const after = n.getFrontmostApp();
625
- if (strategy === 'strict' && after?.bundleId !== target.bundleId) {
626
- throw new FocusError(buildFocusFailure(target.bundleId, target.windowId ?? null, frontBundleId, after?.bundleId ?? null, true));
627
- }
628
- return { hiddenBundleIds };
629
- }
630
- function focusFailureText(details) {
631
- return JSON.stringify(details);
632
- }
633
- // ── State update helpers ────────────────────────────────────────────────
634
- function updateTargetState(target, establishedBy) {
635
- targetState = {
636
- bundleId: target.bundleId,
637
- windowId: target.windowId,
638
- establishedBy,
639
- establishedAt: Date.now(),
640
- };
641
- }
642
- function trackClickTarget(target) {
643
- if (target.bundleId) {
644
- updateTargetState(target, 'pointer');
645
- }
646
- else {
647
- const front = n.getFrontmostApp();
648
- if (front?.bundleId) {
649
- updateTargetState({ bundleId: front.bundleId }, 'pointer');
650
- }
651
- }
652
- }
653
- // ── Tool category helpers ───────────────────────────────────────────────
654
- function establishedByForTool(tool) {
655
- const keyboardTools = ['type', 'key', 'hold_key'];
656
- const activationTools = ['activate_app', 'activate_window', 'open_application'];
657
- if (keyboardTools.includes(tool))
658
- return 'keyboard';
659
- if (activationTools.includes(tool))
660
- return 'activation';
661
- return 'pointer';
662
- }
663
- // ── v5: Levenshtein distance + similar-labels helper ────────────────────
664
- function levenshtein(a, b) {
665
- if (a === b)
666
- return 0;
667
- if (!a.length)
668
- return b.length;
669
- if (!b.length)
670
- return a.length;
671
- const an = a.length;
672
- const bn = b.length;
673
- const prev = new Array(bn + 1);
674
- const curr = new Array(bn + 1);
675
- for (let j = 0; j <= bn; j++)
676
- prev[j] = j;
677
- for (let i = 1; i <= an; i++) {
678
- curr[0] = i;
679
- for (let j = 1; j <= bn; j++) {
680
- const cost = a[i - 1] === b[j - 1] ? 0 : 1;
681
- curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
682
- }
683
- for (let j = 0; j <= bn; j++)
684
- prev[j] = curr[j];
685
- }
686
- return prev[bn];
687
- }
688
- function similarLabelsError(windowId, label, message, roleFilter) {
689
- const candidates = n.findElement(windowId, roleFilter, undefined, undefined, 200);
690
- const target = (label ?? '').toLowerCase();
691
- const ranked = candidates
692
- .map(e => ({ e, d: levenshtein((e.label ?? '').toLowerCase(), target) }))
693
- .sort((a, b) => a.d - b.d)
694
- .slice(0, 5)
695
- .map(x => ({ role: x.e.role, label: x.e.label, value: x.e.value }));
696
- return {
697
- content: [{
698
- type: 'text',
699
- text: JSON.stringify({ error: message, similar: ranked }),
700
- }],
701
- isError: true,
702
- };
703
- }
704
- // ── v5: Scripting dictionary lookup + cache ─────────────────────────────
705
- async function findAppPath(bundleId) {
706
- if (IS_WINDOWS || IS_LINUX)
707
- return undefined; // mdfind/sdef are macOS-only
708
- // Try mdfind first (fast, indexed). Fall back to NSWorkspace is unnecessary
709
- // — we can simply let `sdef` fail if the app is missing.
710
- const r = await spawnBounded('mdfind', [`kMDItemCFBundleIdentifier == '${bundleId}'`], 5_000);
711
- if (r.code !== 0)
712
- return undefined;
713
- const first = r.stdout.split('\n').map(s => s.trim()).find(s => s.length > 0);
714
- return first;
715
- }
716
- async function getAppDictionary(bundleId, suite) {
717
- const running = n.listRunningApps().find(a => a.bundleId === bundleId);
718
- const pid = running?.pid;
719
- const cached = dictionaryCache.get(bundleId);
720
- let dict;
721
- if (cached && cached.pid === pid) {
722
- dict = cached.dict;
723
- }
724
- else {
725
- const path = await findAppPath(bundleId);
726
- if (!path)
727
- return { error: 'app_not_found' };
728
- const r = await spawnBounded('sdef', [path], 10_000);
729
- if (r.timedOut)
730
- return { error: 'sdef_timeout' };
731
- if (r.code !== 0)
732
- return { error: 'not_scriptable' };
733
- dict = parseSdef(r.stdout, bundleId);
734
- if (pid !== undefined)
735
- dictionaryCache.set(bundleId, { pid, dict });
736
- }
737
- if (suite) {
738
- const filtered = dict.suites.filter(s => s.name === suite);
739
- return { dict: { bundleId, suites: filtered } };
740
- }
741
- // Summarized mode: names only, no properties/descriptions.
742
- const summary = {
743
- bundleId,
744
- suites: dict.suites.map(s => ({
745
- name: s.name,
746
- commands: s.commands.map(c => ({ name: c.name })),
747
- classes: s.classes.map(c => ({ name: c.name })),
748
- })),
749
- };
750
- return { dict: summary };
751
- }
752
- // Cache which PowerShell executable is available
753
- let _psExe;
754
- function getPowerShellExe() {
755
- if (_psExe)
756
- return _psExe;
757
- try {
758
- execFileSync('pwsh', ['-NoProfile', '-Command', 'exit 0'], { timeout: 3000 });
759
- _psExe = 'pwsh';
760
- }
761
- catch {
762
- _psExe = 'powershell';
763
- }
764
- return _psExe;
765
- }
766
- function getMacSpacesBackend() {
767
- const raw = (process.env.COMPUTER_USE_SPACES_BACKEND ?? 'auto').toLowerCase();
768
- if (raw === 'yabai' || raw === 'mission_control' || raw === 'cgs')
769
- return raw;
770
- return 'auto';
771
- }
772
- async function yabaiAvailable() {
773
- if (IS_WINDOWS || IS_LINUX)
774
- return false;
775
- const r = await spawnBounded('yabai', ['--version'], 3_000);
776
- return r.code === 0;
777
- }
778
- async function yabaiJson(args, timeoutMs = 5_000) {
779
- const r = await spawnBounded('yabai', args, timeoutMs);
780
- if (r.code !== 0)
781
- return undefined;
782
- try {
783
- return JSON.parse(r.stdout);
784
- }
785
- catch {
786
- return undefined;
787
- }
788
- }
789
- async function yabaiSpaces() {
790
- const raw = await yabaiJson(['-m', 'query', '--spaces']);
791
- return Array.isArray(raw) ? raw : [];
792
- }
793
- function yabaiFailure(error, reason, extra = {}) {
794
- const normalizedReason = reason.trim() || error;
795
- const requiresScriptingAddition = /scripting[- ]addition/i.test(normalizedReason);
796
- return {
797
- error,
798
- reason: normalizedReason,
799
- backend: 'yabai',
800
- ...(requiresScriptingAddition ? {
801
- requires_scripting_addition: true,
802
- setup_commands: ['sudo yabai --load-sa'],
803
- sip_note: 'On SIP-enabled Macs, yabai scripting-addition setup can require Recovery-mode SIP configuration before load succeeds. Run the command with --load-sa; running yabai as root without that flag is intentionally rejected.',
804
- } : {}),
805
- ...extra,
806
- };
807
- }
808
- async function tryCreateSpaceWithYabai() {
809
- if (!(await yabaiAvailable()))
810
- return undefined;
811
- const before = await yabaiSpaces();
812
- const beforeIds = new Set(before.map(s => s.id).filter(id => typeof id === 'number'));
813
- const r = await spawnBounded('yabai', ['-m', 'space', '--create'], 10_000);
814
- await sleep(700);
815
- const after = await yabaiSpaces();
816
- const created = after.find(s => typeof s.id === 'number' && !beforeIds.has(s.id));
817
- if (r.code !== 0 || !created) {
818
- return yabaiFailure('yabai_create_failed', r.stderr || r.stdout || 'no_new_space_detected', {
819
- before_count: before.length,
820
- after_count: after.length,
821
- });
822
- }
823
- return {
824
- supported: true,
825
- backend: 'yabai',
826
- spaceId: created?.id,
827
- index: created?.index,
828
- uuid: created?.uuid,
829
- created: Boolean(created),
830
- attached: true,
831
- note: 'Created visible Space via yabai.',
832
- };
833
- }
834
- async function tryCreateSpaceWithMissionControl() {
835
- if (IS_WINDOWS || IS_LINUX || typeof n.listSpaces !== 'function')
836
- return undefined;
837
- const before = n.listSpaces();
838
- const beforeList = Array.isArray(before.displays)
839
- ? before.displays.flatMap((d) => Array.isArray(d.spaces) ? d.spaces : [])
840
- : [];
841
- const beforeIds = new Set(beforeList.map((s) => s.id));
842
- const display = n.getDisplaySize();
843
- if (!display.width || !display.height)
844
- return undefined;
845
- const candidates = [
846
- [Math.max(0, display.width - 33), 72],
847
- [Math.max(0, display.width - 60), 85],
848
- [Math.max(0, display.width - 80), 85],
849
- [Math.max(0, display.width - 120), 72],
850
- ];
851
- let afterList = beforeList;
852
- let created;
853
- for (const [clickX, clickY] of candidates) {
854
- await spawnBounded('open', ['-a', 'Mission Control'], 5_000);
855
- await sleep(1_500);
856
- n.mouseMove(Math.round(display.width / 2), 200);
857
- await sleep(700);
858
- n.mouseMove(clickX, 200);
859
- await sleep(300);
860
- n.mouseMove(clickX, clickY);
861
- await sleep(900);
862
- n.mouseClick(clickX, clickY, 'left', 1);
863
- await sleep(1_400);
864
- n.keyPress('control+up');
865
- await sleep(1_200);
866
- const after = n.listSpaces();
867
- afterList = Array.isArray(after.displays)
868
- ? after.displays.flatMap((d) => Array.isArray(d.spaces) ? d.spaces : [])
869
- : [];
870
- created = afterList.find((s) => !beforeIds.has(s.id));
871
- if (created)
872
- break;
873
- }
874
- if (!created)
875
- return {
876
- error: 'mission_control_create_failed',
877
- backend: 'mission_control',
878
- beforeCount: beforeList.length,
879
- afterCount: afterList.length,
880
- reason: 'space_count_unchanged',
881
- };
882
- return {
883
- supported: true,
884
- backend: 'mission_control',
885
- spaceId: created.id,
886
- uuid: created.uuid,
887
- created: true,
888
- attached: true,
889
- note: 'Created visible Space via Mission Control gesture.',
890
- };
891
- }
892
- async function createMacSpace() {
893
- const backend = getMacSpacesBackend();
894
- if (backend === 'yabai' || backend === 'auto') {
895
- const y = await tryCreateSpaceWithYabai();
896
- if (y && !('error' in y))
897
- return y;
898
- if (backend === 'yabai')
899
- return y ?? { error: 'yabai_unavailable', backend: 'yabai' };
900
- }
901
- if (backend === 'mission_control' || backend === 'auto') {
902
- const mc = await tryCreateSpaceWithMissionControl();
903
- if (mc && !('error' in mc))
904
- return mc;
905
- if (backend === 'mission_control')
906
- return mc ?? { error: 'mission_control_unavailable', backend: 'mission_control' };
907
- }
908
- const cgs = n.createAgentSpace();
909
- return { ...cgs, backend: 'cgs_internal' };
910
- }
911
- async function moveMacWindowToSpace(windowId, spaceId) {
912
- const backend = getMacSpacesBackend();
913
- if (backend === 'yabai' || backend === 'auto') {
914
- if (await yabaiAvailable()) {
915
- const spaces = await yabaiSpaces();
916
- const target = spaces.find(s => s.id === spaceId);
917
- const selector = target?.index ?? spaceId;
918
- try {
919
- n.activateWindow(windowId);
920
- }
921
- catch { /* best effort */ }
922
- await sleep(250);
923
- const r = await spawnBounded('yabai', ['-m', 'window', '--space', String(selector)], 10_000);
924
- if (r.code === 0) {
925
- return { moved: true, verified: true, backend: 'yabai', window_id: windowId, space_id: spaceId, selector };
926
- }
927
- if (backend === 'yabai') {
928
- return { moved: false, ...yabaiFailure('yabai_move_failed', r.stderr || r.stdout || 'yabai_move_failed') };
929
- }
930
- }
931
- else if (backend === 'yabai') {
932
- return { moved: false, reason: 'yabai_unavailable', backend: 'yabai' };
933
- }
934
- }
935
- const r = n.moveWindowToSpace(windowId, spaceId);
936
- return { ...r, backend: 'cgs_internal' };
937
- }
938
- async function destroyMacSpace(spaceId) {
939
- const backend = getMacSpacesBackend();
940
- if (backend === 'yabai' || backend === 'auto') {
941
- if (await yabaiAvailable()) {
942
- const spaces = await yabaiSpaces();
943
- const target = spaces.find(s => s.id === spaceId);
944
- const selector = target?.index ?? spaceId;
945
- const r = await spawnBounded('yabai', ['-m', 'space', String(selector), '--destroy'], 10_000);
946
- if (r.code === 0)
947
- return { destroyed: true, backend: 'yabai', space_id: spaceId, selector };
948
- if (backend === 'yabai') {
949
- return { destroyed: false, ...yabaiFailure('yabai_destroy_failed', r.stderr || r.stdout || 'yabai_destroy_failed') };
950
- }
951
- }
952
- else if (backend === 'yabai') {
953
- return { destroyed: false, reason: 'yabai_unavailable', backend: 'yabai' };
954
- }
955
- }
956
- const r = n.destroySpace(spaceId);
957
- return { ...r, backend: 'cgs_internal' };
958
- }
959
- async function runScriptHelper(language, script, timeoutMs) {
960
- if (IS_WINDOWS) {
961
- if (language === 'applescript' || language === 'javascript') {
962
- return {
963
- stdout: '',
964
- stderr: `${language} is not supported on Windows. Use language: "powershell" instead.`,
965
- code: 1,
966
- timedOut: false,
967
- };
968
- }
969
- const exe = getPowerShellExe();
970
- // Use -Command for simple scripts (faster, no encoding overhead)
971
- // Use -EncodedCommand only when script contains quotes or special chars
972
- const needsEncoding = /['"$`\r\n]/.test(script);
973
- if (needsEncoding) {
974
- const buf = Buffer.from(script, 'utf16le');
975
- const encoded = buf.toString('base64');
976
- return spawnBounded(exe, ['-NoProfile', '-NonInteractive', '-EncodedCommand', encoded], timeoutMs);
977
- }
978
- return spawnBounded(exe, ['-NoProfile', '-NonInteractive', '-Command', script], timeoutMs);
979
- }
980
- if (IS_LINUX) {
981
- if (language === 'applescript' || language === 'javascript') {
982
- return {
983
- stdout: '',
984
- stderr: `${language} is not supported on Linux. Use language: "bash" or "powershell" instead.`,
985
- code: 1,
986
- timedOut: false,
987
- };
988
- }
989
- if (language === 'powershell') {
990
- return spawnBounded('pwsh', ['-NoProfile', '-NonInteractive', '-Command', script], timeoutMs);
991
- }
992
- // Default to bash on Linux
993
- return spawnBounded('bash', ['-c', script], timeoutMs);
994
- }
995
- // macOS: osascript
996
- const args = language === 'javascript'
997
- ? ['-l', 'JavaScript', '-e', script]
998
- : ['-e', script];
999
- return spawnBounded('osascript', args, timeoutMs);
1000
- }
1001
- function powershellLiteral(value) {
1002
- return `'${value.replace(/'/g, "''")}'`;
1003
- }
1004
- function powershellEncodedArgs(script) {
1005
- return [
1006
- '-NoProfile',
1007
- '-NonInteractive',
1008
- '-EncodedCommand',
1009
- Buffer.from(script, 'utf16le').toString('base64'),
1010
- ];
1011
- }
1012
- function xmlEscape(value) {
1013
- return value
1014
- .replace(/&/g, '&amp;')
1015
- .replace(/</g, '&lt;')
1016
- .replace(/>/g, '&gt;')
1017
- .replace(/"/g, '&quot;')
1018
- .replace(/'/g, '&apos;');
1019
- }
1020
- // ── Coordinate validation ─────────────────────────────────────────────
1021
- function validateCoordinates(x, y) {
1022
- const display = n.getDisplaySize();
1023
- if (x < 0 || y < 0 || x >= display.width || y >= display.height) {
1024
- throw new Error(`Coordinates (${x}, ${y}) are outside display bounds (${display.width}x${display.height}). ` +
1025
- `Valid range: x=[0, ${display.width - 1}], y=[0, ${display.height - 1}].`);
1026
- }
1027
- }
1028
- // ── Click helper ────────────────────────────────────────────────────────
1029
- async function doClick(tool, coord, button, count, args) {
1030
- const target = resolveTarget(args);
1031
- const strategy = getStrategy(tool, args);
1032
- await ensureFocusV4(target, strategy);
1033
- const [x, y] = coord;
1034
- validateCoordinates(x, y);
1035
- n.mouseMove(x, y);
1036
- await sleep(50); // HID round-trip settle before click
1037
- n.mouseClick(x, y, button, count);
1038
- trackClickTarget(target);
1039
- return ok(`Clicked (${x}, ${y})`);
1040
- }
1041
- // ── Dispatch ────────────────────────────────────────────────────────────
1042
- async function dispatch(tool, args) {
1043
- const coord = (key = 'coordinate') => {
1044
- const v = args[key];
1045
- if (!Array.isArray(v) || v.length < 2 || typeof v[0] !== 'number' || typeof v[1] !== 'number')
1046
- throw new Error(`Invalid ${key}: expected [number, number]`);
1047
- return [v[0], v[1]];
1048
- };
1049
- const str = (key) => {
1050
- if (typeof args[key] !== 'string')
1051
- throw new Error(`Invalid ${key}: expected string`);
1052
- return args[key];
1053
- };
1054
- const num = (key, fallback) => {
1055
- const v = args[key];
1056
- return typeof v === 'number' ? v : fallback;
1057
- };
1058
- // v5.2: Only mutating tools take the session lock and start the pump.
1059
- // Observation tools stay concurrent and cheap.
1060
- const mutates = MUTATING_TOOLS.has(tool);
1061
- let acquired = false;
1062
- if (mutates) {
1063
- try {
1064
- lockPump.acquire();
1065
- acquired = true;
1066
- }
1067
- catch (err) {
1068
- if (err instanceof LockError) {
1069
- return {
1070
- content: [{ type: 'text', text: JSON.stringify({
1071
- error: 'locked_by_pid',
1072
- lockingPid: err.lockingPid,
1073
- }) }],
1074
- isError: true,
1075
- };
1076
- }
1077
- throw err;
1078
- }
1079
- }
1080
- // Reset the prepare_display slot at dispatch entry so one tool call's
1081
- // hidden list doesn't leak into the next.
1082
- pendingHiddenBundleIds = undefined;
1083
- let result;
1084
- try {
1085
- result = await (async () => {
1086
- switch (tool) {
1087
- // ── Screenshot (observation — never mutates TargetState) ──────────
1088
- case 'screenshot': {
1089
- const provider = (typeof args.provider === 'string' ? args.provider : defaultProvider);
1090
- const defaultWidth = PROVIDER_WIDTH[provider] ?? 1024;
1091
- const defaultQuality = PROVIDER_QUALITY[provider] ?? PROVIDER_QUALITY.default;
1092
- const w = typeof args.width === 'number' ? args.width
1093
- : typeof process.env.COMPUTER_USE_WIDTH !== 'undefined' ? parseInt(process.env.COMPUTER_USE_WIDTH)
1094
- : defaultWidth;
1095
- const q = typeof args.quality === 'number' ? args.quality
1096
- : typeof process.env.COMPUTER_USE_QUALITY !== 'undefined' ? parseInt(process.env.COMPUTER_USE_QUALITY)
1097
- : defaultQuality;
1098
- // Explicit target_window_id takes precedence over target_app.
1099
- let windowId = typeof args.target_window_id === 'number' ? args.target_window_id : undefined;
1100
- let app = windowId ? undefined : (typeof args.target_app === 'string' && args.target_app.length > 0 ? args.target_app : undefined);
1101
- // v5 auto-target: when neither explicit target is set and the session
1102
- // has an established windowId, use it (read-only — does not mutate
1103
- // bundleId/establishedBy). Stale windowIds fall back to full screen
1104
- // after clearing only the windowId field.
1105
- if (windowId === undefined && app === undefined && targetState?.windowId != null) {
1106
- const stateWin = n.getWindow(targetState.windowId);
1107
- if (stateWin?.isOnScreen) {
1108
- windowId = targetState.windowId;
1109
- }
1110
- else if (targetState) {
1111
- targetState = {
1112
- bundleId: targetState.bundleId,
1113
- windowId: undefined,
1114
- establishedBy: targetState.establishedBy,
1115
- establishedAt: targetState.establishedAt,
1116
- };
1117
- }
1118
- }
1119
- if (!visionEnabled) {
1120
- const front = n.getFrontmostApp();
1121
- const display = n.getDisplaySize();
1122
- return ok(`Screen: ${display.width}×${display.height} | Frontmost: ${front?.bundleId ?? 'unknown'} (${front?.displayName ?? ''})`);
1123
- }
1124
- const r = n.takeScreenshot(w, app, q, _lastHash, windowId);
1125
- if (r.unchanged && _lastResult)
1126
- return _lastResult;
1127
- if (!r.base64)
1128
- throw new Error('Screenshot capture missing image payload');
1129
- _lastHash = r.hash;
1130
- _lastResult = {
1131
- content: [
1132
- { type: 'image', data: r.base64, mimeType: r.mimeType },
1133
- { type: 'text', text: `${r.width}x${r.height}` },
1134
- ]
1135
- };
1136
- return _lastResult;
1137
- }
1138
- // ── Zoom (crop a region at full resolution) ──────────────────────────
1139
- case 'zoom': {
1140
- const region = args.region;
1141
- if (!Array.isArray(region) || region.length !== 4) {
1142
- throw new Error('zoom requires region: [x1, y1, x2, y2]');
1143
- }
1144
- const [x1, y1, x2, y2] = region;
1145
- if (x1 >= x2 || y1 >= y2) {
1146
- throw new Error(`Invalid region: x1(${x1}) must be < x2(${x2}), y1(${y1}) must be < y2(${y2})`);
1147
- }
1148
- // Capture full-res as PNG (lossless, fast)
1149
- const fullRes = n.takeScreenshot(undefined, undefined, 0, undefined, undefined);
1150
- if (!fullRes.base64)
1151
- throw new Error('Screenshot capture failed');
1152
- const q = typeof args.quality === 'number' ? args.quality : 0;
1153
- // Crop the region at full resolution using native Rust
1154
- const cropped = n.cropImage(fullRes.base64, x1, y1, x2, y2, q);
1155
- return {
1156
- content: [
1157
- { type: 'image', data: cropped.base64, mimeType: cropped.mimeType },
1158
- { type: 'text', text: `${cropped.width}x${cropped.height} (zoomed from ${fullRes.width}x${fullRes.height})` },
1159
- ],
1160
- };
1161
- }
1162
- // ── Clicks ───────────────────────────────────────────────────────────
1163
- // NB: we `return await` instead of `return` so the lock/pump in the
1164
- // outer try/finally stay held until doClick actually resolves.
1165
- // `return promise` fires the finally on return, not on resolution.
1166
- case 'left_click': return await doClick(tool, coord(), 'left', 1, args);
1167
- case 'right_click': return await doClick(tool, coord(), 'right', 1, args);
1168
- case 'middle_click': return await doClick(tool, coord(), 'middle', 1, args);
1169
- case 'double_click': return await doClick(tool, coord(), 'left', 2, args);
1170
- case 'triple_click': return await doClick(tool, coord(), 'left', 3, args);
1171
- case 'mouse_move': {
1172
- const target = resolveTarget(args);
1173
- const strategy = getStrategy(tool, args);
1174
- await ensureFocusV4(target, strategy);
1175
- const [x, y] = coord();
1176
- validateCoordinates(x, y);
1177
- n.mouseMove(x, y);
1178
- if (target.bundleId)
1179
- updateTargetState(target, 'pointer');
1180
- return ok(`Moved to (${x}, ${y})`);
1181
- }
1182
- case 'left_click_drag': {
1183
- const target = resolveTarget(args);
1184
- const strategy = getStrategy(tool, args);
1185
- await ensureFocusV4(target, strategy);
1186
- const to = coord();
1187
- const from = args.start_coordinate ? coord('start_coordinate') : undefined;
1188
- if (from) {
1189
- n.mouseMove(from[0], from[1]);
1190
- await sleep(50);
1191
- }
1192
- n.mouseButton('press', from?.[0] ?? to[0], from?.[1] ?? to[1]);
1193
- await sleep(50);
1194
- // Ease-out-cubic at 60fps, distance-proportional duration, max 500ms
1195
- const sx = from?.[0] ?? to[0], sy = from?.[1] ?? to[1];
1196
- const dist = Math.hypot(to[0] - sx, to[1] - sy);
1197
- const durationMs = Math.min(dist / 2, 500);
1198
- const frames = Math.max(Math.floor(durationMs / 16), 1);
1199
- for (let i = 1; i <= frames; i++) {
1200
- const t = i / frames;
1201
- const eased = 1 - Math.pow(1 - t, 3);
1202
- n.mouseDrag(Math.round(sx + (to[0] - sx) * eased), Math.round(sy + (to[1] - sy) * eased));
1203
- if (i < frames)
1204
- await sleep(16);
1205
- }
1206
- await sleep(50);
1207
- n.mouseButton('release', to[0], to[1]);
1208
- trackClickTarget(target);
1209
- return ok(`Dragged to (${to[0]}, ${to[1]})`);
1210
- }
1211
- case 'left_mouse_down': {
1212
- const target = resolveTarget(args);
1213
- const strategy = getStrategy(tool, args);
1214
- await ensureFocusV4(target, strategy);
1215
- const [x, y] = coord();
1216
- n.mouseButton('press', x, y);
1217
- if (target.bundleId)
1218
- updateTargetState(target, 'pointer');
1219
- return ok('Mouse down');
1220
- }
1221
- case 'left_mouse_up': {
1222
- const target = resolveTarget(args);
1223
- const strategy = getStrategy(tool, args);
1224
- await ensureFocusV4(target, strategy);
1225
- const [x, y] = coord();
1226
- n.mouseButton('release', x, y);
1227
- if (target.bundleId)
1228
- updateTargetState(target, 'pointer');
1229
- return ok('Mouse up');
1230
- }
1231
- case 'cursor_position': {
1232
- const p = n.cursorPosition();
1233
- return ok(`(${p.x}, ${p.y})`);
1234
- }
1235
- case 'scroll': {
1236
- const target = resolveTarget(args);
1237
- const strategy = getStrategy(tool, args);
1238
- await ensureFocusV4(target, strategy);
1239
- const [x, y] = coord();
1240
- const dir = str('direction');
1241
- const amt = num('amount', 3);
1242
- n.mouseMove(x, y);
1243
- await sleep(15);
1244
- const dx = dir === 'left' ? -amt : dir === 'right' ? amt : 0;
1245
- const dy = dir === 'up' ? -amt : dir === 'down' ? amt : 0;
1246
- n.mouseScroll(dy, dx);
1247
- if (target.bundleId)
1248
- updateTargetState(target, 'pointer');
1249
- return ok(`Scrolled ${dir} ${amt}`);
1250
- }
1251
- // ── Keyboard ─────────────────────────────────────────────────────────
1252
- case 'type': {
1253
- const target = resolveTarget(args);
1254
- const strategy = getStrategy(tool, args);
1255
- await ensureFocusV4(target, strategy);
1256
- const text = str('text');
1257
- // caret_position: move caret before typing
1258
- const caretPos = typeof args.caret_position === 'string' ? args.caret_position : 'idle';
1259
- if (caretPos === 'start') {
1260
- n.keyPress('home');
1261
- await sleep(30);
1262
- }
1263
- else if (caretPos === 'end') {
1264
- n.keyPress('end');
1265
- await sleep(30);
1266
- }
1267
- // clear: select all + delete before typing
1268
- if (args.clear === true || args.clear === 'true') {
1269
- n.keyPress((IS_WINDOWS || IS_LINUX) ? 'ctrl+a' : 'command+a');
1270
- await sleep(30);
1271
- n.keyPress('delete');
1272
- await sleep(30);
1273
- }
1274
- if (text.length > 100) {
1275
- // Clipboard-based typing: faster and more reliable for long text
1276
- if ((IS_WINDOWS || IS_LINUX) && n.readClipboard && n.writeClipboard) {
1277
- let saved;
1278
- try {
1279
- saved = n.readClipboard();
1280
- }
1281
- catch { /* ignore */ }
1282
- try {
1283
- n.writeClipboard(text);
1284
- n.keyPress(IS_LINUX ? 'ctrl+v' : 'ctrl+v');
1285
- await sleep(100);
1286
- }
1287
- finally {
1288
- if (typeof saved === 'string') {
1289
- try {
1290
- n.writeClipboard(saved);
1291
- }
1292
- catch { /* ignore */ }
1293
- }
1294
- }
1295
- }
1296
- else {
1297
- let saved;
1298
- try {
1299
- saved = execFileSync('pbpaste', []).toString();
1300
- }
1301
- catch { /* ignore */ }
1302
- try {
1303
- execFileSync('pbcopy', [], { input: text });
1304
- const verify = execFileSync('pbpaste', []).toString();
1305
- if (verify === text) {
1306
- n.keyPress('command+v');
1307
- await sleep(100);
1308
- }
1309
- else {
1310
- n.typeText(text);
1311
- }
1312
- }
1313
- finally {
1314
- if (typeof saved === 'string') {
1315
- try {
1316
- execFileSync('pbcopy', [], { input: saved });
1317
- }
1318
- catch { /* ignore */ }
1319
- }
1320
- }
1321
- }
1322
- }
1323
- else {
1324
- n.typeText(text);
1325
- }
1326
- if (target.bundleId)
1327
- updateTargetState(target, 'keyboard');
1328
- // press_enter: submit after typing
1329
- if (args.press_enter === true || args.press_enter === 'true') {
1330
- n.keyPress('return');
1331
- await sleep(30);
1332
- }
1333
- return ok('Typed');
1334
- }
1335
- case 'key': {
1336
- const target = resolveTarget(args);
1337
- const strategy = getStrategy(tool, args);
1338
- await ensureFocusV4(target, strategy);
1339
- n.keyPress(str('text'), args.repeat !== undefined ? num('repeat', 1) : undefined);
1340
- if (target.bundleId)
1341
- updateTargetState(target, 'keyboard');
1342
- return ok(`Pressed ${args.text}`);
1343
- }
1344
- case 'hold_key': {
1345
- const target = resolveTarget(args);
1346
- const strategy = getStrategy(tool, args);
1347
- await ensureFocusV4(target, strategy);
1348
- if (!Array.isArray(args.keys) || !args.keys.every(k => typeof k === 'string'))
1349
- throw new Error('Invalid keys: expected string[]');
1350
- n.holdKey(args.keys, num('duration', 1) * 1000);
1351
- if (target.bundleId)
1352
- updateTargetState(target, 'keyboard');
1353
- return ok('Held');
1354
- }
1355
- // ── Clipboard ────────────────────────────────────────────────────────
1356
- case 'read_clipboard': {
1357
- if ((IS_WINDOWS || IS_LINUX) && n.readClipboard) {
1358
- return ok(n.readClipboard());
1359
- }
1360
- const text = execFileSync('pbpaste', []).toString();
1361
- return ok(text);
1362
- }
1363
- case 'write_clipboard': {
1364
- if ((IS_WINDOWS || IS_LINUX) && n.writeClipboard) {
1365
- n.writeClipboard(str('text'));
1366
- return ok('Written');
1367
- }
1368
- execFileSync('pbcopy', [], { input: str('text') });
1369
- return ok('Written');
1370
- }
1371
- // ── New v4 observation tools (never mutate TargetState) ──────────
1372
- case 'get_window': {
1373
- const wid = num('window_id', -1);
1374
- if (wid < 0)
1375
- throw new Error('Invalid window_id: expected number');
1376
- const win = n.getWindow(wid);
1377
- if (!win) {
1378
- return { content: [{ type: 'text', text: `Window not found: ${wid}` }], isError: true };
1379
- }
1380
- return ok(JSON.stringify(win));
1381
- }
1382
- case 'get_cursor_window': {
1383
- const win = n.getCursorWindow();
1384
- return ok(JSON.stringify(win));
1385
- }
1386
- // ── New v4 activation tools (mutate TargetState on success) ──────
1387
- case 'activate_app': {
1388
- const bid = str('bundle_id');
1389
- const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined;
1390
- const frontBefore = n.getFrontmostApp();
1391
- const r = n.activateApp(bid, timeoutMs ?? 2000);
1392
- await sleep(80);
1393
- const frontAfter = n.getFrontmostApp();
1394
- const activated = r.activated || frontAfter?.bundleId === bid;
1395
- if (activated) {
1396
- updateTargetState({ bundleId: bid }, 'activation');
1397
- }
1398
- // Check for hidden/not_running reasons
1399
- let reason = null;
1400
- let suggestedRecovery;
1401
- if (!activated) {
1402
- const runningApp = n.listRunningApps().find(a => a.bundleId === bid);
1403
- if (!runningApp) {
1404
- reason = 'not_running';
1405
- }
1406
- else if (runningApp.isHidden) {
1407
- reason = 'hidden';
1408
- suggestedRecovery = 'unhide_app';
1409
- }
1410
- else {
1411
- reason = 'timeout';
1412
- }
1413
- }
1414
- const response = {
1415
- requestedBundleId: bid,
1416
- frontmostBefore: frontBefore?.bundleId ?? null,
1417
- frontmostAfter: frontAfter?.bundleId ?? null,
1418
- activated,
1419
- reason,
1420
- };
1421
- if (suggestedRecovery)
1422
- response.suggestedRecovery = suggestedRecovery;
1423
- return ok(JSON.stringify(response));
1424
- }
1425
- case 'activate_window': {
1426
- const wid = num('window_id', -1);
1427
- if (wid < 0)
1428
- throw new Error('Invalid window_id: expected number');
1429
- const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined;
1430
- // Resolve bundleId from window
1431
- const win = n.getWindow(wid);
1432
- if (!win) {
1433
- return ok(JSON.stringify({
1434
- windowId: wid,
1435
- activated: false,
1436
- frontmostAfter: n.getFrontmostApp()?.bundleId ?? null,
1437
- reason: 'window_not_found',
1438
- }));
1439
- }
1440
- const bundleId = win.bundleId;
1441
- // Recovery sequence for hidden apps
1442
- if (bundleId) {
1443
- const runningApp = n.listRunningApps().find(a => a.bundleId === bundleId);
1444
- if (runningApp?.isHidden) {
1445
- n.unhideApp(bundleId);
1446
- await sleep(100);
1447
- }
1448
- n.activateApp(bundleId, timeoutMs ?? 2000);
1449
- await sleep(80);
1450
- }
1451
- // Raise the specific window
1452
- const result = n.activateWindow(wid, timeoutMs);
1453
- await sleep(80);
1454
- const frontAfter = n.getFrontmostApp();
1455
- const activated = result.activated;
1456
- if (activated && bundleId) {
1457
- updateTargetState({ bundleId, windowId: wid }, 'activation');
1458
- }
1459
- return ok(JSON.stringify({
1460
- windowId: wid,
1461
- activated,
1462
- frontmostAfter: frontAfter?.bundleId ?? null,
1463
- reason: result.reason,
1464
- }));
1465
- }
1466
- // ── Apps ─────────────────────────────────────────────────────────────
1467
- case 'open_application': {
1468
- const bid = str('bundle_id');
1469
- const r = n.activateApp(bid, 3000);
1470
- if (r.activated) {
1471
- updateTargetState({ bundleId: bid }, 'activation');
1472
- }
1473
- await sleep(300);
1474
- return ok(`Opened ${bid} (activated: ${r.activated})`);
1475
- }
1476
- // ── Observation tools (never mutate TargetState) ─────────────────
1477
- case 'get_frontmost_app':
1478
- return ok(JSON.stringify(n.getFrontmostApp()));
1479
- case 'list_windows':
1480
- return ok(JSON.stringify(n.listWindows(typeof args.bundle_id === 'string' ? args.bundle_id : undefined)));
1481
- case 'list_running_apps':
1482
- return ok(JSON.stringify(n.listRunningApps()));
1483
- case 'hide_app':
1484
- return ok(n.hideApp(str('bundle_id')) ? 'Hidden' : 'App not found');
1485
- case 'unhide_app':
1486
- return ok(n.unhideApp(str('bundle_id')) ? 'Unhidden' : 'App not found');
1487
- case 'get_display_size':
1488
- return ok(JSON.stringify(n.getDisplaySize(typeof args.display_id === 'number' ? args.display_id : undefined)));
1489
- case 'list_displays':
1490
- return ok(JSON.stringify(n.listDisplays()));
1491
- case 'wait': {
1492
- await sleep(num('duration', 1) * 1000);
1493
- return ok(`Waited ${args.duration}s`);
1494
- }
1495
- case 'resize_window': {
1496
- if (IS_WINDOWS) {
1497
- const wid = typeof args.window_id === 'number' ? args.window_id : undefined;
1498
- const wname = typeof args.window_name === 'string' ? args.window_name : undefined;
1499
- const wsize = Array.isArray(args.window_size) ? args.window_size : undefined;
1500
- const wloc = Array.isArray(args.window_loc) ? args.window_loc : undefined;
1501
- // Find target HWND
1502
- let targetCmd = '';
1503
- if (wid) {
1504
- targetCmd = `$hwnd = [IntPtr]${wid}`;
1505
- }
1506
- else if (wname) {
1507
- targetCmd = `$hwnd = (Get-Process -Name '${wname.replace(/\.exe$/i, '')}' -ErrorAction SilentlyContinue | Select-Object -First 1).MainWindowHandle; if (-not $hwnd -or $hwnd -eq 0) { $hwnd = (Get-Process | Where-Object { $_.MainWindowTitle -like '*${wname}*' } | Select-Object -First 1).MainWindowHandle }`;
1508
- }
1509
- else {
1510
- targetCmd = `Add-Type -TypeDefinition 'using System;using System.Runtime.InteropServices;public class W{[DllImport("user32.dll")]public static extern IntPtr GetForegroundWindow();}'; $hwnd = [W]::GetForegroundWindow()`;
1511
- }
1512
- let moveCmd = '';
1513
- if (wsize && wloc) {
1514
- moveCmd = `Add-Type -TypeDefinition 'using System;using System.Runtime.InteropServices;public class MW{[DllImport("user32.dll")]public static extern bool MoveWindow(IntPtr h,int x,int y,int w,int ht,bool r);}'; [MW]::MoveWindow($hwnd, ${wloc[0]}, ${wloc[1]}, ${wsize[0]}, ${wsize[1]}, $true)`;
1515
- }
1516
- else if (wsize) {
1517
- moveCmd = `Add-Type -TypeDefinition 'using System;using System.Runtime.InteropServices;public class GR{[DllImport("user32.dll")]public static extern bool GetWindowRect(IntPtr h,out RECT r);[StructLayout(LayoutKind.Sequential)]public struct RECT{public int l,t,r,b;} [DllImport("user32.dll")]public static extern bool MoveWindow(IntPtr h,int x,int y,int w,int ht,bool rp);}'; $r = New-Object GR+RECT; [GR]::GetWindowRect($hwnd, [ref]$r); [GR]::MoveWindow($hwnd, $r.l, $r.t, ${wsize[0]}, ${wsize[1]}, $true)`;
1518
- }
1519
- else if (wloc) {
1520
- moveCmd = `Add-Type -TypeDefinition 'using System;using System.Runtime.InteropServices;public class GR2{[DllImport("user32.dll")]public static extern bool GetWindowRect(IntPtr h,out RECT r);[StructLayout(LayoutKind.Sequential)]public struct RECT{public int l,t,r,b;} [DllImport("user32.dll")]public static extern bool MoveWindow(IntPtr h,int x,int y,int w,int ht,bool rp);}'; $r = New-Object GR2+RECT; [GR2]::GetWindowRect($hwnd, [ref]$r); [GR2]::MoveWindow($hwnd, ${wloc[0]}, ${wloc[1]}, $r.r-$r.l, $r.b-$r.t, $true)`;
1521
- }
1522
- else {
1523
- return { content: [{ type: 'text', text: 'window_size or window_loc required' }], isError: true };
1524
- }
1525
- const ps = `${targetCmd}; if ($hwnd -and $hwnd -ne 0) { ${moveCmd}; 'Resized' } else { 'Window not found' }`;
1526
- const r = await runScriptHelper('powershell', ps, 10000);
1527
- return r.code === 0 ? ok(r.stdout.trim()) : { content: [{ type: 'text', text: r.stderr || r.stdout }], isError: true };
1528
- }
1529
- // macOS: use AppleScript for window resize/move
1530
- const wname = typeof args.window_name === 'string' ? args.window_name : undefined;
1531
- const wsize = Array.isArray(args.window_size) ? args.window_size : undefined;
1532
- const wloc = Array.isArray(args.window_loc) ? args.window_loc : undefined;
1533
- if (!wsize && !wloc) {
1534
- return { content: [{ type: 'text', text: 'window_size or window_loc required' }], isError: true };
1535
- }
1536
- // Build AppleScript to resize/move the window
1537
- let targetClause;
1538
- if (wname) {
1539
- targetClause = `tell application "${wname}"`;
1540
- }
1541
- else {
1542
- // Target the frontmost app
1543
- targetClause = `tell application (path to frontmost application as text)`;
1544
- }
1545
- const parts = [];
1546
- if (wloc)
1547
- parts.push(`set position of front window to {${wloc[0]}, ${wloc[1]}}`);
1548
- if (wsize)
1549
- parts.push(`set size of front window to {${wsize[0]}, ${wsize[1]}}`);
1550
- const script = `${targetClause}\n${parts.join('\n')}\nend tell`;
1551
- const r = await runScriptHelper('applescript', script, 10000);
1552
- return r.code === 0 ? ok('Resized') : { content: [{ type: 'text', text: r.stderr || r.stdout || 'resize failed' }], isError: true };
1553
- }
1554
- case 'snapshot': {
1555
- // Combined tool: screenshot + UI tree + windows + desktops
1556
- const parts = [];
1557
- // Desktop info
1558
- const front = n.getFrontmostApp();
1559
- const wins = n.listWindows();
1560
- const display = n.getDisplaySize();
1561
- const runningApps = n.listRunningApps();
1562
- let desktopInfo = `Display: ${display.width}x${display.height} (scale: ${display.scaleFactor})\n`;
1563
- desktopInfo += `Frontmost: ${front?.bundleId ?? 'unknown'} — ${front?.displayName ?? ''}\n`;
1564
- desktopInfo += `Windows: ${Array.isArray(wins) ? wins.length : 0}\n`;
1565
- desktopInfo += `Running apps: ${Array.isArray(runningApps) ? runningApps.length : 0}`;
1566
- // Windows list
1567
- if (Array.isArray(wins)) {
1568
- const winList = wins.map(w => ` ${w.windowId} | ${w.bundleId} | ${w.title ?? '(no title)'}`).join('\n');
1569
- desktopInfo += `\n\nWindows:\n${winList}`;
1570
- }
1571
- // UI tree for the frontmost window
1572
- let uiTreeText = '';
1573
- if (args.use_vision !== false) {
1574
- const frontWin = Array.isArray(wins) ? wins.find(w => w.isFocused) : null;
1575
- if (frontWin) {
1576
- try {
1577
- const tree = n.getUiTree(frontWin.windowId, 5);
1578
- uiTreeText = `\n\nUI Tree (${frontWin.bundleId}):\n${JSON.stringify(tree).slice(0, 4000)}`;
1579
- }
1580
- catch { /* UI tree may fail */ }
1581
- }
1582
- }
1583
- parts.push({ type: 'text', text: desktopInfo + uiTreeText });
1584
- // Screenshot if use_vision
1585
- if (args.use_vision) {
1586
- const provider = defaultProvider;
1587
- const defaultWidth = PROVIDER_WIDTH[provider] ?? 1024;
1588
- const w = typeof args.width === 'number' ? args.width : defaultWidth;
1589
- const r = n.takeScreenshot(w, undefined, 80, undefined, undefined);
1590
- if (r.base64) {
1591
- const needsAnnotation = args.use_annotation && Array.isArray(wins);
1592
- const gridLines = Array.isArray(args.grid_lines) ? args.grid_lines : undefined;
1593
- if (needsAnnotation || gridLines) {
1594
- // Build annotation data for Rust drawing
1595
- const annData = needsAnnotation
1596
- ? wins
1597
- .filter(win => win.bounds)
1598
- .map(win => ({
1599
- x: Math.round(win.bounds.x * r.width / display.width),
1600
- y: Math.round(win.bounds.y * r.height / display.height),
1601
- width: Math.round(win.bounds.width * r.width / display.width),
1602
- height: Math.round(win.bounds.height * r.height / display.height),
1603
- }))
1604
- : null;
1605
- const annotated = n.annotateImage(r.base64, annData ? JSON.stringify(annData) : null, gridLines?.[0] ?? null, gridLines?.[1] ?? null, 80);
1606
- parts.push({ type: 'image', data: annotated.base64, mimeType: annotated.mimeType });
1607
- parts.push({ type: 'text', text: `${annotated.width}x${annotated.height}` });
1608
- }
1609
- else {
1610
- parts.push({ type: 'image', data: r.base64, mimeType: r.mimeType });
1611
- parts.push({ type: 'text', text: `${r.width}x${r.height}` });
1612
- }
1613
- // Always include text annotations for non-vision models
1614
- if (needsAnnotation) {
1615
- const annText = wins
1616
- .filter(win => win.bounds)
1617
- .map(win => `[${win.bundleId}] (${win.bounds.x},${win.bounds.y}) ${win.bounds.width}x${win.bounds.height}`)
1618
- .join('\n');
1619
- parts.push({ type: 'text', text: `\nAnnotations:\n${annText}` });
1620
- }
1621
- }
1622
- }
1623
- return { content: parts };
1624
- }
1625
- // ── v5: Accessibility observation (never mutate TargetState) ─────
1626
- case 'get_ui_tree': {
1627
- const wid = num('window_id', -1);
1628
- if (wid < 0)
1629
- throw new Error('Invalid window_id: expected number');
1630
- const maxDepth = typeof args.max_depth === 'number' ? args.max_depth : undefined;
1631
- return ok(JSON.stringify(n.getUiTree(wid, maxDepth)));
1632
- }
1633
- case 'get_focused_element': {
1634
- return ok(JSON.stringify(n.getFocusedElement()));
1635
- }
1636
- case 'find_element': {
1637
- const wid = num('window_id', -1);
1638
- if (wid < 0)
1639
- throw new Error('Invalid window_id: expected number');
1640
- const role = typeof args.role === 'string' ? args.role : undefined;
1641
- const label = typeof args.label === 'string' ? args.label : undefined;
1642
- const value = typeof args.value === 'string' ? args.value : undefined;
1643
- if (!role && !label && !value) {
1644
- throw new Error('find_element requires at least one of: role, label, value');
1645
- }
1646
- const maxRes = typeof args.max_results === 'number' ? args.max_results : undefined;
1647
- return ok(JSON.stringify(n.findElement(wid, role, label, value, maxRes)));
1648
- }
1649
- // ── v5: Semantic actions (mutate TargetState on success) ─────────
1650
- case 'click_element': {
1651
- const target = resolveTarget({ window_id: args.window_id });
1652
- if (target.windowId == null)
1653
- throw new Error('click_element requires window_id');
1654
- const strategy = getStrategy(tool, args);
1655
- await ensureFocusV4(target, strategy);
1656
- const role = str('role');
1657
- const label = str('label');
1658
- const result = n.performAction(target.windowId, role, label, 'AXPress');
1659
- if (result.performed) {
1660
- updateTargetState(target, 'pointer');
1661
- return ok(`Clicked ${role} "${label}"`);
1662
- }
1663
- if (result.reason === 'unsupported_action' && result.bounds) {
1664
- // Coordinate-click fallback at element center.
1665
- const cx = Math.round(result.bounds.x + result.bounds.width / 2);
1666
- const cy = Math.round(result.bounds.y + result.bounds.height / 2);
1667
- n.mouseMove(cx, cy);
1668
- await sleep(50);
1669
- n.mouseClick(cx, cy, 'left', 1);
1670
- updateTargetState(target, 'pointer');
1671
- return ok(`Clicked ${role} "${label}" via coordinate fallback (${cx}, ${cy})`);
1672
- }
1673
- if (result.reason === 'disabled') {
1674
- return { content: [{ type: 'text', text: `Element ${role} "${label}" is disabled` }], isError: true };
1675
- }
1676
- return similarLabelsError(target.windowId, label, `No element matches role="${role}" label="${label}"`);
1677
- }
1678
- case 'set_value': {
1679
- const target = resolveTarget({ window_id: args.window_id });
1680
- if (target.windowId == null)
1681
- throw new Error('set_value requires window_id');
1682
- const strategy = getStrategy(tool, args);
1683
- await ensureFocusV4(target, strategy);
1684
- const role = str('role');
1685
- const label = str('label');
1686
- const value = str('value');
1687
- const result = n.setElementValue(target.windowId, role, label, value);
1688
- if (result.set) {
1689
- updateTargetState(target, 'keyboard');
1690
- return ok(`Set ${role} "${label}" = ${JSON.stringify(value)}`);
1691
- }
1692
- if (result.reason === 'read_only') {
1693
- return { content: [{ type: 'text', text: `Element ${role} "${label}" is read-only` }], isError: true };
1694
- }
1695
- if (result.reason === 'not_found') {
1696
- return similarLabelsError(target.windowId, label, `No element matches role="${role}" label="${label}"`);
1697
- }
1698
- return { content: [{ type: 'text', text: `set_value failed: ${result.reason ?? 'unknown'}` }], isError: true };
1699
- }
1700
- case 'press_button': {
1701
- const target = resolveTarget({ window_id: args.window_id });
1702
- if (target.windowId == null)
1703
- throw new Error('press_button requires window_id');
1704
- const strategy = getStrategy(tool, args);
1705
- await ensureFocusV4(target, strategy);
1706
- const label = str('label');
1707
- const result = n.performAction(target.windowId, 'AXButton', label, 'AXPress');
1708
- if (result.performed) {
1709
- updateTargetState(target, 'pointer');
1710
- return ok(`Pressed "${label}"`);
1711
- }
1712
- if (result.reason === 'disabled') {
1713
- return { content: [{ type: 'text', text: `Button "${label}" is disabled` }], isError: true };
1714
- }
1715
- return similarLabelsError(target.windowId, label, `No button matches label="${label}"`, 'AXButton');
1716
- }
1717
- case 'select_menu_item': {
1718
- const bundleId = str('bundle_id');
1719
- const menu = str('menu');
1720
- const item = str('item');
1721
- const submenu = typeof args.submenu === 'string' ? args.submenu : undefined;
1722
- // Menu bar only responds to the frontmost app.
1723
- await ensureFocusV4({ bundleId }, 'strict');
1724
- const result = n.pressMenuItem(bundleId, menu, item, submenu);
1725
- if (result.pressed) {
1726
- updateTargetState({ bundleId }, 'activation');
1727
- return ok(`Selected ${bundleId} → ${menu}${submenu ? ` → ${submenu}` : ''} → ${item}`);
1728
- }
1729
- const bar = n.getMenuBar(bundleId);
1730
- return {
1731
- content: [{
1732
- type: 'text',
1733
- text: JSON.stringify({
1734
- error: result.reason ?? 'menu_item_not_found',
1735
- bundle_id: bundleId, menu, item, submenu,
1736
- availableMenus: bar.map(m => m.title),
1737
- }),
1738
- }],
1739
- isError: true,
1740
- };
1741
- }
1742
- case 'list_menu_bar': {
1743
- if (IS_WINDOWS) {
1744
- return { content: [{ type: 'text', text: 'platform_unsupported: list_menu_bar is macOS-only. Use get_ui_tree to discover menu structure on Windows.' }], isError: true };
1745
- }
1746
- const bundleId = str('bundle_id');
1747
- const bar = n.getMenuBar(bundleId);
1748
- return ok(JSON.stringify(bar));
1749
- }
1750
- case 'fill_form': {
1751
- const target = resolveTarget({ window_id: args.window_id });
1752
- if (target.windowId == null)
1753
- throw new Error('fill_form requires window_id');
1754
- const strategy = getStrategy(tool, args);
1755
- await ensureFocusV4(target, strategy);
1756
- const fields = args.fields;
1757
- if (!Array.isArray(fields))
1758
- throw new Error('fill_form requires fields: array');
1759
- let succeeded = 0;
1760
- const failures = [];
1761
- for (const raw of fields) {
1762
- if (raw == null || typeof raw !== 'object') {
1763
- failures.push({ role: '', label: '', reason: 'invalid_entry' });
1764
- continue;
1765
- }
1766
- const f = raw;
1767
- const role = typeof f.role === 'string' ? f.role : '';
1768
- const label = typeof f.label === 'string' ? f.label : '';
1769
- const value = typeof f.value === 'string' ? f.value : '';
1770
- if (!role) {
1771
- failures.push({ role, label, reason: 'invalid_entry' });
1772
- continue;
1773
- }
1774
- const r = n.setElementValue(target.windowId, role, label, value);
1775
- if (r.set) {
1776
- succeeded++;
1777
- }
1778
- else {
1779
- failures.push({ role, label, reason: r.reason ?? 'not_found' });
1780
- }
1781
- }
1782
- if (succeeded > 0)
1783
- updateTargetState(target, 'keyboard');
1784
- return ok(JSON.stringify({
1785
- succeeded,
1786
- failed: failures.length,
1787
- failures,
1788
- }));
1789
- }
1790
- // ── v5: Scripting bridge (never mutate TargetState) ───────────────
1791
- case 'run_script': {
1792
- const lang = IS_WINDOWS
1793
- ? (args.language === 'powershell' ? 'powershell' : String(args.language ?? 'powershell'))
1794
- : (args.language === 'javascript' ? 'javascript' : 'applescript');
1795
- const script = str('script');
1796
- const requested = typeof args.timeout_ms === 'number' ? args.timeout_ms : 30_000;
1797
- const timeoutMs = Math.max(100, Math.min(requested, 120_000));
1798
- const r = await runScriptHelper(lang, script, timeoutMs);
1799
- if (r.timedOut) {
1800
- return { content: [{ type: 'text', text: `script timed out after ${timeoutMs}ms` }], isError: true };
1801
- }
1802
- if (r.code !== 0) {
1803
- return { content: [{ type: 'text', text: (r.stderr || `script exited with code ${r.code}`).trimEnd() }], isError: true };
1804
- }
1805
- return ok(r.stdout.replace(/\n+$/, ''));
1806
- }
1807
- case 'get_app_dictionary': {
1808
- if (IS_WINDOWS) {
1809
- return { content: [{ type: 'text', text: 'platform_unsupported: get_app_dictionary is macOS-only. Use get_ui_tree to discover UI structure on Windows.' }], isError: true };
1810
- }
1811
- const bundleId = str('bundle_id');
1812
- const suite = typeof args.suite === 'string' ? args.suite : undefined;
1813
- const r = await getAppDictionary(bundleId, suite);
1814
- if ('error' in r) {
1815
- return { content: [{ type: 'text', text: r.error }], isError: true };
1816
- }
1817
- return ok(JSON.stringify(r.dict));
1818
- }
1819
- // ── v5: Strategy advisor + capabilities (never mutate state) ─────
1820
- case 'get_tool_guide': {
1821
- const taskDescription = str('task_description');
1822
- return ok(JSON.stringify(lookupToolGuide(taskDescription)));
1823
- }
1824
- case 'get_app_capabilities': {
1825
- const bundleId = str('bundle_id');
1826
- const running = n.listRunningApps().find(a => a.bundleId === bundleId);
1827
- const wins = n.listWindows(bundleId);
1828
- if (IS_WINDOWS) {
1829
- return ok(JSON.stringify({
1830
- bundle_id: bundleId,
1831
- scriptable: false, // No AppleScript on Windows
1832
- suites: [],
1833
- powershell: true, // PowerShell available via run_script
1834
- accessible: wins.length > 0,
1835
- topLevelCount: wins.length,
1836
- running: Boolean(running),
1837
- hidden: running?.isHidden ?? false,
1838
- }));
1839
- }
1840
- const dictResult = await getAppDictionary(bundleId);
1841
- const scriptable = !('error' in dictResult);
1842
- const suites = scriptable ? dictResult.dict.suites.map(s => s.name) : [];
1843
- return ok(JSON.stringify({
1844
- bundle_id: bundleId,
1845
- scriptable,
1846
- suites,
1847
- accessible: wins.length > 0,
1848
- topLevelCount: wins.length,
1849
- running: Boolean(running),
1850
- hidden: running?.isHidden ?? false,
1851
- }));
1852
- }
1853
- // ── v5: Spaces ─────────────────────────────────────────────────────
1854
- case 'list_spaces': {
1855
- return ok(JSON.stringify(n.listSpaces()));
1856
- }
1857
- case 'get_active_space': {
1858
- return ok(JSON.stringify(n.getActiveSpace()));
1859
- }
1860
- case 'create_agent_space': {
1861
- if (IS_WINDOWS) {
1862
- // Count desktops before
1863
- const beforeSpaces = n.listSpaces();
1864
- const beforeCount = beforeSpaces.displays?.[0]?.spaces?.length ?? 0;
1865
- // Create via keyboard shortcut — Ctrl+Win+D
1866
- n.keyPress('ctrl+win+d');
1867
- await sleep(500);
1868
- // Count after
1869
- const afterSpaces = n.listSpaces();
1870
- const afterCount = afterSpaces.displays?.[0]?.spaces?.length ?? 0;
1871
- const newDesktop = afterSpaces.displays?.[0]?.spaces?.[afterCount - 1];
1872
- return ok(JSON.stringify({
1873
- created: afterCount > beforeCount,
1874
- space_id: newDesktop?.uuid ?? null,
1875
- name: `Desktop ${afterCount}`,
1876
- total_desktops: afterCount,
1877
- note: 'Created via Ctrl+Win+D keyboard shortcut. You are now on the new desktop.',
1878
- }));
1879
- }
1880
- if (cachedAgentSpaceId !== undefined) {
1881
- return ok(JSON.stringify({
1882
- space_id: cachedAgentSpaceId,
1883
- created: false,
1884
- cached: true,
1885
- }));
1886
- }
1887
- const r = await createMacSpace();
1888
- if ('error' in r || r.supported === false) {
1889
- return {
1890
- content: [{
1891
- type: 'text',
1892
- text: JSON.stringify({
1893
- error: 'spaces_api_unavailable',
1894
- reason: r.reason ?? r.error ?? 'api_unavailable',
1895
- backend: r.backend,
1896
- requires_scripting_addition: r.requires_scripting_addition,
1897
- setup_commands: r.setup_commands,
1898
- sip_note: r.sip_note,
1899
- before_count: r.before_count,
1900
- after_count: r.after_count,
1901
- workaround: 'Install/start yabai, grant Accessibility, and install/load yabai scripting-addition for visible Space create/destroy. Alternatively allow Mission Control gesture automation. CGS-only Space creation can be orphaned on SIP-enabled Macs.',
1902
- }),
1903
- }],
1904
- isError: true,
1905
- };
1906
- }
1907
- if (typeof r.spaceId === 'number')
1908
- cachedAgentSpaceId = r.spaceId;
1909
- return ok(JSON.stringify({
1910
- space_id: r.spaceId,
1911
- created: r.created ?? true,
1912
- attached: r.attached ?? false,
1913
- backend: r.backend,
1914
- index: r.index,
1915
- uuid: r.uuid,
1916
- note: r.note,
1917
- }));
1918
- }
1919
- case 'move_window_to_space': {
1920
- const wid = num('window_id', -1);
1921
- const spaceId = num('space_id', -1);
1922
- if (wid < 0)
1923
- throw new Error('Invalid window_id: expected number');
1924
- if (spaceId < 0)
1925
- throw new Error('Invalid space_id: expected number');
1926
- const r = await moveMacWindowToSpace(wid, spaceId);
1927
- if (!r.moved) {
1928
- return {
1929
- content: [{
1930
- type: 'text',
1931
- text: JSON.stringify({
1932
- error: r.reason ?? 'move_failed',
1933
- window_id: wid,
1934
- space_id: spaceId,
1935
- backend: r.backend,
1936
- requires_scripting_addition: r.requires_scripting_addition,
1937
- setup_commands: r.setup_commands,
1938
- sip_note: r.sip_note,
1939
- }),
1940
- }],
1941
- isError: true,
1942
- };
1943
- }
1944
- return ok(JSON.stringify({
1945
- window_id: wid,
1946
- space_id: spaceId,
1947
- moved: true,
1948
- verified: r.verified ?? false,
1949
- backend: r.backend,
1950
- selector: r.selector,
1951
- window_on_screen_before: r.window_on_screen_before,
1952
- window_on_screen_after: r.window_on_screen_after,
1953
- note: r.note,
1954
- }));
1955
- }
1956
- case 'remove_window_from_space': {
1957
- const wid = num('window_id', -1);
1958
- const spaceId = num('space_id', -1);
1959
- if (wid < 0)
1960
- throw new Error('Invalid window_id: expected number');
1961
- if (spaceId < 0)
1962
- throw new Error('Invalid space_id: expected number');
1963
- const r = n.removeWindowFromSpace(wid, spaceId);
1964
- if (!r.removed) {
1965
- return {
1966
- content: [{ type: 'text', text: JSON.stringify({ error: r.reason ?? 'remove_failed' }) }],
1967
- isError: true,
1968
- };
1969
- }
1970
- return ok(JSON.stringify({ window_id: wid, space_id: spaceId, removed: true }));
1971
- }
1972
- case 'destroy_space': {
1973
- if (IS_WINDOWS) {
1974
- // Close current desktop via Ctrl+Win+F4
1975
- // This closes the desktop you're currently on and moves windows to the adjacent one
1976
- const beforeSpaces = n.listSpaces();
1977
- const beforeCount = beforeSpaces.displays?.[0]?.spaces?.length ?? 0;
1978
- n.keyPress('ctrl+win+f4');
1979
- await sleep(500);
1980
- const afterSpaces = n.listSpaces();
1981
- const afterCount = afterSpaces.displays?.[0]?.spaces?.length ?? 0;
1982
- return ok(JSON.stringify({
1983
- destroyed: afterCount < beforeCount,
1984
- remaining_desktops: afterCount,
1985
- note: 'Closed current desktop via Ctrl+Win+F4. Windows moved to adjacent desktop.',
1986
- }));
1987
- }
1988
- const spaceId = num('space_id', -1);
1989
- if (spaceId < 0)
1990
- throw new Error('Invalid space_id: expected number');
1991
- const r = await destroyMacSpace(spaceId);
1992
- if (!r.destroyed) {
1993
- return {
1994
- content: [{ type: 'text', text: JSON.stringify({
1995
- error: r.reason ?? 'destroy_failed',
1996
- backend: r.backend,
1997
- requires_scripting_addition: r.requires_scripting_addition,
1998
- setup_commands: r.setup_commands,
1999
- sip_note: r.sip_note,
2000
- }) }],
2001
- isError: true,
2002
- };
2003
- }
2004
- if (cachedAgentSpaceId === spaceId)
2005
- cachedAgentSpaceId = undefined;
2006
- return ok(JSON.stringify({ space_id: spaceId, destroyed: true, backend: r.backend, selector: r.selector }));
2007
- }
2008
- // ── New Windows-parity tools ──────────────────────────────────────
2009
- case 'filesystem': {
2010
- const mode = str('mode');
2011
- let filePath = str('path');
2012
- // Resolve relative paths from Desktop — use already-imported modules
2013
- if (!path.isAbsolute(filePath)) {
2014
- filePath = path.join(os.homedir(), 'Desktop', filePath);
2015
- }
2016
- let dest = typeof args.destination === 'string' ? args.destination : undefined;
2017
- if (dest && !path.isAbsolute(dest)) {
2018
- dest = path.join(os.homedir(), 'Desktop', dest);
2019
- }
2020
- const encoding = (typeof args.encoding === 'string' ? args.encoding : 'utf-8');
2021
- switch (mode) {
2022
- case 'read': {
2023
- if (!fs.existsSync(filePath))
2024
- return { content: [{ type: 'text', text: `File not found: ${filePath}` }], isError: true };
2025
- const content = fs.readFileSync(filePath, encoding);
2026
- const lines = content.split('\n');
2027
- const offset = typeof args.offset === 'number' ? args.offset : 0;
2028
- const limit = typeof args.limit === 'number' ? args.limit : lines.length;
2029
- return ok(lines.slice(offset, offset + limit).join('\n'));
2030
- }
2031
- case 'write': {
2032
- const content = typeof args.content === 'string' ? args.content : '';
2033
- const dir = path.dirname(filePath);
2034
- if (!fs.existsSync(dir))
2035
- fs.mkdirSync(dir, { recursive: true });
2036
- if (args.append) {
2037
- fs.appendFileSync(filePath, content, encoding);
2038
- }
2039
- else {
2040
- fs.writeFileSync(filePath, content, encoding);
2041
- }
2042
- return ok(`Written to ${filePath}`);
2043
- }
2044
- case 'copy': {
2045
- if (!dest)
2046
- return { content: [{ type: 'text', text: 'destination required for copy' }], isError: true };
2047
- fs.cpSync(filePath, dest, { recursive: true, force: Boolean(args.overwrite) });
2048
- return ok(`Copied ${filePath} → ${dest}`);
2049
- }
2050
- case 'move': {
2051
- if (!dest)
2052
- return { content: [{ type: 'text', text: 'destination required for move' }], isError: true };
2053
- fs.renameSync(filePath, dest);
2054
- return ok(`Moved ${filePath} → ${dest}`);
2055
- }
2056
- case 'delete': {
2057
- if (!fs.existsSync(filePath))
2058
- return { content: [{ type: 'text', text: `Not found: ${filePath}` }], isError: true };
2059
- const stat = fs.statSync(filePath);
2060
- if (stat.isDirectory()) {
2061
- fs.rmSync(filePath, { recursive: Boolean(args.recursive), force: true });
2062
- }
2063
- else {
2064
- fs.unlinkSync(filePath);
2065
- }
2066
- return ok(`Deleted ${filePath}`);
2067
- }
2068
- case 'list': {
2069
- if (!fs.existsSync(filePath))
2070
- return { content: [{ type: 'text', text: `Directory not found: ${filePath}` }], isError: true };
2071
- const entries = fs.readdirSync(filePath, { withFileTypes: true });
2072
- const filtered = entries.filter(e => args.show_hidden || !e.name.startsWith('.'));
2073
- const result = filtered.map(e => `${e.isDirectory() ? 'd' : 'f'} ${e.name}`).join('\n');
2074
- return ok(result || '(empty)');
2075
- }
2076
- case 'search': {
2077
- const pattern = typeof args.pattern === 'string' ? args.pattern : '*';
2078
- // Simple glob: just list recursively and filter
2079
- const results = [];
2080
- const walk = (dir) => {
2081
- try {
2082
- for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
2083
- const full = path.join(dir, e.name);
2084
- if (e.name.includes(pattern.replace(/\*/g, '')) || pattern === '*')
2085
- results.push(full);
2086
- if (e.isDirectory() && args.recursive)
2087
- walk(full);
2088
- }
2089
- }
2090
- catch { /* permission denied etc */ }
2091
- };
2092
- walk(filePath);
2093
- return ok(results.slice(0, 100).join('\n') || 'No matches');
2094
- }
2095
- case 'info': {
2096
- if (!fs.existsSync(filePath))
2097
- return { content: [{ type: 'text', text: `Not found: ${filePath}` }], isError: true };
2098
- const stat = fs.statSync(filePath);
2099
- return ok(JSON.stringify({
2100
- path: filePath, type: stat.isDirectory() ? 'directory' : 'file',
2101
- size: stat.size, created: stat.birthtime.toISOString(),
2102
- modified: stat.mtime.toISOString(),
2103
- }));
2104
- }
2105
- default: return { content: [{ type: 'text', text: `Unknown filesystem mode: ${mode}` }], isError: true };
2106
- }
2107
- }
2108
- case 'process_kill': {
2109
- const mode = str('mode');
2110
- if (mode === 'list') {
2111
- if (IS_WINDOWS) {
2112
- const r = await spawnBounded('powershell', ['-NoProfile', '-Command',
2113
- 'Get-Process | Sort-Object WorkingSet64 -Descending | Select-Object -First 20 Id,ProcessName,@{N="MemMB";E={[math]::Round($_.WorkingSet64/1MB,1)}} | ConvertTo-Json'], 10000);
2114
- return r.code === 0 ? ok(r.stdout) : { content: [{ type: 'text', text: r.stderr }], isError: true };
2115
- }
2116
- else {
2117
- const psArgs = IS_LINUX ? ['aux', '--sort=-%mem'] : ['aux', '-r'];
2118
- const r = await spawnBounded('ps', psArgs, 5000);
2119
- const lines = r.stdout.split('\n').slice(0, 21);
2120
- return ok(lines.join('\n'));
2121
- }
2122
- }
2123
- if (mode === 'kill') {
2124
- const name = typeof args.name === 'string' ? args.name : undefined;
2125
- const pid = typeof args.pid === 'number' ? args.pid : undefined;
2126
- if (!name && !pid)
2127
- return { content: [{ type: 'text', text: 'name or pid required' }], isError: true };
2128
- if (IS_WINDOWS) {
2129
- const target = pid ? `/PID ${pid}` : `/IM ${name}`;
2130
- const flag = args.force ? '/F' : '';
2131
- const r = await spawnBounded('taskkill', [target, flag].filter(Boolean), 10000);
2132
- return r.code === 0 ? ok(r.stdout.trim() || 'Process terminated') : { content: [{ type: 'text', text: r.stderr || r.stdout }], isError: true };
2133
- }
2134
- else {
2135
- const signal = args.force ? 'SIGKILL' : 'SIGTERM';
2136
- if (pid) {
2137
- process.kill(pid, signal);
2138
- return ok(`Sent ${signal} to PID ${pid}`);
2139
- }
2140
- const r = await spawnBounded('pkill', [args.force ? '-9' : '-15', name], 5000);
2141
- return r.code === 0 ? ok(`Killed ${name}`) : { content: [{ type: 'text', text: r.stderr || 'No matching process' }], isError: true };
2142
- }
2143
- }
2144
- return { content: [{ type: 'text', text: `Unknown process mode: ${mode}` }], isError: true };
2145
- }
2146
- case 'registry': {
2147
- if (!IS_WINDOWS)
2148
- return { content: [{ type: 'text', text: 'registry is Windows-only. Use `defaults` via run_script on macOS.' }], isError: true };
2149
- const mode = str('mode');
2150
- const regPath = str('path');
2151
- const name = typeof args.name === 'string' ? args.name : undefined;
2152
- const psExe = getPowerShellExe();
2153
- switch (mode) {
2154
- case 'get': {
2155
- if (!name)
2156
- return { content: [{ type: 'text', text: 'name required for get' }], isError: true };
2157
- const ps = `Get-ItemPropertyValue -Path ${powershellLiteral(regPath)} -Name ${powershellLiteral(name)}`;
2158
- const r = await spawnBounded(psExe, powershellEncodedArgs(ps), 10000);
2159
- return r.code === 0 ? ok(r.stdout.trim()) : { content: [{ type: 'text', text: r.stderr }], isError: true };
2160
- }
2161
- case 'set': {
2162
- if (!name)
2163
- return { content: [{ type: 'text', text: 'name required for set' }], isError: true };
2164
- const val = typeof args.value === 'string' ? args.value : '';
2165
- const type = typeof args.type === 'string' ? args.type : 'String';
2166
- const ps = `New-ItemProperty -Path ${powershellLiteral(regPath)} -Name ${powershellLiteral(name)} -Value ${powershellLiteral(val)} -PropertyType ${type} -Force`;
2167
- const r = await spawnBounded(psExe, powershellEncodedArgs(ps), 10000);
2168
- return r.code === 0 ? ok(`Set ${regPath}\\${name}`) : { content: [{ type: 'text', text: r.stderr }], isError: true };
2169
- }
2170
- case 'delete': {
2171
- const cmd = name
2172
- ? `Remove-ItemProperty -Path ${powershellLiteral(regPath)} -Name ${powershellLiteral(name)} -Force`
2173
- : `Remove-Item -Path ${powershellLiteral(regPath)} -Recurse -Force`;
2174
- const r = await spawnBounded(psExe, powershellEncodedArgs(cmd), 10000);
2175
- return r.code === 0 ? ok(`Deleted ${name ? `${regPath}\\${name}` : regPath}`) : { content: [{ type: 'text', text: r.stderr }], isError: true };
2176
- }
2177
- case 'list': {
2178
- const psPath = powershellLiteral(regPath);
2179
- const ps = `Get-Item -Path ${psPath} | Select-Object -ExpandProperty Property; Get-ChildItem -Path ${psPath} -Name`;
2180
- const r = await spawnBounded(psExe, powershellEncodedArgs(ps), 10000);
2181
- return r.code === 0 ? ok(r.stdout.trim() || '(empty)') : { content: [{ type: 'text', text: r.stderr }], isError: true };
2182
- }
2183
- default: return { content: [{ type: 'text', text: `Unknown registry mode: ${mode}` }], isError: true };
2184
- }
2185
- }
2186
- case 'notification': {
2187
- if (!IS_WINDOWS)
2188
- return { content: [{ type: 'text', text: 'notification is Windows-only. Use osascript via run_script on macOS.' }], isError: true };
2189
- const title = str('title');
2190
- const message = str('message');
2191
- const appId = typeof args.app_id === 'string' ? args.app_id : 'Windows.SystemToastNotification';
2192
- const escapedTitle = xmlEscape(title);
2193
- const escapedMessage = xmlEscape(message);
2194
- const ps = `
2195
- [Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] | Out-Null
2196
- [Windows.Data.Xml.Dom.XmlDocument, Windows.Data.Xml.Dom, ContentType = WindowsRuntime] | Out-Null
2197
- $xml = [Windows.Data.Xml.Dom.XmlDocument]::new()
2198
- $xml.LoadXml(${powershellLiteral(`<toast><visual><binding template="ToastText02"><text id="1">${escapedTitle}</text><text id="2">${escapedMessage}</text></binding></visual></toast>`)})
2199
- $toast = [Windows.UI.Notifications.ToastNotification]::new($xml)
2200
- [Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier(${powershellLiteral(appId)}).Show($toast)
2201
- `;
2202
- const r = await spawnBounded(getPowerShellExe(), powershellEncodedArgs(ps), 10000);
2203
- return r.code === 0 ? ok('Notification sent') : { content: [{ type: 'text', text: r.stderr || 'Failed to send notification' }], isError: true };
2204
- }
2205
- case 'multi_select': {
2206
- const target = resolveTarget(args);
2207
- const strategy = getStrategy(tool, args);
2208
- await ensureFocusV4(target, strategy);
2209
- const locs = (Array.isArray(args.locs) ? args.locs : []);
2210
- // Resolve labels to coordinates via find_element
2211
- if (Array.isArray(args.labels) && args.labels.length > 0 && target.windowId) {
2212
- for (const label of args.labels) {
2213
- try {
2214
- const elements = n.findElement(target.windowId, undefined, label, undefined, 1);
2215
- const arr = Array.isArray(elements) ? elements : JSON.parse(JSON.stringify(elements));
2216
- if (arr.length > 0 && arr[0].bounds) {
2217
- const b = arr[0].bounds;
2218
- locs.push([Math.round(b.x + b.width / 2), Math.round(b.y + b.height / 2)]);
2219
- }
2220
- }
2221
- catch { /* label not found */ }
2222
- }
2223
- }
2224
- if (locs.length === 0) {
2225
- return { content: [{ type: 'text', text: 'No coordinates resolved. Provide locs or valid labels.' }], isError: true };
2226
- }
2227
- // Ctrl-click: hold ctrl for the entire sequence
2228
- if (args.press_ctrl) {
2229
- n.keyPress(IS_WINDOWS ? 'ctrl' : 'command'); // press modifier
2230
- }
2231
- for (let i = 0; i < locs.length; i++) {
2232
- const [x, y] = locs[i];
2233
- n.mouseMove(x, y);
2234
- await sleep(50);
2235
- n.mouseClick(x, y, 'left', 1);
2236
- await sleep(30);
2237
- }
2238
- if (target.bundleId)
2239
- updateTargetState(target, 'pointer');
2240
- return ok(`Selected ${locs.length} elements`);
2241
- }
2242
- case 'multi_edit': {
2243
- const target = resolveTarget(args);
2244
- const strategy = getStrategy(tool, args);
2245
- await ensureFocusV4(target, strategy);
2246
- const locs = (Array.isArray(args.locs) ? args.locs : []);
2247
- // Resolve labels to coordinates via find_element
2248
- if (Array.isArray(args.labels) && args.labels.length > 0 && target.windowId) {
2249
- for (const [label, text] of args.labels) {
2250
- try {
2251
- const elements = n.findElement(target.windowId, undefined, label, undefined, 1);
2252
- const arr = Array.isArray(elements) ? elements : JSON.parse(JSON.stringify(elements));
2253
- if (arr.length > 0 && arr[0].bounds) {
2254
- const b = arr[0].bounds;
2255
- locs.push([Math.round(b.x + b.width / 2), Math.round(b.y + b.height / 2), text]);
2256
- }
2257
- }
2258
- catch { /* label not found */ }
2259
- }
2260
- }
2261
- if (locs.length === 0) {
2262
- return { content: [{ type: 'text', text: 'No coordinates resolved. Provide locs or valid labels.' }], isError: true };
2263
- }
2264
- for (const [x, y, text] of locs) {
2265
- n.mouseMove(x, y);
2266
- await sleep(50);
2267
- n.mouseClick(x, y, 'left', 1);
2268
- await sleep(50);
2269
- n.typeText(text);
2270
- await sleep(30);
2271
- }
2272
- if (target.bundleId)
2273
- updateTargetState(target, 'keyboard');
2274
- return ok(`Edited ${locs.length} fields`);
2275
- }
2276
- case 'scrape': {
2277
- const url = str('url');
2278
- if (args.use_dom) {
2279
- return { content: [{ type: 'text', text: 'use_dom mode requires a browser tab open with the URL. This feature is not yet implemented.' }], isError: true };
2280
- }
2281
- try {
2282
- const resp = await fetch(url, {
2283
- headers: { 'User-Agent': 'computer-use-mcp/6.1.0' },
2284
- signal: AbortSignal.timeout(15000),
2285
- });
2286
- if (!resp.ok) {
2287
- return { content: [{ type: 'text', text: `HTTP ${resp.status}: ${resp.statusText}` }], isError: true };
2288
- }
2289
- const html = await resp.text();
2290
- // Simple HTML-to-text: strip tags, decode entities, collapse whitespace
2291
- const text = html
2292
- .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
2293
- .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
2294
- .replace(/<[^>]+>/g, ' ')
2295
- .replace(/&nbsp;/g, ' ')
2296
- .replace(/&amp;/g, '&')
2297
- .replace(/&lt;/g, '<')
2298
- .replace(/&gt;/g, '>')
2299
- .replace(/&quot;/g, '"')
2300
- .replace(/&#39;/g, "'")
2301
- .replace(/\s+/g, ' ')
2302
- .trim();
2303
- const truncated = text.length > 8000 ? text.slice(0, 8000) + '...' : text;
2304
- return ok(`URL: ${url}\nContent:\n${truncated}`);
2305
- }
2306
- catch (e) {
2307
- const msg = e instanceof Error ? e.message : String(e);
2308
- return { content: [{ type: 'text', text: `Scrape failed: ${msg}` }], isError: true };
2309
- }
2310
- }
2311
- default:
2312
- return { content: [{ type: 'text', text: `Unknown tool: ${tool}` }], isError: true };
2313
- }
2314
- })();
2315
- }
2316
- catch (err) {
2317
- if (err instanceof FocusError) {
2318
- result = { content: [{ type: 'text', text: focusFailureText(err.details) }], isError: true };
2319
- }
2320
- else if (err instanceof WindowNotFoundError) {
2321
- // For input tools with invalid target_window_id, return FocusFailure
2322
- const front = n.getFrontmostApp();
2323
- const failure = {
2324
- error: 'focus_failed',
2325
- requestedBundleId: '',
2326
- requestedWindowId: err.windowId,
2327
- frontmostBefore: front?.bundleId ?? null,
2328
- frontmostAfter: front?.bundleId ?? null,
2329
- targetRunning: false,
2330
- targetHidden: false,
2331
- targetWindowVisible: false,
2332
- activationAttempted: false,
2333
- suggestedRecovery: 'open_application',
2334
- };
2335
- result = { content: [{ type: 'text', text: focusFailureText(failure) }], isError: true };
2336
- }
2337
- else {
2338
- const msg = err instanceof Error ? err.message : String(err);
2339
- result = { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
2340
- }
2341
- }
2342
- finally {
2343
- if (acquired)
2344
- lockPump.release();
2345
- }
2346
- // v5.2: decorate the response with hiddenBundleIds when prepare_display
2347
- // ran. We attach to both the text payload (parseable by agents) and as
2348
- // a top-level field (for strictly-typed callers who care).
2349
- if (pendingHiddenBundleIds != null) {
2350
- result = decorateWithHiddenBundleIds(result, pendingHiddenBundleIds);
2351
- }
2352
- return result;
2353
- }
2354
- return { dispatch };
2355
- }
2356
- function ok(text) {
2357
- return { content: [{ type: 'text', text }] };
2358
- }
2359
- /**
2360
- * Append `hiddenBundleIds` metadata to a dispatch result when the caller
2361
- * used `focus_strategy: "prepare_display"`. We add a trailing text block so
2362
- * the original payload is untouched — agents can parse either block.
2363
- */
2364
- function decorateWithHiddenBundleIds(r, hidden) {
2365
- return {
2366
- ...r,
2367
- content: [
2368
- ...r.content,
2369
- { type: 'text', text: JSON.stringify({ hiddenBundleIds: hidden }) },
2370
- ],
2371
- };
2372
- }