@frontendxlab/opencode-computer-use 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +168 -25
- package/SECURITY.md +6 -0
- package/THIRD_PARTY_NOTICES.md +4 -0
- package/bin/opencode-computer-use-permissions.js +221 -0
- package/bin/opencode-computer-use-repl.js +25 -3
- package/bin/opencode-computer-use-setup.js +113 -95
- package/docs/NATIVE_VALIDATION.md +102 -0
- package/docs/index.html +577 -40
- package/docs/llms-full.txt +146 -31
- package/docs/llms.txt +8 -2
- package/docs/sitemap.xml +1 -1
- package/docs/webmcp.js +2 -2
- package/lib/backend.js +50 -14
- package/lib/json-line-peer.mjs +3 -0
- package/lib/sandbox.mjs +51 -4
- package/package.json +11 -8
package/docs/index.html
CHANGED
|
@@ -61,9 +61,9 @@
|
|
|
61
61
|
"name": "OpenCode Computer Use",
|
|
62
62
|
"description": "A dual-generation OpenCode plugin for local desktop control through a persistent cua_repl JavaScript session.",
|
|
63
63
|
"url": "https://opencode-computer-use.pages.dev/",
|
|
64
|
-
"softwareVersion": "0.1.
|
|
64
|
+
"softwareVersion": "0.1.5",
|
|
65
65
|
"datePublished": "2026-09-25",
|
|
66
|
-
"dateModified": "2026-
|
|
66
|
+
"dateModified": "2026-10-30",
|
|
67
67
|
"isAccessibleForFree": true,
|
|
68
68
|
"downloadUrl": "https://www.npmjs.com/package/@frontendxlab/opencode-computer-use",
|
|
69
69
|
"applicationCategory": "DeveloperApplication",
|
|
@@ -93,7 +93,7 @@
|
|
|
93
93
|
"headline": "OpenCode Computer Use: Codex-style desktop control for OpenCode",
|
|
94
94
|
"description": "Setup, API, security, operating-system, and troubleshooting guidance for the OpenCode Computer Use plugin.",
|
|
95
95
|
"datePublished": "2026-09-25",
|
|
96
|
-
"dateModified": "2026-
|
|
96
|
+
"dateModified": "2026-10-30",
|
|
97
97
|
"url": "https://opencode-computer-use.pages.dev/",
|
|
98
98
|
"inLanguage": "en",
|
|
99
99
|
"articleSection": "Developer tools",
|
|
@@ -146,7 +146,15 @@
|
|
|
146
146
|
"name": "Which operating systems are supported?",
|
|
147
147
|
"acceptedAnswer": {
|
|
148
148
|
"@type": "Answer",
|
|
149
|
-
"text": "
|
|
149
|
+
"text": "The optional native package includes Linux, macOS, and Windows binaries for x64 and arm64. Linux x64 GNOME Wayland smoke checks verified registration, inventory, persistent sessions, and semantic click/setValue on a disposable GTK3 app; capture was unavailable. Other apps and capabilities need separate checks, and macOS, Windows, and Linux arm64 still need native validation."
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"@type": "Question",
|
|
154
|
+
"name": "Can it control another device?",
|
|
155
|
+
"acceptedAnswer": {
|
|
156
|
+
"@type": "Answer",
|
|
157
|
+
"text": "It controls the signed-in desktop on the computer running the OpenCode server and its local MCP child. Connecting from another browser or device does not switch the controlled desktop. Install and configure the package on each target computer; automatic device discovery, pairing, and remote desktop transport are not built in."
|
|
150
158
|
}
|
|
151
159
|
},
|
|
152
160
|
{
|
|
@@ -336,6 +344,29 @@
|
|
|
336
344
|
.matrix td { color: var(--muted); }
|
|
337
345
|
.yes { color: var(--mint); font-weight: 700; }
|
|
338
346
|
.pending { color: var(--signal); font-weight: 700; }
|
|
347
|
+
.capability-intro { max-width: 690px; }
|
|
348
|
+
.capability-legend { display: flex; flex-wrap: wrap; gap: 8px; margin: 17px 0 14px; }
|
|
349
|
+
.capability-legend .legend-item { display: inline-flex; align-items: center; gap: 7px; padding: 6px 9px; border: 1px solid var(--line); border-radius: 999px; color: var(--muted); background: var(--panel); font: 12px var(--mono); }
|
|
350
|
+
.capability-scroll { overflow-x: auto; margin: 16px 0 20px; border: 1px solid var(--line); border-radius: 12px; background: var(--panel); }
|
|
351
|
+
.capability-matrix { width: 100%; min-width: 760px; border-collapse: separate; border-spacing: 0; font-size: 13px; }
|
|
352
|
+
.capability-matrix th, .capability-matrix td { padding: 11px 12px; border-bottom: 1px solid var(--line); vertical-align: middle; }
|
|
353
|
+
.capability-matrix thead th { position: sticky; top: 0; z-index: 2; color: var(--text); background: color-mix(in srgb, var(--panel-2) 96%, transparent); font: 700 11px/1.2 var(--mono); letter-spacing: .06em; text-transform: uppercase; }
|
|
354
|
+
.capability-matrix thead th:not(:first-child):not(:last-child) { width: 88px; text-align: center; }
|
|
355
|
+
.capability-matrix th:first-child, .capability-matrix td:first-child { position: sticky; left: 0; z-index: 1; width: 245px; background: var(--panel); }
|
|
356
|
+
.capability-matrix thead th:first-child { z-index: 3; background: var(--panel-2); }
|
|
357
|
+
.capability-matrix td:nth-child(2), .capability-matrix td:nth-child(3), .capability-matrix td:nth-child(4) { text-align: center; }
|
|
358
|
+
.capability-matrix td:last-child { min-width: 250px; color: var(--muted); line-height: 1.45; }
|
|
359
|
+
.capability-matrix tr:last-child td { border-bottom: 0; }
|
|
360
|
+
.capability-matrix .cap-group th { position: static; padding: 9px 12px; border-bottom: 1px solid var(--line); color: var(--signal-2); background: color-mix(in srgb, var(--signal) 8%, var(--panel)); font: 750 11px/1.2 var(--mono); letter-spacing: .08em; text-align: left; text-transform: uppercase; }
|
|
361
|
+
.cap-name { display: block; color: var(--text); font-weight: 700; line-height: 1.3; }
|
|
362
|
+
.cap-code { display: block; margin-top: 3px; color: var(--faint); font: 11px/1.35 var(--mono); }
|
|
363
|
+
.cap-status { display: inline-flex; width: 30px; height: 30px; align-items: center; justify-content: center; border: 1px solid currentColor; border-radius: 999px; }
|
|
364
|
+
.cap-status svg, .legend-item svg { width: 15px; height: 15px; fill: none; stroke: currentColor; stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; }
|
|
365
|
+
.cap-status.supported, .legend-item.supported { color: var(--mint); }
|
|
366
|
+
.cap-status.conditional, .legend-item.conditional { color: var(--signal-2); }
|
|
367
|
+
.cap-status.blocked, .legend-item.blocked { color: var(--danger); }
|
|
368
|
+
.capability-footnote { padding: 14px 16px; border: 1px solid var(--line); border-radius: 10px; color: var(--muted); background: color-mix(in srgb, var(--panel-2) 82%, transparent); font-size: 13px; }
|
|
369
|
+
.capability-footnote strong { color: var(--text); }
|
|
339
370
|
.faq-list { margin-top: 18px; border-top: 1px solid var(--line); }
|
|
340
371
|
.faq-list > div { padding: 14px 0; border-bottom: 1px solid var(--line); }
|
|
341
372
|
.faq-list h3 { margin: 0 0 5px; }
|
|
@@ -370,6 +401,7 @@
|
|
|
370
401
|
h1 { font-size: 3.45rem; }
|
|
371
402
|
.cards, .builder .row { grid-template-columns: 1fr; }
|
|
372
403
|
.definition-list > div { grid-template-columns: 1fr; gap: 5px; }
|
|
404
|
+
.capability-scroll { margin-inline: -2px; }
|
|
373
405
|
.console { box-shadow: var(--shadow-md); }
|
|
374
406
|
footer { display: block; }
|
|
375
407
|
footer span { display: block; margin-top: 8px; }
|
|
@@ -381,6 +413,11 @@
|
|
|
381
413
|
</style>
|
|
382
414
|
</head>
|
|
383
415
|
<body>
|
|
416
|
+
<svg class="sr" aria-hidden="true" focusable="false">
|
|
417
|
+
<symbol id="lucide-circle-check" viewBox="0 0 24 24"><path d="M22 11.08V12a10 10 0 1 1-5.93-9.14"></path><path d="m9 11 3 3L22 4"></path></symbol>
|
|
418
|
+
<symbol id="lucide-triangle-alert" viewBox="0 0 24 24"><path d="m21.73 18-8-14a2 2 0 0 0-3.46 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3"></path><path d="M12 9v4"></path><path d="M12 17h.01"></path></symbol>
|
|
419
|
+
<symbol id="lucide-circle-x" viewBox="0 0 24 24"><circle cx="12" cy="12" r="10"></circle><path d="m15 9-6 6"></path><path d="m9 9 6 6"></path></symbol>
|
|
420
|
+
</svg>
|
|
384
421
|
<a class="skip" href="#content">Skip to documentation</a>
|
|
385
422
|
<header class="top">
|
|
386
423
|
<div class="top-inner">
|
|
@@ -389,6 +426,7 @@
|
|
|
389
426
|
<a href="#install">Install</a>
|
|
390
427
|
<a href="#first-run">First run</a>
|
|
391
428
|
<a href="#api">API</a>
|
|
429
|
+
<a href="#capabilities">Capabilities</a>
|
|
392
430
|
<a href="#security">Security</a>
|
|
393
431
|
<a href="https://github.com/rashidtvmr/opencode-computer-use">GitHub</a>
|
|
394
432
|
</nav>
|
|
@@ -425,6 +463,7 @@
|
|
|
425
463
|
<a href="#first-run">First run</a>
|
|
426
464
|
<a href="#architecture">Architecture</a>
|
|
427
465
|
<a href="#api">API surface</a>
|
|
466
|
+
<a href="#capabilities">Capabilities</a>
|
|
428
467
|
<a href="#webmcp">WebMCP</a>
|
|
429
468
|
<a href="#options">Options</a>
|
|
430
469
|
<a href="#workflow">Workflow</a>
|
|
@@ -435,7 +474,7 @@
|
|
|
435
474
|
</nav>
|
|
436
475
|
|
|
437
476
|
<article>
|
|
438
|
-
<details class="mobile-toc"><summary>On this page</summary><p><a href="#quick-answer">Quick answer</a> · <a href="#install">Install</a> · <a href="#first-run">First run</a> · <a href="#api">API</a> · <a href="#webmcp">WebMCP</a> · <a href="#security">Security</a> · <a href="#faq">FAQ</a></p></details>
|
|
477
|
+
<details class="mobile-toc"><summary>On this page</summary><p><a href="#quick-answer">Quick answer</a> · <a href="#install">Install</a> · <a href="#first-run">First run</a> · <a href="#api">API</a> · <a href="#capabilities">Capabilities</a> · <a href="#webmcp">WebMCP</a> · <a href="#security">Security</a> · <a href="#faq">FAQ</a></p></details>
|
|
439
478
|
|
|
440
479
|
<section id="quick-answer" data-section>
|
|
441
480
|
<h2>Short answer.</h2>
|
|
@@ -445,7 +484,8 @@
|
|
|
445
484
|
<div><dt>MCP server</dt><dd>The local stdio server named <code>cua_repl</code>.</dd></div>
|
|
446
485
|
<div><dt>Model tools</dt><dd><code>js</code> and <code>js_reset</code>, not raw desktop tools.</dd></div>
|
|
447
486
|
<div><dt>Native runtime</dt><dd><code>open-computer-use</code>, running separately with OS accessibility and capture permissions.</dd></div>
|
|
448
|
-
<div><dt>
|
|
487
|
+
<div><dt>Controlled desktop</dt><dd>The signed-in desktop on the OpenCode/MCP server's computer, including when its client connects from another device.</dd></div>
|
|
488
|
+
<div><dt>Packaged hosts</dt><dd>Linux, macOS, and Windows on x64 and arm64. Native permissions, desktop-session access, and app compatibility need validation on each host.</dd></div>
|
|
449
489
|
</dl>
|
|
450
490
|
</section>
|
|
451
491
|
|
|
@@ -454,18 +494,29 @@
|
|
|
454
494
|
<p>This repository is a standalone OpenCode plugin. It contains the plugin entrypoint, the local MCP server, and the native runtime integration. The plugin registers <code>cua_repl</code>; the model sees only <code>js</code> and <code>js_reset</code>.</p>
|
|
455
495
|
<div class="code"><pre><code id="clone">git clone git@github.com:rashidtvmr/opencode-computer-use.git
|
|
456
496
|
cd opencode-computer-use
|
|
457
|
-
npm
|
|
497
|
+
npm ci --ignore-scripts
|
|
498
|
+
node bin/opencode-computer-use-setup.js --local --v1 --source .
|
|
499
|
+
node bin/opencode-computer-use-permissions.js --strict
|
|
500
|
+
opencode mcp list</code></pre><button class="copy" type="button" data-copy="#clone">Copy</button></div>
|
|
501
|
+
<p><strong>A clone is not registered automatically.</strong> Install its dependencies, then run setup with <code>--source .</code> to load this checkout. Use <code>--v2</code> for OpenCode2. Use <code>--global</code> instead of <code>--local</code> to load the checkout in other projects too. Setup resolves the source to an absolute entrypoint; moving the checkout requires rerunning setup. Restart the relevant OpenCode process after changing its configuration.</p>
|
|
458
502
|
<p>For a published-package install, use:</p>
|
|
503
|
+
<p class="note"><strong>Documentation version: 0.1.5.</strong> Check <code>npm view @frontendxlab/opencode-computer-use version</code> to see the registry release. Published-package commands use that version; the checkout instructions let you test source changes before publication.</p>
|
|
459
504
|
<div class="code"><pre><code id="npm-install">npm install @frontendxlab/opencode-computer-use
|
|
460
505
|
# pnpm install @frontendxlab/opencode-computer-use also works</code></pre><button class="copy" type="button" data-copy="#npm-install">Copy</button></div>
|
|
461
|
-
<p>The package postinstall step detects OpenCode 1 versus OpenCode2 and local versus global installation. It edits JSONC without discarding comments
|
|
506
|
+
<p>The package postinstall step detects OpenCode 1 versus OpenCode2 and local versus global installation. It edits JSONC without discarding comments and avoids duplicate entries. Global setup also edits the selected user's JSONC directly. If detection is ambiguous, automatic installation leaves the configuration unchanged and prints both manual formats. Explicit setup failures return a nonzero exit status; choose the generation and scope below to resolve ambiguity.</p>
|
|
507
|
+
<div class="code"><pre><code id="setup-explicit"># Works from any directory; select the scoped package explicitly
|
|
508
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-setup --local --v1
|
|
509
|
+
# Global setup, or replace --v1 with --v2 for OpenCode2
|
|
510
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-setup --global --v1</code></pre><button class="copy" type="button" data-copy="#setup-explicit">Copy</button></div>
|
|
511
|
+
<p>Add <code>--dry-run</code> to preview the configuration change. The shorthand <code>npx opencode-computer-use-setup</code> is suitable when that binary is already available in the installed project; it does not identify the scoped package for a fresh download. In a clone, use the <code>node bin/...</code> commands above. Setup skips automatic package-development and CI installs.</p>
|
|
462
512
|
<p>Recent pnpm releases can require explicit lifecycle-script approval. If pnpm reports <code>ERR_PNPM_IGNORED_BUILDS</code>, run <code>pnpm approve-builds @frontendxlab/opencode-computer-use</code> once, then rerun <code>pnpm install</code>. If scripts are disabled, run <code>pnpm exec opencode-computer-use-setup</code> manually. CI environments skip setup automatically.</p>
|
|
513
|
+
<p><strong>The v0.1.5 checkout includes install-time desktop onboarding.</strong> When package lifecycle scripts are allowed, <code>postinstall</code> first configures OpenCode and then runs <code>opencode-computer-use-permissions --install</code>. On macOS it runs the native doctor and opens the Open Computer Use permission onboarding when Accessibility or Screen Recording is missing; Apple still requires you to approve the protected toggles. On GNOME Linux it best-effort enables toolkit accessibility before the AT-SPI/session check. Windows provides interactive-session guidance. Set <code>OPENCODE_COMPUTER_USE_SKIP_PERMISSIONS=1</code> to opt out.</p>
|
|
463
514
|
<div class="tabs" role="tablist" aria-label="OpenCode generation">
|
|
464
515
|
<button role="tab" id="tab-v1" aria-controls="panel-v1" aria-selected="true">OpenCode 1</button>
|
|
465
516
|
<button role="tab" id="tab-v2" aria-controls="panel-v2" aria-selected="false" tabindex="-1">OpenCode2</button>
|
|
466
517
|
</div>
|
|
467
518
|
<div id="panel-v1" role="tabpanel" aria-labelledby="tab-v1">
|
|
468
|
-
<p>
|
|
519
|
+
<p>The published package uses the V1 <code>plugin</code> list below. For a checkout, use setup with <code>--source .</code> so OpenCode loads its actual entrypoint.</p>
|
|
469
520
|
<div class="code"><pre><code id="config-v1">{
|
|
470
521
|
"$schema": "https://opencode.ai/config.json",
|
|
471
522
|
"plugin": [
|
|
@@ -476,7 +527,7 @@ npm install</code></pre><button class="copy" type="button" data-copy="#clone">Co
|
|
|
476
527
|
<div class="code"><pre><code id="verify">opencode mcp list</code></pre><button class="copy" type="button" data-copy="#verify">Copy</button></div>
|
|
477
528
|
</div>
|
|
478
529
|
<div id="panel-v2" role="tabpanel" aria-labelledby="tab-v2" hidden>
|
|
479
|
-
<p>
|
|
530
|
+
<p>The published package uses the V2 <code>plugins</code> shape below. For a checkout, use setup with <code>--v2 --source .</code>.</p>
|
|
480
531
|
<div class="code"><pre><code id="config-v2">{
|
|
481
532
|
"$schema": "https://opencode.ai/config.json",
|
|
482
533
|
"plugins": [
|
|
@@ -499,7 +550,7 @@ var chrome = await cua.getApp("Google Chrome");
|
|
|
499
550
|
await chrome.getAXState({ textLimit: 12000 });</code></pre><button class="copy" type="button" data-copy="#discovery">Copy</button></div>
|
|
500
551
|
<div class="builder" aria-label="Build a first cua_repl call">
|
|
501
552
|
<div class="row">
|
|
502
|
-
<label for="app-name">App name or
|
|
553
|
+
<label for="app-name">App name or ID<input id="app-name" type="text" value="Google Chrome" spellcheck="false"></label>
|
|
503
554
|
<fieldset><legend>Action</legend><div class="segments" role="radiogroup" aria-label="Action">
|
|
504
555
|
<label><input type="radio" name="action-kind" value="state" checked>Read state</label>
|
|
505
556
|
<label><input type="radio" name="action-kind" value="screenshot">Screenshot</label>
|
|
@@ -518,6 +569,9 @@ await chrome.getAXState({ textLimit: 12000 });</code></pre><button class="copy"
|
|
|
518
569
|
<li><strong>cua_repl MCP server</strong><br>Provides a persistent JavaScript evaluation surface, output bounds, request validation, and worker isolation.</li>
|
|
519
570
|
<li><strong>Native runtime</strong><br>Runs <code>open-computer-use</code> as a separate child process. OS accessibility, screen capture, and input permissions remain there.</li>
|
|
520
571
|
</ol>
|
|
572
|
+
<h3>Which device is controlled?</h3>
|
|
573
|
+
<p>The plugin launches its native runtime on the same computer as the OpenCode server. An attached terminal or browser on another laptop or phone still controls that server computer's desktop. The adapter has no built-in device pairing, remote URL, or cross-device discovery.</p>
|
|
574
|
+
<p>Install dependencies and configure the plugin separately on each target desktop, then grant that host's permissions in its signed-in graphical session. Copying a configuration with another machine's absolute paths, or copying <code>node_modules</code> between operating systems, does not provide that setup. A service, container, WSL instance, or detached SSH session also does not automatically inherit the host user's graphical desktop.</p>
|
|
521
575
|
<div class="note">The JavaScript VM is a capability guard, not a complete security boundary. OpenCode approvals and operating-system permissions remain the enforcement layer.</div>
|
|
522
576
|
</section>
|
|
523
577
|
|
|
@@ -525,19 +579,318 @@ await chrome.getAXState({ textLimit: 12000 });</code></pre><button class="copy"
|
|
|
525
579
|
<h2>The Codex-shaped API.</h2>
|
|
526
580
|
<p>The model receives a small async API inside persistent JavaScript:</p>
|
|
527
581
|
<ul class="cards">
|
|
528
|
-
<li><strong>cua.getState(options?)</strong><span>
|
|
529
|
-
<li><strong>cua.listApps(options?)</strong><span>Return
|
|
530
|
-
<li><strong>cua.getApp(
|
|
582
|
+
<li><strong>cua.getState(options?)</strong><span>Return the permitted app inventory as { apps } and emit it by default. Use { emit: false } to suppress inventory output.</span></li>
|
|
583
|
+
<li><strong>cua.listApps(options?)</strong><span>Return the permitted app records as an array and emit them by default. Use { emit: false } to suppress inventory output.</span></li>
|
|
584
|
+
<li><strong>cua.getApp(nameOrID)</strong><span>Bind an app and emit its initial accessibility state.</span></li>
|
|
531
585
|
<li><strong>app.getAXState(options?)</strong><span>Read a fresh semantic tree before choosing an element index.</span></li>
|
|
532
586
|
<li><strong>app.getScreenshot(options?)</strong><span>Return local image bytes and optionally emit them with nodeRepl.</span></li>
|
|
587
|
+
<li><strong>app.getAXStateAndScreenshot(options?)</strong><span>Read accessibility state and any available screenshot together.</span></li>
|
|
533
588
|
<li><strong>app.click(target, options?)</strong><span>Use an element index first, or a bounded coordinate when necessary.</span></li>
|
|
589
|
+
<li><strong>app.scroll(elementIndex, direction, pages?)</strong><span>Scroll an accessible element using its current snapshot index.</span></li>
|
|
590
|
+
<li><strong>app.drag(from, to)</strong><span>Drag between coordinate pairs when the native desktop supports it.</span></li>
|
|
534
591
|
<li><strong>app.typeText(text)</strong><span>Send literal text through the native runtime.</span></li>
|
|
535
592
|
<li><strong>app.pressKey(key)</strong><span>Send a named key or supported modifier chord.</span></li>
|
|
593
|
+
<li><strong>app.setValue(elementIndex, value)</strong><span>Set the value of an accessible control when its native interface supports it.</span></li>
|
|
536
594
|
<li><strong>app.performSecondaryAction(index, action)</strong><span>Invoke an advertised accessibility action such as a context action.</span></li>
|
|
537
595
|
</ul>
|
|
538
596
|
<p>Use <code>nodeRepl.write(value)</code> for extra text and <code>await nodeRepl.emitImage(bytesOrDataUrl)</code> for local images. Both are bounded before crossing the worker boundary.</p>
|
|
539
597
|
</section>
|
|
540
598
|
|
|
599
|
+
<section id="capabilities" data-section>
|
|
600
|
+
<h2>What it can — and cannot — do.</h2>
|
|
601
|
+
<p class="capability-intro">This is the complete model-facing computer-use surface of the current <code>cua_repl</code> adapter, plus its intentional exclusions. A green check means the adapter/native runtime exposes that path; it does <strong>not</strong> mean every application or desktop session will accept it.</p>
|
|
602
|
+
<div class="capability-legend" aria-label="Capability status legend">
|
|
603
|
+
<span class="legend-item supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg> Supported</span>
|
|
604
|
+
<span class="legend-item conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg> Conditional</span>
|
|
605
|
+
<span class="legend-item blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg> Not exposed / blocked</span>
|
|
606
|
+
</div>
|
|
607
|
+
<div class="capability-scroll" role="region" aria-label="Computer use capabilities by operating system" tabindex="0">
|
|
608
|
+
<table class="capability-matrix">
|
|
609
|
+
<thead><tr><th scope="col">Capability</th><th scope="col">macOS</th><th scope="col">Linux</th><th scope="col">Windows</th><th scope="col">What that means</th></tr></thead>
|
|
610
|
+
<tbody>
|
|
611
|
+
<tr class="cap-group"><th colspan="5">Observe & understand</th></tr>
|
|
612
|
+
<tr>
|
|
613
|
+
<td><span class="cap-name">List visible applications</span><span class="cap-code">cua.listApps()</span></td>
|
|
614
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
615
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
616
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
617
|
+
<td>Discovers apps visible to the native accessibility runtime in the signed-in desktop session.</td>
|
|
618
|
+
</tr>
|
|
619
|
+
<tr>
|
|
620
|
+
<td><span class="cap-name">Read desktop / app inventory</span><span class="cap-code">cua.getState()</span></td>
|
|
621
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
622
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
623
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
624
|
+
<td>Returns the current structured app inventory with bounded output.</td>
|
|
625
|
+
</tr>
|
|
626
|
+
<tr>
|
|
627
|
+
<td><span class="cap-name">Bind an existing application</span><span class="cap-code">cua.getApp(nameOrID)</span></td>
|
|
628
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
629
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
630
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
631
|
+
<td>Binds an already running or visible app. It does not launch a new application.</td>
|
|
632
|
+
</tr>
|
|
633
|
+
<tr>
|
|
634
|
+
<td><span class="cap-name">Read accessibility / UI tree</span><span class="cap-code">app.getAXState()</span></td>
|
|
635
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
636
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
637
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
638
|
+
<td>macOS uses Accessibility, Linux AT-SPI, and Windows UI Automation. Some Wayland/app combinations expose incomplete trees.</td>
|
|
639
|
+
</tr>
|
|
640
|
+
<tr>
|
|
641
|
+
<td><span class="cap-name">Capture an app screenshot</span><span class="cap-code">app.getScreenshot()</span></td>
|
|
642
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
643
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
644
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
645
|
+
<td>Depends on capture permission plus the active compositor/capture backend.</td>
|
|
646
|
+
</tr>
|
|
647
|
+
<tr>
|
|
648
|
+
<td><span class="cap-name">Read state + screenshot together</span><span class="cap-code">app.getAXStateAndScreenshot()</span></td>
|
|
649
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
650
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
651
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
652
|
+
<td>Combines semantic state and image capture; state can still be useful when capture is unavailable.</td>
|
|
653
|
+
</tr>
|
|
654
|
+
<tr>
|
|
655
|
+
<td><span class="cap-name">Keep variables and app bindings</span><span class="cap-code">persistent js session</span></td>
|
|
656
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
657
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
658
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
659
|
+
<td>Top-level variables and app bindings survive across model turns until js_reset or a timeout reset.</td>
|
|
660
|
+
</tr>
|
|
661
|
+
<tr>
|
|
662
|
+
<td><span class="cap-name">Emit text and images to the model</span><span class="cap-code">nodeRepl.write / emitImage</span></td>
|
|
663
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
664
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
665
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
666
|
+
<td>Text and image output are bounded before crossing the worker/MCP boundary.</td>
|
|
667
|
+
</tr>
|
|
668
|
+
<tr class="cap-group"><th colspan="5">Interact with an app</th></tr>
|
|
669
|
+
<tr>
|
|
670
|
+
<td><span class="cap-name">Click a semantic element</span><span class="cap-code">app.click(elementIndex)</span></td>
|
|
671
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
672
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
673
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
674
|
+
<td>Preferred input path. Requires a current actionable accessibility element.</td>
|
|
675
|
+
</tr>
|
|
676
|
+
<tr>
|
|
677
|
+
<td><span class="cap-name">Click a coordinate</span><span class="cap-code">app.click([x, y])</span></td>
|
|
678
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
679
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
680
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
681
|
+
<td>Use when semantic state is insufficient. System/global-pointer fallback is separately gated.</td>
|
|
682
|
+
</tr>
|
|
683
|
+
<tr>
|
|
684
|
+
<td><span class="cap-name">Type literal text</span><span class="cap-code">app.typeText(text)</span></td>
|
|
685
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
686
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
687
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
688
|
+
<td>Requires a writable/focused target and an input path accepted by the host desktop.</td>
|
|
689
|
+
</tr>
|
|
690
|
+
<tr>
|
|
691
|
+
<td><span class="cap-name">Press keys / supported chords</span><span class="cap-code">app.pressKey(key)</span></td>
|
|
692
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
693
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
694
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
695
|
+
<td>Host input restrictions, focus, and compositor behavior still apply.</td>
|
|
696
|
+
</tr>
|
|
697
|
+
<tr>
|
|
698
|
+
<td><span class="cap-name">Scroll an accessibility element</span><span class="cap-code">app.scroll(index, direction, pages)</span></td>
|
|
699
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
700
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
701
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
702
|
+
<td>Semantic element scrolling is exposed. Coordinate-only scrolling is intentionally rejected.</td>
|
|
703
|
+
</tr>
|
|
704
|
+
<tr>
|
|
705
|
+
<td><span class="cap-name">Drag between coordinates</span><span class="cap-code">app.drag([x1,y1], [x2,y2])</span></td>
|
|
706
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
707
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
708
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
709
|
+
<td>Coordinate/input-backend dependent; verify the resulting UI state rather than assuming success.</td>
|
|
710
|
+
</tr>
|
|
711
|
+
<tr>
|
|
712
|
+
<td><span class="cap-name">Set an accessibility value</span><span class="cap-code">app.setValue(index, value)</span></td>
|
|
713
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
714
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
715
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
716
|
+
<td>Only works when the target element/backend advertises a writable accessibility value.</td>
|
|
717
|
+
</tr>
|
|
718
|
+
<tr>
|
|
719
|
+
<td><span class="cap-name">Run a secondary accessibility action</span><span class="cap-code">app.performSecondaryAction()</span></td>
|
|
720
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
721
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
722
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
723
|
+
<td>Only actions advertised by the accessibility element/backend can be invoked.</td>
|
|
724
|
+
</tr>
|
|
725
|
+
<tr>
|
|
726
|
+
<td><span class="cap-name">Use global pointer fallback</span><span class="cap-code">allowGlobalPointerFallbacks</span></td>
|
|
727
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
728
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
729
|
+
<td><span class="cap-status conditional" aria-label="Conditional" title="Conditional"><svg aria-hidden="true"><use href="#lucide-triangle-alert"></use></svg></span></td>
|
|
730
|
+
<td>Off by default. Must be explicitly enabled because it can move the real pointer and change focus.</td>
|
|
731
|
+
</tr>
|
|
732
|
+
<tr class="cap-group"><th colspan="5">Session, safety & recovery</th></tr>
|
|
733
|
+
<tr>
|
|
734
|
+
<td><span class="cap-name">Run a read-only desktop session</span><span class="cap-code">safetyMode: read-only</span></td>
|
|
735
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
736
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
737
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
738
|
+
<td>Mutating CUA methods are blocked by the adapter before native execution.</td>
|
|
739
|
+
</tr>
|
|
740
|
+
<tr>
|
|
741
|
+
<td><span class="cap-name">Allow / deny specific applications</span><span class="cap-code">appAccess</span></td>
|
|
742
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
743
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
744
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
745
|
+
<td>Exact case-insensitive app name/ID rules are enforced before binding and actions.</td>
|
|
746
|
+
</tr>
|
|
747
|
+
<tr>
|
|
748
|
+
<td><span class="cap-name">Reset JavaScript bindings</span><span class="cap-code">js_reset</span></td>
|
|
749
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
750
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
751
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
752
|
+
<td>Clears REPL variables and bindings without closing apps or resetting their native state.</td>
|
|
753
|
+
</tr>
|
|
754
|
+
<tr>
|
|
755
|
+
<td><span class="cap-name">Recover from CPU-bound JavaScript</span><span class="cap-code">worker timeout + restart</span></td>
|
|
756
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
757
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
758
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
759
|
+
<td>A wedged JS evaluation is terminated in its worker and the JavaScript session is restarted.</td>
|
|
760
|
+
</tr>
|
|
761
|
+
<tr>
|
|
762
|
+
<td><span class="cap-name">Cancel timed-out native requests</span><span class="cap-code">notifications/cancelled</span></td>
|
|
763
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
764
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
765
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
766
|
+
<td>Cancellation is best-effort. After an uncertain mutation, observe state before acting again.</td>
|
|
767
|
+
</tr>
|
|
768
|
+
<tr>
|
|
769
|
+
<td><span class="cap-name">Clean up at session end</span><span class="cap-code">notifications/turn-ended</span></td>
|
|
770
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
771
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
772
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
773
|
+
<td>Shutdown sends best-effort lifecycle cleanup before the native child is closed.</td>
|
|
774
|
+
</tr>
|
|
775
|
+
<tr>
|
|
776
|
+
<td><span class="cap-name">Re-emit CUA usage guidance</span><span class="cap-code">cua.rewriteDocumentation()</span></td>
|
|
777
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
778
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
779
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
780
|
+
<td>Reprints the bounded API guidance inside the persistent JavaScript session.</td>
|
|
781
|
+
</tr>
|
|
782
|
+
<tr>
|
|
783
|
+
<td><span class="cap-name">Use safe JavaScript timers</span><span class="cap-code">setTimeout / clearTimeout</span></td>
|
|
784
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
785
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
786
|
+
<td><span class="cap-status supported" aria-label="Supported" title="Supported"><svg aria-hidden="true"><use href="#lucide-circle-check"></use></svg></span></td>
|
|
787
|
+
<td>Timers exist inside the restricted session and are cancelled when the active evaluation ends or resets.</td>
|
|
788
|
+
</tr>
|
|
789
|
+
<tr class="cap-group"><th colspan="5">Intentionally unavailable through cua_repl</th></tr>
|
|
790
|
+
<tr>
|
|
791
|
+
<td><span class="cap-name">Launch a new application</span><span class="cap-code">no model-facing method</span></td>
|
|
792
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
793
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
794
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
795
|
+
<td><code>cua.getApp()</code> binds an existing/visible app; the adapter exposes no launch method.</td>
|
|
796
|
+
</tr>
|
|
797
|
+
<tr>
|
|
798
|
+
<td><span class="cap-name">Quit / force-close an application</span><span class="cap-code">no model-facing method</span></td>
|
|
799
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
800
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
801
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
802
|
+
<td>No quit, terminate, or force-kill application API is exposed to the model.</td>
|
|
803
|
+
</tr>
|
|
804
|
+
<tr>
|
|
805
|
+
<td><span class="cap-name">Move / resize native windows</span><span class="cap-code">no model-facing method</span></td>
|
|
806
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
807
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
808
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
809
|
+
<td>The current public CUA facade has no window-management methods.</td>
|
|
810
|
+
</tr>
|
|
811
|
+
<tr>
|
|
812
|
+
<td><span class="cap-name">Coordinate-only scrolling</span><span class="cap-code">app.scroll([x,y], ...)</span></td>
|
|
813
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
814
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
815
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
816
|
+
<td>The adapter explicitly rejects coordinate scroll; bind a semantic element instead.</td>
|
|
817
|
+
</tr>
|
|
818
|
+
<tr>
|
|
819
|
+
<td><span class="cap-name">Read / write clipboard directly</span><span class="cap-code">no clipboard API</span></td>
|
|
820
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
821
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
822
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
823
|
+
<td>There is no direct clipboard API. Normal app keyboard shortcuts may still work when the app/OS accepts them.</td>
|
|
824
|
+
</tr>
|
|
825
|
+
<tr>
|
|
826
|
+
<td><span class="cap-name">Use browser DOM / CDP directly</span><span class="cap-code">no DOM or CDP API</span></td>
|
|
827
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
828
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
829
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
830
|
+
<td>Browser windows are controlled like desktop apps through accessibility/screenshot state, not page DOM or Chrome DevTools Protocol.</td>
|
|
831
|
+
</tr>
|
|
832
|
+
<tr>
|
|
833
|
+
<td><span class="cap-name">Record screen video or audio</span><span class="cap-code">screenshot only</span></td>
|
|
834
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
835
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
836
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
837
|
+
<td>The model-facing surface can request still screenshots; it does not expose continuous screen, camera, microphone, or audio recording.</td>
|
|
838
|
+
</tr>
|
|
839
|
+
<tr>
|
|
840
|
+
<td><span class="cap-name">Run arbitrary shell commands</span><span class="cap-code">run_shell / exec</span></td>
|
|
841
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
842
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
843
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
844
|
+
<td>Deliberately omitted from the model-facing CUA API.</td>
|
|
845
|
+
</tr>
|
|
846
|
+
<tr>
|
|
847
|
+
<td><span class="cap-name">Read / write filesystem from JS</span><span class="cap-code">fs / require / process</span></td>
|
|
848
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
849
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
850
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
851
|
+
<td>Node built-ins, require, process, and filesystem access are removed from the sandbox.</td>
|
|
852
|
+
</tr>
|
|
853
|
+
<tr>
|
|
854
|
+
<td><span class="cap-name">Make network requests from JS</span><span class="cap-code">fetch / sockets</span></td>
|
|
855
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
856
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
857
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
858
|
+
<td><code>fetch</code> and arbitrary networking are unavailable inside the model-facing VM.</td>
|
|
859
|
+
</tr>
|
|
860
|
+
<tr>
|
|
861
|
+
<td><span class="cap-name">Use dynamic imports / code generation</span><span class="cap-code">import() / eval / Function</span></td>
|
|
862
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
863
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
864
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
865
|
+
<td>Dynamic imports, string code generation, WebAssembly, and host-realm escapes are disabled.</td>
|
|
866
|
+
</tr>
|
|
867
|
+
<tr>
|
|
868
|
+
<td><span class="cap-name">Control lock / login / secure desktop</span><span class="cap-code">OS-owned surfaces</span></td>
|
|
869
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
870
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
871
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
872
|
+
<td>Lock/login screens, Windows UAC secure desktop, and other OS-owned consent surfaces are outside the supported session.</td>
|
|
873
|
+
</tr>
|
|
874
|
+
<tr>
|
|
875
|
+
<td><span class="cap-name">Bypass operating-system permissions</span><span class="cap-code">Accessibility / AT-SPI / UIA / capture</span></td>
|
|
876
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
877
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
878
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
879
|
+
<td>The plugin never elevates itself or bypasses native permission boundaries.</td>
|
|
880
|
+
</tr>
|
|
881
|
+
<tr>
|
|
882
|
+
<td><span class="cap-name">Automate password managers</span><span class="cap-code">native safety denial</span></td>
|
|
883
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
884
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
885
|
+
<td><span class="cap-status blocked" aria-label="Not exposed or blocked" title="Not exposed or blocked"><svg aria-hidden="true"><use href="#lucide-circle-x"></use></svg></span></td>
|
|
886
|
+
<td>The native runtime keeps its password-manager safety denial; secret stores are intentionally out of scope.</td>
|
|
887
|
+
</tr>
|
|
888
|
+
</tbody>
|
|
889
|
+
</table>
|
|
890
|
+
</div>
|
|
891
|
+
<div class="capability-footnote"><strong>Platform reality:</strong> macOS needs Accessibility and Screen Recording permissions. Linux needs a signed-in AT-SPI/D-Bus desktop and some Wayland applications may expose incomplete or non-actionable state. Windows needs a signed-in interactive UI Automation session. macOS and Windows are packaged targets but still require the live certification checklist before their native desktop behavior is described as verified.</div>
|
|
892
|
+
</section>
|
|
893
|
+
|
|
541
894
|
<section id="webmcp" data-section>
|
|
542
895
|
<h2>WebMCP, when available.</h2>
|
|
543
896
|
<p>The documentation page includes a small, read-only WebMCP enhancement based on the current <code>document.modelContext.registerTool()</code> draft. Browsers without the preview API continue normally; no desktop-control or mutating tool is exposed through the page.</p>
|
|
@@ -596,14 +949,86 @@ nodeRepl.write(state);</code></pre><button class="copy" type="button" data-copy=
|
|
|
596
949
|
<caption>Operating-system runtime targets and current validation status</caption>
|
|
597
950
|
<thead><tr><th scope="col">Target</th><th scope="col">Packaged runtime</th><th scope="col">Live status here</th></tr></thead>
|
|
598
951
|
<tbody>
|
|
599
|
-
<tr><td>Linux x64</td><td class="yes">Included</td><td class="yes">
|
|
952
|
+
<tr><td>Linux x64</td><td class="yes">Included</td><td class="yes">GTK semantic smoke passed; capture unavailable</td></tr>
|
|
600
953
|
<tr><td>Linux arm64</td><td class="yes">Included</td><td class="pending">Needs arm64 host</td></tr>
|
|
601
|
-
<tr><td>macOS arm64 and x64</td><td class="yes">Included</td><td class="pending">Needs
|
|
954
|
+
<tr><td>macOS arm64 and x64</td><td class="yes">Included</td><td class="pending">Needs native validation</td></tr>
|
|
602
955
|
<tr><td>Windows arm64 and x64</td><td class="yes">Included</td><td class="pending">Needs UIA validation</td></tr>
|
|
603
956
|
</tbody>
|
|
604
957
|
</table>
|
|
958
|
+
<p><strong>Packaged targets and headless tests are separate from live desktop certification.</strong> CI exercises the JavaScript adapter with a fake native backend on Linux, macOS, and Windows. It does not prove that screenshots, focus, keyboard input, or accessibility actions work on a particular desktop. Use the repository's <code>docs/NATIVE_VALIDATION.md</code> checklist on each target. Android and iOS are not native runtime targets.</p>
|
|
605
959
|
<p>macOS generally needs Accessibility and Screen Recording. Windows needs UI Automation in the interactive desktop session. Linux needs AT-SPI plus the native runtime's input and capture services. A secure desktop, lock screen, UAC surface, or missing permission is not bypassed.</p>
|
|
606
|
-
<
|
|
960
|
+
<h3>Installation walks you through desktop access.</h3>
|
|
961
|
+
<p>The v0.1.5+ npm/pnpm install path runs the same desktop prerequisite helper automatically. It checks the current OS, runs the native <code>doctor</code>, and gives platform-specific guidance. The helper can enable non-protected settings such as GNOME toolkit accessibility, but it never bypasses protected OS consent. On macOS, the native onboarding opens the correct System Settings panes and waits for you to approve <strong>Open Computer Use.app</strong> — not Terminal or OpenCode.</p>
|
|
962
|
+
<p>A successful <code>doctor</code> exit is not a desktop capability certificate. The current Linux command prints session guidance; still verify app discovery, accessibility state, capture, and input separately. The <code>--strict</code> helper reports native-command failures, while the live checklist establishes what actually works.</p>
|
|
963
|
+
<div class="note"><strong>Why there is still a manual step on macOS:</strong> Accessibility and Screen Recording are TCC-protected permissions. Apple does not provide a supported command that silently grants them to an ordinary npm package. Our installer automates detection, opens onboarding, opens the correct Settings destination, and verifies the result; the final privacy toggle remains yours to approve.</div>
|
|
964
|
+
<h3>Manual permission and verification commands.</h3>
|
|
965
|
+
<p>The scoped <code>npm exec</code> helper commands also work outside an installed project. The separate <code>open-computer-use</code> commands below assume its native CLI is on PATH; otherwise prefix them with <code>npm exec --package=open-computer-use@0.3.5 --</code>. In a clone, run <code>node bin/opencode-computer-use-permissions.js --strict</code>.</p>
|
|
966
|
+
<div class="tabs os-permission-tabs" role="tablist" aria-label="Operating-system permission commands">
|
|
967
|
+
<button role="tab" id="tab-os-macos" aria-controls="panel-os-macos" aria-selected="true">macOS</button>
|
|
968
|
+
<button role="tab" id="tab-os-linux" aria-controls="panel-os-linux" aria-selected="false" tabindex="-1">Linux</button>
|
|
969
|
+
<button role="tab" id="tab-os-windows" aria-controls="panel-os-windows" aria-selected="false" tabindex="-1">Windows</button>
|
|
970
|
+
</div>
|
|
971
|
+
<div id="panel-os-macos" role="tabpanel" aria-labelledby="tab-os-macos">
|
|
972
|
+
<p>macOS 14+ requires Accessibility and Screen Recording for <strong>Open Computer Use.app</strong>. The first command is the same checker used by installation.</p>
|
|
973
|
+
<div class="code"><pre><code id="permissions-macos"># Run/re-run our guided installer check
|
|
974
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-permissions --strict
|
|
975
|
+
|
|
976
|
+
# Native status + onboarding UI
|
|
977
|
+
open-computer-use doctor
|
|
978
|
+
|
|
979
|
+
# Open the exact System Settings panes manually if needed
|
|
980
|
+
open "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
|
|
981
|
+
open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
|
|
982
|
+
|
|
983
|
+
# Verify after approving both toggles
|
|
984
|
+
open-computer-use doctor
|
|
985
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#permissions-macos">Copy</button></div>
|
|
986
|
+
<p class="note">If <code>doctor</code> still reports a missing permission after you enable it, close/reopen the affected app or Open Computer Use and run the check again. Do not use <code>tccutil</code> as a grant command: it can reset consent, not silently approve it.</p>
|
|
987
|
+
</div>
|
|
988
|
+
<div id="panel-os-linux" role="tabpanel" aria-labelledby="tab-os-linux" hidden>
|
|
989
|
+
<p>Linux has no universal permission toggle. The requirement is a signed-in graphical desktop with AT-SPI2/D-Bus. GNOME also exposes a toolkit-accessibility setting that our install helper enables best-effort.</p>
|
|
990
|
+
<div class="code"><pre><code id="permissions-linux"># Run/re-run our guided installer check
|
|
991
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-permissions --strict
|
|
992
|
+
|
|
993
|
+
# Native runtime/session notes
|
|
994
|
+
open-computer-use doctor
|
|
995
|
+
|
|
996
|
+
# GNOME only: manually enable and verify toolkit accessibility
|
|
997
|
+
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
998
|
+
gsettings get org.gnome.desktop.interface toolkit-accessibility
|
|
999
|
+
|
|
1000
|
+
# Inspect the session the runtime inherited
|
|
1001
|
+
printf "desktop=%s session=%s\n" "$XDG_CURRENT_DESKTOP" "$XDG_SESSION_TYPE"
|
|
1002
|
+
test -n "$DBUS_SESSION_BUS_ADDRESS" && echo "D-Bus session available" || echo "D-Bus session not exported; runtime will try discovery"
|
|
1003
|
+
|
|
1004
|
+
# Verify real desktop visibility
|
|
1005
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#permissions-linux">Copy</button></div>
|
|
1006
|
+
<p class="note">On Wayland, AT-SPI semantic actions can work while screenshots or coordinate input remain compositor-dependent. If a workflow needs reliable coordinate injection, X11 may behave more predictably; otherwise prefer element-index actions and verify state after every mutation.</p>
|
|
1007
|
+
</div>
|
|
1008
|
+
<div id="panel-os-windows" role="tabpanel" aria-labelledby="tab-os-windows" hidden>
|
|
1009
|
+
<p>Windows UI Automation does not have a separate privacy permission switch. The important requirement is that OpenCode/Open Computer Use runs in the signed-in user desktop rather than a Windows service or session 0.</p>
|
|
1010
|
+
<div class="code"><pre><code id="permissions-windows"># PowerShell
|
|
1011
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-permissions --strict
|
|
1012
|
+
open-computer-use doctor
|
|
1013
|
+
|
|
1014
|
+
# Confirm this shell belongs to an interactive user session
|
|
1015
|
+
Write-Host "SESSIONNAME=$env:SESSIONNAME"
|
|
1016
|
+
(Get-Process -Id $PID).SessionId
|
|
1017
|
+
|
|
1018
|
+
# Verify UI Automation can see desktop applications
|
|
1019
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#permissions-windows">Copy</button></div>
|
|
1020
|
+
<p class="note">A normal process cannot automate the UAC secure desktop. Elevated target applications can also be restricted from a lower-integrity automation process. Prefer ordinary user-session apps and keep OpenCode at the same integrity level as the target when automation is required.</p>
|
|
1021
|
+
</div>
|
|
1022
|
+
<h3>How to turn conditional actions into reliable ones.</h3>
|
|
1023
|
+
<ul class="cards">
|
|
1024
|
+
<li><strong>Fix the host first</strong><span>Run the guided permission check and make sure the native runtime can actually list the target app before debugging clicks or typing.</span></li>
|
|
1025
|
+
<li><strong>Prefer semantic actions</strong><span>Use a fresh accessibility element index for click, scroll, set-value, and secondary actions. This is more portable than remembered coordinates.</span></li>
|
|
1026
|
+
<li><strong>Observe after mutations</strong><span>Read a new accessibility state after navigation, dialogs, rerenders, or an uncertain timeout. Never assume a reported input event changed the UI.</span></li>
|
|
1027
|
+
<li><strong>Use screenshots as a fallback</strong><span>When the accessibility tree is incomplete, a screenshot can recover visual context if the OS capture backend permits it.</span></li>
|
|
1028
|
+
<li><strong>Opt into pointer fallback only when needed</strong><span><code>allowGlobalPointerFallbacks</code> can recover coordinate-only cases, but it may move the real pointer or change focus and stays off by default.</span></li>
|
|
1029
|
+
<li><strong>Choose the right Linux session</strong><span>AT-SPI works best when the process can reach the logged-in desktop bus. Wayland may restrict capture/input; semantic actions are the preferred path.</span></li>
|
|
1030
|
+
</ul>
|
|
1031
|
+
<div class="note"><strong>Native smoke check, 2026-10-30:</strong> Ubuntu GNOME Wayland on Linux x64, Node 26.13.0, OpenCode 1.19.3, and native runtime 0.3.5 passed MCP registration, app inventory, persistent bindings, and reset checks. In a disposable GTK3 app, semantic <code>app.click</code> changed a counter from 0 to 1 and <code>app.setValue</code> changed an entry; fresh accessibility snapshots verified both results. <code>app.getScreenshot</code> returned <code>Screenshot unavailable</code>. Keyboard and coordinate input were not validated. Other app/session combinations can still expose incomplete accessibility trees; this is scoped GTK evidence, not certification of every Linux app or capability.</div>
|
|
607
1032
|
</section>
|
|
608
1033
|
|
|
609
1034
|
<section id="security" data-section>
|
|
@@ -624,11 +1049,119 @@ nodeRepl.write(state);</code></pre><button class="copy" type="button" data-copy=
|
|
|
624
1049
|
|
|
625
1050
|
<section id="troubleshooting" data-section>
|
|
626
1051
|
<h2>When the session is quiet.</h2>
|
|
627
|
-
<
|
|
1052
|
+
<p>Start with the same three checks on every platform. They separate plugin registration problems from native desktop permission/session problems before you retry an action.</p>
|
|
1053
|
+
<div class="code"><pre><code id="troubleshooting-first-checks">opencode mcp list
|
|
1054
|
+
npm exec --package=@frontendxlab/opencode-computer-use -- opencode-computer-use-permissions --strict
|
|
1055
|
+
npm exec --package=open-computer-use@0.3.5 -- open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#troubleshooting-first-checks">Copy</button></div>
|
|
1056
|
+
|
|
1057
|
+
<h3>Permission failures by operating system.</h3>
|
|
1058
|
+
<div class="tabs troubleshooting-os-tabs" role="tablist" aria-label="Permission troubleshooting by operating system">
|
|
1059
|
+
<button role="tab" id="tab-trouble-macos" aria-controls="panel-trouble-macos" aria-selected="true">macOS</button>
|
|
1060
|
+
<button role="tab" id="tab-trouble-linux" aria-controls="panel-trouble-linux" aria-selected="false" tabindex="-1">Linux</button>
|
|
1061
|
+
<button role="tab" id="tab-trouble-windows" aria-controls="panel-trouble-windows" aria-selected="false" tabindex="-1">Windows</button>
|
|
1062
|
+
</div>
|
|
1063
|
+
|
|
1064
|
+
<div id="panel-trouble-macos" role="tabpanel" aria-labelledby="tab-trouble-macos">
|
|
1065
|
+
<details open>
|
|
1066
|
+
<summary><code>doctor</code> says Accessibility is missing</summary>
|
|
1067
|
+
<p><strong>Likely cause:</strong> <strong>Open Computer Use.app</strong> has not been approved in Privacy & Security → Accessibility, or the permission was granted to Terminal/OpenCode instead of the app-agent that performs the real automation.</p>
|
|
1068
|
+
<p><strong>Fix:</strong> run <code>open-computer-use doctor</code>, use the onboarding UI, or open the Accessibility pane directly. Enable <strong>Open Computer Use</strong>, then restart the Open Computer Use app-agent/OpenCode and rerun <code>doctor</code>.</p>
|
|
1069
|
+
<div class="code"><pre><code id="trouble-macos-accessibility">open-computer-use doctor
|
|
1070
|
+
open "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
|
|
1071
|
+
open-computer-use doctor</code></pre><button class="copy" type="button" data-copy="#trouble-macos-accessibility">Copy</button></div>
|
|
1072
|
+
</details>
|
|
1073
|
+
<details>
|
|
1074
|
+
<summary>Screenshots are missing or <code>screenRecording=missing</code></summary>
|
|
1075
|
+
<p><strong>Likely cause:</strong> Screen Recording / Screen & System Audio Recording has not been granted to <strong>Open Computer Use.app</strong>, or the already-running app-agent has not picked up the new TCC grant yet.</p>
|
|
1076
|
+
<p><strong>Fix:</strong> approve Open Computer Use in the Screen Recording pane, then quit/reopen the app-agent or restart OpenCode before checking again. Accessibility text can still work while screenshots are unavailable.</p>
|
|
1077
|
+
<div class="code"><pre><code id="trouble-macos-screen">open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
|
|
1078
|
+
open-computer-use doctor
|
|
1079
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#trouble-macos-screen">Copy</button></div>
|
|
1080
|
+
</details>
|
|
1081
|
+
<details>
|
|
1082
|
+
<summary>System Settings shows permission granted, but onboarding still says Allow</summary>
|
|
1083
|
+
<p><strong>Likely cause:</strong> stale TCC/app-agent state, an old installed copy, or a development build with a different signing identity/path.</p>
|
|
1084
|
+
<p><strong>Fix:</strong> first restart Open Computer Use and rerun <code>doctor</code>. Confirm you approved the current <strong>Open Computer Use.app</strong>, not <strong>Open Computer Use (Dev)</strong> or Terminal. As a last-resort reset, remove only this app's stored consent and let <code>doctor</code> request it again. These commands <em>reset</em> permission; they do not grant it.</p>
|
|
1085
|
+
<div class="code"><pre><code id="trouble-macos-reset">tccutil reset Accessibility com.ifuryst.opencomputeruse
|
|
1086
|
+
tccutil reset ScreenCapture com.ifuryst.opencomputeruse
|
|
1087
|
+
open-computer-use doctor</code></pre><button class="copy" type="button" data-copy="#trouble-macos-reset">Copy</button></div>
|
|
1088
|
+
</details>
|
|
1089
|
+
<details>
|
|
1090
|
+
<summary>The native binary will not start even after granting permissions</summary>
|
|
1091
|
+
<p><strong>Likely cause:</strong> macOS is older than the native runtime's minimum supported version. Permission changes cannot fix binary compatibility.</p>
|
|
1092
|
+
<p><strong>Fix:</strong> check the OS version. The current macOS runtime requires macOS 14 or newer.</p>
|
|
1093
|
+
<div class="code"><pre><code id="trouble-macos-version">sw_vers -productVersion
|
|
1094
|
+
open-computer-use --version</code></pre><button class="copy" type="button" data-copy="#trouble-macos-version">Copy</button></div>
|
|
1095
|
+
</details>
|
|
1096
|
+
</div>
|
|
1097
|
+
|
|
1098
|
+
<div id="panel-trouble-linux" role="tabpanel" aria-labelledby="tab-trouble-linux" hidden>
|
|
1099
|
+
<details open>
|
|
1100
|
+
<summary><code>list_apps</code> is empty or AT-SPI cannot see desktop apps</summary>
|
|
1101
|
+
<p><strong>Likely cause:</strong> the command is running outside the signed-in graphical session, toolkit accessibility is disabled, or the process cannot reach the user's D-Bus/AT-SPI session.</p>
|
|
1102
|
+
<p><strong>Fix:</strong> run OpenCode from the logged-in desktop user, not a pure SSH shell, service, CI job, or unrelated user session. On GNOME, enable toolkit accessibility and verify the session variables. The native runtime also tries to discover missing session variables automatically.</p>
|
|
1103
|
+
<div class="code"><pre><code id="trouble-linux-atspi">gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
1104
|
+
gsettings get org.gnome.desktop.interface toolkit-accessibility
|
|
1105
|
+
printf "desktop=%s session=%s\n" "$XDG_CURRENT_DESKTOP" "$XDG_SESSION_TYPE"
|
|
1106
|
+
printf "runtime=%s\n" "$XDG_RUNTIME_DIR"
|
|
1107
|
+
test -n "$DBUS_SESSION_BUS_ADDRESS" && echo "D-Bus session available"
|
|
1108
|
+
open-computer-use doctor
|
|
1109
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#trouble-linux-atspi">Copy</button></div>
|
|
1110
|
+
</details>
|
|
1111
|
+
<details>
|
|
1112
|
+
<summary>Screenshot is black, omitted, or unavailable on Wayland</summary>
|
|
1113
|
+
<p><strong>Likely cause:</strong> the compositor blocks the runtime's best-effort capture path. This is not an AT-SPI permission toggle and there is no universal Linux command that grants unrestricted screen capture.</p>
|
|
1114
|
+
<p><strong>Fix:</strong> continue with accessibility text/semantic actions when possible. If the workflow requires reliable coordinate/screenshot automation, test the same workflow in an X11 session or a compositor/environment whose capture path works for the runtime.</p>
|
|
1115
|
+
</details>
|
|
1116
|
+
<details>
|
|
1117
|
+
<summary>Click/type/drag reports success but the application does not change</summary>
|
|
1118
|
+
<p><strong>Likely cause:</strong> AT-SPI exposed a pseudo-window or incomplete action tree, or Wayland rejected synthetic coordinate/key input. A native event being accepted does not prove the application consumed it.</p>
|
|
1119
|
+
<p><strong>Fix:</strong> take a fresh state, prefer element-index accessibility actions, <code>setValue</code>, and advertised secondary actions over coordinates. Verify state after every mutation. If only coordinate synthesis is failing, compare the same workflow under X11 before treating it as an adapter bug.</p>
|
|
1120
|
+
</details>
|
|
1121
|
+
<details>
|
|
1122
|
+
<summary>Error mentions <code>Accessible.is_text</code> or <code>is_editable_text</code></summary>
|
|
1123
|
+
<p><strong>Likely cause:</strong> an older Linux native runtime is using PyGObject helpers that are not available on that distribution.</p>
|
|
1124
|
+
<p><strong>Fix:</strong> update/reinstall the current native runtime. The current package line detects AT-SPI <code>Text</code>/<code>EditableText</code> through standard interfaces.</p>
|
|
1125
|
+
<div class="code"><pre><code id="trouble-linux-upgrade">npm install open-computer-use@0.3.5
|
|
1126
|
+
open-computer-use --version
|
|
1127
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#trouble-linux-upgrade">Copy</button></div>
|
|
1128
|
+
</details>
|
|
1129
|
+
</div>
|
|
1130
|
+
|
|
1131
|
+
<div id="panel-trouble-windows" role="tabpanel" aria-labelledby="tab-trouble-windows" hidden>
|
|
1132
|
+
<details open>
|
|
1133
|
+
<summary><code>list_apps</code> is empty even though <code>doctor</code> runs</summary>
|
|
1134
|
+
<p><strong>Likely cause:</strong> Open Computer Use is running in a service, session 0, detached SSH context, or another non-interactive session. UI Automation only sees windows in the signed-in interactive desktop.</p>
|
|
1135
|
+
<p><strong>Fix:</strong> start OpenCode/Open Computer Use from the user's normal desktop session and confirm the current PowerShell process has a user session ID.</p>
|
|
1136
|
+
<div class="code"><pre><code id="trouble-windows-session">open-computer-use doctor
|
|
1137
|
+
Write-Host "SESSIONNAME=$env:SESSIONNAME"
|
|
1138
|
+
(Get-Process -Id $PID).SessionId
|
|
1139
|
+
open-computer-use call list_apps</code></pre><button class="copy" type="button" data-copy="#trouble-windows-session">Copy</button></div>
|
|
1140
|
+
</details>
|
|
1141
|
+
<details>
|
|
1142
|
+
<summary>The target app is elevated and clicks/typing fail</summary>
|
|
1143
|
+
<p><strong>Likely cause:</strong> Windows integrity-level boundaries can block a normal process from driving a higher-integrity application.</p>
|
|
1144
|
+
<p><strong>Fix:</strong> prefer running the target as a normal user. If automation genuinely requires an elevated target, run the automation host at the same integrity level only for that explicit task and keep the elevated session narrow.</p>
|
|
1145
|
+
</details>
|
|
1146
|
+
<details>
|
|
1147
|
+
<summary>A UAC prompt, lock screen, or sign-in screen cannot be controlled</summary>
|
|
1148
|
+
<p><strong>Cause:</strong> these are Windows secure-desktop / OS-owned surfaces. This is an intentional security boundary, not a missing UI Automation permission.</p>
|
|
1149
|
+
<p><strong>Fix:</strong> approve or dismiss the UAC prompt manually and return to the normal user desktop. The plugin will not attempt to bypass the secure desktop.</p>
|
|
1150
|
+
</details>
|
|
1151
|
+
<details>
|
|
1152
|
+
<summary>App is found but an action does nothing</summary>
|
|
1153
|
+
<p><strong>Likely cause:</strong> the app does not expose the expected UI Automation pattern or ignores the Win32 message fallback while backgrounded/hidden.</p>
|
|
1154
|
+
<p><strong>Fix:</strong> make sure the target has a visible, non-minimized top-level window, refresh the accessibility state, and prefer semantic element actions over coordinates. Verify the resulting state instead of trusting the input call alone.</p>
|
|
1155
|
+
</details>
|
|
1156
|
+
</div>
|
|
1157
|
+
|
|
1158
|
+
<h3>General failures.</h3>
|
|
1159
|
+
<details><summary><code>cua_repl</code> is not listed</summary><p>A clone alone does not register a plugin, and installing dependencies for package development deliberately skips automatic setup. From the checkout run <code>npm ci --ignore-scripts</code>, then <code>node bin/opencode-computer-use-setup.js --local --v1 --source .</code>. Choose <code>--v2</code> for OpenCode2 or <code>--global</code> to use the checkout outside this project. For a published package, use the scoped <code>npm exec</code> setup command in the install section.</p><p>Check the selected configuration path printed by setup, including any <code>OPENCODE_CONFIG</code> override. Restart the OpenCode server/TUI using that configuration and run <code>opencode mcp list</code> from the same project. Loading the plugin exposes MCP tools; it does not add a slash command. Check for an existing disabled or stale <code>cua_repl</code> entry, because the plugin preserves existing server entries by default. Avoid duplicate registration through both explicit configuration and automatic discovery.</p></details>
|
|
1160
|
+
<details><summary>It works on one computer but not another</summary><p>Run setup and the permission check on each target desktop. The native runtime runs on the OpenCode server's computer, even when you connect from a different device. Reinstall dependencies for that operating system and architecture, regenerate checkout paths with <code>--source</code>, and run in the correct signed-in graphical session. There is no built-in remote-device pairing or transport. Wayland, macOS privacy permissions, and Windows session/integrity boundaries still apply.</p></details>
|
|
1161
|
+
<details><summary>A Windows PATH launcher fails to start</summary><p>Use the packaged native runtime or set <code>OPEN_COMPUTER_USE_NATIVE_COMMAND</code> to the absolute <code>open-computer-use.exe</code> path. Standard npm global <code>.cmd</code> shims are resolved to the adjacent package's native executable. Other shell shims are not executed; the diagnostic explains which native runtime path is required.</p></details>
|
|
628
1162
|
<details><summary>Node cannot be found</summary><p>Set <code>OPENCODE_NODE</code> to an absolute Node executable before starting OpenCode, or make <code>node</code> available on the server PATH.</p></details>
|
|
629
|
-
<details><summary>The app
|
|
630
|
-
<details><summary>
|
|
631
|
-
<details><summary>Linux input returns success but nothing changes</summary><p>Check the desktop session type and native runtime prerequisites. A pseudo-window or Wayland compositor can expose state without accepting synthetic input. Observe again before retrying a mutating action.</p></details>
|
|
1163
|
+
<details><summary>The app is not found</summary><p>Run <code>open-computer-use call list_apps</code>, use the exact returned app name or ID, and confirm the app already has a visible, non-minimized window in the same graphical session.</p></details>
|
|
1164
|
+
<details><summary>Element index stopped working after navigation</summary><p>Element indexes belong to a particular accessibility snapshot. Fetch a fresh state after navigation, dialogs, rerenders, or focus changes before acting again.</p></details>
|
|
632
1165
|
<details><summary>Global pointer fallback is rejected</summary><p>This is the expected safety gate. Enable <code>allowGlobalPointerFallbacks</code> only for an explicit diagnostic session where moving the real pointer and changing focus are acceptable.</p></details>
|
|
633
1166
|
</section>
|
|
634
1167
|
|
|
@@ -637,7 +1170,8 @@ nodeRepl.write(state);</code></pre><button class="copy" type="button" data-copy=
|
|
|
637
1170
|
<div class="faq-list">
|
|
638
1171
|
<div><h3>Is OpenCode Computer Use a plugin, an MCP server, or both?</h3><p>Both layers are present. The OpenCode plugin loads the local package and registers an MCP server named cua_repl. That server exposes only js and js_reset to the model.</p></div>
|
|
639
1172
|
<div><h3>What do js and js_reset do?</h3><p>js evaluates JavaScript in the persistent session, so variables and app bindings survive between calls. js_reset discards those bindings but does not reset native application state.</p></div>
|
|
640
|
-
<div><h3>Which operating systems are supported?</h3><p>
|
|
1173
|
+
<div><h3>Which operating systems are supported?</h3><p>The optional native package includes Linux, macOS, and Windows binaries for x64 and arm64. Linux x64 GNOME Wayland smoke checks verified registration, inventory, persistent sessions, and semantic click/setValue on a disposable GTK3 app; capture was unavailable. Other apps and capabilities need separate checks, and macOS, Windows, and Linux arm64 still need native validation.</p></div>
|
|
1174
|
+
<div><h3>Can it control another device?</h3><p>It controls the signed-in desktop on the computer running the OpenCode server and its local MCP child. Connecting from another browser or device does not switch the controlled desktop. Install and configure the package on each target computer; automatic device discovery, pairing, and remote desktop transport are not built in.</p></div>
|
|
641
1175
|
<div><h3>Does the plugin bypass operating-system permissions?</h3><p>No. It cannot control a lock screen, login screen, or UAC secure desktop. OpenCode approvals and operating-system permissions remain the enforcement layer.</p></div>
|
|
642
1176
|
<div><h3>Does the model get raw desktop tools or an arbitrary shell?</h3><p>No. The model sees only js and js_reset, not raw desktop tools or run_shell. The restricted JavaScript session also blocks filesystem, process, fetch, dynamic import, and code-generation access.</p></div>
|
|
643
1177
|
<div><h3>How do I verify the installation?</h3><p>Restart OpenCode or reopen the TUI, run opencode mcp list, and confirm that cua_repl is connected before starting a desktop task.</p></div>
|
|
@@ -651,7 +1185,7 @@ nodeRepl.write(state);</code></pre><button class="copy" type="button" data-copy=
|
|
|
651
1185
|
<strong>Default policy</strong>
|
|
652
1186
|
<p>Local stdio only. Global pointer fallback off. Native OS permissions enforced. Read-only and app policies are available as adapter defense in depth.</p>
|
|
653
1187
|
<strong>Updated</strong>
|
|
654
|
-
<p>Published September 25, 2026. Updated September
|
|
1188
|
+
<p>Published September 25, 2026. Updated September 27, 2026.</p>
|
|
655
1189
|
<strong>WebMCP</strong>
|
|
656
1190
|
<p>Preview support registers read-only documentation tools when the browser exposes the WebMCP API. Normal browsers remain fully supported.</p>
|
|
657
1191
|
<strong>Repository</strong>
|
|
@@ -698,23 +1232,26 @@ nodeRepl.write(state);</code></pre><button class="copy" type="button" data-copy=
|
|
|
698
1232
|
});
|
|
699
1233
|
});
|
|
700
1234
|
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
1235
|
+
document.querySelectorAll('[role="tablist"]').forEach((tabList) => {
|
|
1236
|
+
const tabs = [...tabList.querySelectorAll('[role="tab"]')];
|
|
1237
|
+
const selectTab = (tab) => {
|
|
1238
|
+
tabs.forEach((item) => {
|
|
1239
|
+
const active = item === tab;
|
|
1240
|
+
item.setAttribute("aria-selected", String(active));
|
|
1241
|
+
item.tabIndex = active ? 0 : -1;
|
|
1242
|
+
const panel = document.getElementById(item.getAttribute("aria-controls"));
|
|
1243
|
+
if (panel) panel.hidden = !active;
|
|
1244
|
+
});
|
|
1245
|
+
tab.focus();
|
|
1246
|
+
};
|
|
1247
|
+
tabs.forEach((tab, index) => {
|
|
1248
|
+
tab.addEventListener("click", () => selectTab(tab));
|
|
1249
|
+
tab.addEventListener("keydown", (event) => {
|
|
1250
|
+
if (event.key === "ArrowRight") selectTab(tabs[(index + 1) % tabs.length]);
|
|
1251
|
+
if (event.key === "ArrowLeft") selectTab(tabs[(index - 1 + tabs.length) % tabs.length]);
|
|
1252
|
+
if (event.key === "Home") selectTab(tabs[0]);
|
|
1253
|
+
if (event.key === "End") selectTab(tabs.at(-1));
|
|
1254
|
+
});
|
|
718
1255
|
});
|
|
719
1256
|
});
|
|
720
1257
|
|