@qilin-agent/experimental-cua-macos 3.1.11 → 3.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +10 -10
- package/README.md +8 -8
- package/README.zh.md +8 -8
- package/lib/index.js +2 -2
- package/lib/types/guidance.d.ts +1 -1
- package/package.json +34 -34
package/README.i18n.yaml
CHANGED
|
@@ -9,17 +9,17 @@
|
|
|
9
9
|
en: 484138923ad88162
|
|
10
10
|
zh: ef28709a39185bcd
|
|
11
11
|
/qilin-agent-experimental-cua-macos/summary:
|
|
12
|
-
en:
|
|
13
|
-
zh:
|
|
12
|
+
en: d97ee165cb46092b
|
|
13
|
+
zh: 7c9c1544e2becdd9
|
|
14
14
|
/qilin-agent-experimental-cua-macos/table-of-contents:
|
|
15
15
|
en: d152484eb41ac6b4
|
|
16
16
|
zh: 7b30b973a849247d
|
|
17
17
|
/qilin-agent-experimental-cua-macos/use-this-package:
|
|
18
|
-
en:
|
|
19
|
-
zh:
|
|
18
|
+
en: 121bb255387f36fc
|
|
19
|
+
zh: c4e1564c4b101d4f
|
|
20
20
|
/qilin-agent-experimental-cua-macos/use-this-package/connect-an-external-execution-environment:
|
|
21
|
-
en:
|
|
22
|
-
zh:
|
|
21
|
+
en: c29eb87c56477f8d
|
|
22
|
+
zh: b1f1404bd7e6c383
|
|
23
23
|
/qilin-agent-experimental-cua-macos/understand-the-implementation:
|
|
24
24
|
en: 8b9ec208ca057a4a
|
|
25
25
|
zh: e8dbb45891ad15b8
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
en: ab7cba4a42334ab9
|
|
34
34
|
zh: f77cd9d84624b6c2
|
|
35
35
|
/qilin-agent-experimental-cua-macos/model-experience/native-application-operations/what-the-model-sees:
|
|
36
|
-
en:
|
|
37
|
-
zh:
|
|
36
|
+
en: a5105b903c5c061d
|
|
37
|
+
zh: 9ec1e453902d2050
|
|
38
38
|
/qilin-agent-experimental-cua-macos/model-experience/native-application-operations/what-the-model-sees/operation-guidance-verbatim:
|
|
39
39
|
en: 2dbc22ffd1f8a717
|
|
40
40
|
zh: d4458872e75305d9
|
|
@@ -45,8 +45,8 @@
|
|
|
45
45
|
en: a2c9acb5a4320dda
|
|
46
46
|
zh: e8d1586af60215c4
|
|
47
47
|
/qilin-agent-experimental-cua-macos/known-limitations-and-deferred-work:
|
|
48
|
-
en:
|
|
49
|
-
zh:
|
|
48
|
+
en: eef503875eb620cd
|
|
49
|
+
zh: a7ee9cb149a59823
|
|
50
50
|
/qilin-agent-experimental-cua-macos/known-limitations-and-deferred-work/dev-note:
|
|
51
51
|
en: b836f01bd0bceb4a
|
|
52
52
|
zh: 8122fa08b2aa95fa
|
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ English | [中文](README.zh.md)
|
|
|
9
9
|
|
|
10
10
|
## Summary
|
|
11
11
|
|
|
12
|
-
Inspect macOS permissions, select running applications and windows, read accessibility trees, capture screenshots and send input. The object API can be retained by an external persistent execution environment. The Swift helper uses macOS frameworks directly and adapts selected MIT-licensed Cua Driver behavior. This package owns neither a
|
|
12
|
+
Inspect macOS permissions, select running applications and windows, read accessibility trees, capture screenshots and send input. The object API can be retained by an external persistent execution environment. The Swift helper uses macOS frameworks directly and adapts selected MIT-licensed Cua Driver behavior. This package owns neither a coding runtime nor variable persistence.
|
|
13
13
|
|
|
14
14
|
## Table of Contents
|
|
15
15
|
|
|
@@ -59,7 +59,7 @@ Keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain th
|
|
|
59
59
|
|
|
60
60
|
Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot.
|
|
61
61
|
|
|
62
|
-
Application consent is enforced before AX reads, window-title reads, screenshots, activation, and input through the CUA tool. User Questions offers task access or denial, plus saved access when the application has a verifiable signature and the composition provides settings persistence. Task access is bound to the selected process handle; saved access is bound to its canonical executable path and designated code-signing requirement. Denial prevents repeated prompts in that task. Removing a saved `allowedApps` entry cancels affected operations and invalidates their helper handles; reauthorization requires fresh selection. Other handles on the same Agent connection also expire, while saved grants remain reusable. Agent/provider disposal joins cleanup of the corresponding native connection. See the [application consent decision](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md). These checks govern CUA calls; persistent
|
|
62
|
+
Application consent is enforced before AX reads, window-title reads, screenshots, activation, and input through the CUA tool. User Questions offers task access or denial, plus saved access when the application has a verifiable signature and the composition provides settings persistence. Task access is bound to the selected process handle; saved access is bound to its canonical executable path and designated code-signing requirement. Denial prevents repeated prompts in that task. Removing a saved `allowedApps` entry cancels affected operations and invalidates their helper handles; reauthorization requires fresh selection. Other handles on the same Agent connection also expire, while saved grants remain reusable. Agent/provider disposal joins cleanup of the corresponding native connection. See the [application consent decision](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md). These checks govern CUA calls; persistent coding and separately authorized shell tools retain their own execution authority.
|
|
63
63
|
|
|
64
64
|
Raw `getState` exposes validated state fields and the complete text for the requested disclosure options. The native helper also returns structured element diagnostics for native checks; the tool projection omits that duplicate array from model text and structured tool values.
|
|
65
65
|
|
|
@@ -67,7 +67,7 @@ AX observations show menu-bar entries by default. Use `app.getAXState({ includeM
|
|
|
67
67
|
|
|
68
68
|
Values and placeholders are separate. Named, stateful or actionable groups and scroll areas retain their own rows. Repeated names, descriptions and help text on one node are shown once. Values, placeholders and URLs keep their distinct meanings even when their text matches; body text shared by distinct nodes remains intact. Writability is explicit on relevant controls, including `AXIncrementor`; `unknown` means the system did not report it. Missing optional state fields do not imply false. UTF-16 ranges outside the observed value are marked unavailable. Custom action aliases omit AppKit target/selector diagnostics; ambiguous aliases require a complete name from `details:true`.
|
|
69
69
|
|
|
70
|
-
Observations emit by default and also return their value. In persistent
|
|
70
|
+
Observations emit by default and also return their value. In persistent coding, use `void await app.getAXState()` for one emitted copy, or `emit:false` and one explicit print. Capturing a screenshot after an emitted tree can fail independently; retain the successful AX output.
|
|
71
71
|
|
|
72
72
|
Screenshot diagnostics use `[cua-screenshot]` records keyed by the tool request ID. Native phase records return over the originating Agent connection and cover window/AX geometry checks, system capture, PNG/Base64 encoding and response writing. Host debug logs add dispatch, reply and cleanup timings. Each request retains at most 8192 bytes of complete diagnostic records, with explicit truncation; screenshot failures append these records to the error for Session export. Records contain stages, timings and error codes, not window titles, AX text or image bytes.
|
|
73
73
|
|
|
@@ -75,7 +75,7 @@ Native preparation serializes compiler checks and publication with the shared fi
|
|
|
75
75
|
|
|
76
76
|
### Connect an external execution environment
|
|
77
77
|
|
|
78
|
-
To run through persistent
|
|
78
|
+
To run through persistent coding, add `--patch packages/experimental/cua-macos/examples/persistent-ptc.cordis.yml` after the macOS overlay. This provider keeps variables for the live Agent and applies the selected file sandbox policy. The caller imports the bridge and supplies the current authorized `tools.cua` binding. Persistent coding receives an installation-owned bridge file URL in its CUA instructions, so the project does not need to install this package. The example below is for an external host that resolves the package from its own installation and supplies `tools`; the package does not install a global object or keep variables alive:
|
|
79
79
|
|
|
80
80
|
```javascript
|
|
81
81
|
import { connectCua } from '@qilin-agent/experimental-cua/bridge'
|
|
@@ -142,14 +142,14 @@ Node roles and metadata share one AX batch per observation. Group and scroll-are
|
|
|
142
142
|
|
|
143
143
|
#### What the model sees
|
|
144
144
|
|
|
145
|
-
The plugin contributes one `cua` operation tool with method-specific parameters. Standard
|
|
145
|
+
The plugin contributes one `cua` operation tool with method-specific parameters. Standard coding-mode presentation exposes it as `tools.cua` through `run_code`. The following package-owned guidance is emitted verbatim:
|
|
146
146
|
|
|
147
147
|
##### Operation guidance (verbatim)
|
|
148
148
|
|
|
149
149
|
```markdown
|
|
150
150
|
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
|
151
151
|
|
|
152
|
-
Initialize the object API once in persistent
|
|
152
|
+
Initialize the object API once in persistent coding. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
|
153
153
|
|
|
154
154
|
Use app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.
|
|
155
155
|
|
|
@@ -188,7 +188,7 @@ Await operations sequentially. After input, read fresh state to verify the effec
|
|
|
188
188
|
|
|
189
189
|
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
|
190
190
|
|
|
191
|
-
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent
|
|
191
|
+
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent coding, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
|
192
192
|
```
|
|
193
193
|
|
|
194
194
|
#### Token effect
|
|
@@ -211,7 +211,7 @@ The queue orders individual native requests. Holding an app object does not rese
|
|
|
211
211
|
- Background input depends on private SkyLight symbols; macOS updates can make them unavailable. Validate each target OS version.
|
|
212
212
|
- Observed failures include dispatched Unicode text not appearing in Blender and background Command+A not selecting all text in AppKit. Inspect application state after input; a successful tool result confirms dispatch only.
|
|
213
213
|
- Native binaries require a local build. Signing, distribution, and system-permission onboarding remain deferred.
|
|
214
|
-
- Persistent
|
|
214
|
+
- Persistent coding is an external dependency assumption, not an implementation in this package. Real-model task success requires separate evaluation beyond native and protocol fixtures.
|
|
215
215
|
|
|
216
216
|
-----
|
|
217
217
|
|
package/README.zh.md
CHANGED
|
@@ -9,7 +9,7 @@ kind: "package-reference"
|
|
|
9
9
|
|
|
10
10
|
## 概述
|
|
11
11
|
|
|
12
|
-
检查 macOS 权限,选择运行中的应用和窗口,读取辅助功能树、截图并发送输入。外部持久执行环境可以保留这些对象。Swift 辅助进程直接使用 macOS 框架,并改写选取的 MIT 许可 Cua Driver 行为。本包不提供
|
|
12
|
+
检查 macOS 权限,选择运行中的应用和窗口,读取辅助功能树、截图并发送输入。外部持久执行环境可以保留这些对象。Swift 辅助进程直接使用 macOS 框架,并改写选取的 MIT 许可 Cua Driver 行为。本包不提供 coding 运行时或变量持久化。
|
|
13
13
|
|
|
14
14
|
## 目录
|
|
15
15
|
|
|
@@ -59,7 +59,7 @@ pnpm qilin headless --patch packages/experimental/cua-macos/examples/macos.cordi
|
|
|
59
59
|
|
|
60
60
|
Agent 销毁和插件卸载会取消并等待完整工具调用,包括图像接入检查和附件存储。已经开始的图像写入必须结束后,销毁才会返回;已取消的调用不会返回成功截图。
|
|
61
61
|
|
|
62
|
-
CUA 工具在读取 AX、窗口标题、截图、激活和输入之前检查应用授权。User Questions 提供任务级允许和拒绝选项;应用签名可验证且组合支持配置持久化时,也可保存授权。任务授权绑定所选进程句柄,长期授权绑定真实可执行路径和指定代码签名要求。拒绝后,同一任务不再反复询问。删除保存的 `allowedApps` 条目会取消相关操作并使原生组件中的句柄失效;重新授权需要重新选择应用。同一 Agent 连接中的其他句柄也会失效,已保存的授权仍可复用。Agent 或提供方销毁时等待对应连接的原生状态清理。设计理由见[应用授权决策](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md)。这些检查约束 CUA 调用;持久
|
|
62
|
+
CUA 工具在读取 AX、窗口标题、截图、激活和输入之前检查应用授权。User Questions 提供任务级允许和拒绝选项;应用签名可验证且组合支持配置持久化时,也可保存授权。任务授权绑定所选进程句柄,长期授权绑定真实可执行路径和指定代码签名要求。拒绝后,同一任务不再反复询问。删除保存的 `allowedApps` 条目会取消相关操作并使原生组件中的句柄失效;重新授权需要重新选择应用。同一 Agent 连接中的其他句柄也会失效,已保存的授权仍可复用。Agent 或提供方销毁时等待对应连接的原生状态清理。设计理由见[应用授权决策](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md)。这些检查约束 CUA 调用;持久 coding 和另行授权的 shell 工具保留各自的执行权限。
|
|
63
63
|
|
|
64
64
|
原始 `getState` 操作返回经过校验的状态字段,以及按本次披露选项生成的完整文本。原生 helper 还返回供原生检查使用的结构化元素诊断数组;工具投影会从模型文本和结构化工具值中移除这份重复数组。
|
|
65
65
|
|
|
@@ -67,7 +67,7 @@ AX 观察默认只展示菜单栏入口。使用 `app.getAXState({ includeMenus:
|
|
|
67
67
|
|
|
68
68
|
实际值与占位文字分别展示。有名称、状态或可用动作的分组和滚动区域保留独立行。同一节点中重复的名称、描述和帮助只显示一次。实际值、占位文字和 URL 即使文字相同也保留各自含义,不删除不同节点共有的正文。相关控件明确展示可写性,包括 `AXIncrementor`;`unknown` 表示系统未报告。缺少可选状态字段不代表 false。超出当前值长度的 UTF-16 选区标为不可用。自定义动作简称省略 AppKit 的 target/selector 诊断信息;同名动作需要使用 `details:true` 返回的完整名称。
|
|
69
69
|
|
|
70
|
-
观察方法默认自动输出,同时返回结果。在 persistent
|
|
70
|
+
观察方法默认自动输出,同时返回结果。在 persistent coding 中使用 `void await app.getAXState()` 输出一次,或指定 `emit:false` 后自行打印一次。AX 输出后的截图可能单独失败,已成功取得的 AX 输出仍保留。
|
|
71
71
|
|
|
72
72
|
截图诊断使用带工具请求 ID 的 `[cua-screenshot]` 记录。原生阶段记录经本次 Agent 连接返回,覆盖窗口及 AX 几何检查、系统截图、PNG/Base64 编码和响应写出。Host 调试日志补充发送、响应和清理耗时。每次请求最多保留 8192 字节完整诊断记录,超限会明确标记;截图失败时附在错误中,保留到 Session trace。记录只含阶段、耗时和错误码,不含窗口标题、AX 正文或图片内容。
|
|
73
73
|
|
|
@@ -75,7 +75,7 @@ AX 观察默认只展示菜单栏入口。使用 `app.getAXState({ includeMenus:
|
|
|
75
75
|
|
|
76
76
|
### 接入外部执行环境
|
|
77
77
|
|
|
78
|
-
使用 persistent
|
|
78
|
+
使用 persistent coding 时,在 macOS 配置后追加 `--patch packages/experimental/cua-macos/examples/persistent-ptc.cordis.yml`。该 provider 为存活的 Agent 保留变量,并遵守当前选择的文件沙箱策略。调用方导入适配模块,并提供当前执行中已授权的 `tools.cua` 绑定。persistent coding 会在 CUA 指引中收到安装目录内 bridge 的文件 URL,项目无须安装本包。下面的例子用于能从自身安装目录解析本包、且已提供 `tools` 的外部宿主;本包不安装全局对象,也不负责保留变量:
|
|
79
79
|
|
|
80
80
|
```javascript
|
|
81
81
|
import { connectCua } from '@qilin-agent/experimental-cua/bridge'
|
|
@@ -142,14 +142,14 @@ AppKit sheet 可以通过其窗口 ID 单独选择,也会出现在所属文档
|
|
|
142
142
|
|
|
143
143
|
#### 模型可见内容
|
|
144
144
|
|
|
145
|
-
插件提供一个 `cua` 操作工具,定义各方法的参数。标准
|
|
145
|
+
插件提供一个 `cua` 操作工具,定义各方法的参数。标准 coding 呈现通过 `run_code` 将它暴露为 `tools.cua`。以下是本包逐字输出的指导文本:
|
|
146
146
|
|
|
147
147
|
##### 操作指导文本(原文)
|
|
148
148
|
|
|
149
149
|
```markdown
|
|
150
150
|
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
|
151
151
|
|
|
152
|
-
Initialize the object API once in persistent
|
|
152
|
+
Initialize the object API once in persistent coding. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
|
153
153
|
|
|
154
154
|
Use app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.
|
|
155
155
|
|
|
@@ -188,7 +188,7 @@ Await operations sequentially. After input, read fresh state to verify the effec
|
|
|
188
188
|
|
|
189
189
|
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
|
190
190
|
|
|
191
|
-
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent
|
|
191
|
+
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent coding, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
|
192
192
|
```
|
|
193
193
|
|
|
194
194
|
#### Token 影响
|
|
@@ -211,7 +211,7 @@ Use selectText only on controls advertising selectionWritable. It selects a uniq
|
|
|
211
211
|
- 后台输入依赖私有 SkyLight 符号,macOS 升级可能使其不可用,需在目标系统版本验证。
|
|
212
212
|
- 实测 Blender Unicode 输入曾出现分派成功但界面未显示文本,后台 AppKit 的 Command+A 曾未选中全文。输入后必须检查应用状态;工具返回成功只表示已分派。
|
|
213
213
|
- 原生二进制需要本地构建。签名、分发和系统权限引导留待后续。
|
|
214
|
-
- 持久
|
|
214
|
+
- 持久 coding 是对外部运行环境的假设,本包不实现它。真实模型任务成功率需要在原生测试和协议测试之外单独评估。
|
|
215
215
|
|
|
216
216
|
-----
|
|
217
217
|
|
package/lib/index.js
CHANGED
|
@@ -473,7 +473,7 @@ function createSharedNativeChannel(owner, config) {
|
|
|
473
473
|
/** Instructions for raw operation calls and optional object-API bindings. */
|
|
474
474
|
const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
|
475
475
|
|
|
476
|
-
Initialize the object API once in persistent
|
|
476
|
+
Initialize the object API once in persistent coding. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
|
477
477
|
|
|
478
478
|
Use app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.
|
|
479
479
|
|
|
@@ -512,7 +512,7 @@ Await operations sequentially. After input, read fresh state to verify the effec
|
|
|
512
512
|
|
|
513
513
|
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
|
514
514
|
|
|
515
|
-
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent
|
|
515
|
+
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent coding, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`;
|
|
516
516
|
//#endregion
|
|
517
517
|
//#region lib/types/index.js
|
|
518
518
|
/** macOS native CUA provider using the shared application and window API. */
|
package/lib/types/guidance.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** macOS operation guidance supplied with the shared CUA object API. */
|
|
2
2
|
/** Instructions for raw operation calls and optional object-API bindings. */
|
|
3
|
-
export declare const GUIDANCE = "Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.\n\nInitialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.\n\nUse app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.\n\nCuaScreenshot contains { bytes: Uint8Array, mimeType: \"image/png\" | \"image/jpeg\" | \"image/webp\" | \"image/gif\", pixelWidth: number, pixelHeight: number, windowId: number }. Use mimeType to choose a file extension or declare an image; do not assume PNG.\n\nObject API signatures (options default to { emit: true }):\n\n- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }\n- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>\n- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>\n- cua.getState(options?) -> { apps, browsers: [] }\n- cua.getApp(selector: string) -> App\n- app.activate() -> { dispatched: true, verified: false }\n- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>\n- app.getWindow(windowId: number) -> Window\n- app.typeText(text: string), app.pressKey(chord: string) -> action result\n- app.paste(text: string, options?: { format?: \"text\" | \"html\" }) -> action result\n\nApp and Window methods:\n\n- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }\n- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string\n- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot\n- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }\n- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }\n- click(target: number | [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", clickCount?: 1 | 2, modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n- setValue(elementIndex: number, value: string) -> action result\n- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: \"text\" | \"cursor_before\" | \"cursor_after\" }) -> action result\n- performSecondaryAction(elementIndex: number, action: string) -> action result\n- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result\n- drag(from: [x, y], to: [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n\nRaw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.\n\nAwait operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.\n\nHover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.\n\nUse selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.";
|
|
3
|
+
export declare const GUIDANCE = "Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.\n\nInitialize the object API once in persistent coding. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.\n\nUse app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.\n\nCuaScreenshot contains { bytes: Uint8Array, mimeType: \"image/png\" | \"image/jpeg\" | \"image/webp\" | \"image/gif\", pixelWidth: number, pixelHeight: number, windowId: number }. Use mimeType to choose a file extension or declare an image; do not assume PNG.\n\nObject API signatures (options default to { emit: true }):\n\n- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }\n- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>\n- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>\n- cua.getState(options?) -> { apps, browsers: [] }\n- cua.getApp(selector: string) -> App\n- app.activate() -> { dispatched: true, verified: false }\n- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>\n- app.getWindow(windowId: number) -> Window\n- app.typeText(text: string), app.pressKey(chord: string) -> action result\n- app.paste(text: string, options?: { format?: \"text\" | \"html\" }) -> action result\n\nApp and Window methods:\n\n- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }\n- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string\n- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot\n- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }\n- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }\n- click(target: number | [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", clickCount?: 1 | 2, modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n- setValue(elementIndex: number, value: string) -> action result\n- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: \"text\" | \"cursor_before\" | \"cursor_after\" }) -> action result\n- performSecondaryAction(elementIndex: number, action: string) -> action result\n- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result\n- drag(from: [x, y], to: [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n\nRaw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.\n\nAwait operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.\n\nHover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.\n\nUse selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent coding, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.";
|
|
4
4
|
//# sourceMappingURL=guidance.d.ts.map
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qilin-agent/experimental-cua-macos",
|
|
3
3
|
"description": "macOS native provider for shared Computer Use application and window objects",
|
|
4
|
-
"version": "3.1.
|
|
4
|
+
"version": "3.1.12",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
@@ -41,48 +41,48 @@
|
|
|
41
41
|
"license": "MIT",
|
|
42
42
|
"peerDependencies": {
|
|
43
43
|
"@qilin-agent/kylin": "~4.0.7",
|
|
44
|
-
"@qilin-agent/computer-use": "3.1.
|
|
45
|
-
"@qilin-agent/system-prompt": "3.1.
|
|
46
|
-
"@qilin-agent/tools": "3.1.
|
|
47
|
-
"@qilin-agent/agent": "3.1.
|
|
48
|
-
"@qilin-agent/subprocess": "3.1.
|
|
49
|
-
"@qilin-agent/user-questions": "3.1.
|
|
44
|
+
"@qilin-agent/computer-use": "3.1.12",
|
|
45
|
+
"@qilin-agent/system-prompt": "3.1.12",
|
|
46
|
+
"@qilin-agent/tools": "3.1.12",
|
|
47
|
+
"@qilin-agent/agent": "3.1.12",
|
|
48
|
+
"@qilin-agent/subprocess": "3.1.12",
|
|
49
|
+
"@qilin-agent/user-questions": "3.1.12"
|
|
50
50
|
},
|
|
51
51
|
"dependencies": {
|
|
52
52
|
"zod": "^4.4.3",
|
|
53
|
-
"@qilin-agent/atomic-write": "3.1.
|
|
54
|
-
"@qilin-agent/experimental-cua": "3.1.
|
|
53
|
+
"@qilin-agent/atomic-write": "3.1.12",
|
|
54
|
+
"@qilin-agent/experimental-cua": "3.1.12"
|
|
55
55
|
},
|
|
56
56
|
"devDependencies": {
|
|
57
|
-
"@qilin-agent/working-directory": "3.1.
|
|
57
|
+
"@qilin-agent/working-directory": "3.1.12",
|
|
58
58
|
"@qilin-agent/kylin": "~4.0.7",
|
|
59
59
|
"@qilin-agent/kylin-plugin-include": "~1.0.12",
|
|
60
60
|
"@qilin-agent/kylin-plugin-loader": "~1.0.8",
|
|
61
|
-
"@qilin-agent/agent": "3.1.
|
|
62
|
-
"@qilin-agent/agent-loop": "3.1.
|
|
63
|
-
"@qilin-agent/agent-loop-testkit": "3.1.
|
|
64
|
-
"@qilin-agent/attachment": "3.1.
|
|
65
|
-
"@qilin-agent/attachment-local": "3.1.
|
|
66
|
-
"@qilin-agent/computer-use": "3.1.
|
|
67
|
-
"@qilin-agent/llm": "3.1.
|
|
68
|
-
"@qilin-agent/session": "3.1.
|
|
69
|
-
"@qilin-agent/session-projection": "3.1.
|
|
70
|
-
"@qilin-agent/system-prompt": "3.1.
|
|
71
|
-
"@qilin-agent/tools": "3.1.
|
|
72
|
-
"@qilin-agent/subprocess": "3.1.
|
|
73
|
-
"@qilin-agent/subprocess-local": "3.1.
|
|
74
|
-
"@qilin-agent/fs-local": "3.1.
|
|
75
|
-
"@qilin-agent/llm-deepseek": "3.1.
|
|
76
|
-
"@qilin-agent/
|
|
77
|
-
"@qilin-agent/sandbox-local": "3.1.
|
|
78
|
-
"@qilin-agent/sandbox-policy": "3.1.
|
|
61
|
+
"@qilin-agent/agent": "3.1.12",
|
|
62
|
+
"@qilin-agent/agent-loop": "3.1.12",
|
|
63
|
+
"@qilin-agent/agent-loop-testkit": "3.1.12",
|
|
64
|
+
"@qilin-agent/attachment": "3.1.12",
|
|
65
|
+
"@qilin-agent/attachment-local": "3.1.12",
|
|
66
|
+
"@qilin-agent/computer-use": "3.1.12",
|
|
67
|
+
"@qilin-agent/llm": "3.1.12",
|
|
68
|
+
"@qilin-agent/session": "3.1.12",
|
|
69
|
+
"@qilin-agent/session-projection": "3.1.12",
|
|
70
|
+
"@qilin-agent/system-prompt": "3.1.12",
|
|
71
|
+
"@qilin-agent/tools": "3.1.12",
|
|
72
|
+
"@qilin-agent/subprocess": "3.1.12",
|
|
73
|
+
"@qilin-agent/subprocess-local": "3.1.12",
|
|
74
|
+
"@qilin-agent/fs-local": "3.1.12",
|
|
75
|
+
"@qilin-agent/llm-deepseek": "3.1.12",
|
|
76
|
+
"@qilin-agent/coding-runtime-node": "3.1.12",
|
|
77
|
+
"@qilin-agent/sandbox-local": "3.1.12",
|
|
78
|
+
"@qilin-agent/sandbox-policy": "3.1.12",
|
|
79
79
|
"sharp": "^0.35.5",
|
|
80
|
-
"@qilin-agent/
|
|
81
|
-
"@qilin-agent/user-questions": "3.1.
|
|
82
|
-
"@qilin-agent/settings": "3.1.
|
|
83
|
-
"@qilin-agent/app-boot": "3.1.
|
|
84
|
-
"@qilin-agent/config-editor": "3.1.
|
|
85
|
-
"@qilin-agent/hmr": "3.1.
|
|
80
|
+
"@qilin-agent/coding-runtime": "3.1.12",
|
|
81
|
+
"@qilin-agent/user-questions": "3.1.12",
|
|
82
|
+
"@qilin-agent/settings": "3.1.12",
|
|
83
|
+
"@qilin-agent/app-boot": "3.1.12",
|
|
84
|
+
"@qilin-agent/config-editor": "3.1.12",
|
|
85
|
+
"@qilin-agent/hmr": "3.1.12",
|
|
86
86
|
"@qilin-agent/kylin-plugin-timer": "~1.1.9"
|
|
87
87
|
},
|
|
88
88
|
"scripts": {
|