@qilin-agent/experimental-cua-macos 0.0.0-stage → 3.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +52 -0
- package/README.md +225 -2
- package/README.zh.md +226 -0
- package/lib/index.js +573 -0
- package/lib/types/guidance.d.ts +4 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/native-build.d.ts +47 -0
- package/lib/types/shared-native.d.ts +13 -0
- package/native/LICENSE.cua-driver +21 -0
- package/native/Package.swift +13 -0
- package/native/README.md +130 -0
- package/native/README.zh.md +131 -0
- package/native/Sources/CuaNative/AXAccess.swift +525 -0
- package/native/Sources/CuaNative/AXHover.swift +37 -0
- package/native/Sources/CuaNative/AXTextFormatter.swift +120 -0
- package/native/Sources/CuaNative/AXWorker.swift +489 -0
- package/native/Sources/CuaNative/ApplicationAuthorization.swift +69 -0
- package/native/Sources/CuaNative/ApplicationDiscovery.swift +92 -0
- package/native/Sources/CuaNative/ClipboardPaste.swift +138 -0
- package/native/Sources/CuaNative/KeyboardInput.swift +260 -0
- package/native/Sources/CuaNative/Main.swift +37 -0
- package/native/Sources/CuaNative/NativePlatform.swift +118 -0
- package/native/Sources/CuaNative/NativeProtocol.generated.swift +33 -0
- package/native/Sources/CuaNative/PIDEventPoster.swift +85 -0
- package/native/Sources/CuaNative/Protocol.swift +149 -0
- package/native/Sources/CuaNative/RequestAdmission.swift +58 -0
- package/native/Sources/CuaNative/RequestCoordinator.swift +251 -0
- package/native/Sources/CuaNative/RequestScheduler.swift +107 -0
- package/native/Sources/CuaNative/Runtime.swift +170 -0
- package/native/Sources/CuaNative/ScreenshotTrace.swift +46 -0
- package/native/Sources/CuaNative/SharedHelperProxy.swift +124 -0
- package/native/Sources/CuaNative/SharedHelperRuntime.swift +289 -0
- package/native/Sources/CuaNative/SharedHelperServer.swift +188 -0
- package/native/Sources/CuaNative/TextSelection.swift +38 -0
- package/native/Sources/CuaNative/WindowCapture.swift +165 -0
- package/native/Sources/CuaNative/WindowGeometry.swift +54 -0
- package/native/Sources/CuaNative/WindowPointerInput.swift +300 -0
- package/native/Sources/CuaNative/WindowSpaces.swift +133 -0
- package/native/protocol.schema.json +1030 -0
- package/native/tests/swift/AXFailureTests.swift +166 -0
- package/native/tests/swift/AXTextFormatterTests.swift +128 -0
- package/native/tests/swift/ApplicationDiscoveryTests.swift +59 -0
- package/native/tests/swift/ClipboardPasteTests.swift +136 -0
- package/native/tests/swift/ForegroundTests.swift +42 -0
- package/native/tests/swift/InputMappingTests.swift +34 -0
- package/native/tests/swift/PermissionIdentityTests.swift +24 -0
- package/native/tests/swift/PointerTimingTests.swift +10 -0
- package/native/tests/swift/ProtocolTests.swift +118 -0
- package/native/tests/swift/ScreenshotTraceTests.swift +23 -0
- package/native/tests/swift/SharedHelperTests.swift +90 -0
- package/native/tests/swift/SharedSocketTests.swift +57 -0
- package/native/tests/swift/SystemHoverTests.swift +27 -0
- package/native/tests/swift/TextSelectionTests.swift +47 -0
- package/native/tests/swift/WindowCaptureTests.swift +80 -0
- package/native/tests/swift/WindowGeometryTests.swift +105 -0
- package/native/tests/swift/WindowSpacesTests.swift +26 -0
- package/native/tests/test_swift.py +27 -0
- package/native/upstream.json +67 -0
- package/package.json +88 -3
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { SubprocessRuntime } from '@qilin-agent/subprocess';
|
|
2
|
+
/** Architecture names used by the platform-specific native payload. */
|
|
3
|
+
export type NativeArchitecture = 'arm64' | 'x64';
|
|
4
|
+
/**
|
|
5
|
+
* Validate the host before preparing a platform-specific executable.
|
|
6
|
+
* @param platform - Node host platform.
|
|
7
|
+
* @param architecture - Node host architecture.
|
|
8
|
+
* @returns Supported macOS architecture.
|
|
9
|
+
* @throws When the current host has no native implementation.
|
|
10
|
+
*/
|
|
11
|
+
export declare function nativeHostArchitecture(platform: string, architecture: string): NativeArchitecture;
|
|
12
|
+
/** Explicit compiler ownership and resource limits for one preparation. */
|
|
13
|
+
export interface NativeBuildSpec {
|
|
14
|
+
packageRoot: string;
|
|
15
|
+
architecture: NativeArchitecture;
|
|
16
|
+
subprocess: Pick<SubprocessRuntime, 'spawn'>;
|
|
17
|
+
signal: AbortSignal;
|
|
18
|
+
timeoutMs: number;
|
|
19
|
+
graceMs: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Locate the platform-specific executable owned by the CUA package.
|
|
23
|
+
* @param packageRoot - Directory containing the CUA package manifest.
|
|
24
|
+
* @param architecture - Architecture of the process that will execute the helper.
|
|
25
|
+
* @returns Absolute executable path.
|
|
26
|
+
*/
|
|
27
|
+
export declare function nativeExecutablePath(packageRoot: string, architecture: NativeArchitecture): string;
|
|
28
|
+
/**
|
|
29
|
+
* Recognize the repository-owned package location before allowing local compilation.
|
|
30
|
+
* @param packageRoot - Real directory containing this package.
|
|
31
|
+
* @returns True only for a CUA package in a QILIN source checkout.
|
|
32
|
+
*/
|
|
33
|
+
export declare function isSourcePackage(packageRoot: string): Promise<boolean>;
|
|
34
|
+
/**
|
|
35
|
+
* Build the package payload and retain an immutable executable for source-runtime processes.
|
|
36
|
+
* @param spec - Checkout, target, managed process provider and cancellation/deadline limits.
|
|
37
|
+
* @returns Content-addressed runtime path after compilation and complete process cleanup.
|
|
38
|
+
* @throws When compilation, cancellation, resource cleanup or artifact publication fails.
|
|
39
|
+
*/
|
|
40
|
+
export declare function buildNativeExecutable(spec: NativeBuildSpec): Promise<string>;
|
|
41
|
+
/**
|
|
42
|
+
* Prepare source builds on activation; installed packages must already carry their native payload.
|
|
43
|
+
* @param spec - Native preparation inputs for this host.
|
|
44
|
+
* @returns Ready native executable without requiring a compiler for an installed package.
|
|
45
|
+
*/
|
|
46
|
+
export declare function prepareNativeExecutable(spec: NativeBuildSpec): Promise<string>;
|
|
47
|
+
//# sourceMappingURL=native-build.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { SubprocessRuntime } from '@qilin-agent/subprocess';
|
|
2
|
+
import type { NativeClientConfig, NativeChannel } from '@qilin-agent/experimental-cua/native-client';
|
|
3
|
+
export type { NativeChannel } from '@qilin-agent/experimental-cua/native-client';
|
|
4
|
+
/**
|
|
5
|
+
* Prepare or join the native process; writes wait for identity-checked readiness.
|
|
6
|
+
* @param owner - Provider owning the short-lived native startup command.
|
|
7
|
+
* @param config - Native runtime identity, operation limits and connection deadlines.
|
|
8
|
+
* @returns One independently disposable Agent connection, without operation replay.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createSharedNativeChannel(owner: {
|
|
11
|
+
subprocess: Pick<SubprocessRuntime, 'spawn'>;
|
|
12
|
+
}, config: NativeClientConfig): NativeChannel;
|
|
13
|
+
//# sourceMappingURL=shared-native.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Cua AI, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// swift-tools-version: 6.0
|
|
2
|
+
import PackageDescription
|
|
3
|
+
|
|
4
|
+
let package = Package(
|
|
5
|
+
name: "QILINCUANative",
|
|
6
|
+
platforms: [.macOS(.v14)],
|
|
7
|
+
products: [.executable(name: "qilin-cua-native", targets: ["CuaNative"])],
|
|
8
|
+
targets: [
|
|
9
|
+
.executableTarget(name: "CuaNative"),
|
|
10
|
+
.testTarget(name: "CuaNativeTests", dependencies: ["CuaNative"], path: "tests/swift"),
|
|
11
|
+
],
|
|
12
|
+
swiftLanguageModes: [.v6]
|
|
13
|
+
)
|
package/native/README.md
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: "Native macOS helper protocol, input semantics, shared-service lifecycle and verification commands."
|
|
3
|
+
kind: "package-reference"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# qilin-cua native helper
|
|
7
|
+
|
|
8
|
+
English | [中文](README.zh.md)
|
|
9
|
+
|
|
10
|
+
## Summary
|
|
11
|
+
|
|
12
|
+
This macOS 14+ JSON-lines helper uses Swift 6, AppKit, Accessibility, CoreGraphics and ScreenCaptureKit. It builds and runs only on darwin: Windows and Linux have no native equivalent in this tree. Windows support is in development. It adapts selected MIT-licensed Cua Driver behavior without loading an external driver. `upstream.json` records the fixed upstream version, adapted Swift files and local modules; `LICENSE.cua-driver` preserves the MIT notice.
|
|
13
|
+
|
|
14
|
+
## Table of Contents
|
|
15
|
+
|
|
16
|
+
- [Build and service](#build-and-service)
|
|
17
|
+
- [Request protocol](#request-protocol)
|
|
18
|
+
- [Observation and input](#observation-and-input)
|
|
19
|
+
- [Verification](#verification)
|
|
20
|
+
|
|
21
|
+
-----
|
|
22
|
+
|
|
23
|
+
<a id="build-and-service"></a>
|
|
24
|
+
## Build and service
|
|
25
|
+
|
|
26
|
+
Build with `swift build --package-path packages/experimental/cua-macos/native --configuration release`. The binary is `.build/release/qilin-cua-native` beneath this directory. The Host runs `--ensure-shared` as a managed startup command; the serving process then belongs to its connected clients and exits after the last connection finishes cleanup. The `.build/` directory is not committed.
|
|
27
|
+
|
|
28
|
+
The shared service uses an owner-only Unix socket under `/tmp/qilin-cua-<uid>`, keyed by the executable SHA-256. A version lock elects one process; an additional real-executable-path lock excludes a different build at the same path. The first frame is protocol-1 `hello` with the runtime key and this connection’s byte, AX, operation, screenshot, event-spacing and cleanup limits. `ready` reports the verified runtime key, PID and instance ID. Every connection owns an independent AXWorker. The stdin/stdout entry is a diagnostic proxy to this service.
|
|
29
|
+
|
|
30
|
+
AppKit runs on the main event loop. Blocking AX and input use each connection’s serial actor; all connections share a FIFO for complete desktop operations. Socket writes do not hold that operation slot. A request cancellation waits for native work to stop; it does not cancel the connection’s lifetime latch. Screenshot timeout ends only the request. An unfinished system capture retains the single capture slot until its real callback arrives, rejects new captures with CAPTURE_BUSY and leaves AX/input available. Only an operation or cleanup that cannot stop can invalidate the shared process. The proxy accepts `--max-request-bytes`, `--max-response-bytes`, `--ax-timeout-ms`, `--timeout-ms`, `--screenshot-timeout-ms`, `--shutdown-grace-ms` and `--text-event-interval-ms`; the Host sets the same limits in hello. An AX deadline bounds an individual system call. Cancellation can follow partial input, and uncertain input is never replayed automatically.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
-----
|
|
34
|
+
|
|
35
|
+
<a id="request-protocol"></a>
|
|
36
|
+
## Request protocol
|
|
37
|
+
|
|
38
|
+
Operation frames remain `{ "id": "opaque request id", "method": "method name", "params": {} }`. The matching reply contains exactly one of `result` or `error: {code, message}`; malformed envelopes return `INVALID_REQUEST` with a null ID. A connection sends at most one operation at a time. `{ "type": "cancel", "id": "request id" }` cancels that request; a late cancel produces no extra reply. `protocol.schema.json` defines operation fields.
|
|
39
|
+
|
|
40
|
+
Screenshot diagnostics use `type: "diagnostic"` frames on the originating connection. Records contain requestId, stage, elapsedMs since receipt and optional errorCode, without window titles, AX text or image bytes. The stdin/stdout proxy forwards them to stderr with `[cua-screenshot]`; operation replies remain on stdout. The Host retains a bounded diagnostic buffer and includes it in screenshot failures.
|
|
41
|
+
|
|
42
|
+
Successful `activate` and `performSecondaryAction` AXRaise results include `foreground` alongside `{dispatched: true, verified: false}`. It records `requestId`, `operation`, `targetPid`, nullable `targetWindowId`, `activationRequested`, `raiseRequested`, and immediate `before`/`after` observations. `frontPid` comes from WindowServer; `focusedWindowId` is the target app's AX-focused window; `frontWindowId` and `frontWindowPid` identify the first visible normal window. Unknown values are null. Observations are sequential, not atomic or proof of causality. Tool-owned requests use the logged tool call ID. Failed operations retain the existing error response and may have partial effects.
|
|
43
|
+
|
|
44
|
+
| Method | Result |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `getCapabilities` | `{platform: "macos", permissions: {accessibility, screenRecording}, sessionLocked, methods, coordinateSpace: "window-points", browser: false}`. Permission values are boolean preflight observations; no system prompt opens. |
|
|
47
|
+
| `listApps` | Array of `{pid, name, bundleId}` for running regular applications. |
|
|
48
|
+
| `findApps` | Installed matches `{id, displayName, path, isRunning, pid?}` from standard application directories or an absolute application path; discovery never launches them. |
|
|
49
|
+
| `getApp` | `{appHandle, pid, name, bundleId, authorization}` selected by exact name, bundle ID, decimal PID or application path. A stopped selection has `pid: null` and waits for Host consent. Ambiguous names fail. |
|
|
50
|
+
| `launchApp` | Host-only continuation after consent; starts the selected application without requesting activation, waits for launch completion and binds the handle to its verified process identity. |
|
|
51
|
+
| `listWindows` | Owned normal windows with `{windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace}`. Space membership can be null. |
|
|
52
|
+
| `getState` | `{appHandle, pid, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`. `windowId` is null for an app-wide observation. Elements include index, role, title, value, description, identifier, bounds, actions, and enabled state. Element bounds are null or [x, y, width, height] in screen points; window input uses local points. These structured element diagnostics are retained by native checks and omitted by the model tool projection. Unavailable optional descriptive properties are null, and an unavailable action list is empty; permission, communication and window-discovery failures still reject the observation. |
|
|
53
|
+
| `hover`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: true, verified: false}` records native dispatch. Read fresh state to verify the intended application change. |
|
|
54
|
+
| `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`. PNG pixel dimensions can differ from window point dimensions on Retina displays. |
|
|
55
|
+
|
|
56
|
+
Every app-targeted operation requires `appHandle` from `getApp` in the same helper. The helper pins the PID and kernel process start time, and invalidates the handle and all of its snapshots after process exit or PID reuse. Calling `getApp` can acquire the replacement process without reactivating old handles. A random per-helper namespace prevents handles from a terminated helper matching those of its replacement. Repeated acquisition of the same live process shares its handle.
|
|
57
|
+
|
|
58
|
+
Stable error codes are `INVALID_REQUEST` (malformed envelope), `INVALID_ARGUMENT` (invalid parameters), `UNSUPPORTED` (unknown operation), `APP_UNAVAILABLE`, `AMBIGUOUS_APP`, `STALE_APP`, `STALE_SNAPSHOT`, `WINDOW_UNAVAILABLE`, `WINDOW_CHANGED`, `PERMISSION_DENIED`, `FOREGROUND_REQUIRED` (hover target is not the unobscured foreground window), `PANEL_NOT_READY` (system panel recipient unavailable before input), and `NATIVE_ERROR` (remaining operating-system failures). An error never promises that a partially dispatched input was rolled back. The host must not retry input automatically.
|
|
59
|
+
|
|
60
|
+
Application selection verifies the executable path and code-signing requirement before returning an authorization identity to the Host. Unsigned running applications receive a process-lifetime identity; stopped unsigned applications use an executable digest and cannot receive persistent consent. The Host obtains human consent before launch, observation or input.
|
|
61
|
+
|
|
62
|
+
Protocol declarations are generated from [the shared protocol](../../cua/src/protocol.ts). Run `pnpm exec tsx packages/experimental/cua-macos/native/generate-protocol.ts` from the repository root; package tests verify generated JSON and Swift field/enum tables. Model-submittable parameters exclude Host resource bounds and the private launch continuation.
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
-----
|
|
66
|
+
|
|
67
|
+
<a id="observation-and-input"></a>
|
|
68
|
+
## Observation and input
|
|
69
|
+
|
|
70
|
+
`getState` optionally scopes AX traversal to an owned window or nested sheet `windowId`. A new or failed refresh invalidates the previous snapshot for that app/window scope, while other window scopes remain usable. When an owned CG window has no matching AX surface, getState returns a new snapshot with empty tree/elements and no truncation; its coordinates and screenshots remain available. It rechecks process lifetime, window ownership, the unlocked session and Accessibility permission after traversal. Window-discovery APIDisabled and communication errors remain failures; unsupported or absent optional AX attributes can produce an empty observation. Missing or foreign-owned windows never produce an empty success. An AX element invalidated during traversal rejects the whole read with `STALE_SNAPSHOT`; no partial snapshot is published. Callers may read again after the application changes, without repeating input. Element actions require the exact snapshot ID. Native references never cross JSON. An optional snapshot ID on coordinate actions must belong to that window or an app-wide observation. Window-list refresh releases snapshots for closed windows. Native process identity, snapshot generation, and window ownership checks do not lock the desktop against simultaneous user or application changes.
|
|
71
|
+
|
|
72
|
+
Element clicks prefer advertised AX actions and otherwise use pointer delivery. Coordinate click, scroll and drag validate window ownership and bounds, then use background PID/window-directed primitives. Each event uses SkyLight when available, otherwise the public PID-targeted API. A posted event is never repeated through the other API. Coordinates are macOS points from the top-left of the entire window, including its title bar. Positive `deltaLinesY` reveals content above; negative reveals below. Positive `deltaLinesX` reveals content left; negative reveals right. Each axis accepts an integer from -10 to 10, and at least one must be nonzero. The helper sends line-unit scroll events directly. Click and drag accept left/right/middle buttons and command/control/option/shift/fn modifiers. Only left clicks support double-click. Modified element clicks use pointer delivery. Drag uses one button press/move/release in one window; callers explicitly supply durationMs (1..5000) and steps (1..240). The movement delay budget is divided with fractional precision across the segments; fixed press/release settling and scheduler latency are additional. No implicit fallback sends global desktop input.
|
|
73
|
+
|
|
74
|
+
Capture and pointer delivery read CG bounds before and after an exact-window AX frame query. Movement during the query returns WINDOW_CHANGED. Valid AX/CG frames must have equal outward-rounded integral point rectangles; a known mismatch returns WINDOW_CHANGED before mouse movement, priming or button delivery. The helper preserves both coordinate sources rather than substituting AX geometry for CG geometry. Missing or unusable AX geometry remains unknown, while AX communication, permission and invalid-element errors propagate. Capture without Accessibility permission retains its CG/SCK checks. Advertised AX actions bypass this pointer preflight; an AX action error does not trigger pointer input. These reads do not lock the window against later movement, and no geometry check activates or raises it.
|
|
75
|
+
|
|
76
|
+
`typeText` and `pressKey` target the selected process's current focused control, not an exact window. Key chords send PID-targeted modifier transitions before the base key and release modifiers in reverse order; these events do not use the global HID queue. Background pointer actions may change that keyboard focus. Keypad names such as KP_0 and KP_Enter preserve their physical key codes. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each down/up event is followed by the configured text interval; dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue.
|
|
77
|
+
|
|
78
|
+
The responsible host/helper needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable.
|
|
79
|
+
|
|
80
|
+
Observation reads each non-layout node’s descriptive properties and children through `AXUIElementCopyMultipleAttributeValues`. Per-attribute errors retain the individual-read rules; unsupported batch APIs use individual reads. Values are retained only during that observation, and input reads live attributes. Application state can change between nodes; a batch does not make the whole tree atomic.
|
|
81
|
+
|
|
82
|
+
System file-panel keyboard routing resolves the remote recipient through the selected app’s focused AX sheet. Only Apple’s Open and Save Panel Service is accepted, and its focused ancestry must still contain the identified remote window. Element pointer fallbacks resolve an app-owned window or sheet through AX parents when AXWindow is unavailable. AX actions retain priority over pointer input; visual click markers need no system-cursor movement.
|
|
83
|
+
|
|
84
|
+
Request modifier arrays accept command/control/option/shift/fn. The object API normalizes key-chord aliases before dispatch.
|
|
85
|
+
|
|
86
|
+
`selectText` requires a fresh element snapshot and writable AXSelectedTextRange. It matches one literal occurrence, narrowed by immediately adjacent prefix/suffix, and selects text or places the cursor before/after it. Matching and reported ranges use UTF-16 offsets. Ambiguous or missing text fails; the helper rechecks the value and selected range without rewriting content. AX text includes available writable, disabled, selected and selection-range information; AXAccess owns indices and native references while AXTextFormatter renders the observed values.
|
|
87
|
+
|
|
88
|
+
`paste` publishes one plain-text or HTML clipboard format, dispatches the application's enabled standard Paste menu command, and waits for a clipboard data request. It then restores the preserved formats only while its ownership marker and change count still match. The Host serializes its pastes across helpers with a file lock. maxClipboardBytes bounds both input and preserved contents; unsupported menus and oversized data fail before dispatch. A later copy is preserved. Cancellation restores owned clipboard data when possible; forced termination cannot guarantee restoration, and a data request does not prove the target completed editing. After dispatch, `pasteReadTimeoutMs` bounds waiting for clipboard consumption. Expiry returns `PASTE_NOT_CONSUMED`, closes the promised data provider, and restores the original clipboard only while still owned. The failure does not prove the app will never process its queued Paste command; inspect the target before deciding whether to retry.
|
|
89
|
+
|
|
90
|
+
`hover` accepts an observed element index or a window-local point and never presses a mouse button. Disabled controls may be hovered. No request completion, AX read or screenshot restores the previous point; subsequent hover calls can enter revealed content. Click sends its button events at the target without an off-window priming click. Individual requests are serialized, but no application is reserved between calls. Other input or application changes can interrupt the interaction.
|
|
91
|
+
|
|
92
|
+
`hover` first uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Revealed descendants retain the parent UI; leaving restores it once. An AX action failure propagates without trying another input route. Without an applicable AX route, hover moves the visible system cursor through the system input queue after checking the foreground PID and AppKit mouse hit testing at the point. Input-transparent windows and transparent regions do not block the target. A failed check returns `FOREGROUND_REQUIRED` without moving the cursor. The result identifies `hoverRoute` as `accessibility` or `system-pointer` and reports `systemPointerMoved`. Hover does not activate, raise or restore focus. Other input or window changes can race the check and dispatch.
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
-----
|
|
96
|
+
|
|
97
|
+
<a id="verification"></a>
|
|
98
|
+
## Verification
|
|
99
|
+
|
|
100
|
+
Use `python3 packages/experimental/cua-macos/native/tests/test_swift.py` for offline unit checks with the selected Apple toolchain’s Swift Testing framework/runtime paths and `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` for JSONL admission, ordered replies and EOF shutdown without desktop operations. Application and protocol e2e checks accept `--helper` to select a separately built comparison binary.
|
|
101
|
+
|
|
102
|
+
Run `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants are required. `--without-desktop` limits the check to protocol and process lifetime.
|
|
103
|
+
|
|
104
|
+
Run `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py` after building and unlocking the Mac to verify an owned window whose application removes and later restores its AX roots. The check requires existing Accessibility/Screen Recording grants, rejects stale element indices, captures the AXless window, verifies a coordinate click in the fixture state file, and checks restoration. It never changes system permission settings.
|
|
105
|
+
|
|
106
|
+
Run `python3 packages/experimental/cua-macos/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence` to compare complete observations of the same owned AppKit window with 35 and 400 synthetic rows. Each helper receives one warmup and three alternating measured requests. All result fields must match except helper-generated app and snapshot IDs. The report keeps raw timings, byte counts, hashes and replies; it sets no CI speed threshold and makes no memory claim. Fixture startup can activate the application. Run without other desktop automation or compilation jobs.
|
|
107
|
+
|
|
108
|
+
Use `--panels-only` to isolate the system file-panel checks. Fixture windows can join application sets so Stage Manager does not shrink them during background input. Use `--hover-only` with `e2e.py` or `electron_e2e.py` to verify hover independently of keyboard and file-panel scenarios. Add `--alternate-hover` to `e2e.py` for AX reveal, descendant entry, click and exit with sampled system-pointer stability. `--input-overlays` verifies that pass-through overlays allow hover and input-blocking overlays reject it without moving the pointer.
|
|
109
|
+
|
|
110
|
+
Run `python3 packages/experimental/cua-macos/native/tests/e2e.py` after building to exercise an isolated two-window AppKit fixture. It independently records button counts, Unicode text, actual scroll position, right-click callbacks, and drag release position. Scroll, right-button down/up, and drag down/up counters must each increase exactly once per requested gesture. The check also verifies a real NSOpenPanel sheet, path-entry keyboard routing and the selected file URL. It covers scoped snapshot replacement, screenshot dimensions, stale app cleanup and EOF shutdown. `--artifacts DIRECTORY` writes fixture-only screenshots and a private protocol transcript; the transcript contains the running-app inventory and is not a publication artifact.
|
|
111
|
+
|
|
112
|
+
For a Blender keyboard check, run `python3 packages/experimental/cua-macos/native/tests/blender_keyboard_e2e.py --blender /absolute/path/to/Blender.app/Contents/MacOS/Blender`. It launches one owned factory-startup GUI with private user directories, observes actual Text Editor text, selection and received modifier flags, and checks Command+A replacement, Shift+Left selection and ordinary input after modifier release. It requires an unlocked macOS session and Accessibility permission. The test activates its own instance; run it without concurrent desktop automation. `--helper` selects a previous compatible binary for a negative control; `--artifacts` selects the parent of a new private evidence directory. No Blender runtime is downloaded.
|
|
113
|
+
|
|
114
|
+
For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `--reference-helper`, `--candidate-helper`, `--blender` and `--artifacts`. It types identical synthetic 200/2000-scalar strings into fresh Blender Text Editors and requires full text equality, including Chinese, emoji and newlines. Repeated `--reference-arg` / `--candidate-arg` options select helper startup flags. Raw request and verified-effect timings include observer publication latency and exclude startup; these manual measurements have no CI speed threshold.
|
|
115
|
+
|
|
116
|
+
For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript.
|
|
117
|
+
|
|
118
|
+
Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures and keypad identities through an isolated AppKit receiver. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results.
|
|
119
|
+
|
|
120
|
+
Run `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence` to verify that an owned AppKit app accepts a background Paste action without reading its data, reports `PASTE_NOT_CONSUMED`, then consumes the same paste in the foreground. The test requires an unlocked Mac and Accessibility permission, activates only its own two applications, and records received actions and text independently of helper replies.
|
|
121
|
+
|
|
122
|
+
-----
|
|
123
|
+
|
|
124
|
+
## Further Exploration
|
|
125
|
+
|
|
126
|
+
- [macOS provider](../README.md)
|
|
127
|
+
|
|
128
|
+
## Dev Note
|
|
129
|
+
|
|
130
|
+
None.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: "macOS 原生辅助程序的请求协议、输入语义、共享服务和验证命令。"
|
|
3
|
+
kind: "package-reference"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# qilin-cua 原生辅助程序
|
|
7
|
+
|
|
8
|
+
[English](README.md) | 中文
|
|
9
|
+
|
|
10
|
+
## 摘要
|
|
11
|
+
|
|
12
|
+
该 JSON-lines 辅助程序支持 macOS 14 及以上,使用 Swift 6、AppKit、辅助功能、CoreGraphics 和 ScreenCaptureKit。它只在 darwin 上构建和运行,Windows 与 Linux 在本仓库中没有对应的原生实现。Windows 支持正在开发中。程序改写了 MIT 许可的 Cua Driver 部分行为,不加载外部驱动。`upstream.json` 记录固定的上游版本、改写的 Swift 文件和本地模块;`LICENSE.cua-driver` 保留 MIT 声明。
|
|
13
|
+
|
|
14
|
+
## 目录
|
|
15
|
+
|
|
16
|
+
- [构建与服务](#build-and-service)
|
|
17
|
+
- [请求协议](#request-protocol)
|
|
18
|
+
- [观察与输入](#observation-and-input)
|
|
19
|
+
- [验证](#verification)
|
|
20
|
+
|
|
21
|
+
-----
|
|
22
|
+
|
|
23
|
+
<a id="build-and-service"></a>
|
|
24
|
+
## 构建与服务
|
|
25
|
+
|
|
26
|
+
使用 `swift build --package-path packages/experimental/cua-macos/native --configuration release` 构建。二进制位于本目录下的 `.build/release/qilin-cua-native`。Host 将 `--ensure-shared` 作为受管启动命令运行,服务进程随后归属于已连接的客户端,最后一条连接完成清理后退出。`.build/` 不提交到仓库。
|
|
27
|
+
|
|
28
|
+
共享服务在 `/tmp/qilin-cua-<uid>` 下使用仅所有者可访问的 Unix socket,以可执行文件的 SHA-256 区分实例。版本锁选出一个进程,另一个真实可执行路径锁排除同一路径的不同构建。首帧为协议版本 1 的 `hello`,包含运行时标识以及该连接的字节、AX、操作、截图、事件间隔和清理限制。`ready` 返回已核实的运行时标识、PID 和实例 ID。每条连接拥有独立的 AXWorker。stdin/stdout 入口是该服务的诊断代理。
|
|
29
|
+
|
|
30
|
+
AppKit 在主事件循环上运行。阻塞式 AX 和输入使用每条连接的串行 actor,所有连接共享用于完整桌面操作的 FIFO。socket 写出不占用操作槽位。请求取消会等待原生工作停止,不取消连接的生命期控制器。截图超时只结束该请求。未完成的系统捕获保留唯一捕获槽位,直至真实回调到达;期间新截图返回 CAPTURE_BUSY,AX 和输入仍可用。只有无法停止的操作或清理才可能使共享进程失效。代理接受 `--max-request-bytes`、`--max-response-bytes`、`--ax-timeout-ms`、`--timeout-ms`、`--screenshot-timeout-ms`、`--shutdown-grace-ms` 和 `--text-event-interval-ms`;Host 在 hello 中设置相同限制。AX 超时约束单次系统调用。取消可能发生在部分输入之后,结果不确定的输入不自动重放。
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
-----
|
|
34
|
+
|
|
35
|
+
<a id="request-protocol"></a>
|
|
36
|
+
## 请求协议
|
|
37
|
+
|
|
38
|
+
操作帧为 `{ "id": "opaque request id", "method": "method name", "params": {} }`。对应回复只包含 `result` 或 `error: {code, message}` 中的一项;无效信封返回 `INVALID_REQUEST`,ID 为 null。每条连接同时最多发送一个操作。`{ "type": "cancel", "id": "request id" }` 取消该请求,迟到的取消不会产生额外回复。`protocol.schema.json` 定义操作字段。
|
|
39
|
+
|
|
40
|
+
截图诊断通过原始连接发送 `type: "diagnostic"` 帧。记录包含 requestId、stage、从接收开始计算的 elapsedMs 和可选的 errorCode,不含窗口标题、AX 文本或图片字节。stdin/stdout 代理在 stderr 上以 `[cua-screenshot]` 前缀转发诊断,操作回复仍写入 stdout。Host 保留有上限的诊断缓冲区,并将其附在截图失败中。
|
|
41
|
+
|
|
42
|
+
成功的 `activate` 和 `performSecondaryAction` AXRaise 结果在 `{dispatched: true, verified: false}` 之外包含 `foreground`,记录 `requestId`、`operation`、`targetPid`、可空的 `targetWindowId`、`activationRequested`、`raiseRequested` 和即时的 `before`/`after` 观察。`frontPid` 来自 WindowServer,`focusedWindowId` 为目标应用的 AX 焦点窗口,`frontWindowId` 和 `frontWindowPid` 标识最前面的可见普通窗口。未知值为 null。观察依次取得,不具备原子性,也不证明因果关系。工具发起的请求使用日志中的工具调用 ID。失败操作保留原有错误响应,可能已产生部分效果。
|
|
43
|
+
|
|
44
|
+
| 方法 | 结果 |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `getCapabilities` | `{platform: "macos", permissions: {accessibility, screenRecording}, sessionLocked, methods, coordinateSpace: "window-points", browser: false}`。权限值是布尔型预检结果,不打开系统提示。 |
|
|
47
|
+
| `listApps` | 运行中的普通应用数组,元素为 `{pid, name, bundleId}`。 |
|
|
48
|
+
| `findApps` | 从标准应用目录或绝对应用路径返回已安装匹配项 `{id, displayName, path, isRunning, pid?}`;发现过程不启动应用。 |
|
|
49
|
+
| `getApp` | 按精确名称、bundle ID、十进制 PID 或应用路径选择,返回 `{appHandle, pid, name, bundleId, authorization}`。未运行的选择具有 `pid: null`,等待 Host 授权。名称歧义会失败。 |
|
|
50
|
+
| `launchApp` | 授权后的 Host 专用续接操作;不请求激活,等待应用启动完成后将句柄绑定到已核实的进程身份。 |
|
|
51
|
+
| `listWindows` | 应用所属普通窗口,包含 `{windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace}`。Space 归属可为 null。 |
|
|
52
|
+
| `getState` | `{appHandle, pid, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`。应用级观察的 `windowId` 为 null。元素包含 index、role、title、value、description、identifier、bounds、actions 和 enabled 状态。元素边界为 null 或屏幕 point 单位的 [x, y, width, height],窗口输入使用局部 point。这些结构化元素诊断供原生检查保留,模型工具投影会移除它们。不可用的可选描述属性为 null,无法取得的动作列表为空;权限、通信和窗口发现失败仍会拒绝观察。 |
|
|
53
|
+
| `hover`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: true, verified: false}` 记录原生分派。需要读取新状态以核实预期的应用变化。 |
|
|
54
|
+
| `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`。Retina 显示器上 PNG 像素尺寸可能不同于窗口 point 尺寸。 |
|
|
55
|
+
|
|
56
|
+
每个面向应用的操作都要求使用同一 helper 的 `getApp` 返回的 `appHandle`。helper 固定 PID 和内核进程启动时间,进程退出或 PID 被复用后使句柄及其全部快照失效。再次调用 `getApp` 可取得替代进程,不会重新激活旧句柄。随机的 helper 命名空间防止已终止 helper 的句柄与替代实例重合。重复获取同一存活进程会共享句柄。
|
|
57
|
+
|
|
58
|
+
稳定错误码包括 `INVALID_REQUEST`(信封无效)、`INVALID_ARGUMENT`(参数无效)、`UNSUPPORTED`(操作未知)、`APP_UNAVAILABLE`、`AMBIGUOUS_APP`、`STALE_APP`、`STALE_SNAPSHOT`、`WINDOW_UNAVAILABLE`、`WINDOW_CHANGED`、`PERMISSION_DENIED`、`FOREGROUND_REQUIRED`(悬停目标不是未被遮挡的前台窗口)、`PANEL_NOT_READY`(输入前系统面板接收方尚不可用)及 `NATIVE_ERROR`(其余操作系统失败)。错误不保证已分派的部分输入被回滚,Host 不得自动重试输入。
|
|
59
|
+
|
|
60
|
+
应用选择先核实可执行路径和代码签名要求,再向 Host 返回授权身份。未签名的运行中应用使用进程生命期身份,未启动的未签名应用使用可执行文件摘要,不能获得持久授权。Host 在启动、观察和输入前取得用户许可。
|
|
61
|
+
|
|
62
|
+
协议声明从[共享协议](../../cua/src/protocol.ts)生成。在仓库根目录运行 `pnpm exec tsx packages/experimental/cua-macos/native/generate-protocol.ts`,包测试检查生成的 JSON 和 Swift 字段/枚举表。模型参数不包含 Host 资源上限和私有启动续接操作。
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
-----
|
|
66
|
+
|
|
67
|
+
<a id="observation-and-input"></a>
|
|
68
|
+
## 观察与输入
|
|
69
|
+
|
|
70
|
+
`getState` 可将 AX 遍历限定到应用所属窗口或嵌套 sheet 的 `windowId`。新的刷新或失败的刷新使同一应用/窗口范围的旧快照失效,其他窗口范围仍可使用。应用所属 CG 窗口没有匹配 AX 界面时,getState 返回空 tree/elements、truncated 为 false 的新快照,仍可截图和坐标输入。遍历后重新检查进程生命期、窗口归属、会话未锁定及辅助功能权限。窗口发现中的 APIDisabled 和通信错误仍会失败;不支持或缺失的可选 AX 属性可以产生空观察。不存在或属于其他应用的窗口不会返回空成功结果。遍历中 AX 元素失效会以 `STALE_SNAPSHOT` 拒绝整次读取,不发布部分树。应用变化后可以重新观察,但不重复输入。元素动作要求精确的快照 ID,原生引用不跨 JSON 传递。坐标动作的可选快照 ID 必须属于该窗口或应用级观察。刷新窗口列表会释放已关闭窗口的快照。进程身份、快照代次和窗口归属检查不会锁住桌面,用户或应用仍可同时改变状态。
|
|
71
|
+
|
|
72
|
+
元素点击优先使用声明的 AX 动作,否则通过鼠标事件发送。坐标点击、滚动和拖动核实窗口归属和边界后,使用后台 PID/窗口定向输入。事件优先使用 SkyLight,不可用时使用公开的 PID 定向 API;已发送的事件不通过另一 API 重发。坐标是整个窗口左上角起算的 macOS point,包含标题栏。`deltaLinesY` 为正时显示上方内容,为负时显示下方内容;`deltaLinesX` 为正时显示左侧内容,为负时显示右侧内容。每个轴接受 -10 到 10 的整数,至少一个轴非零。helper 直接发送按行计量的滚轮事件。点击和拖动支持左、右、中键及 command/control/option/shift/fn 修饰键,只有左键支持双击。带修饰键的元素点击使用鼠标输入。拖动在一个窗口中完成一次按下、移动、抬起,调用方显式提供 durationMs(1..5000)和 steps(1..240)。移动延迟按小数精度分配到各段,按下和抬起的固定等待及调度延迟另计。不存在隐式改用全局桌面输入的后备路径。
|
|
73
|
+
|
|
74
|
+
截图和鼠标输入在精确窗口的 AX 边界查询前后读取 CG 边界,查询期间移动会返回 WINDOW_CHANGED。有效的 AX/CG 边界向外取整后必须一致;已知不一致会在移动、预备事件或按键发送前返回 WINDOW_CHANGED。helper 保留两个坐标来源,不用 AX 几何替换 CG 几何。缺失或无效的 AX 几何仍表示未知,AX 通信、权限和无效元素错误继续传递。无辅助功能权限的截图保留 CG/SCK 检查。声明的 AX 动作不经过鼠标预检,AX 动作失败也不改用鼠标输入。这些读取不会锁住窗口以阻止后续移动,几何检查不激活或抬起窗口。
|
|
75
|
+
|
|
76
|
+
`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件,不指定窗口。组合键先向 PID 发送修饰键变化,再发送主键,并逆序释放修饰键,不使用全局 HID 队列。后台鼠标操作可能改变键盘焦点。KP_0 和 KP_Enter 等数字键盘名称保留物理键码。文本逐个 Unicode 字符输入,不使用剪贴板。LF、CR 和 CRLF 发送一次 Return,制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。每次按下和抬起后等待配置的文本事件间隔,分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势,无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。
|
|
77
|
+
|
|
78
|
+
负责执行的 host/helper 需要 macOS 辅助功能权限以读取 AX 和输入,需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。
|
|
79
|
+
|
|
80
|
+
观察通过 `AXUIElementCopyMultipleAttributeValues` 读取每个非布局节点的描述属性和子节点。逐属性错误保留单独读取的规则;不支持批量 API 时使用单独读取。值只在本次观察期间保留,输入读取即时属性。应用状态可能在节点之间变化,批量读取不使整棵树具备原子性。
|
|
81
|
+
|
|
82
|
+
系统文件面板的键盘路由通过所选应用的 AX 焦点 sheet 解析远端接收方。只接受 Apple 的 Open and Save Panel Service,且其焦点祖先链仍须包含已识别的远端窗口。AXWindow 不可用时,元素鼠标后备路径沿 AX 父节点解析应用所属窗口或 sheet。AX 动作保持优先,可视点击标记不需要系统鼠标移动。
|
|
83
|
+
|
|
84
|
+
请求中的修饰键数组接受 command/control/option/shift/fn。对象 API 在分派前统一组合键别名。
|
|
85
|
+
|
|
86
|
+
`selectText` 要求新近元素快照和可写 AXSelectedTextRange。它匹配唯一的字面文本,可用紧邻的 prefix/suffix 缩小范围,然后选中文本或将光标放在前后。匹配和返回范围采用 UTF-16 偏移。歧义或缺失文本会失败,helper 不改写内容,并重新核实值和选区。AX 文本包含可用的可写性、禁用状态、选中状态和选区信息;AXAccess 持有索引和原生引用,AXTextFormatter 渲染已观察的值。
|
|
87
|
+
|
|
88
|
+
`paste` 提供一种纯文本或 HTML 剪贴板格式,调用应用已启用的标准 Paste 菜单命令,并等待剪贴板数据请求。随后仅在所有权标记和修改计数仍匹配时恢复原有格式。Host 通过文件锁串行化不同 helper 的粘贴。maxClipboardBytes 同时限制输入和保留内容,不支持的菜单和过大的数据在分派前失败。后续复制会被保留。取消会尽可能恢复仍持有的剪贴板数据,强制结束不保证恢复,数据请求也不证明目标已完成编辑。 分派完成后,`pasteReadTimeoutMs` 限制等待剪贴板读取的时间。到期返回 `PASTE_NOT_CONSUMED`,关闭延迟数据提供方,并仅在仍持有剪贴板时恢复原内容。该失败不能证明应用以后不会处理已排队的 Paste 命令;决定是否重试前先检查目标内容。
|
|
89
|
+
|
|
90
|
+
`hover` 接受已观察的元素索引或窗口内坐标,不按下鼠标按钮,也可用于禁用控件。调用结束、AX 读取和截图都不恢复原位置,后续 hover 调用可进入新出现的内容。点击只在目标位置发送按键事件,不先在窗口外点击。单次请求串行执行,不在调用之间保留应用。其他输入或应用变化可能中断交互。
|
|
91
|
+
|
|
92
|
+
`hover` 优先查找同一窗口内声明了 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入展开区域的子控件时保留父控件界面,离开时恢复一次。AX 动作失败直接返回错误,不尝试其他输入路径。没有可用 AX 路径时,`hover` 核实前台 PID,并通过 AppKit 查询目标点实际接收鼠标输入的窗口后,使用系统输入队列移动可见鼠标。不接收鼠标输入的窗口和透明区域不会阻挡目标。检查失败返回 `FOREGROUND_REQUIRED`,不移动鼠标。结果中的 `hoverRoute` 标明 `accessibility` 或 `system-pointer`,`systemPointerMoved` 标明是否移动系统鼠标。悬停不激活、不抬起窗口,也不恢复焦点。其他输入或窗口变化可能与检查和分派交错。
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
-----
|
|
96
|
+
|
|
97
|
+
<a id="verification"></a>
|
|
98
|
+
## 验证
|
|
99
|
+
|
|
100
|
+
使用 `python3 packages/experimental/cua-macos/native/tests/test_swift.py` 通过选定 Apple 工具链的 Swift Testing 框架和运行时路径执行离线单元检查;使用 `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` 检查 JSONL 接纳、有序回复和 EOF 退出,不执行桌面操作。应用与协议端到端检查接受 `--helper` 以选择单独构建的对比二进制。
|
|
101
|
+
|
|
102
|
+
运行 `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper`,通过测试自建的 AppKit fixture 核实并发启动、共享服务 PID、独立句柄、同一应用交接、请求级截图期限、连接清理和信号退出。需要已有辅助功能和屏幕录制权限。`--without-desktop` 将检查限定为协议与进程生命期。
|
|
103
|
+
|
|
104
|
+
构建并解锁 Mac 后运行 `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py`,验证应用移除并恢复 AX 根节点的窗口。检查需要已有辅助功能/屏幕录制权限,会拒绝过期元素索引,捕获无 AX 窗口,通过 fixture 状态文件核实坐标点击,并检查恢复。不会修改系统权限设置。
|
|
105
|
+
|
|
106
|
+
运行 `python3 packages/experimental/cua-macos/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence`,对比同一自建 AppKit 窗口中 35 行和 400 行合成内容的完整观察。每个 helper 先预热一次,再交替测量三次。除 helper 生成的应用和快照 ID 外,结果字段必须一致。报告保留原始耗时、字节数、哈希和回复,不设置 CI 性能阈值,也不声称内存表现。fixture 启动可能激活应用。运行时避免其他桌面自动化或编译任务。
|
|
107
|
+
|
|
108
|
+
使用 `--panels-only` 单独检查系统文件面板。fixture 窗口可加入应用集合,避免 Stage Manager 在后台输入时缩小它们。使用 `e2e.py` 或 `electron_e2e.py` 的 `--hover-only`,可独立于键盘和文件面板场景验证悬停。为 `e2e.py` 加上 `--alternate-hover` 可验证 AX 展开、进入子控件、点击和移出,并采样确认系统指针不变。`--input-overlays` 验证穿透鼠标的覆盖窗口允许悬停、拦截输入的覆盖窗口拒绝悬停且不移动指针。
|
|
109
|
+
|
|
110
|
+
构建后运行 `python3 packages/experimental/cua-macos/native/tests/e2e.py`,操作一个隔离的双窗口 AppKit fixture。fixture 独立记录按钮计数、Unicode 文本、实际滚动位置、右键回调和拖动释放位置。滚动、右键按下/抬起和拖动按下/抬起计数每次手势必须恰好增加一次。检查还验证真实 NSOpenPanel sheet、路径输入的键盘路由和选中文件 URL,并覆盖范围内快照替换、截图尺寸、过期应用清理和 EOF 退出。`--artifacts DIRECTORY` 写出仅包含 fixture 的截图和私有协议记录;协议记录包含运行中应用清单,不用于公开发布。
|
|
111
|
+
|
|
112
|
+
Blender 键盘检查使用 `python3 packages/experimental/cua-macos/native/tests/blender_keyboard_e2e.py --blender /absolute/path/to/Blender.app/Contents/MacOS/Blender`。它以独立用户目录启动一个采用出厂配置的 GUI,观察 Text Editor 的实际文本、选区和修饰键标志,检查 Command+A 替换、Shift+Left 选区及修饰键释放后的普通输入。需要解锁的 macOS 会话和辅助功能权限。测试会激活自建实例,运行时避免并发桌面自动化。`--helper` 选择以前兼容的二进制作为反例对照,`--artifacts` 指定新建私有证据目录的父目录。不下载 Blender 运行时。
|
|
113
|
+
|
|
114
|
+
文本节奏对比使用 `native/tests/text_input_benchmark.py`,参数为 `--reference-helper`、`--candidate-helper`、`--blender` 和 `--artifacts`。它在新建的 Blender Text Editor 中输入相同的 200/2000 字符合成文本,要求包括中文、emoji 和换行在内的完整文本一致。重复的 `--reference-arg` / `--candidate-arg` 选项选择 helper 启动参数。原始请求和已核实效果的耗时包含观察发布延迟,不含启动耗时;这些手动测量没有 CI 性能阈值。
|
|
115
|
+
|
|
116
|
+
隔离的 Electron/Chromium 检查使用 `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`。省略 `--electron` 时使用桌面工作区已安装的运行时,检查不下载 Electron。渲染器禁用 Node 集成,启用上下文隔离和 Chromium 沙箱,使用独立 userData/sessionData/log 目录及本地 fixture 页面。渲染器的点击/输入事件通过主进程写出独立观察,测试动作仍通过原生 helper 执行。AX 轮询等待 Chromium 辅助功能启用后的真实控件。原生和 Electron 子进程组由测试持有,在删除临时文件前停止。可选产物包含仅有 fixture 的截图和私有请求记录。
|
|
117
|
+
|
|
118
|
+
运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`,通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、中键/修饰键手势及数字键盘身份。需要已有辅助功能/屏幕录制权限,接收方实际状态与分派结果分开记录。
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
运行 `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence`,验证自有 AppKit 应用在后台接收 Paste 动作但不读取数据时返回 `PASTE_NOT_CONSUMED`,以及前台读取同一次粘贴后成功。测试要求 Mac 已解锁并已授予辅助功能权限,只激活自有的两个应用,并独立记录收到的动作和文本以核对 helper 回复。
|
|
122
|
+
|
|
123
|
+
-----
|
|
124
|
+
|
|
125
|
+
## 延伸阅读
|
|
126
|
+
|
|
127
|
+
- [macOS 提供方](../README.zh.md)
|
|
128
|
+
|
|
129
|
+
## 开发备注
|
|
130
|
+
|
|
131
|
+
无。
|