@deepseek-ai/dsh-experimental-cua-macos 0.2.2-alpha.2 → 0.2.2-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.i18n.yaml +6 -6
  2. package/README.md +14 -9
  3. package/README.zh.md +14 -9
  4. package/cua-helper.json +16 -0
  5. package/lib/index.js +190 -98
  6. package/lib/types/index.d.ts +61 -3
  7. package/lib/types/native-build.d.ts +12 -12
  8. package/lib/types/permission-guide.d.ts +23 -0
  9. package/native/DSHComputerUse.icns +0 -0
  10. package/native/Info.plist +24 -0
  11. package/native/LICENSE.computer-use-icon +18 -0
  12. package/native/README.md +24 -6
  13. package/native/README.zh.md +24 -6
  14. package/native/Sources/CuaNative/Main.swift +40 -27
  15. package/native/Sources/CuaNative/NativeProtocol.generated.swift +2 -2
  16. package/native/Sources/CuaNative/PermissionGuide.swift +404 -0
  17. package/native/Sources/CuaNative/Permissions.swift +79 -19
  18. package/native/Sources/CuaNative/RequestCoordinator.swift +10 -3
  19. package/native/Sources/CuaNative/SharedHelperProxy.swift +7 -7
  20. package/native/Sources/CuaNative/SharedHelperRuntime.swift +106 -51
  21. package/native/Sources/CuaNative/SharedHelperServer.swift +2 -2
  22. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/CodeResources +0 -0
  23. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/Info.plist +24 -0
  24. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
  25. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/Resources/DSHComputerUse.icns +0 -0
  26. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/Resources/LICENSE.computer-use-icon +18 -0
  27. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/Resources/LICENSE.cua-driver +21 -0
  28. package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/_CodeSignature/CodeResources +150 -0
  29. package/native/bin/darwin-arm64/current.json +1 -0
  30. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/CodeResources +0 -0
  31. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/Info.plist +24 -0
  32. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
  33. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/Resources/DSHComputerUse.icns +0 -0
  34. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/Resources/LICENSE.computer-use-icon +18 -0
  35. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/Resources/LICENSE.cua-driver +21 -0
  36. package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/_CodeSignature/CodeResources +150 -0
  37. package/native/bin/darwin-x64/current.json +1 -0
  38. package/native/build_app.py +85 -0
  39. package/native/protocol.schema.json +38 -2
  40. package/native/tests/swift/SharedHelperTests.swift +52 -0
  41. package/native/tests/swift/SystemPermissionsTests.swift +245 -0
  42. package/native/upstream.json +1 -0
  43. package/package.json +53 -46
@@ -1,3 +1,4 @@
1
+ import Schema from '@deepseek-ai/schemastery';
1
2
  import type { Context } from '@deepseek-ai/cordis';
2
3
  import { CuaConfig } from '@deepseek-ai/dsh-cua';
3
4
  export { GUIDANCE } from './guidance.ts';
@@ -6,9 +7,66 @@ export declare const name = "experimental-cua-macos";
6
7
  /** Services required by the shared CUA executor and the native launcher. */
7
8
  export declare const inject: string[];
8
9
  /** Native preparation, operation bounds, and profile-local application grants. */
9
- export type Config = CuaConfig;
10
- /** Validate the shared configuration before preparing the macOS helper. */
11
- export declare const Config: typeof CuaConfig;
10
+ export interface Config extends CuaConfig {
11
+ /** Interval in milliseconds between permission and position checks while the native guide is open. */
12
+ permissionPollIntervalMs: number;
13
+ }
14
+ /** Validate shared resource bounds and macOS guide configuration before preparing the helper. */
15
+ export declare const Config: Schema<{
16
+ agentPreset?: string | false | null;
17
+ allowedApps?: ({
18
+ id?: string | null;
19
+ name?: string | null;
20
+ executablePath?: string | null;
21
+ } & import("@deepseek-ai/cosmokit").Dict<any, string>)[] | null;
22
+ helperPath?: string | null;
23
+ buildTimeoutMs?: number | null;
24
+ timeoutMs?: number | null;
25
+ connectTimeoutMs?: number | null;
26
+ screenshotTimeoutMs?: number | null;
27
+ axTimeoutMs?: number | null;
28
+ maxAppMatches?: number | null;
29
+ maxClipboardBytes?: number | null;
30
+ pasteReadTimeoutMs?: number | null;
31
+ maxRequestBytes?: number | null;
32
+ dragDurationMs?: number | null;
33
+ maxClickHoldMs?: number | null;
34
+ dragSteps?: number | null;
35
+ maxResponseBytes?: number | null;
36
+ textEventIntervalMs?: number | null;
37
+ graceMs?: number | null;
38
+ maxElements?: number | null;
39
+ maxDepth?: number | null;
40
+ } & import("@deepseek-ai/cosmokit").Dict & {
41
+ permissionPollIntervalMs?: number | null;
42
+ }, {
43
+ agentPreset: string | false;
44
+ allowedApps: Schemastery.ObjectT<NoInfer<{
45
+ id: Schema<string, string, "defined">;
46
+ name: Schema<string, string, "defined">;
47
+ executablePath: Schema<string, string, "plain">;
48
+ }>>[];
49
+ helperPath: string;
50
+ buildTimeoutMs: number;
51
+ timeoutMs: number;
52
+ connectTimeoutMs: number;
53
+ screenshotTimeoutMs: number;
54
+ axTimeoutMs: number;
55
+ maxAppMatches: number;
56
+ maxClipboardBytes: number;
57
+ pasteReadTimeoutMs: number;
58
+ maxRequestBytes: number;
59
+ dragDurationMs: number;
60
+ maxClickHoldMs: number;
61
+ dragSteps: number;
62
+ maxResponseBytes: number;
63
+ textEventIntervalMs: number;
64
+ graceMs: number;
65
+ maxElements: number;
66
+ maxDepth: number;
67
+ } & import("@deepseek-ai/cosmokit").Dict & {
68
+ permissionPollIntervalMs: number;
69
+ }, "plain">;
12
70
  /**
13
71
  * Mount logged native operations with separate Agent connections to the shared helper.
14
72
  * @param ctx - Host context with computer-use, subprocess and tool services.
@@ -2,7 +2,7 @@ import type { SubprocessRuntime } from '@deepseek-ai/dsh-subprocess';
2
2
  /** Architecture names used by the platform-specific native payload. */
3
3
  export type NativeArchitecture = 'arm64' | 'x64';
4
4
  /**
5
- * Validate the host before preparing a platform-specific executable.
5
+ * Validate the host before preparing a platform-specific application.
6
6
  * @param platform - Node host platform.
7
7
  * @param architecture - Node host architecture.
8
8
  * @returns Supported macOS architecture.
@@ -19,29 +19,29 @@ export interface NativeBuildSpec {
19
19
  graceMs: number;
20
20
  }
21
21
  /**
22
- * Locate the platform-specific executable owned by the CUA package.
23
- * @param packageRoot - Directory containing the CUA package manifest.
24
- * @param architecture - Architecture of the process that will execute the helper.
25
- * @returns Absolute executable path.
22
+ * Resolve the immutable application selected by a packaged or source-build receipt.
23
+ * @param directory - Architecture-specific directory containing current.json and application generations.
24
+ * @returns Executable inside the selected application bundle.
25
+ * @throws When the receipt is invalid or the application executable is missing.
26
26
  */
27
- export declare function nativeExecutablePath(packageRoot: string, architecture: NativeArchitecture): string;
27
+ export declare function nativeExecutablePath(directory: string): Promise<string>;
28
28
  /**
29
29
  * Recognize the repository-owned package location before allowing local compilation.
30
- * @param packageRoot - Real directory containing this package.
30
+ * @param packageRoot - Real directory containing the CUA package manifest.
31
31
  * @returns True only for a CUA package in a DSH source checkout.
32
32
  */
33
33
  export declare function isSourcePackage(packageRoot: string): Promise<boolean>;
34
34
  /**
35
- * Build the package payload and retain an immutable executable for source-runtime processes.
35
+ * Build and sign an immutable helper application with the selected Apple developer tools.
36
36
  * @param spec - Checkout, target, managed process provider and cancellation/deadline limits.
37
- * @returns Content-addressed runtime path after compilation and complete process cleanup.
38
- * @throws When compilation, cancellation, resource cleanup or artifact publication fails.
37
+ * @returns Executable inside the prepared application, after complete compiler cleanup.
38
+ * @throws When preparation, cancellation, resource cleanup or receipt validation fails.
39
39
  */
40
40
  export declare function buildNativeExecutable(spec: NativeBuildSpec): Promise<string>;
41
41
  /**
42
- * Prepare source builds on activation; installed packages must already carry their native payload.
42
+ * Prepare source builds on activation; installed packages carry their signed application.
43
43
  * @param spec - Native preparation inputs for this host.
44
- * @returns Ready native executable without requiring a compiler for an installed package.
44
+ * @returns Executable inside a ready application without compiling or re-signing an installed package.
45
45
  */
46
46
  export declare function prepareNativeExecutable(spec: NativeBuildSpec): Promise<string>;
47
47
  //# sourceMappingURL=native-build.d.ts.map
@@ -0,0 +1,23 @@
1
+ /** macOS permission guide copy and private native request parameters. */
2
+ import { z } from 'zod';
3
+ import type { CuaBackend } from '@deepseek-ai/dsh-cua';
4
+ /** Native-only fields for opening the selected privacy pane. */
5
+ export declare const PermissionSettingsParameters: z.ZodObject<{
6
+ settingsMessage: z.ZodString;
7
+ }, z.core.$strip>;
8
+ /** Native-only guide fields, validated again by the Swift request parser. */
9
+ export declare const PermissionGuideParameters: z.ZodObject<{
10
+ settingsMessage: z.ZodString;
11
+ instruction: z.ZodString;
12
+ backLabel: z.ZodString;
13
+ finderLabel: z.ZodString;
14
+ unavailableMessage: z.ZodString;
15
+ pollIntervalMs: z.ZodNumber;
16
+ }, z.core.$strip>;
17
+ /**
18
+ * Resolve native permission copy from the Client language; unsupported languages use English.
19
+ * @param pollIntervalMs Interval between native guide permission and position checks.
20
+ * @returns Provider-owned parameters for the selected permission action.
21
+ */
22
+ export declare function createPermissionParameters(pollIntervalMs: number): CuaBackend['permissionParameters'];
23
+ //# sourceMappingURL=permission-guide.d.ts.map
Binary file
@@ -0,0 +1,24 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3
+ <plist version="1.0">
4
+ <dict>
5
+ <key>CFBundleIdentifier</key>
6
+ <string>com.deepseek.harness.computer-use</string>
7
+ <key>CFBundleExecutable</key>
8
+ <string>dsh-cua-native</string>
9
+ <key>CFBundleName</key>
10
+ <string>DSH Computer Use</string>
11
+ <key>CFBundleDisplayName</key>
12
+ <string>DSH Computer Use</string>
13
+ <key>CFBundleIconFile</key>
14
+ <string>DSHComputerUse.icns</string>
15
+ <key>CFBundlePackageType</key>
16
+ <string>APPL</string>
17
+ <key>CFBundleVersion</key>
18
+ <string>1</string>
19
+ <key>LSMinimumSystemVersion</key>
20
+ <string>14.0</string>
21
+ <key>LSUIElement</key>
22
+ <true/>
23
+ </dict>
24
+ </plist>
@@ -0,0 +1,18 @@
1
+ Computer Use icon adapted from Lucide mouse-pointer-click.
2
+ Source artwork: packages/experimental/cua-bundle/icon.svg
3
+ https://lucide.dev/icons/mouse-pointer-click
4
+
5
+ ISC License
6
+ Copyright (c) 2026 Lucide Icons and Contributors
7
+ https://github.com/lucide-icons/lucide/blob/main/LICENSE
8
+
9
+ Permission to use, copy, modify, and/or distribute this software for any
10
+ purpose with or without fee is hereby granted, provided that the above
11
+ copyright notice and this permission notice appear in all copies.
12
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15
+ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
package/native/README.md CHANGED
@@ -23,7 +23,21 @@ This macOS 14+ JSON-lines helper uses Swift 6, AppKit, Accessibility, CoreGraphi
23
23
  <a id="build-and-service"></a>
24
24
  ## Build and service
25
25
 
26
- Build with `swift build --package-path packages/experimental/cua-macos/native --configuration release`. The binary is `.build/release/dsh-cua-native` beneath this directory. The Host runs `--ensure-shared` as a managed startup command; the serving process then belongs to its connected clients and exits after the last connection finishes cleanup. The `.build/` directory is not committed.
26
+ Run `pnpm --dir packages/experimental/cua-macos run build:native` to publish a signed `DSH Computer Use.app` under `native/bin/darwin-<arch>`. The `current.json` receipt selects a complete content-addressed generation. Signing defaults to ad hoc; release packagers can pass `--identity` for Developer ID signing and must notarize the distribution. The Host runs the application executable with `--ensure-shared`, which launches the background service through LaunchServices. The service exits after its last connection finishes cleanup.
27
+
28
+ Generations in `native/.build/prepared/<arch>` and `native/bin/darwin-<arch>` are retained without automatic garbage collection. Use a fresh `--output-directory` for release staging, or clean the packaging output before building while no client uses it, so old generations do not enter the tarball. Secure signing timestamps can create distinct generations for identical sources.
29
+
30
+ The manual **Package CUA helper · macOS** [workflow](../../../../.github/workflows/cua-helper-package.yml) builds both architectures on `dsh-mac-package-trial`. Select a trusted `source_ref` and machine `preset`; credentials use the existing [macOS packaging configuration](../../../../apps/desktop/packaging/README.md#macos-actions). Packaging does not require a macOS desktop login or Rosetta. `pnpm exec tsx scripts/cua-helper/package.ts --out <new-directory>` runs the same preparation from the repository root. It signs, notarizes and staples each complete application before selecting its final content digest, then verifies architectures, signatures, tickets, Gatekeeper and file integrity from the extracted `cua-helper.tgz`. Packaging does not launch the helper. Run `python3 scripts/cua-helper/smoke.py <helper-executable>` separately in a logged-in desktop session to check capabilities, helper identity and cleanup; x64 execution on Apple Silicon requires Rosetta. The smoke does not request permissions or inspect applications. Failed or timed-out commands and unreadable startup replies report captured stdout and stderr; invalid encoding is replaced only in diagnostic text, while protocol parsing remains strict. A successful package build does not certify runtime behavior.
31
+
32
+ The workflow retains `cua-helper.tgz` and `cua-helper.json`, which binds the product version, source commit, Team ID and both application digests. [Release publish](../../../../.github/workflows/release-publish.yml) consumes this artifact before packing the provider for npm; ordinary PR release rehearsals have no signing credentials. Staging rejects a source/version mismatch, incomplete architectures, stale generations, links, changed bytes and missing executable permissions. npm publication rechecks the exact provider tarball before any registry write. A re-signed release has different bytes; preserve the retained artifact when retrying a partially published version.
33
+
34
+ In **Release publish (dsh)**, leave `cua_run_id` and `cua_run_attempt` empty to build a new helper. To reuse an existing artifact, set `cua_run_id` to a run of **Package CUA helper · macOS** or **Release publish (dsh)** in this repository. `cua_run_attempt` selects an exact attempt; empty selects the latest attempt once. The selected attempt must have completed with a successful CUA packaging job and an unexpired artifact; a later npm publication failure does not invalidate that job. Failed packaging, missing or expired artifacts fail without choosing an older attempt. The hosted pack job downloads the selected artifact by ID, then applies the same source commit, product version, architecture and file-integrity checks as a fresh build. This does not support reuse across commits or product versions and does not contact the Mac runner. Artifacts are retained for 14 days.
35
+
36
+ If npm publication fails after uploading some packages, rerun only the **Publish to npm** job while its `dsh-npm-tarballs` artifact is retained (7 days). It reuses the exact tarballs without rebuilding or signing; already published identical bytes are skipped. Do not rerun all jobs for that retry. If a new dispatch must repack the same commit and version, copy the original `cua_run_id` and `cua_run_attempt` from the pack job summary so the signed helper bytes stay unchanged. Matching commit and version alone do not make a newly signed helper byte-identical. If original artifacts are unavailable or any rebuilt package differs, stop: an existing npm version cannot be replaced with different content.
37
+
38
+ Startup errors retain their code and message through an owner-only, per-attempt failure file and the launcher’s JSON error frame. Launch-lock and readiness waits share one deadline; operating-system launch time consumes that budget. Readiness must name the newly launched PID. A failed new instance is force-terminated and awaited before the launcher reports failure; this cleanup has a separate bound equal to the startup timeout. Existing shared services are not terminated by a failed attaching launcher.
39
+
40
+ The application icon is generated from the [Computer Use artwork](../../cua-bundle/icon.svg) with `pnpm --dir packages/experimental/cua-macos run build:icon`. The native icon adds an opaque white rounded tile beneath the artwork and keeps its outer corners transparent. System permission prompts and the DSH permission page use this application icon.
27
41
 
28
42
  The shared service uses an owner-only Unix socket under `/tmp/dsh-cua-<uid>`, keyed by the executable SHA-256. A version lock elects one process; an additional real-executable-path lock excludes a different build at the same path. The first frame is protocol-1 `hello` with the runtime key and this connection’s byte, AX, operation, screenshot, event-spacing and cleanup limits. `ready` reports the verified runtime key, PID and instance ID. Every connection owns an independent AXWorker. The stdin/stdout entry is a diagnostic proxy to this service.
29
43
 
@@ -77,9 +91,9 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX
77
91
 
78
92
  `typeText` and `pressKey` target the selected process's current focused control, not an exact window. Key chords send PID-targeted modifier transitions before the base key and release modifiers in reverse order; these events do not use the global HID queue. Background pointer actions may change that keyboard focus. Keypad names such as KP_0 and KP_Enter preserve their physical key codes. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each down/up event is followed by the configured text interval; dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue.
79
93
 
80
- The responsible host/helper needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable.
94
+ `DSH Computer Use.app` needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable.
81
95
 
82
- Host-only permission controls are `permissionStatus {}`, `requestPermission {permission}`, and `openPermissionSettings {permission}`, where permission is `accessibility` or `screenRecording`. They are excluded from model tools and from `getCapabilities.methods`. Permission status includes observed helper/parent process paths; these paths do not identify the TCC responsible application. The operating-system prompt names the application and asks for the user’s decision. The helper identifies the responsible process with the optional libSystem SPI `responsibility_get_pid_responsible_for_pid`, then captures its application name and icon before publishing readiness. The function is resolved dynamically, following the same signature used by [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm). If the symbol, process, or application metadata is unavailable, attribution stays unknown; it is not inferred from the nearest GUI ancestor. Permission checks remain independent of this metadata.
96
+ Host-only permission controls are `permissionStatus {}`, `requestPermission` and `openPermissionSettings {permission}`, where permission is `accessibility` or `screenRecording`. Requests carry localized `instruction`, `backLabel`, `finderLabel`, `unavailableMessage`, `settingsMessage`, and an integer `pollIntervalMs` of 100–2147483647. The native guide substitutes its application name for `{application}`. Status reports permission observations, process diagnostics, and `launchApplication` metadata. These operations remain absent from the model tool and `getCapabilities.methods`. The helper dynamically resolves `responsibility_get_pid_responsible_for_pid` with the signature used by [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm), capturing the responsible bundle before readiness and falling back to its own bundle when unavailable. Parent paths do not determine authorization ownership.
83
97
 
84
98
  Observation reads each non-layout node’s descriptive properties and children through `AXUIElementCopyMultipleAttributeValues`. Per-attribute errors retain the individual-read rules; unsupported batch APIs use individual reads. Values are retained only during that observation, and input reads live attributes. Application state can change between nodes; a batch does not make the whole tree atomic.
85
99
 
@@ -101,13 +115,17 @@ Request modifier arrays accept command/control/option/shift/fn. The object API n
101
115
  <a id="verification"></a>
102
116
  ## Verification
103
117
 
104
- Use `python3 packages/experimental/cua-macos/native/tests/test_swift.py` for offline unit checks with the selected Apple toolchain’s Swift Testing framework/runtime paths and `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` for JSONL admission, ordered replies and EOF shutdown without desktop operations. Application and protocol e2e checks accept `--helper` to select a separately built comparison binary.
118
+ Desktop checks require grants for `DSH Computer Use.app`; source rebuilds with ad-hoc signatures can require authorization again. `--helper` names the executable inside that application. Bare executables are supported only as old-version comparison inputs, not as current helper overrides.
119
+
120
+ Run `python3 packages/experimental/cua-macos/native/tests/startup_e2e.py` to check immediate exit, typed startup errors, stale readiness rejection and termination of stalled new instances through LaunchServices. It uses owned applications and requests no system permissions. Publication unit tests mock signing; `build:native` invokes real `codesign --verify --strict`, and `launcher_identity_e2e.py` verifies LaunchServices attribution.
121
+
122
+ Use `python3 packages/experimental/cua-macos/native/tests/test_swift.py` for offline unit checks with the selected Apple toolchain’s Swift Testing framework/runtime paths and `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` for JSONL admission, ordered replies and EOF shutdown without desktop operations. Application and protocol e2e checks accept `--helper` to select a separately built comparison binary. `python3 packages/experimental/cua-macos/native/tests/test_build_app.py` covers application publication, signing failure and corrupted-generation rejection.
105
123
 
106
- Run `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants are required. `--without-desktop` limits the check to protocol and process lifetime.
124
+ Run `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants for `DSH Computer Use.app` are required. `--without-desktop` limits the check to protocol and process lifetime.
107
125
 
108
126
  Run `python3 packages/experimental/cua-macos/native/tests/application_consent_e2e.py --helper /absolute/helper` in an unlocked macOS GUI session with the Apple Swift compiler available. It owns and cleans up its AppKit bundles and standalone GUI processes, checks metadata replacement, cold and direct launches, shared bundle IDs, persistent path identity across standalone restarts, and selection after a running executable is deleted. The default identity check requires no Accessibility or Screen Recording grants. Add `--desktop` to read intact fixtures’ controls, verify AX button presses in their state files, capture their windows, and record observation results after executable deletion; this requires existing grants for both permissions.
109
127
 
110
- Run `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py --helper /absolute/helper` in a GUI login session to verify a Launch Services application that starts a nested GUI launcher and then the helper. The helper identifies the outer responsible application and retains its name and icon after the launchers exit. This check owns both applications and the helper, cleans up its Launch Services registrations, and does not request Accessibility or Screen Recording access.
128
+ Run `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py` in a GUI login session to verify that the helper owns its permission identity even when started through two nested GUI launchers. Concurrent connections report the same helper name and icon. The check owns its applications, cleans up LaunchServices registrations, and requests no Accessibility or Screen Recording access.
111
129
 
112
130
  Run `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py` after building and unlocking the Mac to verify an owned window whose application removes and later restores its AX roots. The check requires existing Accessibility/Screen Recording grants, rejects stale element indices, captures the AXless window, verifies a coordinate click in the fixture state file, and checks restoration. It never changes system permission settings.
113
131
 
@@ -23,7 +23,21 @@ kind: "package-reference"
23
23
  <a id="build-and-service"></a>
24
24
  ## 构建与服务
25
25
 
26
- 使用 `swift build --package-path packages/experimental/cua-macos/native --configuration release` 构建。二进制位于本目录下的 `.build/release/dsh-cua-native`。Host 将 `--ensure-shared` 作为受管启动命令运行,服务进程随后归属于已连接的客户端,最后一条连接完成清理后退出。`.build/` 不提交到仓库。
26
+ 运行 `pnpm --dir packages/experimental/cua-macos run build:native`,在 `native/bin/darwin-<arch>` 下生成带签名的 `DSH Computer Use.app`,`current.json` 选择完整的内容摘要目录。默认使用临时签名;正式打包可传入 `--identity` 指定 Developer ID,并由发布流程完成公证。Host 运行应用内的 `--ensure-shared` 入口,经 LaunchServices 后台启动服务。最后一条连接完成清理后,服务退出。
27
+
28
+ `native/.build/prepared/<arch>` 和 `native/bin/darwin-<arch>` 中的各代应用不会自动清理。正式打包时使用全新的 `--output-directory`,或确认没有客户端使用后清理打包输出再构建,避免旧应用进入 tarball。安全签名时间戳可能让相同源码生成不同代应用。
29
+
30
+ 手动工作流 **Package CUA helper · macOS** 的[配置](../../../../.github/workflows/cua-helper-package.yml)在 `dsh-mac-package-trial` 构建两种架构。选择可信的 `source_ref` 和机器 `preset`;凭据沿用[macOS 打包配置](../../../../apps/desktop/packaging/README.zh.md#macos-actions)。打包无需 macOS 桌面登录或 Rosetta。在仓库根目录运行 `pnpm exec tsx scripts/cua-helper/package.ts --out <new-directory>` 可执行相同准备。完整应用签名、公证和贴票后才确定最终内容摘要,再从解包后的 `cua-helper.tgz` 验证架构、签名、票据、Gatekeeper 和文件完整性。打包不启动 helper。在已登录的桌面会话中单独运行 `python3 scripts/cua-helper/smoke.py <helper-executable>` 检查能力、helper 身份和退出清理;Apple Silicon 上执行 x64 版本需要 Rosetta。该检查不申请权限或观察应用。命令失败、超时或启动回复无法解析时,会输出捕获的 stdout 和 stderr;无效编码只在诊断文本中替换,协议解析仍严格校验。打包成功不代表运行验收通过。
31
+
32
+ 工作流保留 `cua-helper.tgz` 和 `cua-helper.json`,后者记录产品版本、源码 commit、Team ID 和两份应用摘要。[Release publish](../../../../.github/workflows/release-publish.yml)在生成 provider 的 npm 包前接入该产物;普通 PR 发布检查不使用签名凭据。准备步骤拒绝源码或版本不匹配、架构不全、旧应用目录、链接、字节变化和执行权限缺失。npm 发布在写入注册表前重新检查实际 provider tarball。重新签名会改变产物字节,重试部分发布的版本时需保留原产物。
33
+
34
+ 在 **Release publish (dsh)** 中,`cua_run_id` 和 `cua_run_attempt` 都留空时重新构建 helper。复用已有产物时,将 `cua_run_id` 填为本仓库 **Package CUA helper · macOS** 或 **Release publish (dsh)** 的运行 ID。`cua_run_attempt` 指定准确的 attempt;留空时选定最新一次。所选 attempt 必须已结束,CUA 打包 job 成功,且产物未过期;后续 npm 发布失败不影响该打包 job 的结果。打包失败、产物缺失或过期时直接失败,不选择更早的 attempt。托管 runner 上的 pack job 按产物 ID 下载,再执行与新构建相同的源码 commit、产品版本、架构及文件完整性校验。此方式不支持跨 commit 或产品版本复用,也不访问 Mac runner。产物保留 14 天。
35
+
36
+ npm 发布在上传部分包后失败时,只重跑 **Publish to npm** job,复用仍在保留期内的 `dsh-npm-tarballs`(7 天)。该 job 不构建或签名,直接复用原 tarball;已发布且字节相同的包会被跳过。此次重试不要重跑全部 job。若必须新建发布运行,以同一 commit 和版本重新打包,应从 pack job 的摘要复制原 `cua_run_id` 和 `cua_run_attempt`,保持已签名 helper 的字节不变。commit 和版本一致不代表重新签名后字节也相同。原产物不可用或任何重新打包的内容不同时,应停止发布;不能用不同内容替换已有 npm 版本。
37
+
38
+ 启动错误通过当前用户专用、每次启动独立的错误文件和 launcher 的 JSON 错误帧保留错误码与消息。等待启动锁和就绪状态共用一个期限,系统启动应用的耗时计入该预算。就绪记录必须指向本次启动的 PID。新实例启动失败后,launcher 强制终止并等待其退出,再报告失败;清理另有与启动超时等长的期限。连接已有服务失败时不会终止该服务。
39
+
40
+ 应用图标通过 `pnpm --dir packages/experimental/cua-macos run build:icon` 从 [Computer Use 图形](../../cua-bundle/icon.svg)生成。原生图标在图形下添加不透明的白色圆角底板,外侧四角保持透明。系统授权提示和 DSH 权限页使用这个应用图标。
27
41
 
28
42
  共享服务在 `/tmp/dsh-cua-<uid>` 下使用仅所有者可访问的 Unix socket,以可执行文件的 SHA-256 区分实例。版本锁选出一个进程,另一个真实可执行路径锁排除同一路径的不同构建。首帧为协议版本 1 的 `hello`,包含运行时标识以及该连接的字节、AX、操作、截图、事件间隔和清理限制。`ready` 返回已核实的运行时标识、PID 和实例 ID。每条连接拥有独立的 AXWorker。stdin/stdout 入口是该服务的诊断代理。
29
43
 
@@ -77,9 +91,9 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后
77
91
 
78
92
  `typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件,不指定窗口。组合键先向 PID 发送修饰键变化,再发送主键,并逆序释放修饰键,不使用全局 HID 队列。后台鼠标操作可能改变键盘焦点。KP_0 和 KP_Enter 等数字键盘名称保留物理键码。文本逐个 Unicode 字符输入,不使用剪贴板。LF、CR 和 CRLF 发送一次 Return,制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。每次按下和抬起后等待配置的文本事件间隔,分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势,无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。
79
93
 
80
- 负责执行的 host/helper 需要 macOS 辅助功能权限以读取 AX 和输入,需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。
94
+ `DSH Computer Use.app` 需要 macOS 辅助功能权限以读取 AX 和输入,需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。
81
95
 
82
- 仅供 Host 使用的权限操作为 `permissionStatus {}`、`requestPermission {permission}` 和 `openPermissionSettings {permission}`,其中 permission 为 `accessibility` 或 `screenRecording`。这些操作不出现在模型工具及 `getCapabilities.methods` 中。权限状态包含观察到的 helper 和父进程路径,这些路径不能确定 TCC 归属的应用。系统授权提示负责指出应用并询问用户决定。helper 使用可选的 libSystem 内部接口 `responsibility_get_pid_responsible_for_pid` 查询责任进程,在报告就绪前记录对应应用的名称和图标。接口通过动态加载解析,函数签名与 [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm) 使用的一致。符号、进程或应用信息不可用时,归属保持未知,不根据最近的 GUI 祖先推断。权限检查独立于这些信息。
96
+ 仅供 Host 使用的权限操作为 `permissionStatus {}`、`requestPermission` 和 `openPermissionSettings {permission}`,其中 permission 为 `accessibility` 或 `screenRecording`。请求携带本地化的 `instruction`、`backLabel`、`finderLabel`、`unavailableMessage`、`settingsMessage`,以及 100–2147483647 的整数 `pollIntervalMs`。原生引导将 `{application}` 替换为自身应用名。状态返回权限观测、进程诊断和 `launchApplication` 信息。这些操作不出现在模型工具及 `getCapabilities.methods` 中。helper 动态解析 `responsibility_get_pid_responsible_for_pid`,函数签名与 [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm) 一致,在报告就绪前记录责任 bundle,无法获取时使用自身 bundle。父进程路径不决定授权归属。
83
97
 
84
98
  观察通过 `AXUIElementCopyMultipleAttributeValues` 读取每个非布局节点的描述属性和子节点。逐属性错误保留单独读取的规则;不支持批量 API 时使用单独读取。值只在本次观察期间保留,输入读取即时属性。应用状态可能在节点之间变化,批量读取不使整棵树具备原子性。
85
99
 
@@ -101,13 +115,17 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后
101
115
  <a id="verification"></a>
102
116
  ## 验证
103
117
 
104
- 使用 `python3 packages/experimental/cua-macos/native/tests/test_swift.py` 通过选定 Apple 工具链的 Swift Testing 框架和运行时路径执行离线单元检查;使用 `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` 检查 JSONL 接纳、有序回复和 EOF 退出,不执行桌面操作。应用与协议端到端检查接受 `--helper` 以选择单独构建的对比二进制。
118
+ 桌面检查要求给 `DSH Computer Use.app` 授权;临时签名的源码构建变化后可能需要重新授权。`--helper` 指向该应用内的可执行文件。裸二进制仅用于旧版本对比,不支持作为当前 helper 的覆盖路径。
119
+
120
+ 运行 `python3 packages/experimental/cua-macos/native/tests/startup_e2e.py`,通过 LaunchServices 检查启动即退出、原始错误传递、过期就绪记录拒绝,以及卡住的新实例退出。检查使用自建应用,不申请系统权限。发布单元测试模拟签名;`build:native` 调用真实的 `codesign --verify --strict`,`launcher_identity_e2e.py` 验证 LaunchServices 授权归属。
121
+
122
+ 使用 `python3 packages/experimental/cua-macos/native/tests/test_swift.py` 通过选定 Apple 工具链的 Swift Testing 框架和运行时路径执行离线单元检查;使用 `python3 packages/experimental/cua-macos/native/tests/protocol_e2e.py` 检查 JSONL 接纳、有序回复和 EOF 退出,不执行桌面操作。应用与协议端到端检查接受 `--helper` 以选择单独构建的对比二进制。 `python3 packages/experimental/cua-macos/native/tests/test_build_app.py` 验证应用发布、签名失败和损坏副本拒绝。
105
123
 
106
- 运行 `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper`,通过测试自建的 AppKit fixture 核实并发启动、共享服务 PID、独立句柄、同一应用交接、请求级截图期限、连接清理和信号退出。需要已有辅助功能和屏幕录制权限。`--without-desktop` 将检查限定为协议与进程生命期。
124
+ 运行 `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper`,通过测试自建的 AppKit fixture 核实并发启动、共享服务 PID、独立句柄、同一应用交接、请求级截图期限、连接清理和信号退出。需要已经给 `DSH Computer Use.app` 授予辅助功能和屏幕录制权限。`--without-desktop` 将检查限定为协议与进程生命期。
107
125
 
108
126
  在已解锁的 macOS GUI 会话中运行 `python3 packages/experimental/cua-macos/native/tests/application_consent_e2e.py --helper /absolute/helper`,需要可用的 Apple Swift 编译器。它创建并清理自己的 AppKit 应用和独立 GUI 进程,检查元数据替换、冷启动与直接启动、共用 Bundle ID、独立程序重启后的持久路径身份,以及删除运行中可执行文件后的选择行为。默认身份检查不需要辅助功能或屏幕录制权限。添加 `--desktop` 可读取完整测试应用的控件,通过状态文件验证 AX 按钮点击,截取窗口,并记录删除可执行文件后的观察结果;此模式要求已有两项系统权限。
109
127
 
110
- 在 GUI 登录会话中运行 `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py --helper /absolute/helper`,验证由 Launch Services 启动的应用先启动另一个 GUI 启动器,再启动 helper。helper 识别外层责任应用,并在启动器退出后保留其名称和图标。检查只使用测试自建的两个应用和 helper,结束后清理对应的 Launch Services 注册,不申请辅助功能或屏幕录制权限。
128
+ 在 GUI 登录会话中运行 `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py`,验证两个嵌套 GUI 启动器启动 helper 后,授权归属仍为 helper 自己,多条连接得到相同名称和图标。检查自行创建应用并清理 LaunchServices 注册,不申请辅助功能或屏幕录制权限。
111
129
 
112
130
  构建并解锁 Mac 后运行 `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py`,验证应用移除并恢复 AX 根节点的窗口。检查需要已有辅助功能/屏幕录制权限,会拒绝过期元素索引,捕获无 AX 窗口,通过 fixture 状态文件核实坐标点击,并检查恢复。不会修改系统权限设置。
113
131
 
@@ -1,37 +1,50 @@
1
- // Only server mode owns AppKit; launcher and stdio clients connect to its socket.
1
+ // Server mode owns the AppKit event loop; launchers use NSWorkspace and clients use the service socket.
2
2
  import AppKit
3
3
  import Foundation
4
4
  import Darwin
5
+ import Dispatch
5
6
 
6
7
  @main enum CuaNativeMain {
7
8
  @MainActor static func main() {
8
- do {
9
- let arguments = Array(CommandLine.arguments.dropFirst())
10
- if arguments.first == "--ensure-shared" {
11
- try SharedHelperLauncher.ensure(SharedHelperOptions(arguments: arguments))
12
- return
13
- }
14
- if arguments.first == "--serve" {
15
- guard setsid() >= 0 else { throw SharedSocket.failure("Creating shared helper session") }
9
+ let arguments = Array(CommandLine.arguments.dropFirst())
10
+ if arguments.first == "--serve" {
11
+ do {
16
12
  let options = try SharedHelperOptions(arguments: arguments)
17
- let app = NSApplication.shared
18
- app.setActivationPolicy(.prohibited)
19
- guard let server = try SharedHelperServer(options: options) else { return }
20
- try server.start()
21
- withExtendedLifetime(server) { app.run() }
22
- return
23
- }
24
- let configuration = try HelperConfiguration(arguments: arguments)
25
- try SharedHelperProxy.run(configuration: configuration)
26
- } catch {
27
- let diagnostic = (error as? NativeFailure)?.message ?? error.localizedDescription
28
- if CommandLine.arguments.dropFirst().first == "--serve" {
29
- let failure = JSONValue.object(["type": .string("error"), "code": .string((error as? NativeFailure)?.code ?? "NATIVE_STARTUP"),
30
- "message": .string(diagnostic)])
31
- if let data = try? JSONEncoder().encode(failure) { try? FileHandle.standardOutput.write(contentsOf: data + Data([10])) }
32
- }
33
- if let data = "dsh-cua-native: \(diagnostic)\n".data(using: .utf8) { try? FileHandle.standardError.write(contentsOf: data) }
34
- exit(1)
13
+ do {
14
+ let app = NSApplication.shared
15
+ app.setActivationPolicy(.accessory)
16
+ guard let server = try SharedHelperServer(options: options) else { return }
17
+ try server.start()
18
+ withExtendedLifetime(server) { app.run() }
19
+ } catch {
20
+ do { try options.recordStartupFailure(error) }
21
+ catch { fputs("dsh-cua-native: could not record startup failure: \(error)\n", stderr) }
22
+ fail(error)
23
+ }
24
+ } catch { fail(error) }
25
+ return
26
+ }
27
+ Task { @MainActor in
28
+ do {
29
+ if arguments.first == "--ensure-shared" {
30
+ try await SharedHelperLauncher.ensure(SharedHelperOptions(arguments: arguments))
31
+ } else {
32
+ try await SharedHelperProxy.run(configuration: HelperConfiguration(arguments: arguments))
33
+ }
34
+ exit(0)
35
+ } catch { fail(error) }
36
+ }
37
+ dispatchMain()
38
+ }
39
+
40
+ private static func fail(_ error: Error) -> Never {
41
+ let diagnostic = (error as? NativeFailure)?.message ?? error.localizedDescription
42
+ do {
43
+ let failure = JSONValue.object(["type": .string("error"), "code": .string((error as? NativeFailure)?.code ?? "NATIVE_STARTUP"),
44
+ "message": .string(diagnostic)])
45
+ if let data = try? JSONEncoder().encode(failure) { try? FileHandle.standardOutput.write(contentsOf: data + Data([10])) }
35
46
  }
47
+ if let data = "dsh-cua-native: \(diagnostic)\n".data(using: .utf8) { try? FileHandle.standardError.write(contentsOf: data) }
48
+ exit(1)
36
49
  }
37
50
  }
@@ -14,12 +14,12 @@ enum NativeProtocol {
14
14
  "launchApp": ["appHandle"],
15
15
  "listApps": [],
16
16
  "listWindows": ["appHandle"],
17
- "openPermissionSettings": ["permission"],
17
+ "openPermissionSettings": ["permission", "settingsMessage"],
18
18
  "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"],
19
19
  "performSecondaryAction": ["action", "appHandle", "elementIndex", "snapshotId"],
20
20
  "permissionStatus": [],
21
21
  "pressKey": ["appHandle", "key", "modifiers"],
22
- "requestPermission": ["permission"],
22
+ "requestPermission": ["backLabel", "finderLabel", "instruction", "permission", "pollIntervalMs", "settingsMessage", "unavailableMessage"],
23
23
  "screenshot": ["appHandle", "windowId"],
24
24
  "scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
25
25
  "selectText": ["appHandle", "elementIndex", "prefix", "selectionType", "snapshotId", "suffix", "text"],