@deepseek-ai/dsh-experimental-cua-macos 0.2.2-alpha.3 → 0.2.2-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +14 -14
- package/README.md +102 -67
- package/README.zh.md +102 -67
- package/cua-helper.json +7 -5
- package/lib/index.js +165 -110
- package/lib/types/ax-limits.d.ts +6 -0
- package/lib/types/ax-limits.d.ts.map +1 -0
- package/lib/types/ax-limits.js.map +1 -0
- package/lib/types/guidance.d.ts +4 -3
- package/lib/types/guidance.d.ts.map +1 -0
- package/lib/types/guidance.js.map +1 -0
- package/lib/types/held-keys.d.ts +9 -0
- package/lib/types/held-keys.d.ts.map +1 -0
- package/lib/types/held-keys.js.map +1 -0
- package/lib/types/index.d.ts +82 -53
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/native-build.d.ts +16 -2
- package/lib/types/native-build.d.ts.map +1 -0
- package/lib/types/native-build.js.map +1 -0
- package/lib/types/permission-guide.d.ts.map +1 -0
- package/lib/types/permission-guide.js.map +1 -0
- package/lib/types/shared-native.d.ts.map +1 -0
- package/lib/types/shared-native.js.map +1 -0
- package/native/README.md +52 -30
- package/native/README.zh.md +52 -30
- package/native/Sources/CuaNative/AXAccess.swift +241 -51
- package/native/Sources/CuaNative/AXHover.swift +3 -3
- package/native/Sources/CuaNative/AXWorker.swift +118 -97
- package/native/Sources/CuaNative/HeldKeys.swift +114 -0
- package/native/Sources/CuaNative/KeyboardInput.swift +83 -64
- package/native/Sources/CuaNative/NativePlatform.swift +16 -6
- package/native/Sources/CuaNative/NativeProtocol.generated.swift +16 -11
- package/native/Sources/CuaNative/Permissions.swift +1 -0
- package/native/Sources/CuaNative/Protocol.swift +11 -3
- package/native/Sources/CuaNative/RequestAdmission.swift +11 -0
- package/native/Sources/CuaNative/RequestCoordinator.swift +15 -17
- package/native/Sources/CuaNative/RequestScheduler.swift +1 -0
- package/native/Sources/CuaNative/SharedHelperRuntime.swift +4 -3
- package/native/Sources/CuaNative/StandaloneApplication.swift +86 -0
- package/native/Sources/CuaNative/WindowCapture.swift +2 -0
- package/native/Sources/CuaNative/WindowGeometry.swift +2 -2
- package/native/Sources/CuaNative/WindowPointerInput.swift +2 -2
- package/native/bin/darwin-arm64/current.json +1 -1
- package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/CodeResources +0 -0
- package/native/bin/darwin-arm64/d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
- package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/CodeResources +0 -0
- package/native/bin/darwin-x64/b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
- package/native/bin/darwin-x64/current.json +1 -1
- package/native/build_app.py +72 -16
- package/native/protocol.schema.json +373 -84
- package/native/tests/swift/AXFailureTests.swift +67 -3
- package/native/tests/swift/AXTextFormatterTests.swift +0 -10
- package/native/tests/swift/ForegroundTests.swift +6 -33
- package/native/tests/swift/KeyboardChordTests.swift +137 -0
- package/native/tests/swift/ListActionsTests.swift +38 -0
- package/native/tests/swift/PerformActionTests.swift +106 -0
- package/native/tests/swift/ProtocolTests.swift +32 -2
- package/native/tests/swift/SharedHelperTests.swift +1 -1
- package/native/tests/swift/StandaloneApplicationTests.swift +136 -0
- package/native/upstream.json +1 -1
- package/package.json +51 -48
- package/src/ax-limits.ts +2 -0
- package/src/guidance.ts +21 -0
- package/src/held-keys.ts +42 -0
- package/src/index.ts +90 -0
- package/src/native-build.ts +113 -0
- package/src/permission-guide.ts +110 -0
- package/src/shared-native.ts +294 -0
- package/native/bin/darwin-arm64/007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
- package/native/bin/darwin-x64/9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2/DSH Computer Use.app/Contents/MacOS/dsh-cua-native +0 -0
- /package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/Info.plist +0 -0
- /package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/Resources/DSHComputerUse.icns +0 -0
- /package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/Resources/LICENSE.computer-use-icon +0 -0
- /package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/Resources/LICENSE.cua-driver +0 -0
- /package/native/bin/darwin-arm64/{007deaf0f4b6b473af03a201a5042208f8249172c41c27894bc09241e5e3f19b → d896b6b6e45a717a7dbc7aa52d0b34eb70f1558c472b16b1272f80678b2ed4be}/DSH Computer Use.app/Contents/_CodeSignature/CodeResources +0 -0
- /package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/Info.plist +0 -0
- /package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/Resources/DSHComputerUse.icns +0 -0
- /package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/Resources/LICENSE.computer-use-icon +0 -0
- /package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/Resources/LICENSE.cua-driver +0 -0
- /package/native/bin/darwin-x64/{9f6e6dedbc05279505e36b140ddd74a7d143b31838e17ba22ad3115fc14c55b2 → b68fcdb1474dad2b8694e1aa78d53bc963e6263fb3d12953502537a44df7f646}/DSH Computer Use.app/Contents/_CodeSignature/CodeResources +0 -0
package/lib/index.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { isAbsolute, join, resolve } from "node:path";
|
|
2
2
|
import { fileURLToPath } from "node:url";
|
|
3
3
|
import Schema from "@deepseek-ai/schemastery";
|
|
4
|
+
import { installCua } from "@deepseek-ai/dsh-cua";
|
|
5
|
+
import { CuaConfigFields } from "@deepseek-ai/dsh-cua/config";
|
|
4
6
|
import { z } from "zod";
|
|
5
|
-
import { CuaConfig, installCua } from "@deepseek-ai/dsh-cua";
|
|
6
7
|
import { constants } from "node:fs";
|
|
7
8
|
import { access, lstat, mkdir, readFile, realpath } from "node:fs/promises";
|
|
8
|
-
import {
|
|
9
|
-
import { createHash } from "node:crypto";
|
|
9
|
+
import { buildNativeExecutable, sourceNativePackage } from "@deepseek-ai/dsh-cua/native-build";
|
|
10
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
10
11
|
import { Socket } from "node:net";
|
|
11
12
|
import { userInfo } from "node:os";
|
|
12
13
|
import { PassThrough, Writable } from "node:stream";
|
|
13
14
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
15
|
+
import { NativeError } from "@deepseek-ai/dsh-cua/native-client";
|
|
16
|
+
import { RUN_CODE_NAME } from "@deepseek-ai/dsh-tools";
|
|
17
|
+
import { COMMON_GUIDANCE } from "@deepseek-ai/dsh-cua/guidance";
|
|
14
18
|
z.object({ settingsMessage: z.string().min(1) }).extend({
|
|
15
19
|
instruction: z.string().min(1),
|
|
16
20
|
backLabel: z.string().min(1),
|
|
@@ -130,17 +134,20 @@ const NativeApplication = z.object({ bundle: z.string().regex(/^[a-f0-9]{64}$/)
|
|
|
130
134
|
* @throws When the current host has no native implementation.
|
|
131
135
|
*/
|
|
132
136
|
function nativeHostArchitecture(platform, architecture) {
|
|
133
|
-
if (platform !== "darwin" || architecture !== "arm64" && architecture !== "x64") throw new Error("Computer
|
|
137
|
+
if (platform !== "darwin" || architecture !== "arm64" && architecture !== "x64") throw new Error("The macOS Computer Use provider requires macOS on Apple silicon or Intel");
|
|
134
138
|
return architecture;
|
|
135
139
|
}
|
|
136
140
|
/**
|
|
137
141
|
* Resolve the immutable application selected by a packaged or source-build receipt.
|
|
138
142
|
* @param directory - Architecture-specific directory containing current.json and application generations.
|
|
143
|
+
* @param variant - Development or release application filename selected by the build or installation.
|
|
139
144
|
* @returns Executable inside the selected application bundle.
|
|
140
145
|
* @throws When the receipt is invalid or the application executable is missing.
|
|
141
146
|
*/
|
|
142
|
-
async function nativeExecutablePath(directory) {
|
|
143
|
-
const
|
|
147
|
+
async function nativeExecutablePath(directory, variant) {
|
|
148
|
+
const receipt = NativeApplication.parse(JSON.parse(await readFile(join(directory, "current.json"), "utf8")));
|
|
149
|
+
const application = variant === "dev" ? "DSH Computer Use Dev.app" : "DSH Computer Use.app";
|
|
150
|
+
const executable = join(directory, receipt.bundle, application, "Contents", "MacOS", "dsh-cua-native");
|
|
144
151
|
await access(executable, constants.X_OK);
|
|
145
152
|
return executable;
|
|
146
153
|
}
|
|
@@ -150,79 +157,63 @@ async function nativeExecutablePath(directory) {
|
|
|
150
157
|
* @returns True only for a CUA package in a DSH source checkout.
|
|
151
158
|
*/
|
|
152
159
|
async function isSourcePackage(packageRoot) {
|
|
153
|
-
|
|
154
|
-
if (join(root, "packages/experimental/cua-macos") !== packageRoot) return false;
|
|
155
|
-
try {
|
|
156
|
-
const manifest = JSON.parse(await readFile(join(root, "package.json"), "utf8"));
|
|
157
|
-
return typeof manifest === "object" && manifest !== null && "name" in manifest && manifest.name === "@deepseek-ai/dsh-root";
|
|
158
|
-
} catch (error) {
|
|
159
|
-
if (error.code === "ENOENT") return false;
|
|
160
|
-
throw error;
|
|
161
|
-
}
|
|
160
|
+
return sourceNativePackage(packageRoot, "packages/experimental/cua-macos");
|
|
162
161
|
}
|
|
163
162
|
/**
|
|
164
|
-
* Build and sign an immutable helper application with the selected Apple developer tools.
|
|
163
|
+
* Build and sign an immutable development helper application with the selected Apple developer tools.
|
|
164
|
+
* The development variant has its own bundle identifier and the signature selected by `spec.sourceSigning`.
|
|
165
165
|
* @param spec - Checkout, target, managed process provider and cancellation/deadline limits.
|
|
166
166
|
* @returns Executable inside the prepared application, after complete compiler cleanup.
|
|
167
167
|
* @throws When preparation, cancellation, resource cleanup or receipt validation fails.
|
|
168
168
|
*/
|
|
169
|
-
async function buildNativeExecutable(spec) {
|
|
170
|
-
spec.signal.throwIfAborted();
|
|
169
|
+
async function buildNativeExecutable$1(spec) {
|
|
171
170
|
const nativeRoot = join(spec.packageRoot, "native");
|
|
172
171
|
const directory = join(nativeRoot, ".build", "prepared", spec.architecture);
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
try {
|
|
199
|
-
outcome = await handle.done;
|
|
200
|
-
} finally {
|
|
201
|
-
try {
|
|
202
|
-
handle.terminate();
|
|
203
|
-
} finally {
|
|
204
|
-
if (!await handle.waitForExit()) throw new Error("Computer use preparation did not release its managed process range");
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
signal.throwIfAborted();
|
|
208
|
-
if (outcome.exitCode !== 0 || outcome.signal !== null) {
|
|
209
|
-
const diagnostics = `${handle.collected.stdout?.readFrom(0).text ?? ""}${handle.collected.stderr?.readFrom(0).text ?? ""}`;
|
|
210
|
-
throw new Error(`Computer use native preparation failed (${String(outcome.exitCode)}, ${String(outcome.signal)}): ${diagnostics}`);
|
|
211
|
-
}
|
|
212
|
-
return nativeExecutablePath(directory);
|
|
213
|
-
}, {
|
|
214
|
-
waitMs: spec.timeoutMs,
|
|
215
|
-
signal
|
|
172
|
+
const signing = spec.sourceSigning;
|
|
173
|
+
const signingArgs = typeof signing === "string" ? ["--identity", "-"] : [
|
|
174
|
+
"--identity",
|
|
175
|
+
signing.identity,
|
|
176
|
+
"--keychain",
|
|
177
|
+
signing.keychain
|
|
178
|
+
];
|
|
179
|
+
return buildNativeExecutable({
|
|
180
|
+
...spec,
|
|
181
|
+
lockPath: join(nativeRoot, ".build", "application-build"),
|
|
182
|
+
argv: [
|
|
183
|
+
"/usr/bin/xcrun",
|
|
184
|
+
"python3",
|
|
185
|
+
join(nativeRoot, "build_app.py"),
|
|
186
|
+
"--scratch-path",
|
|
187
|
+
join(nativeRoot, ".build", `runtime-${spec.architecture}`),
|
|
188
|
+
"--output-directory",
|
|
189
|
+
directory,
|
|
190
|
+
"--arch",
|
|
191
|
+
spec.architecture === "x64" ? "x86_64" : "arm64",
|
|
192
|
+
"--variant",
|
|
193
|
+
"dev",
|
|
194
|
+
...signingArgs
|
|
195
|
+
],
|
|
196
|
+
resolveExecutable: () => nativeExecutablePath(directory, "dev")
|
|
216
197
|
});
|
|
217
198
|
}
|
|
218
199
|
/**
|
|
219
200
|
* Prepare source builds on activation; installed packages carry their signed application.
|
|
201
|
+
* A package inside an Electron `app.asar` archive reads its application from the sibling `app.asar.unpacked`
|
|
202
|
+
* directory, which the Desktop installer populates so LaunchServices and process spawning can open the bundle.
|
|
220
203
|
* @param spec - Native preparation inputs for this host.
|
|
221
204
|
* @returns Executable inside a ready application without compiling or re-signing an installed package.
|
|
205
|
+
* @throws When an installed package carries no application receipt, naming the directory that was expected to hold it.
|
|
222
206
|
*/
|
|
223
207
|
async function prepareNativeExecutable(spec) {
|
|
224
|
-
if (await isSourcePackage(spec.packageRoot)) return buildNativeExecutable(spec);
|
|
225
|
-
const
|
|
208
|
+
if (await isSourcePackage(spec.packageRoot)) return buildNativeExecutable$1(spec);
|
|
209
|
+
const directory = join(spec.packageRoot.replace(/([\\/])app\.asar(?=[\\/])/u, "$1app.asar.unpacked"), "native", "bin", `darwin-${spec.architecture}`);
|
|
210
|
+
let executable;
|
|
211
|
+
try {
|
|
212
|
+
executable = await nativeExecutablePath(directory, "release");
|
|
213
|
+
} catch (error) {
|
|
214
|
+
if (error.code !== "ENOENT" || error.path !== join(directory, "current.json")) throw error;
|
|
215
|
+
throw new Error(`Computer use helper application is missing from ${directory}; this installation does not include the signed helper`, { cause: error });
|
|
216
|
+
}
|
|
226
217
|
spec.signal.throwIfAborted();
|
|
227
218
|
return executable;
|
|
228
219
|
}
|
|
@@ -559,54 +550,108 @@ function createSharedNativeChannel(owner, config) {
|
|
|
559
550
|
return new SharedNativeChannel(owner, config);
|
|
560
551
|
}
|
|
561
552
|
//#endregion
|
|
553
|
+
//#region lib/types/held-keys.js
|
|
554
|
+
/** Directional input belongs to the enclosing run_code, including while its worker calls other tools. */
|
|
555
|
+
/**
|
|
556
|
+
* Keep native key ownership alive until its program returns, throws, or is interrupted.
|
|
557
|
+
* @param ctx - Provider scope owning the tool execution listener.
|
|
558
|
+
* @returns Handler for authorized directional keyboard requests.
|
|
559
|
+
*/
|
|
560
|
+
function installHeldKeys(ctx) {
|
|
561
|
+
const runs = /* @__PURE__ */ new Map();
|
|
562
|
+
ctx.on("tools/execute", async (execution, next) => {
|
|
563
|
+
if (execution.name !== RUN_CODE_NAME) return next();
|
|
564
|
+
const run = { id: randomUUID() };
|
|
565
|
+
runs.set(execution.token, run);
|
|
566
|
+
let cleanup;
|
|
567
|
+
const release = () => {
|
|
568
|
+
runs.delete(execution.token);
|
|
569
|
+
if (run.client) cleanup ??= run.client.requestCleanup("releaseHeldKeys", { runId: run.id });
|
|
570
|
+
return cleanup;
|
|
571
|
+
};
|
|
572
|
+
const onAbort = () => {
|
|
573
|
+
release()?.catch(() => void 0);
|
|
574
|
+
};
|
|
575
|
+
const signal = execution.signal;
|
|
576
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
577
|
+
try {
|
|
578
|
+
return await next();
|
|
579
|
+
} finally {
|
|
580
|
+
signal.removeEventListener("abort", onAbort);
|
|
581
|
+
await release();
|
|
582
|
+
}
|
|
583
|
+
});
|
|
584
|
+
return (client, params, execution) => {
|
|
585
|
+
const run = execution.parent && runs.get(execution.parent);
|
|
586
|
+
if (!run) throw new NativeError("INVALID_ARGUMENT", "Use sendKeyEvent inside an active run_code execution");
|
|
587
|
+
run.client = client;
|
|
588
|
+
return client.request("sendKeyEvent", {
|
|
589
|
+
...params,
|
|
590
|
+
runId: run.id
|
|
591
|
+
}, execution.signal, execution.callId);
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
//#endregion
|
|
595
|
+
//#region lib/types/ax-limits.js
|
|
596
|
+
/** Largest traversal budgets accepted by the macOS helper. */
|
|
597
|
+
const AX_TRAVERSAL_MAX = {
|
|
598
|
+
maxElements: 1e5,
|
|
599
|
+
maxDepth: 512
|
|
600
|
+
};
|
|
601
|
+
//#endregion
|
|
562
602
|
//#region lib/types/guidance.js
|
|
563
|
-
/**
|
|
564
|
-
/**
|
|
565
|
-
const
|
|
566
|
-
|
|
567
|
-
|
|
603
|
+
/** Platform setup guidance prefixed to the shared application and window instructions. */
|
|
604
|
+
/** macOS operation levels, permissions, selectors, and native measurement units. */
|
|
605
|
+
const PLATFORM_GUIDANCE = "macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.";
|
|
606
|
+
const methodDetails = new Map([
|
|
607
|
+
["getAccessibilityTree", "macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current."],
|
|
608
|
+
["mouseMove", "Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again."],
|
|
609
|
+
["click", "click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying."]
|
|
610
|
+
]);
|
|
611
|
+
/** Native operation instructions; the shared Host adds guidance for declared capabilities. */
|
|
612
|
+
const GUIDANCE = `${PLATFORM_GUIDANCE}
|
|
568
613
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }
|
|
576
|
-
- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>
|
|
577
|
-
- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>
|
|
578
|
-
- cua.getState(options?) -> { apps, browsers: [] }
|
|
579
|
-
- cua.getApp(selector: string) -> App
|
|
580
|
-
- app.activate() -> { dispatched: true, verified: false }
|
|
581
|
-
- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>
|
|
582
|
-
- app.getWindow(windowId: number) -> Window
|
|
583
|
-
- app.typeText(text: string), app.pressKey(chord: string) -> action result
|
|
584
|
-
- app.paste(text: string, options?: { format?: "text" | "html" }) -> action result
|
|
585
|
-
|
|
586
|
-
App and Window methods:
|
|
587
|
-
|
|
588
|
-
- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }
|
|
589
|
-
- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string
|
|
590
|
-
- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot
|
|
591
|
-
- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }
|
|
592
|
-
- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }
|
|
593
|
-
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, holdMs?: number, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
|
594
|
-
- setValue(elementIndex: number, value: string) -> action result
|
|
595
|
-
- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: "text" | "cursor_before" | "cursor_after" }) -> action result
|
|
596
|
-
- performSecondaryAction(elementIndex: number, action: string) -> action result
|
|
597
|
-
- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result
|
|
598
|
-
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
|
599
|
-
|
|
600
|
-
Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.
|
|
601
|
-
|
|
602
|
-
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
|
603
|
-
|
|
604
|
-
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
|
605
|
-
|
|
606
|
-
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Optional holdMs is a nonnegative integer in milliseconds for each press, including each press of a double-click. Zero releases without an additional hold. The configured maximum is stated below. It uses pointer input even for AX targets; omission preserves ordinary clicks. The operation deadline still applies, and cancellation or timeout releases the held button. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`;
|
|
614
|
+
${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method) => {
|
|
615
|
+
if (method === "pressKey") return `${entry}
|
|
616
|
+
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.`;
|
|
617
|
+
const detail = methodDetails.get(method);
|
|
618
|
+
return detail ? `${entry} ${detail}` : entry;
|
|
619
|
+
})}`;
|
|
607
620
|
//#endregion
|
|
608
621
|
//#region lib/types/index.js
|
|
609
622
|
/** macOS native CUA provider using the shared application and window API. */
|
|
623
|
+
const CAPABILITIES = {
|
|
624
|
+
platform: "macos",
|
|
625
|
+
systemPointer: true,
|
|
626
|
+
methods: [
|
|
627
|
+
"activate",
|
|
628
|
+
"click",
|
|
629
|
+
"drag",
|
|
630
|
+
"findApps",
|
|
631
|
+
"getApp",
|
|
632
|
+
"getCapabilities",
|
|
633
|
+
"getState",
|
|
634
|
+
"invoke",
|
|
635
|
+
"listActions",
|
|
636
|
+
"listApps",
|
|
637
|
+
"listWindows",
|
|
638
|
+
"mouseMove",
|
|
639
|
+
"paste",
|
|
640
|
+
"performAction",
|
|
641
|
+
"pressKey",
|
|
642
|
+
"screenshot",
|
|
643
|
+
"scroll",
|
|
644
|
+
"selectText",
|
|
645
|
+
"sendKeyEvent",
|
|
646
|
+
"setValue",
|
|
647
|
+
"show",
|
|
648
|
+
"typeText"
|
|
649
|
+
],
|
|
650
|
+
appLevelAccessibility: true,
|
|
651
|
+
keyboard: ["app"],
|
|
652
|
+
browser: false,
|
|
653
|
+
coordinateSpace: "screenshot-pixels"
|
|
654
|
+
};
|
|
610
655
|
/** Loader plugin identity. */
|
|
611
656
|
const name = "experimental-cua-macos";
|
|
612
657
|
/** Services required by the shared CUA executor and the native launcher. */
|
|
@@ -617,9 +662,17 @@ const inject = [
|
|
|
617
662
|
"subprocess",
|
|
618
663
|
"agents"
|
|
619
664
|
];
|
|
620
|
-
/** Validate shared
|
|
621
|
-
const Config = Schema.
|
|
622
|
-
|
|
665
|
+
/** Validate shared operation bounds and macOS source signing before preparing the helper. */
|
|
666
|
+
const Config = Schema.object({
|
|
667
|
+
...CuaConfigFields,
|
|
668
|
+
maxElements: CuaConfigFields.maxElements.max(AX_TRAVERSAL_MAX.maxElements).default(5e3),
|
|
669
|
+
maxDepth: CuaConfigFields.maxDepth.max(AX_TRAVERSAL_MAX.maxDepth).default(50),
|
|
670
|
+
permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1e3),
|
|
671
|
+
sourceSigning: Schema.union([Schema.const("ad-hoc"), Schema.object({
|
|
672
|
+
identity: Schema.string().pattern(/^[^-\s]/).required(),
|
|
673
|
+
keychain: Schema.string().pattern(/^\/.+/).required()
|
|
674
|
+
})]).default("ad-hoc")
|
|
675
|
+
});
|
|
623
676
|
/**
|
|
624
677
|
* Mount logged native operations with separate Agent connections to the shared helper.
|
|
625
678
|
* @param ctx - Host context with computer-use, subprocess and tool services.
|
|
@@ -629,8 +682,9 @@ const description = "Inspect macOS permissions, select an application and window
|
|
|
629
682
|
async function apply(ctx, config) {
|
|
630
683
|
const backend = {
|
|
631
684
|
createChannel: createSharedNativeChannel,
|
|
685
|
+
sendKeyEvent: installHeldKeys(ctx),
|
|
632
686
|
guidance: GUIDANCE,
|
|
633
|
-
|
|
687
|
+
capabilities: CAPABILITIES,
|
|
634
688
|
permissionParameters: createPermissionParameters(config.permissionPollIntervalMs)
|
|
635
689
|
};
|
|
636
690
|
if (config.helperPath !== void 0) {
|
|
@@ -646,6 +700,7 @@ async function apply(ctx, config) {
|
|
|
646
700
|
const prepared = prepareNativeExecutable({
|
|
647
701
|
packageRoot: resolve(fileURLToPath(new URL("..", import.meta.url))),
|
|
648
702
|
architecture,
|
|
703
|
+
sourceSigning: config.sourceSigning,
|
|
649
704
|
subprocess: ctx.subprocess,
|
|
650
705
|
timeoutMs: config.buildTimeoutMs,
|
|
651
706
|
graceMs: config.graceMs,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ax-limits.d.ts","sourceRoot":"","sources":["../../src/ax-limits.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,eAAO,MAAM,gBAAgB;;;CAAmD,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ax-limits.js","sourceRoot":"","sources":["../../src/ax-limits.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAW,CAAA"}
|
package/lib/types/guidance.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
/** macOS operation
|
|
2
|
-
|
|
3
|
-
export declare const GUIDANCE = "Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 999 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.\n\nInitialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.\n\nUse app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.\n\nCuaScreenshot contains { bytes: Uint8Array, mimeType: \"image/png\" | \"image/jpeg\" | \"image/webp\" | \"image/gif\", pixelWidth: number, pixelHeight: number, windowId: number }. Use mimeType to choose a file extension or declare an image; do not assume PNG.\n\nObject API signatures (options default to { emit: true }):\n\n- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }\n- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>\n- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>\n- cua.getState(options?) -> { apps, browsers: [] }\n- cua.getApp(selector: string) -> App\n- app.activate() -> { dispatched: true, verified: false }\n- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>\n- app.getWindow(windowId: number) -> Window\n- app.typeText(text: string), app.pressKey(chord: string) -> action result\n- app.paste(text: string, options?: { format?: \"text\" | \"html\" }) -> action result\n\nApp and Window methods:\n\n- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }\n- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string\n- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot\n- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }\n- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }\n- click(target: number | [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", clickCount?: 1 | 2, holdMs?: number, modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n- setValue(elementIndex: number, value: string) -> action result\n- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: \"text\" | \"cursor_before\" | \"cursor_after\" }) -> action result\n- performSecondaryAction(elementIndex: number, action: string) -> action result\n- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result\n- drag(from: [x, y], to: [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n\nRaw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.\n\nAwait operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input.\n\nHover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.\n\nUse selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Optional holdMs is a nonnegative integer in milliseconds for each press, including each press of a double-click. Zero releases without an additional hold. The configured maximum is stated below. It uses pointer input even for AX targets; omission preserves ordinary clicks. The operation deadline still applies, and cancellation or timeout releases the held button. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.";
|
|
1
|
+
/** macOS operation levels, permissions, selectors, and native measurement units. */
|
|
2
|
+
export declare const PLATFORM_GUIDANCE = "macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.";
|
|
3
|
+
/** Native operation instructions; the shared Host adds guidance for declared capabilities. */
|
|
4
|
+
export declare const GUIDANCE: string;
|
|
4
5
|
//# sourceMappingURL=guidance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"guidance.d.ts","sourceRoot":"","sources":["../../src/guidance.ts"],"names":[],"mappings":"AAGA,oFAAoF;AACpF,eAAO,MAAM,iBAAiB,yzBAAyzB,CAAA;AAQv1B,8FAA8F;AAC9F,eAAO,MAAM,QAAQ,QAOjB,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"guidance.js","sourceRoot":"","sources":["../../src/guidance.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,oFAAoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,szBAAszB,CAAA;AAEv1B,MAAM,aAAa,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,sBAAsB,EAAE,glBAAglB,CAAC;IAC1mB,CAAC,WAAW,EAAE,ySAAyS,CAAC;IACxT,CAAC,OAAO,EAAE,0qBAA0qB,CAAC;CACtrB,CAAC,CAAA;AAEF,8FAA8F;AAC9F,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,iBAAiB;;EAE1C,eAAe,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;IACxE,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,GAAG,KAAK;ymBAC6jB,CAAA;IACvmB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;AAC9C,CAAC,CAAC,EAAE,CAAA"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import type { CuaBackend } from '@deepseek-ai/dsh-cua';
|
|
3
|
+
/**
|
|
4
|
+
* Keep native key ownership alive until its program returns, throws, or is interrupted.
|
|
5
|
+
* @param ctx - Provider scope owning the tool execution listener.
|
|
6
|
+
* @returns Handler for authorized directional keyboard requests.
|
|
7
|
+
*/
|
|
8
|
+
export declare function installHeldKeys(ctx: Context): NonNullable<CuaBackend['sendKeyEvent']>;
|
|
9
|
+
//# sourceMappingURL=held-keys.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"held-keys.d.ts","sourceRoot":"","sources":["../../src/held-keys.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAItD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CA6BrF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"held-keys.js","sourceRoot":"","sources":["../../src/held-keys.ts"],"names":[],"mappings":"AAAA,yGAAyG;AACzG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAGxC,OAAO,EAAE,WAAW,EAAqB,MAAM,oCAAoC,CAAA;AACnF,OAAO,EAAE,aAAa,EAA2B,MAAM,wBAAwB,CAAA;AAE/E;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,GAAY;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAA6D,CAAA;IACjF,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE;QAChD,IAAI,SAAS,CAAC,IAAI,KAAK,aAAa;YAAE,OAAO,IAAI,EAAE,CAAA;QACnD,MAAM,GAAG,GAA0C,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,CAAA;QACvE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAC9B,IAAI,OAAqC,CAAA;QACzC,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAC5B,IAAI,GAAG,CAAC,MAAM;gBAAE,OAAO,KAAK,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;YAC3F,OAAO,OAAO,CAAA;QAChB,CAAC,CAAA;QACD,iEAAiE;QACjE,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,KAAK,OAAO,EAAE,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA,CAAC,CAAC,CAAA;QAChE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;QAC/B,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACzD,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAA;QACrB,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC5C,MAAM,OAAO,EAAE,CAAA;QACjB,CAAC;IACH,CAAC,CAAC,CAAA;IACF,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;QAC1D,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,WAAW,CAAC,kBAAkB,EAAE,sDAAsD,CAAC,CAAA;QAC3G,GAAG,CAAC,MAAM,GAAG,MAAM,CAAA;QACnB,OAAO,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;IACzG,CAAC,CAAA;AACH,CAAC"}
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,72 +1,101 @@
|
|
|
1
|
-
import Schema from '@deepseek-ai/schemastery';
|
|
2
1
|
import type { Context } from '@deepseek-ai/cordis';
|
|
3
|
-
import
|
|
2
|
+
import Schema from '@deepseek-ai/schemastery';
|
|
3
|
+
import { type CuaConfig } from '@deepseek-ai/dsh-cua';
|
|
4
|
+
import { type SourceSigning } from './native-build.ts';
|
|
4
5
|
export { GUIDANCE } from './guidance.ts';
|
|
5
6
|
/** Loader plugin identity. */
|
|
6
7
|
export declare const name = "experimental-cua-macos";
|
|
7
8
|
/** Services required by the shared CUA executor and the native launcher. */
|
|
8
9
|
export declare const inject: string[];
|
|
9
|
-
/** Native preparation, operation bounds, and profile-local application grants. */
|
|
10
|
+
/** Native preparation, source-build signing, operation bounds, and profile-local application grants. */
|
|
10
11
|
export interface Config extends CuaConfig {
|
|
11
12
|
/** Interval in milliseconds between permission and position checks while the native guide is open. */
|
|
12
13
|
permissionPollIntervalMs: number;
|
|
14
|
+
/**
|
|
15
|
+
* Source-build signing: `ad-hoc` (default) binds grants to the code hash;
|
|
16
|
+
* an object selects an existing certificate in an explicitly configured keychain.
|
|
17
|
+
* Installed packages and `helperPath` keep their own signature.
|
|
18
|
+
*/
|
|
19
|
+
sourceSigning: SourceSigning;
|
|
13
20
|
}
|
|
14
|
-
/** Validate shared
|
|
15
|
-
export declare const Config: Schema<{
|
|
16
|
-
|
|
17
|
-
|
|
21
|
+
/** Validate shared operation bounds and macOS source signing before preparing the helper. */
|
|
22
|
+
export declare const Config: Schema<Schemastery.ObjectS<NoInfer<{
|
|
23
|
+
maxElements: Schema<number, number, "defined">;
|
|
24
|
+
maxDepth: Schema<number, number, "defined">;
|
|
25
|
+
permissionPollIntervalMs: Schema<number, number, "defined">;
|
|
26
|
+
sourceSigning: Schema<"ad-hoc" | ({
|
|
27
|
+
identity?: string | null;
|
|
28
|
+
keychain?: string | null;
|
|
29
|
+
} & import("@deepseek-ai/cosmokit").Dict), "ad-hoc" | Schemastery.ObjectT<NoInfer<{
|
|
30
|
+
identity: Schema<string, string, "defined">;
|
|
31
|
+
keychain: Schema<string, string, "defined">;
|
|
32
|
+
}>>, "defined">;
|
|
33
|
+
requireSystemPointerApproval: Schema<boolean, boolean, "defined">;
|
|
34
|
+
agentPreset: Schema<string | false, string | false, "defined">;
|
|
35
|
+
allowedApps: Schema<({
|
|
18
36
|
id?: string | null;
|
|
19
37
|
name?: string | null;
|
|
20
38
|
executablePath?: string | null;
|
|
21
|
-
} & import("@deepseek-ai/cosmokit").Dict
|
|
22
|
-
helperPath?: string | null;
|
|
23
|
-
buildTimeoutMs?: number | null;
|
|
24
|
-
timeoutMs?: number | null;
|
|
25
|
-
connectTimeoutMs?: number | null;
|
|
26
|
-
screenshotTimeoutMs?: number | null;
|
|
27
|
-
axTimeoutMs?: number | null;
|
|
28
|
-
maxAppMatches?: number | null;
|
|
29
|
-
maxClipboardBytes?: number | null;
|
|
30
|
-
pasteReadTimeoutMs?: number | null;
|
|
31
|
-
maxRequestBytes?: number | null;
|
|
32
|
-
dragDurationMs?: number | null;
|
|
33
|
-
maxClickHoldMs?: number | null;
|
|
34
|
-
dragSteps?: number | null;
|
|
35
|
-
maxResponseBytes?: number | null;
|
|
36
|
-
textEventIntervalMs?: number | null;
|
|
37
|
-
graceMs?: number | null;
|
|
38
|
-
maxElements?: number | null;
|
|
39
|
-
maxDepth?: number | null;
|
|
40
|
-
} & import("@deepseek-ai/cosmokit").Dict & {
|
|
41
|
-
permissionPollIntervalMs?: number | null;
|
|
42
|
-
}, {
|
|
43
|
-
agentPreset: string | false;
|
|
44
|
-
allowedApps: Schemastery.ObjectT<NoInfer<{
|
|
39
|
+
} & import("@deepseek-ai/cosmokit").Dict)[], Schemastery.ObjectT<NoInfer<{
|
|
45
40
|
id: Schema<string, string, "defined">;
|
|
46
41
|
name: Schema<string, string, "defined">;
|
|
47
42
|
executablePath: Schema<string, string, "plain">;
|
|
48
|
-
}>>[]
|
|
49
|
-
helperPath: string
|
|
50
|
-
buildTimeoutMs: number
|
|
51
|
-
timeoutMs: number
|
|
52
|
-
connectTimeoutMs: number
|
|
53
|
-
screenshotTimeoutMs: number
|
|
54
|
-
axTimeoutMs: number
|
|
55
|
-
maxAppMatches: number
|
|
56
|
-
maxClipboardBytes: number
|
|
57
|
-
pasteReadTimeoutMs: number
|
|
58
|
-
maxRequestBytes: number
|
|
59
|
-
dragDurationMs: number
|
|
60
|
-
maxClickHoldMs: number
|
|
61
|
-
dragSteps: number
|
|
62
|
-
maxResponseBytes: number
|
|
63
|
-
textEventIntervalMs: number
|
|
64
|
-
graceMs: number
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
permissionPollIntervalMs: number
|
|
69
|
-
|
|
43
|
+
}>>[], "defined">;
|
|
44
|
+
helperPath: Schema<string, string, "plain">;
|
|
45
|
+
buildTimeoutMs: Schema<number, number, "defined">;
|
|
46
|
+
timeoutMs: Schema<number, number, "defined">;
|
|
47
|
+
connectTimeoutMs: Schema<number, number, "defined">;
|
|
48
|
+
screenshotTimeoutMs: Schema<number, number, "defined">;
|
|
49
|
+
axTimeoutMs: Schema<number, number, "defined">;
|
|
50
|
+
maxAppMatches: Schema<number, number, "defined">;
|
|
51
|
+
maxClipboardBytes: Schema<number, number, "defined">;
|
|
52
|
+
pasteReadTimeoutMs: Schema<number, number, "defined">;
|
|
53
|
+
maxRequestBytes: Schema<number, number, "defined">;
|
|
54
|
+
dragDurationMs: Schema<number, number, "defined">;
|
|
55
|
+
maxClickHoldMs: Schema<number, number, "defined">;
|
|
56
|
+
dragSteps: Schema<number, number, "defined">;
|
|
57
|
+
maxResponseBytes: Schema<number, number, "defined">;
|
|
58
|
+
textEventIntervalMs: Schema<number, number, "defined">;
|
|
59
|
+
graceMs: Schema<number, number, "defined">;
|
|
60
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
61
|
+
maxElements: Schema<number, number, "defined">;
|
|
62
|
+
maxDepth: Schema<number, number, "defined">;
|
|
63
|
+
permissionPollIntervalMs: Schema<number, number, "defined">;
|
|
64
|
+
sourceSigning: Schema<"ad-hoc" | ({
|
|
65
|
+
identity?: string | null;
|
|
66
|
+
keychain?: string | null;
|
|
67
|
+
} & import("@deepseek-ai/cosmokit").Dict), "ad-hoc" | Schemastery.ObjectT<NoInfer<{
|
|
68
|
+
identity: Schema<string, string, "defined">;
|
|
69
|
+
keychain: Schema<string, string, "defined">;
|
|
70
|
+
}>>, "defined">;
|
|
71
|
+
requireSystemPointerApproval: Schema<boolean, boolean, "defined">;
|
|
72
|
+
agentPreset: Schema<string | false, string | false, "defined">;
|
|
73
|
+
allowedApps: Schema<({
|
|
74
|
+
id?: string | null;
|
|
75
|
+
name?: string | null;
|
|
76
|
+
executablePath?: string | null;
|
|
77
|
+
} & import("@deepseek-ai/cosmokit").Dict)[], Schemastery.ObjectT<NoInfer<{
|
|
78
|
+
id: Schema<string, string, "defined">;
|
|
79
|
+
name: Schema<string, string, "defined">;
|
|
80
|
+
executablePath: Schema<string, string, "plain">;
|
|
81
|
+
}>>[], "defined">;
|
|
82
|
+
helperPath: Schema<string, string, "plain">;
|
|
83
|
+
buildTimeoutMs: Schema<number, number, "defined">;
|
|
84
|
+
timeoutMs: Schema<number, number, "defined">;
|
|
85
|
+
connectTimeoutMs: Schema<number, number, "defined">;
|
|
86
|
+
screenshotTimeoutMs: Schema<number, number, "defined">;
|
|
87
|
+
axTimeoutMs: Schema<number, number, "defined">;
|
|
88
|
+
maxAppMatches: Schema<number, number, "defined">;
|
|
89
|
+
maxClipboardBytes: Schema<number, number, "defined">;
|
|
90
|
+
pasteReadTimeoutMs: Schema<number, number, "defined">;
|
|
91
|
+
maxRequestBytes: Schema<number, number, "defined">;
|
|
92
|
+
dragDurationMs: Schema<number, number, "defined">;
|
|
93
|
+
maxClickHoldMs: Schema<number, number, "defined">;
|
|
94
|
+
dragSteps: Schema<number, number, "defined">;
|
|
95
|
+
maxResponseBytes: Schema<number, number, "defined">;
|
|
96
|
+
textEventIntervalMs: Schema<number, number, "defined">;
|
|
97
|
+
graceMs: Schema<number, number, "defined">;
|
|
98
|
+
}>>, "plain">;
|
|
70
99
|
/**
|
|
71
100
|
* Mount logged native operations with separate Agent connections to the shared helper.
|
|
72
101
|
* @param ctx - Host context with computer-use, subprocess and tool services.
|