@qilin-agent/experimental-cua-macos 0.0.0-stage → 3.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +52 -0
- package/README.md +225 -2
- package/README.zh.md +226 -0
- package/lib/index.js +573 -0
- package/lib/types/guidance.d.ts +4 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/native-build.d.ts +47 -0
- package/lib/types/shared-native.d.ts +13 -0
- package/native/LICENSE.cua-driver +21 -0
- package/native/Package.swift +13 -0
- package/native/README.md +130 -0
- package/native/README.zh.md +131 -0
- package/native/Sources/CuaNative/AXAccess.swift +525 -0
- package/native/Sources/CuaNative/AXHover.swift +37 -0
- package/native/Sources/CuaNative/AXTextFormatter.swift +120 -0
- package/native/Sources/CuaNative/AXWorker.swift +489 -0
- package/native/Sources/CuaNative/ApplicationAuthorization.swift +69 -0
- package/native/Sources/CuaNative/ApplicationDiscovery.swift +92 -0
- package/native/Sources/CuaNative/ClipboardPaste.swift +138 -0
- package/native/Sources/CuaNative/KeyboardInput.swift +260 -0
- package/native/Sources/CuaNative/Main.swift +37 -0
- package/native/Sources/CuaNative/NativePlatform.swift +118 -0
- package/native/Sources/CuaNative/NativeProtocol.generated.swift +33 -0
- package/native/Sources/CuaNative/PIDEventPoster.swift +85 -0
- package/native/Sources/CuaNative/Protocol.swift +149 -0
- package/native/Sources/CuaNative/RequestAdmission.swift +58 -0
- package/native/Sources/CuaNative/RequestCoordinator.swift +251 -0
- package/native/Sources/CuaNative/RequestScheduler.swift +107 -0
- package/native/Sources/CuaNative/Runtime.swift +170 -0
- package/native/Sources/CuaNative/ScreenshotTrace.swift +46 -0
- package/native/Sources/CuaNative/SharedHelperProxy.swift +124 -0
- package/native/Sources/CuaNative/SharedHelperRuntime.swift +289 -0
- package/native/Sources/CuaNative/SharedHelperServer.swift +188 -0
- package/native/Sources/CuaNative/TextSelection.swift +38 -0
- package/native/Sources/CuaNative/WindowCapture.swift +165 -0
- package/native/Sources/CuaNative/WindowGeometry.swift +54 -0
- package/native/Sources/CuaNative/WindowPointerInput.swift +300 -0
- package/native/Sources/CuaNative/WindowSpaces.swift +133 -0
- package/native/protocol.schema.json +1030 -0
- package/native/tests/swift/AXFailureTests.swift +166 -0
- package/native/tests/swift/AXTextFormatterTests.swift +128 -0
- package/native/tests/swift/ApplicationDiscoveryTests.swift +59 -0
- package/native/tests/swift/ClipboardPasteTests.swift +136 -0
- package/native/tests/swift/ForegroundTests.swift +42 -0
- package/native/tests/swift/InputMappingTests.swift +34 -0
- package/native/tests/swift/PermissionIdentityTests.swift +24 -0
- package/native/tests/swift/PointerTimingTests.swift +10 -0
- package/native/tests/swift/ProtocolTests.swift +118 -0
- package/native/tests/swift/ScreenshotTraceTests.swift +23 -0
- package/native/tests/swift/SharedHelperTests.swift +90 -0
- package/native/tests/swift/SharedSocketTests.swift +57 -0
- package/native/tests/swift/SystemHoverTests.swift +27 -0
- package/native/tests/swift/TextSelectionTests.swift +47 -0
- package/native/tests/swift/WindowCaptureTests.swift +80 -0
- package/native/tests/swift/WindowGeometryTests.swift +105 -0
- package/native/tests/swift/WindowSpacesTests.swift +26 -0
- package/native/tests/test_swift.py +27 -0
- package/native/upstream.json +67 -0
- package/package.json +88 -3
package/lib/index.js
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { CuaConfig, installCua } from "@qilin-agent/experimental-cua";
|
|
4
|
+
import { constants } from "node:fs";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { access, lstat, mkdir, readFile, realpath } from "node:fs/promises";
|
|
7
|
+
import { withFileLock, writeFileAtomic } from "@qilin-agent/atomic-write";
|
|
8
|
+
import { Socket } from "node:net";
|
|
9
|
+
import { userInfo } from "node:os";
|
|
10
|
+
import { PassThrough, Writable } from "node:stream";
|
|
11
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
//#region lib/types/native-build.js
|
|
14
|
+
/** Source-checkout preparation and installed native executable resolution. */
|
|
15
|
+
/**
|
|
16
|
+
* Validate the host before preparing a platform-specific executable.
|
|
17
|
+
* @param platform - Node host platform.
|
|
18
|
+
* @param architecture - Node host architecture.
|
|
19
|
+
* @returns Supported macOS architecture.
|
|
20
|
+
* @throws When the current host has no native implementation.
|
|
21
|
+
*/
|
|
22
|
+
function nativeHostArchitecture(platform, architecture) {
|
|
23
|
+
if (platform !== "darwin" || architecture !== "arm64" && architecture !== "x64") throw new Error("Computer Use requires macOS on Apple silicon or Intel; other hosts have no native helper path. Windows support is in development");
|
|
24
|
+
return architecture;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Locate the platform-specific executable owned by the CUA package.
|
|
28
|
+
* @param packageRoot - Directory containing the CUA package manifest.
|
|
29
|
+
* @param architecture - Architecture of the process that will execute the helper.
|
|
30
|
+
* @returns Absolute executable path.
|
|
31
|
+
*/
|
|
32
|
+
function nativeExecutablePath(packageRoot, architecture) {
|
|
33
|
+
return join(packageRoot, "native", "bin", `darwin-${architecture}`, "qilin-cua-native");
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Recognize the repository-owned package location before allowing local compilation.
|
|
37
|
+
* @param packageRoot - Real directory containing this package.
|
|
38
|
+
* @returns True only for a CUA package in a QILIN source checkout.
|
|
39
|
+
*/
|
|
40
|
+
async function isSourcePackage(packageRoot) {
|
|
41
|
+
const root = resolve(packageRoot, "../../..");
|
|
42
|
+
if (join(root, "packages/experimental/cua-macos") !== packageRoot) return false;
|
|
43
|
+
try {
|
|
44
|
+
const manifest = JSON.parse(await readFile(join(root, "package.json"), "utf8"));
|
|
45
|
+
return typeof manifest === "object" && manifest !== null && "name" in manifest && manifest.name === "@qilin-agent/root";
|
|
46
|
+
} catch (error) {
|
|
47
|
+
if (error.code === "ENOENT") return false;
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Build the package payload and retain an immutable executable for source-runtime processes.
|
|
53
|
+
* @param spec - Checkout, target, managed process provider and cancellation/deadline limits.
|
|
54
|
+
* @returns Content-addressed runtime path after compilation and complete process cleanup.
|
|
55
|
+
* @throws When compilation, cancellation, resource cleanup or artifact publication fails.
|
|
56
|
+
*/
|
|
57
|
+
async function buildNativeExecutable(spec) {
|
|
58
|
+
spec.signal.throwIfAborted();
|
|
59
|
+
const executable = nativeExecutablePath(spec.packageRoot, spec.architecture);
|
|
60
|
+
await mkdir(dirname(executable), { recursive: true });
|
|
61
|
+
const signal = AbortSignal.any([spec.signal, AbortSignal.timeout(spec.timeoutMs)]);
|
|
62
|
+
return withFileLock(executable, () => compileNative(spec, signal, executable), {
|
|
63
|
+
waitMs: spec.timeoutMs,
|
|
64
|
+
signal
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
async function compileNative(spec, signal, executable) {
|
|
68
|
+
const nativeRoot = join(spec.packageRoot, "native");
|
|
69
|
+
const scratch = join(nativeRoot, ".build", `runtime-${spec.architecture}`);
|
|
70
|
+
const handle = spec.subprocess.spawn({
|
|
71
|
+
argv: [
|
|
72
|
+
"/usr/bin/xcrun",
|
|
73
|
+
"swift",
|
|
74
|
+
"build",
|
|
75
|
+
"--package-path",
|
|
76
|
+
nativeRoot,
|
|
77
|
+
"--scratch-path",
|
|
78
|
+
scratch,
|
|
79
|
+
"--configuration",
|
|
80
|
+
"release",
|
|
81
|
+
"--product",
|
|
82
|
+
"qilin-cua-native",
|
|
83
|
+
"--arch",
|
|
84
|
+
spec.architecture === "x64" ? "x86_64" : "arm64"
|
|
85
|
+
],
|
|
86
|
+
cwd: spec.packageRoot,
|
|
87
|
+
signal,
|
|
88
|
+
graceMs: spec.graceMs,
|
|
89
|
+
stdio: {
|
|
90
|
+
stdin: "ignore",
|
|
91
|
+
stdout: { maxBytes: 65536 },
|
|
92
|
+
stderr: { maxBytes: 65536 }
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
let outcome;
|
|
96
|
+
try {
|
|
97
|
+
outcome = await handle.done;
|
|
98
|
+
} finally {
|
|
99
|
+
try {
|
|
100
|
+
handle.terminate();
|
|
101
|
+
} finally {
|
|
102
|
+
await handle.waitForExit();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
signal.throwIfAborted();
|
|
106
|
+
if (outcome.exitCode !== 0 || outcome.signal !== null) {
|
|
107
|
+
const diagnostics = `${handle.collected.stdout?.readFrom(0).text ?? ""}${handle.collected.stderr?.readFrom(0).text ?? ""}`;
|
|
108
|
+
throw new Error(`Computer Use native preparation failed (${String(outcome.exitCode)}, ${String(outcome.signal)}): ${diagnostics}`);
|
|
109
|
+
}
|
|
110
|
+
const built = await readFile(join(scratch, "release", "qilin-cua-native"));
|
|
111
|
+
let previous;
|
|
112
|
+
try {
|
|
113
|
+
previous = await readFile(executable);
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (error.code !== "ENOENT") throw error;
|
|
116
|
+
}
|
|
117
|
+
signal.throwIfAborted();
|
|
118
|
+
if (!previous?.equals(built)) await writeFileAtomic(executable, built, { mode: 493 });
|
|
119
|
+
const digest = createHash("sha256").update(built).digest("hex");
|
|
120
|
+
const runtime = join(nativeRoot, ".build", "prepared", spec.architecture, digest, "qilin-cua-native");
|
|
121
|
+
await mkdir(dirname(runtime), { recursive: true });
|
|
122
|
+
let retained;
|
|
123
|
+
try {
|
|
124
|
+
retained = await readFile(runtime);
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (error.code !== "ENOENT") throw error;
|
|
127
|
+
}
|
|
128
|
+
if (retained && !retained.equals(built)) throw new Error("Computer Use retained executable does not match its content digest");
|
|
129
|
+
signal.throwIfAborted();
|
|
130
|
+
if (!retained) await writeFileAtomic(runtime, built, { mode: 493 });
|
|
131
|
+
await access(runtime, constants.X_OK);
|
|
132
|
+
return runtime;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Prepare source builds on activation; installed packages must already carry their native payload.
|
|
136
|
+
* @param spec - Native preparation inputs for this host.
|
|
137
|
+
* @returns Ready native executable without requiring a compiler for an installed package.
|
|
138
|
+
*/
|
|
139
|
+
async function prepareNativeExecutable(spec) {
|
|
140
|
+
if (await isSourcePackage(spec.packageRoot)) return buildNativeExecutable(spec);
|
|
141
|
+
const executable = nativeExecutablePath(spec.packageRoot, spec.architecture);
|
|
142
|
+
await access(executable, constants.X_OK);
|
|
143
|
+
spec.signal.throwIfAborted();
|
|
144
|
+
return executable;
|
|
145
|
+
}
|
|
146
|
+
//#endregion
|
|
147
|
+
//#region lib/types/shared-native.js
|
|
148
|
+
/** Per-Agent byte connections to the per-user native screenshot process. */
|
|
149
|
+
const Ready = z.object({
|
|
150
|
+
type: z.literal("ready"),
|
|
151
|
+
protocol: z.literal(1),
|
|
152
|
+
runtimeKey: z.string(),
|
|
153
|
+
pid: z.number().int().positive(),
|
|
154
|
+
instanceId: z.string().min(1)
|
|
155
|
+
}).strict();
|
|
156
|
+
const Refusal = z.object({
|
|
157
|
+
type: z.literal("error"),
|
|
158
|
+
code: z.string().min(1),
|
|
159
|
+
message: z.string()
|
|
160
|
+
}).strict();
|
|
161
|
+
const handshakeByteLimit = 8192;
|
|
162
|
+
function transportError(code, message, cause) {
|
|
163
|
+
return Object.assign(new Error(message, { cause }), { code });
|
|
164
|
+
}
|
|
165
|
+
function asError(error) {
|
|
166
|
+
return error instanceof Error ? error : transportError("NATIVE_CONNECT", "Native connection failed", error);
|
|
167
|
+
}
|
|
168
|
+
async function untilAbort(operation, signal) {
|
|
169
|
+
const cancelled = Promise.withResolvers();
|
|
170
|
+
const abort = () => {
|
|
171
|
+
cancelled.reject(signal.reason);
|
|
172
|
+
};
|
|
173
|
+
if (signal.aborted) abort();
|
|
174
|
+
else signal.addEventListener("abort", abort, { once: true });
|
|
175
|
+
try {
|
|
176
|
+
return await Promise.race([operation, cancelled.promise]);
|
|
177
|
+
} finally {
|
|
178
|
+
signal.removeEventListener("abort", abort);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
var SharedNativeChannel = class {
|
|
182
|
+
owner;
|
|
183
|
+
config;
|
|
184
|
+
stdin;
|
|
185
|
+
stdout = new PassThrough();
|
|
186
|
+
done;
|
|
187
|
+
ready = Promise.withResolvers();
|
|
188
|
+
closed = Promise.withResolvers();
|
|
189
|
+
startup = new AbortController();
|
|
190
|
+
startupTimer;
|
|
191
|
+
launcher;
|
|
192
|
+
launcherExit;
|
|
193
|
+
launcherFinished = false;
|
|
194
|
+
socket;
|
|
195
|
+
socketClosed;
|
|
196
|
+
connected = false;
|
|
197
|
+
closing = false;
|
|
198
|
+
cleanupTimer;
|
|
199
|
+
cleanupFailure;
|
|
200
|
+
constructor(owner, config) {
|
|
201
|
+
this.owner = owner;
|
|
202
|
+
this.config = config;
|
|
203
|
+
this.stdin = new Writable({
|
|
204
|
+
write: (chunk, encoding, callback) => {
|
|
205
|
+
this.ready.promise.then((socket) => {
|
|
206
|
+
if (this.closing) callback(transportError("NATIVE_CLOSED", "Native connection is closing"));
|
|
207
|
+
else try {
|
|
208
|
+
socket.write(chunk, encoding, callback);
|
|
209
|
+
} catch (error) {
|
|
210
|
+
callback(asError(error));
|
|
211
|
+
}
|
|
212
|
+
}, callback);
|
|
213
|
+
},
|
|
214
|
+
final: (callback) => {
|
|
215
|
+
this.close();
|
|
216
|
+
callback();
|
|
217
|
+
},
|
|
218
|
+
destroy: (error, callback) => {
|
|
219
|
+
this.close();
|
|
220
|
+
callback(error);
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
this.stdin.on("error", () => {});
|
|
224
|
+
this.stdout.on("error", () => {});
|
|
225
|
+
this.ready.promise.catch(() => {});
|
|
226
|
+
this.closed.promise.catch(() => {});
|
|
227
|
+
this.startupTimer = setTimeout(() => {
|
|
228
|
+
this.startup.abort(transportError("NATIVE_CONNECT_TIMEOUT", `Native connection exceeded ${config.connectTimeoutMs}ms`));
|
|
229
|
+
}, config.connectTimeoutMs);
|
|
230
|
+
this.done = this.run();
|
|
231
|
+
this.done.catch(() => {});
|
|
232
|
+
}
|
|
233
|
+
close() {
|
|
234
|
+
if (this.closing) return;
|
|
235
|
+
this.closing = true;
|
|
236
|
+
clearTimeout(this.startupTimer);
|
|
237
|
+
if (!this.connected) {
|
|
238
|
+
this.startup.abort(transportError("NATIVE_CLOSED", "Native connection closed before it became ready"));
|
|
239
|
+
this.socket?.destroy();
|
|
240
|
+
if (!this.launcherFinished) this.launcher?.terminate();
|
|
241
|
+
} else {
|
|
242
|
+
this.socket?.end();
|
|
243
|
+
this.cleanupTimer = setTimeout(() => {
|
|
244
|
+
this.cleanupFailure = transportError("NATIVE_CLEANUP", `Native connection did not finish cleanup within ${this.config.graceMs}ms`);
|
|
245
|
+
this.socket?.destroy(this.cleanupFailure);
|
|
246
|
+
}, this.config.graceMs);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
waitForClose() {
|
|
250
|
+
return this.closed.promise;
|
|
251
|
+
}
|
|
252
|
+
async run() {
|
|
253
|
+
let failure;
|
|
254
|
+
try {
|
|
255
|
+
const socket = await this.connect(await this.prepare());
|
|
256
|
+
this.startup.signal.throwIfAborted();
|
|
257
|
+
clearTimeout(this.startupTimer);
|
|
258
|
+
this.connected = true;
|
|
259
|
+
this.ready.resolve(socket);
|
|
260
|
+
await this.socketClosed;
|
|
261
|
+
} catch (error) {
|
|
262
|
+
failure = asError(this.startup.signal.aborted ? this.startup.signal.reason : error);
|
|
263
|
+
this.ready.reject(failure);
|
|
264
|
+
if (!this.closing) this.stdout.destroy(failure);
|
|
265
|
+
} finally {
|
|
266
|
+
clearTimeout(this.startupTimer);
|
|
267
|
+
clearTimeout(this.cleanupTimer);
|
|
268
|
+
this.closing = true;
|
|
269
|
+
this.socket?.destroy();
|
|
270
|
+
try {
|
|
271
|
+
await this.cleanup();
|
|
272
|
+
} catch (error) {
|
|
273
|
+
this.cleanupFailure ??= asError(error);
|
|
274
|
+
}
|
|
275
|
+
this.stdin.destroy();
|
|
276
|
+
if (this.cleanupFailure) {
|
|
277
|
+
this.stdout.destroy(this.cleanupFailure);
|
|
278
|
+
this.closed.reject(this.cleanupFailure);
|
|
279
|
+
} else {
|
|
280
|
+
this.stdout.end();
|
|
281
|
+
this.closed.resolve();
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (this.cleanupFailure) throw this.cleanupFailure;
|
|
285
|
+
if (failure) throw failure;
|
|
286
|
+
}
|
|
287
|
+
async prepare() {
|
|
288
|
+
const helperPath = await realpath(this.config.helperPath);
|
|
289
|
+
const runtimeKey = createHash("sha256").update(await readFile(helperPath, { signal: this.startup.signal })).digest("hex");
|
|
290
|
+
const uid = userInfo().uid;
|
|
291
|
+
const directory = `/tmp/qilin-cua-${uid}`;
|
|
292
|
+
try {
|
|
293
|
+
await mkdir(directory, { mode: 448 });
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) throw error;
|
|
296
|
+
}
|
|
297
|
+
const stat = await lstat(directory);
|
|
298
|
+
if (!stat.isDirectory() || stat.uid !== uid || (stat.mode & 511) !== 448) throw transportError("NATIVE_RUNTIME_DIRECTORY", "Native runtime directory must be a real, owner-only directory owned by the current user");
|
|
299
|
+
return {
|
|
300
|
+
helperPath,
|
|
301
|
+
runtimeKey,
|
|
302
|
+
directory
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
async ensureShared({ helperPath, runtimeKey, directory }) {
|
|
306
|
+
this.startup.signal.throwIfAborted();
|
|
307
|
+
this.launcherFinished = false;
|
|
308
|
+
this.launcher = this.owner.subprocess.spawn({
|
|
309
|
+
argv: [
|
|
310
|
+
helperPath,
|
|
311
|
+
"--ensure-shared",
|
|
312
|
+
"--runtime-key",
|
|
313
|
+
runtimeKey,
|
|
314
|
+
"--runtime-directory",
|
|
315
|
+
directory,
|
|
316
|
+
"--startup-timeout-ms",
|
|
317
|
+
String(this.config.connectTimeoutMs)
|
|
318
|
+
],
|
|
319
|
+
cwd: this.config.cwd,
|
|
320
|
+
graceMs: this.config.graceMs,
|
|
321
|
+
signal: this.startup.signal,
|
|
322
|
+
stdio: {
|
|
323
|
+
stdin: "ignore",
|
|
324
|
+
stdout: { maxBytes: 8192 },
|
|
325
|
+
stderr: { maxBytes: 8192 }
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
this.launcherExit = Promise.allSettled([this.launcher.done, this.launcher.waitForExit()]);
|
|
329
|
+
const [outcome, range] = await untilAbort(this.launcherExit, this.startup.signal);
|
|
330
|
+
this.launcherFinished = true;
|
|
331
|
+
if (range.status === "rejected") throw range.reason;
|
|
332
|
+
if (!range.value) throw transportError("NATIVE_CLEANUP", "Native launcher process range did not close");
|
|
333
|
+
if (outcome.status === "rejected") throw outcome.reason;
|
|
334
|
+
if (outcome.value.exitCode !== 0 || outcome.value.signal !== null) {
|
|
335
|
+
const stderr = this.launcher.collected.stderr?.readFrom(0).text ?? "";
|
|
336
|
+
throw transportError("NATIVE_STARTUP", `Native launcher failed (exit ${outcome.value.exitCode}, signal ${outcome.value.signal})${stderr ? `: ${stderr}` : ""}`);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
async connect(location) {
|
|
340
|
+
for (;;) {
|
|
341
|
+
await this.ensureShared(location);
|
|
342
|
+
const socket = await this.connectOnce(join(location.directory, `${location.runtimeKey}.sock`), location.runtimeKey);
|
|
343
|
+
if (socket) return socket;
|
|
344
|
+
await setTimeout$1(25, void 0, { signal: this.startup.signal });
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
async connectOnce(path, runtimeKey) {
|
|
348
|
+
this.startup.signal.throwIfAborted();
|
|
349
|
+
const socket = new Socket();
|
|
350
|
+
this.socket = socket;
|
|
351
|
+
const handshake = Promise.withResolvers();
|
|
352
|
+
const ended = Promise.withResolvers();
|
|
353
|
+
this.socketClosed = ended.promise;
|
|
354
|
+
ended.promise.catch(() => {});
|
|
355
|
+
let peerEnded = false;
|
|
356
|
+
let socketError;
|
|
357
|
+
let bytes = Buffer.alloc(0);
|
|
358
|
+
const admission = { status: "pending" };
|
|
359
|
+
const abort = () => {
|
|
360
|
+
socket.destroy(asError(this.startup.signal.reason));
|
|
361
|
+
};
|
|
362
|
+
this.startup.signal.addEventListener("abort", abort, { once: true });
|
|
363
|
+
socket.on("error", (error) => {
|
|
364
|
+
socketError = error;
|
|
365
|
+
handshake.reject(error);
|
|
366
|
+
});
|
|
367
|
+
socket.on("end", () => {
|
|
368
|
+
peerEnded = true;
|
|
369
|
+
});
|
|
370
|
+
socket.on("close", () => {
|
|
371
|
+
this.startup.signal.removeEventListener("abort", abort);
|
|
372
|
+
const error = socketError ?? (peerEnded ? void 0 : transportError("NATIVE_EOF", "Native socket closed without cleanup acknowledgement"));
|
|
373
|
+
handshake.reject(error ?? transportError("NATIVE_EOF", "Native socket ended before its handshake"));
|
|
374
|
+
if (error) ended.reject(error);
|
|
375
|
+
else ended.resolve();
|
|
376
|
+
});
|
|
377
|
+
socket.once("connect", () => {
|
|
378
|
+
const { maxRequestBytes, maxResponseBytes, axTimeoutMs, textEventIntervalMs, screenshotTimeoutMs, timeoutMs, graceMs } = this.config;
|
|
379
|
+
socket.write(JSON.stringify({
|
|
380
|
+
type: "hello",
|
|
381
|
+
protocol: 1,
|
|
382
|
+
runtimeKey,
|
|
383
|
+
config: {
|
|
384
|
+
maxRequestBytes,
|
|
385
|
+
maxResponseBytes,
|
|
386
|
+
axTimeoutMs,
|
|
387
|
+
textEventIntervalMs,
|
|
388
|
+
screenshotTimeoutMs,
|
|
389
|
+
timeoutMs,
|
|
390
|
+
graceMs
|
|
391
|
+
}
|
|
392
|
+
}) + "\n");
|
|
393
|
+
});
|
|
394
|
+
const receive = (chunk) => {
|
|
395
|
+
const newline = chunk.indexOf(10);
|
|
396
|
+
const length = newline < 0 ? chunk.length : newline + 1;
|
|
397
|
+
if (bytes.length + length > handshakeByteLimit) {
|
|
398
|
+
admission.status = "refused";
|
|
399
|
+
socket.destroy(transportError("NATIVE_PROTOCOL", "Native handshake exceeded its byte limit"));
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
bytes = Buffer.concat([bytes, chunk.subarray(0, length)]);
|
|
403
|
+
if (newline < 0) return;
|
|
404
|
+
try {
|
|
405
|
+
const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, -1)));
|
|
406
|
+
const refusal = Refusal.safeParse(value);
|
|
407
|
+
if (refusal.success) {
|
|
408
|
+
admission.status = "refused";
|
|
409
|
+
socket.destroy(transportError(refusal.data.code, refusal.data.message));
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
if (Ready.parse(value).runtimeKey !== runtimeKey) throw transportError("NATIVE_PROTOCOL", "Native handshake returned a different runtime identity");
|
|
413
|
+
admission.status = "accepted";
|
|
414
|
+
socket.removeListener("data", receive);
|
|
415
|
+
this.startup.signal.removeEventListener("abort", abort);
|
|
416
|
+
if (chunk.length > length) this.stdout.write(chunk.subarray(length));
|
|
417
|
+
socket.pipe(this.stdout, { end: false });
|
|
418
|
+
handshake.resolve();
|
|
419
|
+
} catch (error) {
|
|
420
|
+
admission.status = "refused";
|
|
421
|
+
socket.destroy(transportError("NATIVE_PROTOCOL", "Native helper refused or returned an invalid handshake", error));
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
socket.on("data", receive);
|
|
425
|
+
socket.connect(path);
|
|
426
|
+
try {
|
|
427
|
+
await untilAbort(handshake.promise, this.startup.signal);
|
|
428
|
+
return socket;
|
|
429
|
+
} catch (error) {
|
|
430
|
+
socket.destroy();
|
|
431
|
+
await ended.promise.catch(() => {});
|
|
432
|
+
if (admission.status === "pending" && error instanceof Error && "code" in error && [
|
|
433
|
+
"ENOENT",
|
|
434
|
+
"ECONNREFUSED",
|
|
435
|
+
"ECONNRESET",
|
|
436
|
+
"NATIVE_EOF"
|
|
437
|
+
].includes(String(error.code))) return void 0;
|
|
438
|
+
throw error;
|
|
439
|
+
} finally {
|
|
440
|
+
if (admission.status !== "accepted") socket.removeListener("data", receive);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
async cleanup() {
|
|
444
|
+
if (!this.launcherFinished) this.launcher?.terminate();
|
|
445
|
+
const controller = new AbortController();
|
|
446
|
+
const timer = setTimeout(() => {
|
|
447
|
+
controller.abort(transportError("NATIVE_CLEANUP", `Native startup cleanup exceeded ${this.config.graceMs}ms`));
|
|
448
|
+
}, this.config.graceMs);
|
|
449
|
+
try {
|
|
450
|
+
const [, launcher] = await untilAbort(Promise.allSettled([Promise.resolve(this.socketClosed), Promise.resolve(this.launcherExit)]), controller.signal);
|
|
451
|
+
if (launcher.status === "fulfilled" && launcher.value) {
|
|
452
|
+
const range = launcher.value[1];
|
|
453
|
+
if (range.status === "rejected") throw range.reason;
|
|
454
|
+
if (!range.value) throw transportError("NATIVE_CLEANUP", "Native launcher process range did not close");
|
|
455
|
+
}
|
|
456
|
+
} finally {
|
|
457
|
+
clearTimeout(timer);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
};
|
|
461
|
+
/**
|
|
462
|
+
* Prepare or join the native process; writes wait for identity-checked readiness.
|
|
463
|
+
* @param owner - Provider owning the short-lived native startup command.
|
|
464
|
+
* @param config - Native runtime identity, operation limits and connection deadlines.
|
|
465
|
+
* @returns One independently disposable Agent connection, without operation replay.
|
|
466
|
+
*/
|
|
467
|
+
function createSharedNativeChannel(owner, config) {
|
|
468
|
+
return new SharedNativeChannel(owner, config);
|
|
469
|
+
}
|
|
470
|
+
//#endregion
|
|
471
|
+
//#region lib/types/guidance.js
|
|
472
|
+
/** macOS operation guidance supplied with the shared CUA object API. */
|
|
473
|
+
/** Instructions for raw operation calls and optional object-API bindings. */
|
|
474
|
+
const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
|
475
|
+
|
|
476
|
+
Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
|
477
|
+
|
|
478
|
+
Use app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.
|
|
479
|
+
|
|
480
|
+
CuaScreenshot contains { bytes: Uint8Array, mimeType: "image/png" | "image/jpeg" | "image/webp" | "image/gif", pixelWidth: number, pixelHeight: number, windowId: number }. Use mimeType to choose a file extension or declare an image; do not assume PNG.
|
|
481
|
+
|
|
482
|
+
Object API signatures (options default to { emit: true }):
|
|
483
|
+
|
|
484
|
+
- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }
|
|
485
|
+
- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>
|
|
486
|
+
- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>
|
|
487
|
+
- cua.getState(options?) -> { apps, browsers: [] }
|
|
488
|
+
- cua.getApp(selector: string) -> App
|
|
489
|
+
- app.activate() -> { dispatched: true, verified: false }
|
|
490
|
+
- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>
|
|
491
|
+
- app.getWindow(windowId: number) -> Window
|
|
492
|
+
- app.typeText(text: string), app.pressKey(chord: string) -> action result
|
|
493
|
+
- app.paste(text: string, options?: { format?: "text" | "html" }) -> action result
|
|
494
|
+
|
|
495
|
+
App and Window methods:
|
|
496
|
+
|
|
497
|
+
- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }
|
|
498
|
+
- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string
|
|
499
|
+
- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot
|
|
500
|
+
- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }
|
|
501
|
+
- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }
|
|
502
|
+
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
|
503
|
+
- setValue(elementIndex: number, value: string) -> action result
|
|
504
|
+
- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: "text" | "cursor_before" | "cursor_after" }) -> action result
|
|
505
|
+
- performSecondaryAction(elementIndex: number, action: string) -> action result
|
|
506
|
+
- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result
|
|
507
|
+
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
|
508
|
+
|
|
509
|
+
Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.
|
|
510
|
+
|
|
511
|
+
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
|
512
|
+
|
|
513
|
+
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
|
514
|
+
|
|
515
|
+
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`;
|
|
516
|
+
//#endregion
|
|
517
|
+
//#region lib/types/index.js
|
|
518
|
+
/** macOS native CUA provider using the shared application and window API. */
|
|
519
|
+
/** Loader plugin identity. */
|
|
520
|
+
const name = "experimental-cua-macos";
|
|
521
|
+
/** Services required by the shared CUA executor and the native launcher. */
|
|
522
|
+
const inject = [
|
|
523
|
+
"computerUse",
|
|
524
|
+
"tools",
|
|
525
|
+
"systemPrompt",
|
|
526
|
+
"subprocess",
|
|
527
|
+
"agents"
|
|
528
|
+
];
|
|
529
|
+
/** Validate the shared configuration before preparing the macOS helper. */
|
|
530
|
+
const Config = CuaConfig;
|
|
531
|
+
const description = "Inspect macOS permissions, select an application and window, read AX state, capture screenshots or dispatch input. Use appHandle from getApp; verify actions by reading fresh state.";
|
|
532
|
+
/**
|
|
533
|
+
* Mount logged native operations with separate Agent connections to the shared helper.
|
|
534
|
+
* @param ctx - Host context with computer-use, subprocess and tool services.
|
|
535
|
+
* @param config - Validated native preparation and operation bounds.
|
|
536
|
+
* @returns Completion after native preparation and tool registration.
|
|
537
|
+
*/
|
|
538
|
+
async function apply(ctx, config) {
|
|
539
|
+
if (config.helperPath !== void 0) {
|
|
540
|
+
if (!isAbsolute(config.helperPath)) throw new Error("qilin-cua helperPath must be absolute");
|
|
541
|
+
installCua(ctx, config, {
|
|
542
|
+
helperPath: config.helperPath,
|
|
543
|
+
createChannel: createSharedNativeChannel,
|
|
544
|
+
guidance: GUIDANCE,
|
|
545
|
+
description
|
|
546
|
+
});
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const architecture = nativeHostArchitecture(process.platform, process.arch);
|
|
550
|
+
const controller = new AbortController();
|
|
551
|
+
const prepared = prepareNativeExecutable({
|
|
552
|
+
packageRoot: resolve(fileURLToPath(new URL("..", import.meta.url))),
|
|
553
|
+
architecture,
|
|
554
|
+
subprocess: ctx.subprocess,
|
|
555
|
+
timeoutMs: config.buildTimeoutMs,
|
|
556
|
+
graceMs: config.graceMs,
|
|
557
|
+
signal: controller.signal
|
|
558
|
+
});
|
|
559
|
+
ctx.effect(() => async () => {
|
|
560
|
+
controller.abort();
|
|
561
|
+
await prepared.catch(() => {});
|
|
562
|
+
}, "qilin-cua.native-preparation");
|
|
563
|
+
const helperPath = await prepared;
|
|
564
|
+
controller.signal.throwIfAborted();
|
|
565
|
+
installCua(ctx, config, {
|
|
566
|
+
helperPath,
|
|
567
|
+
createChannel: createSharedNativeChannel,
|
|
568
|
+
guidance: GUIDANCE,
|
|
569
|
+
description
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
//#endregion
|
|
573
|
+
export { Config, GUIDANCE, apply, inject, name };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** macOS operation guidance supplied with the shared CUA object API. */
|
|
2
|
+
/** Instructions for raw operation calls and optional object-API bindings. */
|
|
3
|
+
export declare const GUIDANCE = "Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.\n\nInitialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.\n\nUse app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.\n\nCuaScreenshot contains { bytes: Uint8Array, mimeType: \"image/png\" | \"image/jpeg\" | \"image/webp\" | \"image/gif\", pixelWidth: number, pixelHeight: number, windowId: number }. Use mimeType to choose a file extension or declare an image; do not assume PNG.\n\nObject API signatures (options default to { emit: true }):\n\n- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }\n- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>\n- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>\n- cua.getState(options?) -> { apps, browsers: [] }\n- cua.getApp(selector: string) -> App\n- app.activate() -> { dispatched: true, verified: false }\n- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>\n- app.getWindow(windowId: number) -> Window\n- app.typeText(text: string), app.pressKey(chord: string) -> action result\n- app.paste(text: string, options?: { format?: \"text\" | \"html\" }) -> action result\n\nApp and Window methods:\n\n- show() -> { dispatched: true, verified: false, windowId: number | null, foregroundChanged: boolean | null }\n- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string\n- getScreenshot(options?: { emit?: boolean }) -> CuaScreenshot\n- getAXStateAndScreenshot(options?) -> { state: string, screenshot: CuaScreenshot }\n- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }\n- click(target: number | [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", clickCount?: 1 | 2, modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n- setValue(elementIndex: number, value: string) -> action result\n- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: \"text\" | \"cursor_before\" | \"cursor_after\" }) -> action result\n- performSecondaryAction(elementIndex: number, action: string) -> action result\n- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result\n- drag(from: [x, y], to: [x, y], options?: { mouseButton?: \"left\" | \"right\" | \"middle\", modifiers?: Array<\"command\" | \"control\" | \"option\" | \"shift\" | \"fn\">, relative?: boolean }) -> action result\n\nRaw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.\n\nAwait operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.\n\nHover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.\n\nUse selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Paste follows the target app's focused text input. Foreground requirements vary by application; for example, ChatGPT/Codex was observed to accept background Paste without reading clipboard data. PASTE_NOT_CONSUMED means the command was dispatched but no clipboard read was observed before the configured deadline. Inspect the target contents, explicitly activate the app and focus the intended input, then observe again before deciding whether to retry; a delayed paste may still run. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.";
|
|
4
|
+
//# sourceMappingURL=guidance.d.ts.map
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Context } from '@qilin-agent/kylin';
|
|
2
|
+
import { CuaConfig } from '@qilin-agent/experimental-cua';
|
|
3
|
+
export { GUIDANCE } from './guidance.ts';
|
|
4
|
+
/** Loader plugin identity. */
|
|
5
|
+
export declare const name = "experimental-cua-macos";
|
|
6
|
+
/** Services required by the shared CUA executor and the native launcher. */
|
|
7
|
+
export declare const inject: string[];
|
|
8
|
+
/** Native preparation, operation bounds, and profile-local application grants. */
|
|
9
|
+
export type Config = CuaConfig;
|
|
10
|
+
/** Validate the shared configuration before preparing the macOS helper. */
|
|
11
|
+
export declare const Config: typeof CuaConfig;
|
|
12
|
+
/**
|
|
13
|
+
* Mount logged native operations with separate Agent connections to the shared helper.
|
|
14
|
+
* @param ctx - Host context with computer-use, subprocess and tool services.
|
|
15
|
+
* @param config - Validated native preparation and operation bounds.
|
|
16
|
+
* @returns Completion after native preparation and tool registration.
|
|
17
|
+
*/
|
|
18
|
+
export declare function apply(ctx: Context, config: Config): Promise<void>;
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|