automify 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +236 -36
- package/examples/browser-with-safety.js +7 -10
- package/examples/cli-qemu.js +28 -0
- package/examples/desktop-qemu.js +41 -0
- package/package.json +5 -2
- package/scripts/generate-argument-reference.js +3 -1
- package/scripts/qemu-image.js +151 -0
- package/src/index.d.ts +368 -10
- package/src/index.js +18 -38
- package/src/lib/adapter-toolkit.js +8 -4
- package/src/lib/anthropic-model-adapter.js +24 -13
- package/src/lib/argument-reference.js +60 -8
- package/src/lib/automify.js +96 -0
- package/src/lib/cli-automify.js +41 -2
- package/src/lib/computer-automify.js +45 -26
- package/src/lib/docker-cli-automify.js +2 -6
- package/src/lib/docker-desktop-computer.js +7 -13
- package/src/lib/file-data.js +6 -6
- package/src/lib/init.js +14 -3
- package/src/lib/local-desktop-computer.js +2 -1
- package/src/lib/openai-responses-client.js +10 -3
- package/src/lib/presets.js +50 -2
- package/src/lib/qemu-cli-automify.js +555 -0
- package/src/lib/qemu-desktop-computer.js +681 -0
- package/src/lib/qemu-runtime.js +654 -0
- package/src/lib/runtime.js +23 -2
- package/src/lib/screen-recording.js +184 -0
- package/src/lib/task.js +564 -0
- package/src/lib/virtual-shared-folder.js +3 -1
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { execFile, spawn as spawnProcess } from "node:child_process";
|
|
3
|
+
import { promisify } from "node:util";
|
|
4
|
+
|
|
5
|
+
import { acquireAdapterLock } from "./adapter-locks.js";
|
|
6
|
+
import { AutomifyError } from "./errors.js";
|
|
7
|
+
import { applyVirtualDesktopPreset } from "./presets.js";
|
|
8
|
+
import { assertKnownOptions, debugLog, normalizeLogFile, writeDebugLogFile } from "./runtime.js";
|
|
9
|
+
import { prepareVirtualSharedFolder } from "./virtual-shared-folder.js";
|
|
10
|
+
import {
|
|
11
|
+
buildQemuArgs,
|
|
12
|
+
defaultQemuBaseImagePath,
|
|
13
|
+
defaultQemuCommand,
|
|
14
|
+
getAvailablePort,
|
|
15
|
+
installCommand,
|
|
16
|
+
mountSharedFolderCommand,
|
|
17
|
+
prepareDefaultQemuImage,
|
|
18
|
+
positiveInteger,
|
|
19
|
+
shellQuote,
|
|
20
|
+
sleep,
|
|
21
|
+
sshArgs,
|
|
22
|
+
stopQemuProcess,
|
|
23
|
+
uniquePackages,
|
|
24
|
+
waitForSsh
|
|
25
|
+
} from "./qemu-runtime.js";
|
|
26
|
+
|
|
27
|
+
const execFileAsync = promisify(execFile);
|
|
28
|
+
|
|
29
|
+
const DEFAULT_ENVIRONMENT = "linux";
|
|
30
|
+
const DEFAULT_DISPLAY = ":99";
|
|
31
|
+
const DEFAULT_WIDTH = 1440;
|
|
32
|
+
const DEFAULT_HEIGHT = 900;
|
|
33
|
+
const DEFAULT_DEPTH = 24;
|
|
34
|
+
export const DEFAULT_QEMU_DESKTOP_PACKAGES = [
|
|
35
|
+
"xvfb",
|
|
36
|
+
"openbox",
|
|
37
|
+
"xterm",
|
|
38
|
+
"x11-utils",
|
|
39
|
+
"xdotool",
|
|
40
|
+
"imagemagick",
|
|
41
|
+
"scrot",
|
|
42
|
+
"ca-certificates"
|
|
43
|
+
];
|
|
44
|
+
const DEFAULT_INSTRUCTIONS = [
|
|
45
|
+
"You are controlling an isolated Linux desktop running in a QEMU virtual machine.",
|
|
46
|
+
"Orient from the screenshot first: identify the active app, visible window, focused field, current page, and the specific target required by the task before acting.",
|
|
47
|
+
"Use deterministic entry points. For a website or web app, use the browser address bar only when a browser is clearly focused; otherwise use a visible app icon, terminal command, launcher/search mechanism, or provided startup app. For app content, use visible search, filters, or navigation controls.",
|
|
48
|
+
"Do not open or use Alt+Tab or other cyclic app/window switchers unless the task explicitly asks to switch to the previous app.",
|
|
49
|
+
"Do not click as a probe. Click only when the screenshot shows a specific visible target and the purpose of that click is clear from the task or current UI.",
|
|
50
|
+
"Treat the environment as ephemeral unless the task explicitly says the virtual machine disk is persistent.",
|
|
51
|
+
"After any action that launches an app, navigates, submits input, changes windows, or might trigger loading, use the next screenshot to decide the next step."
|
|
52
|
+
].join("\n");
|
|
53
|
+
|
|
54
|
+
const QEMU_DESKTOP_OPTION_KEYS = new Set([
|
|
55
|
+
"preset",
|
|
56
|
+
"vm",
|
|
57
|
+
"qemuCommand",
|
|
58
|
+
"qemuImgCommand",
|
|
59
|
+
"qemuImageCacheDir",
|
|
60
|
+
"qemuImageUrl",
|
|
61
|
+
"defaultImageCache",
|
|
62
|
+
"createCloudInitServer",
|
|
63
|
+
"fetchImpl",
|
|
64
|
+
"image",
|
|
65
|
+
"diskImage",
|
|
66
|
+
"diskFormat",
|
|
67
|
+
"vmName",
|
|
68
|
+
"existingVM",
|
|
69
|
+
"keepVM",
|
|
70
|
+
"start",
|
|
71
|
+
"ssh",
|
|
72
|
+
"sshCommand",
|
|
73
|
+
"sshKeygenCommand",
|
|
74
|
+
"sshHost",
|
|
75
|
+
"sshPort",
|
|
76
|
+
"sshUser",
|
|
77
|
+
"sshKeyPath",
|
|
78
|
+
"sshOptions",
|
|
79
|
+
"sshTimeoutMs",
|
|
80
|
+
"sudo",
|
|
81
|
+
"display",
|
|
82
|
+
"viewport",
|
|
83
|
+
"displayWidth",
|
|
84
|
+
"displayHeight",
|
|
85
|
+
"displayDepth",
|
|
86
|
+
"environment",
|
|
87
|
+
"instructions",
|
|
88
|
+
"desktop",
|
|
89
|
+
"startupCommand",
|
|
90
|
+
"windowManagerCommand",
|
|
91
|
+
"installDependencies",
|
|
92
|
+
"desktopPackages",
|
|
93
|
+
"additionalAptPackages",
|
|
94
|
+
"memory",
|
|
95
|
+
"cpus",
|
|
96
|
+
"accel",
|
|
97
|
+
"machine",
|
|
98
|
+
"cpu",
|
|
99
|
+
"firmware",
|
|
100
|
+
"network",
|
|
101
|
+
"networkDevice",
|
|
102
|
+
"extraQemuArgs",
|
|
103
|
+
"shared",
|
|
104
|
+
"sharedFolder",
|
|
105
|
+
"sharedFiles",
|
|
106
|
+
"files",
|
|
107
|
+
"sharedMode",
|
|
108
|
+
"sharedTag",
|
|
109
|
+
"sharedSecurityModel",
|
|
110
|
+
"waitMs",
|
|
111
|
+
"startupTimeoutMs",
|
|
112
|
+
"qemuTimeoutMs",
|
|
113
|
+
"commandTimeoutMs",
|
|
114
|
+
"screenshotMaxBuffer",
|
|
115
|
+
"screenshotSettleMs",
|
|
116
|
+
"execFile",
|
|
117
|
+
"spawn",
|
|
118
|
+
"silent",
|
|
119
|
+
"debug",
|
|
120
|
+
"logFile",
|
|
121
|
+
"onUnknownAction"
|
|
122
|
+
]);
|
|
123
|
+
export const VIRTUAL_DESKTOP_COMPUTER_OPTION_KEYS = QEMU_DESKTOP_OPTION_KEYS;
|
|
124
|
+
|
|
125
|
+
const QEMU_VM_KEYS = new Set([
|
|
126
|
+
"qemu",
|
|
127
|
+
"qemuCommand",
|
|
128
|
+
"qemuImgCommand",
|
|
129
|
+
"imageCacheDir",
|
|
130
|
+
"imageUrl",
|
|
131
|
+
"defaultImageCache",
|
|
132
|
+
"image",
|
|
133
|
+
"diskImage",
|
|
134
|
+
"diskFormat",
|
|
135
|
+
"name",
|
|
136
|
+
"existing",
|
|
137
|
+
"keep",
|
|
138
|
+
"memory",
|
|
139
|
+
"cpus",
|
|
140
|
+
"accel",
|
|
141
|
+
"machine",
|
|
142
|
+
"cpu",
|
|
143
|
+
"firmware",
|
|
144
|
+
"network",
|
|
145
|
+
"networkDevice",
|
|
146
|
+
"extraArgs",
|
|
147
|
+
"timeoutMs"
|
|
148
|
+
]);
|
|
149
|
+
const QEMU_DESKTOP_KEYS = new Set([
|
|
150
|
+
"startupCommand",
|
|
151
|
+
"windowManagerCommand",
|
|
152
|
+
"packages",
|
|
153
|
+
"additionalAptPackages",
|
|
154
|
+
"installDependencies"
|
|
155
|
+
]);
|
|
156
|
+
const QEMU_SSH_KEYS = new Set(["command", "host", "port", "user", "keyPath", "options", "timeoutMs", "sudo"]);
|
|
157
|
+
|
|
158
|
+
export async function createVirtualDesktopComputer(options = {}) {
|
|
159
|
+
options = normalizeVirtualDesktopOptions(options);
|
|
160
|
+
const releaseLock = await acquireQemuDesktopLock(options);
|
|
161
|
+
debugVirtualDesktop(options, "create", {
|
|
162
|
+
image: options.image ?? null,
|
|
163
|
+
vmName: options.vmName,
|
|
164
|
+
start: options.start !== false
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
try {
|
|
168
|
+
const session = new QemuDesktopSession(options);
|
|
169
|
+
await session.prepareSharedFolder();
|
|
170
|
+
if (options.start !== false) {
|
|
171
|
+
await session.start();
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const computer = {
|
|
175
|
+
displayWidth: session.width,
|
|
176
|
+
displayHeight: session.height,
|
|
177
|
+
environment: options.environment ?? DEFAULT_ENVIRONMENT,
|
|
178
|
+
instructions: options.instructions ?? DEFAULT_INSTRUCTIONS,
|
|
179
|
+
|
|
180
|
+
async execute(action, context) {
|
|
181
|
+
await session.execute(action, context);
|
|
182
|
+
},
|
|
183
|
+
|
|
184
|
+
async screenshot(context) {
|
|
185
|
+
return session.screenshot(context);
|
|
186
|
+
},
|
|
187
|
+
|
|
188
|
+
async close() {
|
|
189
|
+
try {
|
|
190
|
+
await session.close();
|
|
191
|
+
} finally {
|
|
192
|
+
await releaseLock?.();
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
|
|
196
|
+
session
|
|
197
|
+
};
|
|
198
|
+
computer.sharedFolder = session.sharedFolder?.data;
|
|
199
|
+
return computer;
|
|
200
|
+
} catch (error) {
|
|
201
|
+
await releaseLock?.();
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export class QemuDesktopSession {
|
|
207
|
+
constructor(options = {}) {
|
|
208
|
+
options = normalizeVirtualDesktopOptions(options);
|
|
209
|
+
this.options = options;
|
|
210
|
+
this.qemu = options.qemuCommand ?? defaultQemuCommand();
|
|
211
|
+
this.sshCommand = options.sshCommand ?? "ssh";
|
|
212
|
+
this.execFile = options.execFile ?? execFileAsync;
|
|
213
|
+
this.spawn = options.spawn ?? spawnProcess;
|
|
214
|
+
this.image = options.image;
|
|
215
|
+
this.extraQemuArgs = options.extraQemuArgs ?? [];
|
|
216
|
+
this.originalSshUser = options.sshUser;
|
|
217
|
+
this.originalSshKeyPath = options.sshKeyPath;
|
|
218
|
+
this.originalSudo = options.sudo;
|
|
219
|
+
this.defaultImage = null;
|
|
220
|
+
this.preparedPackages = new Set();
|
|
221
|
+
this.usesDefaultImage = !this.image && !options.existingVM;
|
|
222
|
+
this.name = options.vmName ?? `automify-vm-desktop-${randomUUID()}`;
|
|
223
|
+
this.display = normalizeDisplay(options.display ?? DEFAULT_DISPLAY);
|
|
224
|
+
this.width = positiveInteger(options.displayWidth) ?? DEFAULT_WIDTH;
|
|
225
|
+
this.height = positiveInteger(options.displayHeight) ?? DEFAULT_HEIGHT;
|
|
226
|
+
this.depth = positiveInteger(options.displayDepth) ?? DEFAULT_DEPTH;
|
|
227
|
+
this.sshPort = positiveInteger(options.sshPort);
|
|
228
|
+
this.sharedFolder = null;
|
|
229
|
+
this.started = false;
|
|
230
|
+
this.created = false;
|
|
231
|
+
this.process = null;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async prepareSharedFolder() {
|
|
235
|
+
if (this.sharedFolder) return this.sharedFolder;
|
|
236
|
+
this.sharedFolder = await prepareVirtualSharedFolder(
|
|
237
|
+
{ ...this.options, keepContainer: this.options.keepVM },
|
|
238
|
+
{
|
|
239
|
+
prefix: "automify-qemu-desktop-"
|
|
240
|
+
}
|
|
241
|
+
);
|
|
242
|
+
return this.sharedFolder;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
async start() {
|
|
246
|
+
if (this.started) return;
|
|
247
|
+
await this.prepareSharedFolder();
|
|
248
|
+
if (!this.sshPort) this.sshPort = await getAvailablePort();
|
|
249
|
+
|
|
250
|
+
if (this.options.existingVM) {
|
|
251
|
+
debugVirtualDesktop(this.options, "use_existing_vm", { vmName: this.name, sshPort: this.sshPort });
|
|
252
|
+
this.started = true;
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
await this.prepareDefaultImage();
|
|
258
|
+
const args = buildQemuArgs({
|
|
259
|
+
...this.options,
|
|
260
|
+
name: this.name,
|
|
261
|
+
image: this.image,
|
|
262
|
+
sshPort: this.sshPort,
|
|
263
|
+
sharedFolder: this.sharedFolder
|
|
264
|
+
});
|
|
265
|
+
debugVirtualDesktop(this.options, "vm_start", {
|
|
266
|
+
vmName: this.name,
|
|
267
|
+
qemu: this.qemu,
|
|
268
|
+
image: this.image,
|
|
269
|
+
sshPort: this.sshPort,
|
|
270
|
+
width: this.width,
|
|
271
|
+
height: this.height
|
|
272
|
+
});
|
|
273
|
+
this.process = this.spawn(this.qemu, args, {
|
|
274
|
+
stdio: "ignore"
|
|
275
|
+
});
|
|
276
|
+
this.process.unref?.();
|
|
277
|
+
this.created = true;
|
|
278
|
+
this.started = true;
|
|
279
|
+
await waitForSsh(this.execFile, this.sshCommand, this.sshOptions());
|
|
280
|
+
await this.runSsh(this.detachedStartupCommand(), {
|
|
281
|
+
timeout: positiveInteger(this.options.commandTimeoutMs) ?? 30_000
|
|
282
|
+
});
|
|
283
|
+
await this.waitForReady();
|
|
284
|
+
debugVirtualDesktop(this.options, "desktop_ready", { vmName: this.name, display: this.display });
|
|
285
|
+
} catch (error) {
|
|
286
|
+
await this.close();
|
|
287
|
+
throw new AutomifyError("QEMU virtual desktop did not become ready before startupTimeoutMs.", {
|
|
288
|
+
cause: error
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
async prepareDefaultImage() {
|
|
294
|
+
if (!this.usesDefaultImage || this.defaultImage) return;
|
|
295
|
+
debugVirtualDesktop(this.options, "default_image_prepare", {
|
|
296
|
+
vmName: this.name,
|
|
297
|
+
imageUrl: this.options.qemuImageUrl ?? process.env.AUTOMIFY_QEMU_DEFAULT_IMAGE_URL
|
|
298
|
+
});
|
|
299
|
+
const prepared = await prepareDefaultQemuImage({
|
|
300
|
+
execFile: this.execFile,
|
|
301
|
+
fetchImpl: this.options.fetchImpl,
|
|
302
|
+
cacheDir: this.options.qemuImageCacheDir,
|
|
303
|
+
imageUrl: this.options.qemuImageUrl,
|
|
304
|
+
defaultImageCache: this.options.defaultImageCache,
|
|
305
|
+
qemuImgCommand: this.options.qemuImgCommand,
|
|
306
|
+
qemuCommand: this.qemu,
|
|
307
|
+
memory: this.options.memory,
|
|
308
|
+
cpus: this.options.cpus,
|
|
309
|
+
accel: this.options.accel,
|
|
310
|
+
machine: this.options.machine,
|
|
311
|
+
cpu: this.options.cpu,
|
|
312
|
+
firmware: this.options.firmware,
|
|
313
|
+
networkDevice: this.options.networkDevice,
|
|
314
|
+
sshKeygenCommand: this.options.sshKeygenCommand,
|
|
315
|
+
sshCommand: this.sshCommand,
|
|
316
|
+
sshPort: this.sshPort,
|
|
317
|
+
sshTimeoutMs: this.options.sshTimeoutMs,
|
|
318
|
+
startupTimeoutMs: this.options.startupTimeoutMs,
|
|
319
|
+
timeoutMs: this.options.commandTimeoutMs,
|
|
320
|
+
qemuTimeoutMs: this.options.qemuTimeoutMs,
|
|
321
|
+
createCloudInitServer: this.options.createCloudInitServer,
|
|
322
|
+
preparedImageProfile: "desktop",
|
|
323
|
+
preparedPackages: this.options.installDependencies === false ? [] : this.dependencyPackages(),
|
|
324
|
+
spawn: this.spawn,
|
|
325
|
+
vmName: this.name
|
|
326
|
+
});
|
|
327
|
+
this.defaultImage = prepared;
|
|
328
|
+
this.preparedPackages = new Set(prepared.preparedPackages ?? []);
|
|
329
|
+
this.image = prepared.image;
|
|
330
|
+
this.options = {
|
|
331
|
+
...this.options,
|
|
332
|
+
image: prepared.image,
|
|
333
|
+
diskFormat: this.options.diskFormat ?? prepared.diskFormat,
|
|
334
|
+
sshUser: prepared.sshUser,
|
|
335
|
+
sshKeyPath: prepared.sshKeyPath,
|
|
336
|
+
sudo: this.options.sudo ?? prepared.sudo,
|
|
337
|
+
extraQemuArgs: [...prepared.extraQemuArgs, ...this.extraQemuArgs]
|
|
338
|
+
};
|
|
339
|
+
debugVirtualDesktop(this.options, "default_image_ready", {
|
|
340
|
+
vmName: this.name,
|
|
341
|
+
image: prepared.image,
|
|
342
|
+
baseImage: prepared.baseImage
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
detachedStartupCommand() {
|
|
347
|
+
return `nohup sh -lc ${shellQuote(this.startupScript())} >/tmp/automify-desktop-supervisor.log 2>&1 &`;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
startupScript() {
|
|
351
|
+
const windowManager = this.options.windowManagerCommand ?? "openbox";
|
|
352
|
+
const app = this.options.startupCommand;
|
|
353
|
+
const xvfb = [
|
|
354
|
+
"Xvfb",
|
|
355
|
+
shellQuote(this.display),
|
|
356
|
+
"-screen",
|
|
357
|
+
"0",
|
|
358
|
+
shellQuote(`${this.width}x${this.height}x${this.depth}`),
|
|
359
|
+
"-nolisten",
|
|
360
|
+
"tcp"
|
|
361
|
+
].join(" ");
|
|
362
|
+
return [
|
|
363
|
+
this.dependencyInstallScript(),
|
|
364
|
+
mountSharedFolderCommand(this.sharedFolder, this.options),
|
|
365
|
+
`${xvfb} &`,
|
|
366
|
+
"XVFB_PID=$!",
|
|
367
|
+
"sleep 0.2",
|
|
368
|
+
`${windowManager} >/tmp/automify-window-manager.log 2>&1 &`,
|
|
369
|
+
`${app} >/tmp/automify-startup.log 2>&1 &`,
|
|
370
|
+
"wait $XVFB_PID"
|
|
371
|
+
].join("\n");
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
dependencyInstallScript() {
|
|
375
|
+
const packages = this.dependencyPackages().filter((pkg) => !this.preparedPackages.has(pkg));
|
|
376
|
+
return installCommand(packages, this.options);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
dependencyPackages() {
|
|
380
|
+
return uniquePackages([
|
|
381
|
+
...(this.options.desktopPackages ?? DEFAULT_QEMU_DESKTOP_PACKAGES),
|
|
382
|
+
...(this.options.additionalAptPackages ?? [])
|
|
383
|
+
]);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
async waitForReady() {
|
|
387
|
+
const timeoutMs = positiveInteger(this.options.startupTimeoutMs) ?? 60_000;
|
|
388
|
+
const startedAt = Date.now();
|
|
389
|
+
let lastError;
|
|
390
|
+
|
|
391
|
+
while (Date.now() - startedAt < timeoutMs) {
|
|
392
|
+
try {
|
|
393
|
+
await this.exec(["sh", "-lc", "xdpyinfo >/dev/null 2>&1"], {
|
|
394
|
+
timeout: positiveInteger(this.options.sshTimeoutMs) ?? 5_000
|
|
395
|
+
});
|
|
396
|
+
return;
|
|
397
|
+
} catch (error) {
|
|
398
|
+
lastError = error;
|
|
399
|
+
await sleep(250);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
throw lastError ?? new AutomifyError("QEMU virtual desktop readiness check timed out.");
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
async execute(action, context) {
|
|
407
|
+
await this.start();
|
|
408
|
+
if (!action || typeof action !== "object") {
|
|
409
|
+
throw new AutomifyError("QEMU virtual desktop action must be an object.");
|
|
410
|
+
}
|
|
411
|
+
debugVirtualDesktop(this.options, "action", { action });
|
|
412
|
+
|
|
413
|
+
switch (action.type) {
|
|
414
|
+
case "click":
|
|
415
|
+
await this.exec(["xdotool", "mousemove", x(action.x), y(action.y), "click", button(action.button)]);
|
|
416
|
+
break;
|
|
417
|
+
case "double_click":
|
|
418
|
+
await this.exec([
|
|
419
|
+
"xdotool",
|
|
420
|
+
"mousemove",
|
|
421
|
+
x(action.x),
|
|
422
|
+
y(action.y),
|
|
423
|
+
"click",
|
|
424
|
+
"--repeat",
|
|
425
|
+
"2",
|
|
426
|
+
button(action.button)
|
|
427
|
+
]);
|
|
428
|
+
break;
|
|
429
|
+
case "scroll":
|
|
430
|
+
await this.scroll(action);
|
|
431
|
+
break;
|
|
432
|
+
case "keypress":
|
|
433
|
+
await this.exec(["xdotool", "key", ...keys(action.keys ?? [action.key])]);
|
|
434
|
+
break;
|
|
435
|
+
case "type":
|
|
436
|
+
await this.exec(["xdotool", "type", "--clearmodifiers", "--", String(action.text ?? "")]);
|
|
437
|
+
break;
|
|
438
|
+
case "wait":
|
|
439
|
+
await sleep(Math.max(0, Number(action.ms ?? action.duration_ms ?? this.options.waitMs ?? 1000) || 0));
|
|
440
|
+
break;
|
|
441
|
+
case "screenshot":
|
|
442
|
+
break;
|
|
443
|
+
case "move":
|
|
444
|
+
await this.exec(["xdotool", "mousemove", x(action.x), y(action.y)]);
|
|
445
|
+
break;
|
|
446
|
+
case "drag":
|
|
447
|
+
await this.drag(action);
|
|
448
|
+
break;
|
|
449
|
+
default:
|
|
450
|
+
if (typeof this.options.onUnknownAction === "function") {
|
|
451
|
+
await this.options.onUnknownAction(action, context);
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
throw new AutomifyError(`Unsupported QEMU virtual desktop action: ${action.type}`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async scroll(action) {
|
|
459
|
+
const scrollY = Number(action.scroll_y ?? action.delta_y ?? action.deltaY ?? 0);
|
|
460
|
+
const scrollX = Number(action.scroll_x ?? action.delta_x ?? action.deltaX ?? 0);
|
|
461
|
+
const amount = Math.max(1, Math.ceil(Math.max(Math.abs(scrollY), Math.abs(scrollX)) / 120));
|
|
462
|
+
const scrollButton = Math.abs(scrollX) > Math.abs(scrollY) ? (scrollX > 0 ? "7" : "6") : scrollY > 0 ? "5" : "4";
|
|
463
|
+
await this.exec([
|
|
464
|
+
"xdotool",
|
|
465
|
+
"mousemove",
|
|
466
|
+
x(action.x),
|
|
467
|
+
y(action.y),
|
|
468
|
+
"click",
|
|
469
|
+
"--repeat",
|
|
470
|
+
String(amount),
|
|
471
|
+
scrollButton
|
|
472
|
+
]);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
async drag(action) {
|
|
476
|
+
const path = action.path?.length ? action.path : [action, action];
|
|
477
|
+
const start = path[0];
|
|
478
|
+
const end = path.at(-1);
|
|
479
|
+
await this.exec(["xdotool", "mousemove", x(start.x), y(start.y), "mousedown", "1"]);
|
|
480
|
+
for (const point of path.slice(1)) {
|
|
481
|
+
await this.exec(["xdotool", "mousemove", x(point.x), y(point.y)]);
|
|
482
|
+
}
|
|
483
|
+
await this.exec(["xdotool", "mousemove", x(end.x), y(end.y), "mouseup", "1"]);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async screenshot(context) {
|
|
487
|
+
await this.start();
|
|
488
|
+
const startedAt = Date.now();
|
|
489
|
+
if (context?.initial || context?.final) {
|
|
490
|
+
await sleep(this.options.screenshotSettleMs ?? 300);
|
|
491
|
+
}
|
|
492
|
+
const { stdout } = await this.exec(["sh", "-lc", "scrot -o - 2>/dev/null || import -window root -screen png:-"], {
|
|
493
|
+
encoding: "buffer",
|
|
494
|
+
maxBuffer: this.options.screenshotMaxBuffer ?? 20 * 1024 * 1024
|
|
495
|
+
});
|
|
496
|
+
debugVirtualDesktop(this.options, "screenshot", {
|
|
497
|
+
phase: context?.final ? "final" : context?.initial ? "initial" : "step",
|
|
498
|
+
bytes: stdout?.byteLength,
|
|
499
|
+
durationMs: Date.now() - startedAt
|
|
500
|
+
});
|
|
501
|
+
return stdout;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
async exec(args, options = {}) {
|
|
505
|
+
return this.runSsh(`DISPLAY=${shellQuote(this.display)} ${args.map(shellQuote).join(" ")}`, options);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async runSsh(command, options = {}) {
|
|
509
|
+
return this.execFile(this.sshCommand, sshArgs(this.sshOptions(), command), {
|
|
510
|
+
timeout: positiveInteger(this.options.commandTimeoutMs) ?? 30_000,
|
|
511
|
+
...options
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
sshOptions() {
|
|
516
|
+
return {
|
|
517
|
+
...this.options,
|
|
518
|
+
sshPort: this.sshPort
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
async close() {
|
|
523
|
+
if (this.created && !this.options.existingVM && !this.options.keepVM) {
|
|
524
|
+
debugVirtualDesktop(this.options, "vm_close", { vmName: this.name });
|
|
525
|
+
await stopQemuProcess(this.process, positiveInteger(this.options.qemuTimeoutMs) ?? 1500);
|
|
526
|
+
}
|
|
527
|
+
await this.defaultImage?.close();
|
|
528
|
+
this.defaultImage = null;
|
|
529
|
+
this.preparedPackages = new Set();
|
|
530
|
+
if (this.usesDefaultImage) {
|
|
531
|
+
this.image = null;
|
|
532
|
+
this.options = {
|
|
533
|
+
...this.options,
|
|
534
|
+
image: undefined,
|
|
535
|
+
sshUser: this.originalSshUser,
|
|
536
|
+
sshKeyPath: this.originalSshKeyPath,
|
|
537
|
+
sudo: this.originalSudo,
|
|
538
|
+
extraQemuArgs: this.extraQemuArgs
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
await this.sharedFolder?.close();
|
|
542
|
+
this.started = false;
|
|
543
|
+
this.created = false;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
export function defaultVirtualDesktopImage() {
|
|
548
|
+
return process.env.AUTOMIFY_QEMU_IMAGE ?? defaultQemuBaseImagePath();
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function normalizeVirtualDesktopOptions(options = {}) {
|
|
552
|
+
assertKnownOptions("QEMU virtual desktop adapter", options, QEMU_DESKTOP_OPTION_KEYS);
|
|
553
|
+
assertKnownOptions("QEMU virtual desktop vm", options.vm, QEMU_VM_KEYS);
|
|
554
|
+
assertKnownOptions("QEMU virtual desktop desktop", options.desktop, QEMU_DESKTOP_KEYS);
|
|
555
|
+
assertKnownOptions("QEMU virtual desktop ssh", options.ssh, QEMU_SSH_KEYS);
|
|
556
|
+
options = applyVirtualDesktopPreset(options);
|
|
557
|
+
const vm = options.vm ?? {};
|
|
558
|
+
const viewport = options.viewport ?? {};
|
|
559
|
+
const desktop = options.desktop ?? {};
|
|
560
|
+
const ssh = options.ssh ?? {};
|
|
561
|
+
const normalized = {
|
|
562
|
+
...options,
|
|
563
|
+
debug: options.debug ?? false,
|
|
564
|
+
logFile: normalizeLogFile(options.logFile, "QEMU virtual desktop logFile"),
|
|
565
|
+
qemuCommand: options.qemuCommand ?? vm.qemuCommand ?? vm.qemu,
|
|
566
|
+
qemuImgCommand: options.qemuImgCommand ?? vm.qemuImgCommand ?? vm.imgCommand,
|
|
567
|
+
qemuImageCacheDir: options.qemuImageCacheDir ?? vm.imageCacheDir,
|
|
568
|
+
qemuImageUrl: options.qemuImageUrl ?? vm.imageUrl,
|
|
569
|
+
defaultImageCache: options.defaultImageCache ?? vm.defaultImageCache,
|
|
570
|
+
image: options.image ?? options.diskImage ?? vm.image ?? vm.diskImage ?? process.env.AUTOMIFY_QEMU_IMAGE,
|
|
571
|
+
diskFormat: options.diskFormat ?? vm.diskFormat,
|
|
572
|
+
vmName: options.vmName ?? vm.name,
|
|
573
|
+
existingVM: options.existingVM ?? vm.existing,
|
|
574
|
+
keepVM: options.keepVM ?? vm.keep,
|
|
575
|
+
memory: options.memory ?? vm.memory,
|
|
576
|
+
cpus: options.cpus ?? vm.cpus,
|
|
577
|
+
accel: options.accel ?? vm.accel,
|
|
578
|
+
machine: options.machine ?? vm.machine,
|
|
579
|
+
cpu: options.cpu ?? vm.cpu,
|
|
580
|
+
firmware: options.firmware ?? vm.firmware,
|
|
581
|
+
network: options.network ?? vm.network,
|
|
582
|
+
networkDevice: options.networkDevice ?? vm.networkDevice,
|
|
583
|
+
extraQemuArgs: options.extraQemuArgs ?? vm.extraArgs,
|
|
584
|
+
qemuTimeoutMs: options.qemuTimeoutMs ?? vm.timeoutMs,
|
|
585
|
+
sshCommand: options.sshCommand ?? ssh.command,
|
|
586
|
+
sshKeygenCommand: options.sshKeygenCommand,
|
|
587
|
+
sshHost: options.sshHost ?? ssh.host,
|
|
588
|
+
sshPort: options.sshPort ?? ssh.port,
|
|
589
|
+
sshUser: options.sshUser ?? ssh.user,
|
|
590
|
+
sshKeyPath: options.sshKeyPath ?? ssh.keyPath,
|
|
591
|
+
sshOptions: options.sshOptions ?? ssh.options,
|
|
592
|
+
sshTimeoutMs: options.sshTimeoutMs ?? ssh.timeoutMs,
|
|
593
|
+
sudo: options.sudo ?? ssh.sudo,
|
|
594
|
+
displayWidth: options.displayWidth ?? viewport.width,
|
|
595
|
+
displayHeight: options.displayHeight ?? viewport.height,
|
|
596
|
+
displayDepth: options.displayDepth ?? viewport.depth,
|
|
597
|
+
startupCommand: options.startupCommand ?? desktop.startupCommand,
|
|
598
|
+
windowManagerCommand: options.windowManagerCommand ?? desktop.windowManagerCommand,
|
|
599
|
+
desktopPackages: options.desktopPackages ?? desktop.packages,
|
|
600
|
+
additionalAptPackages: options.additionalAptPackages ?? desktop.additionalAptPackages,
|
|
601
|
+
installDependencies: options.installDependencies ?? desktop.installDependencies,
|
|
602
|
+
sharedFolder: options.sharedFolder ?? options.shared,
|
|
603
|
+
files: options.files ?? options.sharedFiles
|
|
604
|
+
};
|
|
605
|
+
validateStartupCommand(normalized);
|
|
606
|
+
return normalized;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
function validateStartupCommand(options) {
|
|
610
|
+
if (typeof options.startupCommand === "string" && options.startupCommand.trim()) return;
|
|
611
|
+
throw new AutomifyError(
|
|
612
|
+
"QEMU virtual desktop startupCommand is required. Pass a non-empty startupCommand or desktop.startupCommand."
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
async function acquireQemuDesktopLock(options) {
|
|
617
|
+
if (!options.vmName) return null;
|
|
618
|
+
return acquireAdapterLock(`qemu-desktop:${options.vmName}`, {
|
|
619
|
+
label: `QEMU virtual desktop ${JSON.stringify(options.vmName)}`
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function debugVirtualDesktop(options, message, details) {
|
|
624
|
+
writeDebugLogFile(options.logFile, "automify:qemu-desktop", message, details, { silent: options.silent });
|
|
625
|
+
debugLog(options.debug, "automify:qemu-desktop", message, details, { silent: options.silent });
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
function keys(values) {
|
|
629
|
+
const normalized = values.map((value) => key(value)).filter(Boolean);
|
|
630
|
+
if (normalized.length === 0) {
|
|
631
|
+
throw new AutomifyError("keypress action did not include any keys.");
|
|
632
|
+
}
|
|
633
|
+
return normalized;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function key(value) {
|
|
637
|
+
const raw = String(value ?? "").trim();
|
|
638
|
+
const lower = raw.toLowerCase().replace(/\s+/g, "_");
|
|
639
|
+
const aliases = {
|
|
640
|
+
alt: "Alt",
|
|
641
|
+
backspace: "BackSpace",
|
|
642
|
+
cmd: "Super",
|
|
643
|
+
command: "Super",
|
|
644
|
+
control: "Control",
|
|
645
|
+
ctrl: "Control",
|
|
646
|
+
delete: "Delete",
|
|
647
|
+
down: "Down",
|
|
648
|
+
enter: "Return",
|
|
649
|
+
esc: "Escape",
|
|
650
|
+
escape: "Escape",
|
|
651
|
+
left: "Left",
|
|
652
|
+
meta: "Super",
|
|
653
|
+
option: "Alt",
|
|
654
|
+
return: "Return",
|
|
655
|
+
right: "Right",
|
|
656
|
+
shift: "Shift",
|
|
657
|
+
space: "space",
|
|
658
|
+
tab: "Tab",
|
|
659
|
+
up: "Up"
|
|
660
|
+
};
|
|
661
|
+
return aliases[lower] ?? raw;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
function button(value) {
|
|
665
|
+
if (value === "right") return "3";
|
|
666
|
+
if (value === "middle") return "2";
|
|
667
|
+
return "1";
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
function x(value) {
|
|
671
|
+
return String(Math.max(0, Math.round(Number(value) || 0)));
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function y(value) {
|
|
675
|
+
return String(Math.max(0, Math.round(Number(value) || 0)));
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
function normalizeDisplay(display) {
|
|
679
|
+
const value = String(display || DEFAULT_DISPLAY).trim();
|
|
680
|
+
return value.startsWith(":") ? value : `:${value}`;
|
|
681
|
+
}
|