automify 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -36
- package/examples/browser-with-safety.js +7 -10
- package/examples/cli-qemu.js +28 -0
- package/examples/desktop-qemu.js +41 -0
- package/package.json +5 -2
- package/scripts/generate-argument-reference.js +3 -1
- package/scripts/qemu-image.js +154 -0
- package/src/index.d.ts +368 -10
- package/src/index.js +18 -38
- package/src/lib/adapter-toolkit.js +8 -4
- package/src/lib/anthropic-model-adapter.js +24 -13
- package/src/lib/argument-reference.js +60 -8
- package/src/lib/automify.js +96 -0
- package/src/lib/cli-automify.js +41 -2
- package/src/lib/computer-automify.js +45 -26
- package/src/lib/docker-cli-automify.js +2 -6
- package/src/lib/docker-desktop-computer.js +7 -13
- package/src/lib/file-data.js +6 -6
- package/src/lib/init.js +14 -3
- package/src/lib/local-desktop-computer.js +2 -1
- package/src/lib/openai-responses-client.js +10 -3
- package/src/lib/presets.js +50 -2
- package/src/lib/qemu-cli-automify.js +568 -0
- package/src/lib/qemu-desktop-computer.js +681 -0
- package/src/lib/qemu-runtime.js +654 -0
- package/src/lib/runtime.js +23 -2
- package/src/lib/screen-recording.js +184 -0
- package/src/lib/task.js +564 -0
- package/src/lib/virtual-shared-folder.js +3 -1
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
import { DEFAULT_QEMU_DESKTOP_PACKAGES } from "../src/lib/qemu-desktop-computer.js";
|
|
6
|
+
import { ensureDefaultQemuImageCache } from "../src/lib/qemu-runtime.js";
|
|
7
|
+
|
|
8
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
const root = resolve(__dirname, "..");
|
|
10
|
+
const args = process.argv.slice(2);
|
|
11
|
+
|
|
12
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
13
|
+
printHelp();
|
|
14
|
+
process.exit(0);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const options = parseArgs(args);
|
|
18
|
+
|
|
19
|
+
console.log("Preparing Automify QEMU default Debian image cache.");
|
|
20
|
+
if (options.desktop) {
|
|
21
|
+
console.log("Desktop cache enabled: the prepared image will include the default QEMU desktop packages.");
|
|
22
|
+
} else if (options.packages.length > 0) {
|
|
23
|
+
console.log("CLI cache enabled: the prepared image will include the requested apt packages.");
|
|
24
|
+
} else {
|
|
25
|
+
console.log("CLI cache enabled: the prepared image will stay minimal for QEMU virtual CLI runs.");
|
|
26
|
+
}
|
|
27
|
+
if (options.defaultImageCache.forceDownload) {
|
|
28
|
+
console.log("Force download enabled: the cached base image will be replaced.");
|
|
29
|
+
}
|
|
30
|
+
if (options.defaultImageCache.forcePrepare) {
|
|
31
|
+
console.log("Force prepare enabled: the cached prepared image will be rebuilt.");
|
|
32
|
+
}
|
|
33
|
+
if (options.defaultImageCache.prepared === false) {
|
|
34
|
+
console.log("Prepared image disabled: only the base Debian qcow2 will be downloaded.");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
try {
|
|
38
|
+
const cache = await ensureDefaultQemuImageCache({
|
|
39
|
+
qemuCommand: options.qemuCommand,
|
|
40
|
+
qemuImgCommand: options.qemuImgCommand,
|
|
41
|
+
imageUrl: options.imageUrl,
|
|
42
|
+
defaultImageCache: options.defaultImageCache,
|
|
43
|
+
preparedImageProfile: options.desktop ? "desktop" : undefined,
|
|
44
|
+
preparedPackages: options.desktop ? uniquePackages([...DEFAULT_QEMU_DESKTOP_PACKAGES, ...options.packages]) : options.packages,
|
|
45
|
+
vmName: "automify-qemu-image"
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
console.log(`Base image: ${cache.baseImage}`);
|
|
49
|
+
if (cache.preparedImage) {
|
|
50
|
+
console.log(`Prepared image: ${cache.preparedImage}`);
|
|
51
|
+
console.log(`SSH key: ${cache.sshKeyPath}`);
|
|
52
|
+
}
|
|
53
|
+
console.log("Automify QEMU image cache is ready.");
|
|
54
|
+
} catch (error) {
|
|
55
|
+
console.error(error?.stack || error?.message || String(error));
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function parseArgs(argv) {
|
|
60
|
+
const result = {
|
|
61
|
+
defaultImageCache: {},
|
|
62
|
+
desktop: false,
|
|
63
|
+
packages: []
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
67
|
+
const arg = argv[index];
|
|
68
|
+
switch (arg) {
|
|
69
|
+
case "--force":
|
|
70
|
+
case "--force-download":
|
|
71
|
+
result.defaultImageCache.forceDownload = true;
|
|
72
|
+
result.defaultImageCache.forcePrepare = true;
|
|
73
|
+
break;
|
|
74
|
+
case "--force-prepare":
|
|
75
|
+
result.defaultImageCache.forcePrepare = true;
|
|
76
|
+
break;
|
|
77
|
+
case "--no-prepare":
|
|
78
|
+
result.defaultImageCache.prepared = false;
|
|
79
|
+
break;
|
|
80
|
+
case "--desktop":
|
|
81
|
+
result.desktop = true;
|
|
82
|
+
break;
|
|
83
|
+
case "--package":
|
|
84
|
+
result.packages.push(requiredValue(argv, ++index, arg));
|
|
85
|
+
break;
|
|
86
|
+
case "--cache-dir":
|
|
87
|
+
result.defaultImageCache.dir = requiredValue(argv, ++index, arg);
|
|
88
|
+
break;
|
|
89
|
+
case "--prepared-cache-dir":
|
|
90
|
+
result.defaultImageCache.preparedDir = requiredValue(argv, ++index, arg);
|
|
91
|
+
break;
|
|
92
|
+
case "--image-url":
|
|
93
|
+
result.imageUrl = requiredValue(argv, ++index, arg);
|
|
94
|
+
break;
|
|
95
|
+
case "--qemu":
|
|
96
|
+
case "--qemu-command":
|
|
97
|
+
result.qemuCommand = requiredValue(argv, ++index, arg);
|
|
98
|
+
break;
|
|
99
|
+
case "--qemu-img":
|
|
100
|
+
case "--qemu-img-command":
|
|
101
|
+
result.qemuImgCommand = requiredValue(argv, ++index, arg);
|
|
102
|
+
break;
|
|
103
|
+
default:
|
|
104
|
+
console.error(`Unknown option: ${arg}`);
|
|
105
|
+
printHelp();
|
|
106
|
+
process.exit(1);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return result;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function uniquePackages(packages) {
|
|
114
|
+
return [...new Set(packages.map((pkg) => String(pkg).trim()).filter(Boolean))];
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function requiredValue(argv, index, flag) {
|
|
118
|
+
const value = argv[index];
|
|
119
|
+
if (!value || value.startsWith("--")) {
|
|
120
|
+
console.error(`${flag} requires a value.`);
|
|
121
|
+
process.exit(1);
|
|
122
|
+
}
|
|
123
|
+
return value;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function printHelp() {
|
|
127
|
+
console.log(`Usage: npx automify-qemu-image [options]
|
|
128
|
+
|
|
129
|
+
Prepare or refresh the default Debian image cache used by Automify QEMU adapters.
|
|
130
|
+
Without --desktop, this pre-warms the minimal QEMU virtual CLI cache.
|
|
131
|
+
|
|
132
|
+
Options:
|
|
133
|
+
--force-download Re-download the base Debian qcow2 and rebuild prepared image
|
|
134
|
+
--force-prepare Rebuild only the prepared image
|
|
135
|
+
--no-prepare Download only the base Debian qcow2
|
|
136
|
+
--desktop Prepare the QEMU desktop cache variant
|
|
137
|
+
--package <name> Add an apt package to the prepared cache
|
|
138
|
+
--cache-dir <path> Cache root for base and prepared images
|
|
139
|
+
--prepared-cache-dir <path>
|
|
140
|
+
Cache directory for prepared images
|
|
141
|
+
--image-url <url> Override the default Debian qcow2 URL
|
|
142
|
+
--qemu <command> Override qemu-system command
|
|
143
|
+
--qemu-img <command> Override qemu-img command
|
|
144
|
+
--help Show this help
|
|
145
|
+
|
|
146
|
+
Examples:
|
|
147
|
+
npx automify-qemu-image # pre-warm the QEMU CLI cache
|
|
148
|
+
npx automify-qemu-image --package coreutils --package nodejs
|
|
149
|
+
npx automify-qemu-image --desktop # pre-warm the QEMU desktop cache
|
|
150
|
+
npx automify-qemu-image --force-download
|
|
151
|
+
npx automify-qemu-image --cache-dir ${root}/.automify-qemu-cache
|
|
152
|
+
npx automify-qemu-image --image-url https://example.com/path/linux.qcow2 --cache-dir ${root}/.automify-qemu-custom
|
|
153
|
+
`);
|
|
154
|
+
}
|
package/src/index.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export type ComputerAction =
|
|
|
13
13
|
| { type: string; [key: string]: unknown };
|
|
14
14
|
|
|
15
15
|
export type Screenshot = string | ArrayBuffer | Uint8Array | Buffer;
|
|
16
|
+
export type ScreenRecordingInput = boolean | string | ScreenRecordingOptions;
|
|
16
17
|
export type DomainRule = string | RegExp | ((url: URL) => boolean);
|
|
17
18
|
export type CommandRule = string | RegExp | ((command: string) => boolean);
|
|
18
19
|
export type DebugLogger = boolean | ((message: string, details?: unknown) => void);
|
|
@@ -88,6 +89,30 @@ export interface JsonOutputOptions {
|
|
|
88
89
|
parse?: boolean;
|
|
89
90
|
}
|
|
90
91
|
|
|
92
|
+
export interface ScreenRecordingOptions {
|
|
93
|
+
enabled?: boolean;
|
|
94
|
+
path?: string;
|
|
95
|
+
fps?: number;
|
|
96
|
+
intervalMs?: number;
|
|
97
|
+
captureIntervalMs?: number;
|
|
98
|
+
framesDir?: string;
|
|
99
|
+
keepFrames?: boolean;
|
|
100
|
+
ffmpegCommand?: string;
|
|
101
|
+
command?: string;
|
|
102
|
+
encodingTimeoutMs?: number;
|
|
103
|
+
timeoutMs?: number;
|
|
104
|
+
execFile?: (...args: unknown[]) => Promise<unknown> | unknown;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface ScreenRecordingResult {
|
|
108
|
+
path: string;
|
|
109
|
+
bytes?: number;
|
|
110
|
+
frames?: number;
|
|
111
|
+
fps?: number;
|
|
112
|
+
startedAt?: string;
|
|
113
|
+
stoppedAt?: string;
|
|
114
|
+
}
|
|
115
|
+
|
|
91
116
|
export function jsonOutput(
|
|
92
117
|
name: string,
|
|
93
118
|
shape: JsonOutputShape | Record<string, unknown>,
|
|
@@ -142,6 +167,8 @@ export interface AutomifyOptions {
|
|
|
142
167
|
finalScreenshot?: string;
|
|
143
168
|
actionScreenshots?: string;
|
|
144
169
|
screenshots?: DoScreenshotsOptions;
|
|
170
|
+
recording?: ScreenRecordingInput;
|
|
171
|
+
screenRecording?: ScreenRecordingInput;
|
|
145
172
|
trace?: boolean;
|
|
146
173
|
silent?: boolean;
|
|
147
174
|
debug?: DebugLogger;
|
|
@@ -161,6 +188,11 @@ export type DockerComputerAutomifyOptions = Omit<AutomifyOptions, "computer"> &
|
|
|
161
188
|
computer?: ComputerAdapter & { session: DockerDesktopSession; sharedFolder?: VirtualSharedFolderData };
|
|
162
189
|
};
|
|
163
190
|
|
|
191
|
+
export type VirtualComputerAutomifyOptions = Omit<AutomifyOptions, "computer"> &
|
|
192
|
+
VirtualDesktopComputerOptions & {
|
|
193
|
+
computer?: ComputerAdapter & { session: QemuDesktopSession; sharedFolder?: VirtualSharedFolderData };
|
|
194
|
+
};
|
|
195
|
+
|
|
164
196
|
export interface LocalComputerAutomifyOptions extends Omit<AutomifyOptions, "computer">, LocalDesktopComputerOptions {
|
|
165
197
|
computer?: ComputerAdapter;
|
|
166
198
|
}
|
|
@@ -175,6 +207,12 @@ export interface LocalDesktopComputerOptions {
|
|
|
175
207
|
actionDelayMs?: number;
|
|
176
208
|
instructions?: string;
|
|
177
209
|
screenshotPath?: string;
|
|
210
|
+
/**
|
|
211
|
+
* Defaults to "local-desktop", giving the local adapter an exclusive
|
|
212
|
+
* cross-process lock. Override only when you know this runner is attached to
|
|
213
|
+
* an independent desktop session or isolated test double.
|
|
214
|
+
*/
|
|
215
|
+
lockResource?: string;
|
|
178
216
|
/**
|
|
179
217
|
* Screenshot pixels per native mouse coordinate. On macOS Retina displays the
|
|
180
218
|
* default is inferred as 2; use 1 for non-Retina external displays.
|
|
@@ -381,7 +419,145 @@ export type DockerDesktopComputerOptions =
|
|
|
381
419
|
| (DockerDesktopComputerBaseOptions & { startupCommand: string })
|
|
382
420
|
| (DockerDesktopComputerBaseOptions & { desktop: DockerDesktopOptions });
|
|
383
421
|
|
|
384
|
-
export
|
|
422
|
+
export interface QemuSshOptions {
|
|
423
|
+
command?: string;
|
|
424
|
+
host?: string;
|
|
425
|
+
port?: number;
|
|
426
|
+
user?: string;
|
|
427
|
+
keyPath?: string;
|
|
428
|
+
options?: string[];
|
|
429
|
+
timeoutMs?: number;
|
|
430
|
+
sudo?: boolean;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
export interface QemuVmOptions {
|
|
434
|
+
qemu?: string;
|
|
435
|
+
qemuCommand?: string;
|
|
436
|
+
qemuImgCommand?: string;
|
|
437
|
+
imageCacheDir?: string;
|
|
438
|
+
imageUrl?: string;
|
|
439
|
+
defaultImageCache?: QemuDefaultImageCacheOptions;
|
|
440
|
+
image?: string;
|
|
441
|
+
diskImage?: string;
|
|
442
|
+
diskFormat?: "qcow2" | "raw" | string;
|
|
443
|
+
name?: string;
|
|
444
|
+
existing?: boolean;
|
|
445
|
+
keep?: boolean;
|
|
446
|
+
memory?: number | string;
|
|
447
|
+
cpus?: number | string;
|
|
448
|
+
accel?: "hvf" | "kvm" | "whpx" | "tcg" | string;
|
|
449
|
+
machine?: string;
|
|
450
|
+
cpu?: string;
|
|
451
|
+
firmware?: string;
|
|
452
|
+
network?: boolean;
|
|
453
|
+
networkDevice?: string;
|
|
454
|
+
extraArgs?: string[];
|
|
455
|
+
timeoutMs?: number;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
export type QemuDefaultImageCacheOptions =
|
|
459
|
+
| boolean
|
|
460
|
+
| {
|
|
461
|
+
/**
|
|
462
|
+
* Cache root for the downloaded base image and prepared image subdirectory.
|
|
463
|
+
*/
|
|
464
|
+
dir?: string;
|
|
465
|
+
/**
|
|
466
|
+
* Cache directory for downloaded base images. Defaults to dir when provided.
|
|
467
|
+
*/
|
|
468
|
+
imageCacheDir?: string;
|
|
469
|
+
/**
|
|
470
|
+
* Cache directory for Automify-ready prepared images. Defaults to `${dir}/prepared`.
|
|
471
|
+
*/
|
|
472
|
+
preparedDir?: string;
|
|
473
|
+
/**
|
|
474
|
+
* Defaults to true. When true, Automify caches a booted Debian image with
|
|
475
|
+
* the automify SSH user already provisioned, then creates runtime overlays
|
|
476
|
+
* from that image.
|
|
477
|
+
*/
|
|
478
|
+
prepared?: boolean;
|
|
479
|
+
/**
|
|
480
|
+
* Re-download the Debian base qcow2 and rebuild the prepared image.
|
|
481
|
+
*/
|
|
482
|
+
forceDownload?: boolean;
|
|
483
|
+
/**
|
|
484
|
+
* Rebuild the prepared image even if one already exists.
|
|
485
|
+
*/
|
|
486
|
+
forcePrepare?: boolean;
|
|
487
|
+
};
|
|
488
|
+
|
|
489
|
+
export interface VirtualDesktopComputerBaseOptions {
|
|
490
|
+
preset?: VirtualDesktopPreset;
|
|
491
|
+
vm?: QemuVmOptions;
|
|
492
|
+
qemuCommand?: string;
|
|
493
|
+
qemuImgCommand?: string;
|
|
494
|
+
qemuImageCacheDir?: string;
|
|
495
|
+
qemuImageUrl?: string;
|
|
496
|
+
defaultImageCache?: QemuDefaultImageCacheOptions;
|
|
497
|
+
image?: string;
|
|
498
|
+
diskImage?: string;
|
|
499
|
+
diskFormat?: "qcow2" | "raw" | string;
|
|
500
|
+
vmName?: string;
|
|
501
|
+
existingVM?: boolean;
|
|
502
|
+
keepVM?: boolean;
|
|
503
|
+
start?: boolean;
|
|
504
|
+
ssh?: QemuSshOptions;
|
|
505
|
+
sshCommand?: string;
|
|
506
|
+
sshKeygenCommand?: string;
|
|
507
|
+
sshHost?: string;
|
|
508
|
+
sshPort?: number;
|
|
509
|
+
sshUser?: string;
|
|
510
|
+
sshKeyPath?: string;
|
|
511
|
+
sshOptions?: string[];
|
|
512
|
+
sshTimeoutMs?: number;
|
|
513
|
+
sudo?: boolean;
|
|
514
|
+
display?: string;
|
|
515
|
+
viewport?: ViewportOptions;
|
|
516
|
+
displayWidth?: number;
|
|
517
|
+
displayHeight?: number;
|
|
518
|
+
displayDepth?: number;
|
|
519
|
+
environment?: ComputerUseEnvironment;
|
|
520
|
+
instructions?: string;
|
|
521
|
+
desktop?: DockerDesktopOptions;
|
|
522
|
+
startupCommand?: string;
|
|
523
|
+
windowManagerCommand?: string;
|
|
524
|
+
installDependencies?: boolean;
|
|
525
|
+
desktopPackages?: string[];
|
|
526
|
+
additionalAptPackages?: string[];
|
|
527
|
+
memory?: number | string;
|
|
528
|
+
cpus?: number | string;
|
|
529
|
+
accel?: "hvf" | "kvm" | "whpx" | "tcg" | string;
|
|
530
|
+
machine?: string;
|
|
531
|
+
cpu?: string;
|
|
532
|
+
firmware?: string;
|
|
533
|
+
network?: boolean;
|
|
534
|
+
networkDevice?: string;
|
|
535
|
+
extraQemuArgs?: string[];
|
|
536
|
+
shared?: VirtualSharedFolderInput;
|
|
537
|
+
sharedFolder?: VirtualSharedFolderInput;
|
|
538
|
+
sharedFiles?: VirtualSharedFileInput[];
|
|
539
|
+
files?: VirtualSharedFileInput[];
|
|
540
|
+
sharedMode?: "virtfs" | "none" | string;
|
|
541
|
+
sharedTag?: string;
|
|
542
|
+
sharedSecurityModel?: string;
|
|
543
|
+
waitMs?: number;
|
|
544
|
+
startupTimeoutMs?: number;
|
|
545
|
+
qemuTimeoutMs?: number;
|
|
546
|
+
commandTimeoutMs?: number;
|
|
547
|
+
screenshotMaxBuffer?: number;
|
|
548
|
+
screenshotSettleMs?: number;
|
|
549
|
+
fetchImpl?: typeof fetch;
|
|
550
|
+
execFile?: (...args: unknown[]) => Promise<{ stdout?: Buffer | string; stderr?: Buffer | string }>;
|
|
551
|
+
spawn?: (...args: unknown[]) => unknown;
|
|
552
|
+
silent?: boolean;
|
|
553
|
+
debug?: DebugLogger;
|
|
554
|
+
logFile?: string;
|
|
555
|
+
onUnknownAction?: (action: ComputerAction, context?: Record<string, unknown>) => Promise<void> | void;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
export type VirtualDesktopComputerOptions =
|
|
559
|
+
| (VirtualDesktopComputerBaseOptions & { startupCommand: string })
|
|
560
|
+
| (VirtualDesktopComputerBaseOptions & { desktop: DockerDesktopOptions });
|
|
385
561
|
|
|
386
562
|
export type VirtualSharedFolderInput =
|
|
387
563
|
| true
|
|
@@ -427,7 +603,17 @@ export class DockerDesktopSession {
|
|
|
427
603
|
close(): Promise<void>;
|
|
428
604
|
}
|
|
429
605
|
|
|
430
|
-
export
|
|
606
|
+
export class QemuDesktopSession {
|
|
607
|
+
constructor(options?: VirtualDesktopComputerOptions);
|
|
608
|
+
readonly name: string;
|
|
609
|
+
readonly display: string;
|
|
610
|
+
readonly width: number;
|
|
611
|
+
readonly height: number;
|
|
612
|
+
start(): Promise<void>;
|
|
613
|
+
execute(action: ComputerAction): Promise<void>;
|
|
614
|
+
screenshot(): Promise<Buffer>;
|
|
615
|
+
close(): Promise<void>;
|
|
616
|
+
}
|
|
431
617
|
|
|
432
618
|
export interface OpenAIProviderConfig {
|
|
433
619
|
type: "openai";
|
|
@@ -486,6 +672,8 @@ export interface InitAutomifyOptions {
|
|
|
486
672
|
finalScreenshot?: string;
|
|
487
673
|
actionScreenshots?: string;
|
|
488
674
|
screenshots?: DoScreenshotsOptions;
|
|
675
|
+
recording?: ScreenRecordingInput;
|
|
676
|
+
screenRecording?: ScreenRecordingInput;
|
|
489
677
|
trace?: boolean;
|
|
490
678
|
silent?: boolean;
|
|
491
679
|
debug?: DebugLogger;
|
|
@@ -525,6 +713,8 @@ export interface DoOptions {
|
|
|
525
713
|
finalScreenshot?: string;
|
|
526
714
|
actionScreenshots?: string;
|
|
527
715
|
screenshots?: DoScreenshotsOptions;
|
|
716
|
+
recording?: ScreenRecordingInput;
|
|
717
|
+
screenRecording?: ScreenRecordingInput;
|
|
528
718
|
trace?: boolean;
|
|
529
719
|
silent?: boolean;
|
|
530
720
|
onSafetyCheck?: (event: {
|
|
@@ -545,6 +735,7 @@ export interface DoScreenshotsOptions {
|
|
|
545
735
|
final?: string;
|
|
546
736
|
actions?: string;
|
|
547
737
|
actionScreenshots?: string;
|
|
738
|
+
recording?: ScreenRecordingInput;
|
|
548
739
|
}
|
|
549
740
|
|
|
550
741
|
export interface DoScreenshotOptions {
|
|
@@ -592,6 +783,8 @@ export interface CliCommandOptions {
|
|
|
592
783
|
export interface AutomifyResult {
|
|
593
784
|
response: Record<string, unknown>;
|
|
594
785
|
steps: Array<Record<string, unknown>>;
|
|
786
|
+
taskSteps?: TaskStepResult[];
|
|
787
|
+
extracts?: Record<string, unknown>;
|
|
595
788
|
trace?: Array<Record<string, unknown>>;
|
|
596
789
|
ok: boolean;
|
|
597
790
|
status: "succeeded";
|
|
@@ -603,6 +796,7 @@ export interface AutomifyResult {
|
|
|
603
796
|
path: string;
|
|
604
797
|
bytes?: number;
|
|
605
798
|
};
|
|
799
|
+
recording?: ScreenRecordingResult;
|
|
606
800
|
}
|
|
607
801
|
|
|
608
802
|
export interface AutomifyCompleteEvent {
|
|
@@ -621,10 +815,80 @@ export interface AutomifyCompleteEvent {
|
|
|
621
815
|
export class Automify {
|
|
622
816
|
constructor(options: AutomifyOptions);
|
|
623
817
|
do(instruction: string, options?: DoOptions): Promise<AutomifyResult>;
|
|
818
|
+
task(options?: TaskRunOptions): AutomifyTask;
|
|
819
|
+
addStep(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
820
|
+
addAct(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
821
|
+
addWait(conditionOrMs?: string | number, options?: TaskStepOptions): AutomifyTask;
|
|
822
|
+
addWaitFor(condition?: string, options?: TaskStepOptions): AutomifyTask;
|
|
823
|
+
addPause(ms: number, options?: TaskStepOptions): AutomifyTask;
|
|
824
|
+
addObserve(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
825
|
+
addExtract(instruction: string, options?: TaskExtractOptions | OutputFormat): AutomifyTask;
|
|
826
|
+
addAssert(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
624
827
|
}
|
|
625
828
|
|
|
626
829
|
export function createAutomify(options: AutomifyOptions): Automify;
|
|
627
830
|
|
|
831
|
+
export type TaskMode = "single" | "sequential";
|
|
832
|
+
export type TaskRunOptions = (DoOptions | CliAutomifyDoOptions) & { mode?: TaskMode };
|
|
833
|
+
|
|
834
|
+
export interface TaskStepOptions {
|
|
835
|
+
label?: string;
|
|
836
|
+
notes?: string;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
export interface TaskExtractOptions extends TaskStepOptions {
|
|
840
|
+
key?: string;
|
|
841
|
+
shape?: JsonOutputShape | Record<string, unknown>;
|
|
842
|
+
schema?: JsonOutputShape | Record<string, unknown>;
|
|
843
|
+
output?: OutputFormat;
|
|
844
|
+
description?: string;
|
|
845
|
+
strict?: boolean;
|
|
846
|
+
parse?: boolean;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
export interface TaskStepResult {
|
|
850
|
+
index: number;
|
|
851
|
+
type: "step" | "wait" | "pause" | "observe" | "extract" | "assert" | string;
|
|
852
|
+
instruction: string;
|
|
853
|
+
label?: string;
|
|
854
|
+
notes?: string;
|
|
855
|
+
status: "succeeded" | "failed";
|
|
856
|
+
durationMs?: number;
|
|
857
|
+
responseId?: string;
|
|
858
|
+
text?: string;
|
|
859
|
+
parsed?: unknown;
|
|
860
|
+
modelSteps?: number;
|
|
861
|
+
error?: string;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
export class AutomifyTask {
|
|
865
|
+
constructor(automify: Automify | CliAutomify, options?: TaskRunOptions);
|
|
866
|
+
addStep(instruction: string, options?: TaskStepOptions): this;
|
|
867
|
+
step(instruction: string, options?: TaskStepOptions): this;
|
|
868
|
+
addAct(instruction: string, options?: TaskStepOptions): this;
|
|
869
|
+
act(instruction: string, options?: TaskStepOptions): this;
|
|
870
|
+
addWait(conditionOrMs?: string | number, options?: TaskStepOptions): this;
|
|
871
|
+
wait(conditionOrMs?: string | number, options?: TaskStepOptions): this;
|
|
872
|
+
addWaitFor(condition?: string, options?: TaskStepOptions): this;
|
|
873
|
+
waitFor(condition?: string, options?: TaskStepOptions): this;
|
|
874
|
+
addPause(ms: number, options?: TaskStepOptions): this;
|
|
875
|
+
pause(ms: number, options?: TaskStepOptions): this;
|
|
876
|
+
addObserve(instruction: string, options?: TaskStepOptions): this;
|
|
877
|
+
observe(instruction: string, options?: TaskStepOptions): this;
|
|
878
|
+
addExtract(instruction: string, options?: TaskExtractOptions | OutputFormat): this;
|
|
879
|
+
extract(instruction: string, options?: TaskExtractOptions | OutputFormat): this;
|
|
880
|
+
addAssert(instruction: string, options?: TaskStepOptions): this;
|
|
881
|
+
assert(instruction: string, options?: TaskStepOptions): this;
|
|
882
|
+
addData(data: Record<string, unknown> | unknown): this;
|
|
883
|
+
withData(data: Record<string, unknown> | unknown): this;
|
|
884
|
+
withOptions(options?: TaskRunOptions): this;
|
|
885
|
+
toInstruction(): string;
|
|
886
|
+
run(options?: TaskRunOptions): Promise<AutomifyResult | CliAutomifyResult>;
|
|
887
|
+
do(options?: TaskRunOptions): Promise<AutomifyResult | CliAutomifyResult>;
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
export function createTask(automify: Automify | CliAutomify, options?: TaskRunOptions): AutomifyTask;
|
|
891
|
+
|
|
628
892
|
export function createComputerAutomify(options: AutomifyOptions): Automify;
|
|
629
893
|
|
|
630
894
|
export interface InitializedAutomify {
|
|
@@ -642,7 +906,10 @@ export interface InitializedAutomify {
|
|
|
642
906
|
localComputer(
|
|
643
907
|
options?: Omit<LocalComputerAutomifyOptions, "openaiApiKey" | "client">
|
|
644
908
|
): Promise<LocalComputerAutomify>;
|
|
645
|
-
|
|
909
|
+
virtualComputer(
|
|
910
|
+
options?: Omit<VirtualComputerAutomifyOptions, "openaiApiKey" | "client">
|
|
911
|
+
): Promise<VirtualComputerAutomify>;
|
|
912
|
+
virtualCli(options?: Omit<VirtualCliAutomifyOptions, "openaiApiKey" | "client">): VirtualCliAutomify;
|
|
646
913
|
computer(options: Omit<AutomifyOptions, "openaiApiKey" | "client">): Automify;
|
|
647
914
|
custom(options: Omit<AutomifyOptions, "openaiApiKey" | "client">): Automify;
|
|
648
915
|
}
|
|
@@ -720,6 +987,12 @@ export class DockerComputerAutomify extends Automify {
|
|
|
720
987
|
close(): Promise<void>;
|
|
721
988
|
}
|
|
722
989
|
|
|
990
|
+
export class VirtualComputerAutomify extends Automify {
|
|
991
|
+
session: QemuDesktopSession;
|
|
992
|
+
sharedFolder?: VirtualSharedFolderData;
|
|
993
|
+
close(): Promise<void>;
|
|
994
|
+
}
|
|
995
|
+
|
|
723
996
|
export class LocalComputerAutomify extends Automify {
|
|
724
997
|
close(): Promise<void>;
|
|
725
998
|
}
|
|
@@ -730,6 +1003,9 @@ export function withBrowserAutomify<T>(
|
|
|
730
1003
|
run: (automify: BrowserAutomify) => Promise<T> | T
|
|
731
1004
|
): Promise<T>;
|
|
732
1005
|
export function createDockerComputerAutomify(options?: DockerComputerAutomifyOptions): Promise<DockerComputerAutomify>;
|
|
1006
|
+
export function createVirtualComputerAutomify(
|
|
1007
|
+
options?: VirtualComputerAutomifyOptions
|
|
1008
|
+
): Promise<VirtualComputerAutomify>;
|
|
733
1009
|
export function createLocalComputerAutomify(options?: LocalComputerAutomifyOptions): Promise<LocalComputerAutomify>;
|
|
734
1010
|
|
|
735
1011
|
export function createLocalDesktopComputer(options?: LocalDesktopComputerOptions): Promise<ComputerAdapter>;
|
|
@@ -740,12 +1016,11 @@ export function createDockerDesktopComputer(
|
|
|
740
1016
|
options?: DockerDesktopComputerOptions
|
|
741
1017
|
): Promise<ComputerAdapter & { session: DockerDesktopSession; sharedFolder?: VirtualSharedFolderData }>;
|
|
742
1018
|
export function createVirtualDesktopComputer(
|
|
743
|
-
options?:
|
|
744
|
-
): Promise<ComputerAdapter & { session:
|
|
1019
|
+
options?: VirtualDesktopComputerOptions
|
|
1020
|
+
): Promise<ComputerAdapter & { session: QemuDesktopSession; sharedFolder?: VirtualSharedFolderData }>;
|
|
745
1021
|
export function defaultDockerDesktopImage(): string;
|
|
746
1022
|
export function defaultVirtualDesktopImage(): string;
|
|
747
1023
|
export function dockerDesktopDockerfile(): string;
|
|
748
|
-
export function virtualDesktopDockerfile(): string;
|
|
749
1024
|
|
|
750
1025
|
export interface CliAutomifyOptions extends Omit<InitAutomifyOptions, "model" | "onComplete"> {
|
|
751
1026
|
preset?: CliPreset;
|
|
@@ -818,6 +1093,8 @@ export type CliAutomifyDoOptions = CliDoOptions;
|
|
|
818
1093
|
export interface CliAutomifyResult {
|
|
819
1094
|
response: Record<string, unknown>;
|
|
820
1095
|
steps: Array<Record<string, unknown>>;
|
|
1096
|
+
taskSteps?: TaskStepResult[];
|
|
1097
|
+
extracts?: Record<string, unknown>;
|
|
821
1098
|
ok: boolean;
|
|
822
1099
|
status: "succeeded";
|
|
823
1100
|
completed: boolean;
|
|
@@ -842,6 +1119,15 @@ export interface CliAutomifyCompleteEvent {
|
|
|
842
1119
|
export class CliAutomify {
|
|
843
1120
|
constructor(options: CliAutomifyOptions);
|
|
844
1121
|
do(instruction: string, options?: CliAutomifyDoOptions): Promise<CliAutomifyResult>;
|
|
1122
|
+
task(options?: TaskRunOptions): AutomifyTask;
|
|
1123
|
+
addStep(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
1124
|
+
addAct(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
1125
|
+
addWait(conditionOrMs?: string | number, options?: TaskStepOptions): AutomifyTask;
|
|
1126
|
+
addWaitFor(condition?: string, options?: TaskStepOptions): AutomifyTask;
|
|
1127
|
+
addPause(ms: number, options?: TaskStepOptions): AutomifyTask;
|
|
1128
|
+
addObserve(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
1129
|
+
addExtract(instruction: string, options?: TaskExtractOptions | OutputFormat): AutomifyTask;
|
|
1130
|
+
addAssert(instruction: string, options?: TaskStepOptions): AutomifyTask;
|
|
845
1131
|
}
|
|
846
1132
|
|
|
847
1133
|
export function createCliAutomify(options: CliAutomifyOptions): CliAutomify;
|
|
@@ -936,7 +1222,60 @@ export interface DockerCliAutomifyOptions extends CliAutomifyOptions {
|
|
|
936
1222
|
execFile?: (...args: unknown[]) => Promise<{ stdout?: Buffer | string; stderr?: Buffer | string }>;
|
|
937
1223
|
}
|
|
938
1224
|
|
|
939
|
-
export
|
|
1225
|
+
export interface VirtualCliAutomifyOptions extends CliAutomifyOptions {
|
|
1226
|
+
preset?: VirtualCliPreset;
|
|
1227
|
+
session?: QemuCliSession;
|
|
1228
|
+
vm?: QemuVmOptions;
|
|
1229
|
+
qemuCommand?: string;
|
|
1230
|
+
qemuImgCommand?: string;
|
|
1231
|
+
qemuImageCacheDir?: string;
|
|
1232
|
+
qemuImageUrl?: string;
|
|
1233
|
+
defaultImageCache?: QemuDefaultImageCacheOptions;
|
|
1234
|
+
image?: string;
|
|
1235
|
+
diskImage?: string;
|
|
1236
|
+
diskFormat?: "qcow2" | "raw" | string;
|
|
1237
|
+
vmName?: string;
|
|
1238
|
+
existingVM?: boolean;
|
|
1239
|
+
keepVM?: boolean;
|
|
1240
|
+
workdir?: string;
|
|
1241
|
+
workspacePath?: string;
|
|
1242
|
+
guestCwd?: string;
|
|
1243
|
+
startupCommand?: string;
|
|
1244
|
+
packages?: string[];
|
|
1245
|
+
additionalAptPackages?: string[];
|
|
1246
|
+
installDependencies?: boolean;
|
|
1247
|
+
memory?: number | string;
|
|
1248
|
+
cpus?: number | string;
|
|
1249
|
+
accel?: "hvf" | "kvm" | "whpx" | "tcg" | string;
|
|
1250
|
+
machine?: string;
|
|
1251
|
+
cpu?: string;
|
|
1252
|
+
firmware?: string;
|
|
1253
|
+
network?: boolean;
|
|
1254
|
+
networkDevice?: string;
|
|
1255
|
+
extraQemuArgs?: string[];
|
|
1256
|
+
ssh?: QemuSshOptions;
|
|
1257
|
+
sshCommand?: string;
|
|
1258
|
+
sshKeygenCommand?: string;
|
|
1259
|
+
sshHost?: string;
|
|
1260
|
+
sshPort?: number;
|
|
1261
|
+
sshUser?: string;
|
|
1262
|
+
sshKeyPath?: string;
|
|
1263
|
+
sshOptions?: string[];
|
|
1264
|
+
sshTimeoutMs?: number;
|
|
1265
|
+
sudo?: boolean;
|
|
1266
|
+
shared?: VirtualSharedFolderInput;
|
|
1267
|
+
sharedFolder?: VirtualSharedFolderInput;
|
|
1268
|
+
sharedFiles?: VirtualSharedFileInput[];
|
|
1269
|
+
files?: VirtualSharedFileInput[];
|
|
1270
|
+
sharedMode?: "virtfs" | "none" | string;
|
|
1271
|
+
sharedTag?: string;
|
|
1272
|
+
sharedSecurityModel?: string;
|
|
1273
|
+
startupTimeoutMs?: number;
|
|
1274
|
+
qemuTimeoutMs?: number;
|
|
1275
|
+
commandMaxBuffer?: number;
|
|
1276
|
+
execFile?: (...args: unknown[]) => Promise<{ stdout?: Buffer | string; stderr?: Buffer | string }>;
|
|
1277
|
+
spawn?: (...args: unknown[]) => unknown;
|
|
1278
|
+
}
|
|
940
1279
|
|
|
941
1280
|
export class DockerCliSession {
|
|
942
1281
|
constructor(options?: DockerCliAutomifyOptions);
|
|
@@ -958,10 +1297,29 @@ export class DockerCliAutomify extends CliAutomify {
|
|
|
958
1297
|
close(): Promise<void>;
|
|
959
1298
|
}
|
|
960
1299
|
|
|
961
|
-
export { DockerCliAutomify as VirtualCliAutomify, DockerCliSession as DockerVirtualCliSession };
|
|
962
|
-
|
|
963
1300
|
export function createDockerCliAutomify(options: DockerCliAutomifyOptions): DockerCliAutomify;
|
|
964
|
-
export
|
|
1301
|
+
export class QemuCliSession {
|
|
1302
|
+
constructor(options?: VirtualCliAutomifyOptions);
|
|
1303
|
+
readonly name: string;
|
|
1304
|
+
readonly cwd: string;
|
|
1305
|
+
readonly sharedFolder?: { data: VirtualSharedFolderData };
|
|
1306
|
+
start(): Promise<void>;
|
|
1307
|
+
run(
|
|
1308
|
+
command: string,
|
|
1309
|
+
options?: { cwd?: string; env?: Record<string, string>; timeoutMs?: number }
|
|
1310
|
+
): Promise<Record<string, unknown>>;
|
|
1311
|
+
close(): Promise<void>;
|
|
1312
|
+
}
|
|
1313
|
+
export { QemuCliSession as QemuVirtualCliSession };
|
|
1314
|
+
|
|
1315
|
+
export class VirtualCliAutomify extends CliAutomify {
|
|
1316
|
+
session: QemuCliSession;
|
|
1317
|
+
readonly sharedFolder?: VirtualSharedFolderData;
|
|
1318
|
+
do(instruction: string, options?: CliAutomifyDoOptions): Promise<CliAutomifyResult>;
|
|
1319
|
+
close(): Promise<void>;
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
export function createVirtualCliAutomify(options: VirtualCliAutomifyOptions): VirtualCliAutomify;
|
|
965
1323
|
|
|
966
1324
|
export class OpenAIResponsesClient {
|
|
967
1325
|
constructor(options: {
|