automify 0.1.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -36
- package/examples/browser-with-safety.js +7 -10
- package/examples/cli-qemu.js +28 -0
- package/examples/desktop-qemu.js +41 -0
- package/package.json +5 -2
- package/scripts/generate-argument-reference.js +3 -1
- package/scripts/qemu-image.js +151 -0
- package/src/index.d.ts +368 -10
- package/src/index.js +18 -38
- package/src/lib/adapter-toolkit.js +8 -4
- package/src/lib/anthropic-model-adapter.js +24 -13
- package/src/lib/argument-reference.js +60 -8
- package/src/lib/automify.js +97 -1
- package/src/lib/cli-automify.js +42 -3
- package/src/lib/computer-automify.js +45 -26
- package/src/lib/docker-cli-automify.js +2 -6
- package/src/lib/docker-desktop-computer.js +7 -13
- package/src/lib/file-data.js +6 -6
- package/src/lib/init.js +14 -3
- package/src/lib/local-desktop-computer.js +2 -1
- package/src/lib/openai-responses-client.js +10 -3
- package/src/lib/presets.js +50 -2
- package/src/lib/qemu-cli-automify.js +555 -0
- package/src/lib/qemu-desktop-computer.js +681 -0
- package/src/lib/qemu-runtime.js +654 -0
- package/src/lib/runtime.js +23 -2
- package/src/lib/screen-recording.js +184 -0
- package/src/lib/task.js +564 -0
- package/src/lib/virtual-shared-folder.js +3 -1
package/README.md
CHANGED
|
@@ -6,33 +6,35 @@
|
|
|
6
6
|
[](./LICENSE)
|
|
7
7
|
[](https://nodejs.org/)
|
|
8
8
|
|
|
9
|
-
`Automify` is a Node.js library for AI computer use and command use across web apps, terminals, native desktops, Docker
|
|
10
|
-
|
|
11
|
-
Created by [Aldo Vincenti](https://aldovincenti.com).
|
|
9
|
+
`Automify` is a Node.js library for AI computer use and command use across web apps, terminals, native desktops, Docker sandboxes, and QEMU-backed virtual machines.
|
|
12
10
|
|
|
13
11
|
Computer use surfaces:
|
|
14
12
|
|
|
15
|
-
| Surface
|
|
16
|
-
|
|
|
17
|
-
| Browser
|
|
18
|
-
| Desktop
|
|
19
|
-
| Docker desktop
|
|
13
|
+
| Surface | Factory | Controlled environment |
|
|
14
|
+
| --------------- | ---------------------------- | --------------------------------------------------------- |
|
|
15
|
+
| Browser | `automify.browser()` | Playwright browser with screenshots and actions |
|
|
16
|
+
| Desktop | `automify.localComputer()` | Native desktop on macOS, Windows, or Linux X11/Xorg hosts |
|
|
17
|
+
| Docker desktop | `automify.dockerComputer()` | Linux desktop inside a running Docker container |
|
|
18
|
+
| Virtual desktop | `automify.virtualComputer()` | Linux desktop inside a QEMU VM |
|
|
20
19
|
|
|
21
20
|
Command use surfaces:
|
|
22
21
|
|
|
23
|
-
| Surface
|
|
24
|
-
|
|
|
25
|
-
| CLI
|
|
26
|
-
| Docker CLI
|
|
22
|
+
| Surface | Factory | What it does |
|
|
23
|
+
| ----------- | ----------------------- | ----------------------------------------------------- |
|
|
24
|
+
| CLI | `automify.cli()` | Terminal automation through model-requested commands |
|
|
25
|
+
| Docker CLI | `automify.dockerCli()` | Containerized terminal automation with running Docker |
|
|
26
|
+
| Virtual CLI | `automify.virtualCli()` | Terminal automation inside a QEMU VM |
|
|
27
27
|
|
|
28
28
|
OpenAI and Anthropic models are supported, and any other model can be plugged in with a custom provider adapter.
|
|
29
29
|
|
|
30
30
|
## What You Get
|
|
31
31
|
|
|
32
|
-
- Computer use for browser, local desktop, Docker desktop, and custom computer adapters.
|
|
33
|
-
- Command use for local CLI and
|
|
32
|
+
- Computer use for browser, local desktop, Docker desktop, QEMU virtual desktop, and custom computer adapters.
|
|
33
|
+
- Command use for local CLI, Docker CLI, and QEMU virtual CLI runs.
|
|
34
34
|
- One `.do()` loop: give the model a task, let it request actions, return a structured result.
|
|
35
|
+
- Step-by-step task builders for longer workflows that should read like a checklist.
|
|
35
36
|
- Structured task input with `data` and structured output with `jsonOutput()`.
|
|
37
|
+
- Screen recording for browser and desktop-style computer runs.
|
|
36
38
|
- Built-in OpenAI and Anthropic support, plus custom model adapters.
|
|
37
39
|
- Practical guardrails: domain allowlists, command policies, screenshot controls, max steps, and hooks.
|
|
38
40
|
|
|
@@ -102,6 +104,42 @@ To run Docker commands without `sudo`, add your user to the `docker` group, then
|
|
|
102
104
|
sudo usermod -aG docker $USER
|
|
103
105
|
```
|
|
104
106
|
|
|
107
|
+
### Optional QEMU Setup
|
|
108
|
+
|
|
109
|
+
QEMU is required only for `automify.virtualCli()`, `automify.virtualComputer()`, and `createVirtualDesktopComputer()`.
|
|
110
|
+
When no image is configured, Automify downloads the official Debian genericcloud qcow2 image into a local cache, then prepares a reusable Automify-ready Debian qcow2 with the `automify` SSH user already provisioned. Runtime VMs boot from short-lived overlays backed by that prepared image. Pass `image` or `vm.image` only when you want to use your own bootable Linux disk image with SSH access.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# Ubuntu
|
|
114
|
+
sudo apt-get install -y qemu-system qemu-utils
|
|
115
|
+
|
|
116
|
+
# macOS
|
|
117
|
+
brew install qemu
|
|
118
|
+
|
|
119
|
+
# Windows
|
|
120
|
+
# Install QEMU from https://www.qemu.org/download/
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Pre-warm or refresh QEMU image caches:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
# Pre-warm the minimal QEMU CLI cache.
|
|
127
|
+
npx automify-qemu-image
|
|
128
|
+
|
|
129
|
+
# Pre-warm the QEMU desktop cache with Xvfb/openbox/xterm/xdotool/scrot.
|
|
130
|
+
npx automify-qemu-image --desktop
|
|
131
|
+
|
|
132
|
+
# Re-download the Debian base image and rebuild the prepared cache.
|
|
133
|
+
npx automify-qemu-image --force-download
|
|
134
|
+
|
|
135
|
+
# Pre-warm an alternate qcow2 URL into its own cache.
|
|
136
|
+
npx automify-qemu-image \
|
|
137
|
+
--image-url https://example.com/path/linux.qcow2 \
|
|
138
|
+
--cache-dir ~/.cache/automify/qemu-custom
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Use `--image-url` for an alternate apt-based Linux cloud qcow2 that Automify should download, boot, and prepare. Use the same URL and cache directory at runtime with `vm.imageUrl` or `qemuImageUrl` plus `defaultImageCache`. If you pass a local disk with `image` or `vm.image`, Automify uses that qcow2 directly instead of preparing it; the image must already boot, accept SSH, and include the desktop packages required for virtual desktop runs.
|
|
142
|
+
|
|
105
143
|
## Quick Start
|
|
106
144
|
|
|
107
145
|
```js
|
|
@@ -217,7 +255,7 @@ const automify = initAutomify({
|
|
|
217
255
|
const cli = automify.dockerCli({
|
|
218
256
|
// Optional: choose resource limits without changing the default image.
|
|
219
257
|
container: { cpus: 1, memory: "1g" },
|
|
220
|
-
// Optional: install
|
|
258
|
+
// Optional: install apt packages before commands run.
|
|
221
259
|
additionalAptPackages: ["coreutils", "nodejs"],
|
|
222
260
|
// Optional: mount a host folder into the container workspace.
|
|
223
261
|
shared: { hostPath: sharedDir, containerPath: "/workspace" }
|
|
@@ -235,6 +273,38 @@ try {
|
|
|
235
273
|
}
|
|
236
274
|
```
|
|
237
275
|
|
|
276
|
+
Use QEMU virtual CLI when command execution should happen inside a real VM. QEMU must be installed. By default, Automify prepares a minimal Debian cloud image automatically; pass `image` or `vm.image` only to use a custom VM disk:
|
|
277
|
+
|
|
278
|
+
```js
|
|
279
|
+
const cli = automify.virtualCli({
|
|
280
|
+
vm: {
|
|
281
|
+
memory: "2g",
|
|
282
|
+
cpus: 2
|
|
283
|
+
},
|
|
284
|
+
additionalAptPackages: ["coreutils"],
|
|
285
|
+
shared: { hostPath: process.cwd(), containerPath: "/workspace" }
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
try {
|
|
289
|
+
await cli.do("Run 'node --version' and summarize the result");
|
|
290
|
+
} finally {
|
|
291
|
+
await cli.close();
|
|
292
|
+
}
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
To reuse an alternate qcow2 URL that you pre-warmed with `automify-qemu-image --image-url`, pass the same URL and cache directory:
|
|
296
|
+
|
|
297
|
+
```js
|
|
298
|
+
const cli = automify.virtualCli({
|
|
299
|
+
vm: {
|
|
300
|
+
imageUrl: "https://example.com/path/linux.qcow2"
|
|
301
|
+
},
|
|
302
|
+
defaultImageCache: {
|
|
303
|
+
dir: "/var/cache/automify-qemu-custom"
|
|
304
|
+
}
|
|
305
|
+
});
|
|
306
|
+
```
|
|
307
|
+
|
|
238
308
|
### Desktop Computer Use
|
|
239
309
|
|
|
240
310
|
Local desktop computer use controls the native desktop on the machine running your Node.js process. It supports macOS, Windows, and Linux through the local desktop adapter. On Linux, local desktop support requires X11/Xorg or Xvfb; Wayland sessions are not supported. It needs native desktop dependencies that are not installed by default, and your OS may ask for permission to control the desktop.
|
|
@@ -255,17 +325,11 @@ winget install --id Kitware.CMake --exact --source winget
|
|
|
255
325
|
xcode-select --install
|
|
256
326
|
brew install cmake
|
|
257
327
|
|
|
258
|
-
#
|
|
328
|
+
# Ubuntu Linux.
|
|
259
329
|
sudo apt-get install -y git build-essential cmake pkg-config libx11-dev libxtst-dev libpng++-dev
|
|
260
|
-
|
|
261
|
-
# Fedora Linux.
|
|
262
|
-
sudo dnf install -y gcc-c++ make cmake libXtst-devel libpng-devel
|
|
263
|
-
|
|
264
|
-
# Arch Linux.
|
|
265
|
-
sudo pacman -S --needed base-devel cmake libxtst libpng
|
|
266
330
|
```
|
|
267
331
|
|
|
268
|
-
On Linux,
|
|
332
|
+
On Linux, the documented local desktop path is Ubuntu. Install the full package list before running `npx automify-install-desktop`; the installer checks for command-line build tools but does not verify every native library package. Linux local desktop capture is X11-based: use Xorg/X11, not Wayland. On headless Ubuntu hosts, also install `xvfb` unless you manage `DISPLAY` yourself. On macOS, install Homebrew first if `brew` is not available, then install CMake with `brew install cmake`. On macOS and Windows, `cmake --version` must work in the terminal where you run `npx automify-install-desktop`. On Windows, the VS Code CMake Tools extension is not enough by itself, and Visual Studio 2026 is not currently recognized by the native build chain used by nut.js.
|
|
269
333
|
|
|
270
334
|
`npx automify-install-desktop` stores the compiled desktop runtime outside `node_modules` in a long-term cache, so normal `npm update` runs do not remove it. If the command is run again and the cached runtime already matches the current platform, CPU architecture, Node ABI, and pinned nut.js/libnut revisions, Automify prints a skip message and exits without rebuilding. Use `npx automify-install-desktop --force` (or `npx automify-install-desktop force`) to rebuild a compatible cache anyway. If a later `npm install` or `npm update` detects that a previously installed desktop runtime no longer matches the current environment, Automify rebuilds it automatically during `postinstall`. Default cache roots are `%LOCALAPPDATA%\automify\desktop-runtime` on Windows, `~/Library/Caches/automify/desktop-runtime` on macOS, and `${XDG_CACHE_HOME:-~/.cache}/automify/desktop-runtime` on Linux. Override with `AUTOMIFY_DESKTOP_RUNTIME_DIR`; disable auto-rebuild with `AUTOMIFY_SKIP_DESKTOP_AUTO_REBUILD=1`.
|
|
271
335
|
|
|
@@ -319,7 +383,25 @@ try {
|
|
|
319
383
|
}
|
|
320
384
|
```
|
|
321
385
|
|
|
322
|
-
|
|
386
|
+
For a real VM-backed Linux desktop, use QEMU. `virtualComputer()` starts a QEMU VM, connects over SSH, and controls an Xvfb desktop inside the guest with `xdotool` and `scrot`. By default, Automify prepares a minimal Debian cloud image automatically; pass `image` or `vm.image` only to use a custom VM disk:
|
|
387
|
+
|
|
388
|
+
```js
|
|
389
|
+
const desktop = await automify.virtualComputer({
|
|
390
|
+
vm: {
|
|
391
|
+
memory: "2g",
|
|
392
|
+
cpus: 2
|
|
393
|
+
},
|
|
394
|
+
desktop: { startupCommand: "xterm" }
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
try {
|
|
398
|
+
await desktop.do("Use the open terminal to run 'uname -a' and summarize the VM system information");
|
|
399
|
+
} finally {
|
|
400
|
+
await desktop.close();
|
|
401
|
+
}
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
Local desktop computer use takes an exclusive cross-process lock until `close()`. Docker desktop locks are scoped to the container name, and QEMU virtual desktop locks are scoped to the VM name.
|
|
323
405
|
|
|
324
406
|
### Custom Computer Use
|
|
325
407
|
|
|
@@ -335,7 +417,7 @@ await automify.computer({ computer }).do("Use the remote app with the supplied t
|
|
|
335
417
|
});
|
|
336
418
|
```
|
|
337
419
|
|
|
338
|
-
Custom computer adapters can expose `environment`, `displayWidth`, and `displayHeight` when they control a fixed remote target. Built-in local and
|
|
420
|
+
Custom computer adapters can expose `environment`, `displayWidth`, and `displayHeight` when they control a fixed remote target. Built-in local, Docker desktop, and QEMU virtual desktop adapters infer or choose those values for you.
|
|
339
421
|
|
|
340
422
|
## Input And Output
|
|
341
423
|
|
|
@@ -360,10 +442,135 @@ const run = await browser.do("Create the lead from data and return the saved rec
|
|
|
360
442
|
});
|
|
361
443
|
```
|
|
362
444
|
|
|
445
|
+
For longer workflows, build the task as ordered steps and run it at the end. This is useful when the task has
|
|
446
|
+
distinct phases, such as navigate, wait, create, verify, and extract:
|
|
447
|
+
|
|
448
|
+
```js
|
|
449
|
+
const run = await browser
|
|
450
|
+
.addStep("Open the contacts page.")
|
|
451
|
+
.addWait("the contacts table is visible")
|
|
452
|
+
.addStep("Create the lead from data.")
|
|
453
|
+
.addExtract("Return the saved record JSON.", {
|
|
454
|
+
key: "lead",
|
|
455
|
+
shape: { id: "string", firstName: "string", lastName: "string" }
|
|
456
|
+
})
|
|
457
|
+
.addData({ firstName: "Ada", lastName: "Lovelace" })
|
|
458
|
+
.run();
|
|
459
|
+
|
|
460
|
+
console.log(run.parsed.lead.id);
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
You can also start from `browser.task()` when you want to keep a reusable builder variable:
|
|
464
|
+
|
|
465
|
+
```js
|
|
466
|
+
const task = browser
|
|
467
|
+
.task({ limits: { steps: 30 } })
|
|
468
|
+
.addStep("Open the billing page.")
|
|
469
|
+
.addWait("the invoice list has finished loading")
|
|
470
|
+
.addObserve("Find the newest unpaid invoice.")
|
|
471
|
+
.addAssert("Confirm the customer name matches the data.")
|
|
472
|
+
.addExtract("Return the invoice id and total.", {
|
|
473
|
+
key: "invoice",
|
|
474
|
+
shape: { id: "string", total: "number" }
|
|
475
|
+
})
|
|
476
|
+
.addExtract("Return audit metadata.", {
|
|
477
|
+
key: "audit",
|
|
478
|
+
shape: { requestId: "string" }
|
|
479
|
+
})
|
|
480
|
+
.withData({ customerName: "Ada Lovelace" });
|
|
481
|
+
|
|
482
|
+
const run = await task.run();
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
Builder steps are converted into one ordered `.do()` instruction, so hooks, screenshots, output parsing, safety
|
|
486
|
+
options, and limits behave the same as a normal run. `addStep()` is the general-purpose method; `addAct()` is an alias
|
|
487
|
+
for action-oriented steps. `addWait("condition")` waits for a visible condition; `addWait(500)` remains supported and
|
|
488
|
+
maps to `addPause(500)`. `addObserve()`, `addExtract()`, and `addAssert()` add readable intent for common phases. When
|
|
489
|
+
`addExtract()` gets `{ key, shape }`, Automify builds the structured output for you, and multiple extracts are returned
|
|
490
|
+
under `run.parsed[key]`. Short aliases without `add` also work: `step()`, `act()`, `wait()`, `waitFor()`, `pause()`,
|
|
491
|
+
`observe()`, `extract()`, and `assert()`.
|
|
492
|
+
|
|
493
|
+
Use `task({ mode: "sequential" })` when each step should be its own model run. Sequential mode preserves the same
|
|
494
|
+
builder API, but executes every non-pause step separately, keeps browser or desktop state between steps, and returns a
|
|
495
|
+
`taskSteps` audit trail in addition to the aggregated action `steps`. `addPause(ms)` is deterministic in this mode and
|
|
496
|
+
does not call the model. `addAssert()` asks the model for a structured pass/fail check and fails the task when the
|
|
497
|
+
assertion is not true.
|
|
498
|
+
|
|
499
|
+
```js
|
|
500
|
+
const run = await browser
|
|
501
|
+
.task({ mode: "sequential" })
|
|
502
|
+
.addStep("Fill the first name field from data.")
|
|
503
|
+
.addStep("Fill the last name field from data.")
|
|
504
|
+
.addStep("Submit the form.")
|
|
505
|
+
.addExtract("Return the saved record JSON.", {
|
|
506
|
+
key: "record",
|
|
507
|
+
shape: { id: "string", firstName: "string", lastName: "string" }
|
|
508
|
+
})
|
|
509
|
+
.addData({ firstName: "Dorothy", lastName: "Vaughan" })
|
|
510
|
+
.run({ recording: "/tmp/automify-sequential.mp4" });
|
|
511
|
+
|
|
512
|
+
console.log(run.taskSteps.length);
|
|
513
|
+
console.log(run.parsed.record.id);
|
|
514
|
+
console.log(run.recording.path);
|
|
515
|
+
```
|
|
516
|
+
|
|
517
|
+
In sequential mode, a run-level `output` applies only to the final non-pause step. Extract outputs still belong on
|
|
518
|
+
`addExtract()` steps; if you define multiple extracts, give each one a `key`. Browser and desktop recordings cover the
|
|
519
|
+
whole sequential task. CLI tasks can run sequentially, but CLI adapters do not record the screen.
|
|
520
|
+
|
|
363
521
|
- `data` is structured JSON for the task.
|
|
364
522
|
- `evaluate` sends images or text files directly to the model.
|
|
365
|
-
- `shared` and `sharedFiles` expose files inside Docker CLI or
|
|
523
|
+
- `shared` and `sharedFiles` expose files inside Docker CLI, Docker desktop, QEMU virtual CLI, or QEMU virtual desktop runs.
|
|
366
524
|
- `jsonOutput()` requests structured JSON and makes parsed output available as `run.parsed`.
|
|
525
|
+
- `limits.steps` controls the maximum model-action turns before `MaxStepsExceededError`. The default is `100`.
|
|
526
|
+
|
|
527
|
+
Visual adapters can record a run by polling screenshots and encoding them with `ffmpeg`:
|
|
528
|
+
|
|
529
|
+
```js
|
|
530
|
+
const run = await browser.do("Run the checkout smoke test.", {
|
|
531
|
+
recording: "/tmp/automify-checkout.mp4"
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
console.log(run.recording.path);
|
|
535
|
+
```
|
|
536
|
+
|
|
537
|
+
Use `recording` or `screenRecording`. Pass a string for the output path, `true` to write a temp MP4, or an object when
|
|
538
|
+
you need control over capture rate and encoding:
|
|
539
|
+
|
|
540
|
+
```js
|
|
541
|
+
const run = await browser.do("Run the checkout smoke test.", {
|
|
542
|
+
screenRecording: {
|
|
543
|
+
path: "/tmp/automify-checkout.mp4",
|
|
544
|
+
fps: 6,
|
|
545
|
+
keepFrames: false
|
|
546
|
+
}
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
console.log(run.recording.frames);
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
Recording works for browser and computer-use adapters. CLI adapters do not produce screen recordings. The host process
|
|
553
|
+
needs `ffmpeg` on PATH unless you pass `screenRecording.ffmpegCommand` or a custom `screenRecording.execFile`.
|
|
554
|
+
|
|
555
|
+
Set max steps on an adapter when most runs need the same limit:
|
|
556
|
+
|
|
557
|
+
```js
|
|
558
|
+
const browser = await automify.browser({
|
|
559
|
+
startUrl: "https://example.com",
|
|
560
|
+
limits: { steps: 250 }
|
|
561
|
+
});
|
|
562
|
+
```
|
|
563
|
+
|
|
564
|
+
Override it for one `.do()` call when a task needs a different limit:
|
|
565
|
+
|
|
566
|
+
```js
|
|
567
|
+
await browser.do("Quick smoke test", {
|
|
568
|
+
limits: { steps: 25 }
|
|
569
|
+
});
|
|
570
|
+
```
|
|
571
|
+
|
|
572
|
+
The older flat option also works: `maxSteps: 250` is equivalent to `limits: { steps: 250 }`. If both are provided,
|
|
573
|
+
`maxSteps` wins.
|
|
367
574
|
|
|
368
575
|
For arrays of objects, the most ergonomic shape is usually an object with a named array property:
|
|
369
576
|
|
|
@@ -470,13 +677,13 @@ Pass `{ parse: false }` if you want Automify to request the Zod-derived JSON Sch
|
|
|
470
677
|
|
|
471
678
|
Before running computer use against real accounts or user data:
|
|
472
679
|
|
|
473
|
-
| Area | Recommendation
|
|
474
|
-
| ------- |
|
|
475
|
-
| Scope | Use dedicated accounts, narrow browser allowlists, command policies, and isolated desktops or containers.
|
|
476
|
-
| Data | Pass task input through `data`; request application output with `jsonOutput()` instead of parsing prose.
|
|
477
|
-
| Safety | Add human approval for sensitive CLI commands, browser actions, or externally visible operations.
|
|
478
|
-
| Privacy | Redact screenshots before model upload when screens can contain secrets or regulated data.
|
|
479
|
-
| Audit | Use `hooks`, `screenshots.actions`, `logFile`, and `trace: true` for workflows that need review.
|
|
680
|
+
| Area | Recommendation |
|
|
681
|
+
| ------- | ------------------------------------------------------------------------------------------------------------- |
|
|
682
|
+
| Scope | Use dedicated accounts, narrow browser allowlists, command policies, and isolated desktops or containers. |
|
|
683
|
+
| Data | Pass task input through `data`; request application output with `jsonOutput()` instead of parsing prose. |
|
|
684
|
+
| Safety | Add human approval for sensitive CLI commands, browser actions, or externally visible operations. |
|
|
685
|
+
| Privacy | Redact screenshots before model upload when screens can contain secrets or regulated data. |
|
|
686
|
+
| Audit | Use `hooks`, `screenshots.actions`, `recording`, `logFile`, and `trace: true` for workflows that need review. |
|
|
480
687
|
|
|
481
688
|
## Providers
|
|
482
689
|
|
|
@@ -528,8 +735,10 @@ Use the adapter toolkit when a custom provider needs to emit computer use action
|
|
|
528
735
|
- `examples/browser-with-safety.js`
|
|
529
736
|
- `examples/cli-basic.js`
|
|
530
737
|
- `examples/cli-docker.js`
|
|
738
|
+
- `examples/cli-qemu.js`
|
|
531
739
|
- `examples/desktop-local.js`
|
|
532
740
|
- `examples/desktop-docker.js`
|
|
741
|
+
- `examples/desktop-qemu.js`
|
|
533
742
|
- `examples/custom-computer.js`
|
|
534
743
|
- `examples/custom-model-adapter.js`
|
|
535
744
|
|
|
@@ -539,16 +748,22 @@ Use the adapter toolkit when a custom provider needs to emit computer use action
|
|
|
539
748
|
npm test
|
|
540
749
|
npm run test:e2e
|
|
541
750
|
OPENAI_API_KEY=... npm run test:live
|
|
751
|
+
npm run test:live:qemu
|
|
752
|
+
npm run test:live:qemu:desktop
|
|
542
753
|
```
|
|
543
754
|
|
|
544
|
-
`npm run test:live` runs `test
|
|
755
|
+
`npm run test:live:qemu` runs only the real QEMU Debian boot smoke test, without OpenAI. `npm run test:live:qemu:desktop` runs the real QEMU desktop smoke test with the default Debian image. Set `AUTOMIFY_QEMU_IMAGE=/path/to/linux.qcow2` only when you want the desktop smoke test or the QEMU live tests to use a custom image. The equivalent direct flags are `RUN_QEMU_DEBIAN_E2E=1 npm run test:e2e` and `RUN_QEMU_DESKTOP_E2E=1 npm run test:e2e`.
|
|
756
|
+
|
|
757
|
+
`npm run test:live` runs `test/e2e/live-openai.e2e.test.js` with `RUN_OPENAI_E2E=1`. By default, it runs the live OpenAI CLI and Docker CLI checks and skips the browser, Docker desktop, and QEMU checks. Set `RUN_OPENAI_BROWSER_E2E=1` to include the live browser demo tests, including task-builder and recording coverage.
|
|
545
758
|
|
|
546
759
|
Run every live test:
|
|
547
760
|
|
|
548
761
|
```bash
|
|
549
762
|
OPENAI_API_KEY=... \
|
|
550
763
|
RUN_OPENAI_BROWSER_E2E=1 \
|
|
551
|
-
|
|
764
|
+
RUN_OPENAI_DOCKER_DESKTOP_E2E=1 \
|
|
765
|
+
RUN_OPENAI_QEMU_CLI_E2E=1 \
|
|
766
|
+
RUN_OPENAI_QEMU_DESKTOP_E2E=1 \
|
|
552
767
|
npm run test:live
|
|
553
768
|
```
|
|
554
769
|
|
|
@@ -558,10 +773,20 @@ The equivalent direct command is:
|
|
|
558
773
|
OPENAI_API_KEY=... \
|
|
559
774
|
RUN_OPENAI_E2E=1 \
|
|
560
775
|
RUN_OPENAI_BROWSER_E2E=1 \
|
|
561
|
-
|
|
776
|
+
RUN_OPENAI_DOCKER_DESKTOP_E2E=1 \
|
|
777
|
+
RUN_OPENAI_QEMU_CLI_E2E=1 \
|
|
778
|
+
RUN_OPENAI_QEMU_DESKTOP_E2E=1 \
|
|
562
779
|
node --test test/e2e/live-openai.e2e.test.js
|
|
563
780
|
```
|
|
564
781
|
|
|
782
|
+
Use `AUTOMIFY_QEMU_DEFAULT_IMAGE_URL` to point the default Debian download at a mirror, and `AUTOMIFY_QEMU_IMAGE_CACHE_DIR` to choose the cache directory. By default, Automify caches the downloaded Debian base image and prepared Automify-ready Debian images on the user's computer. The CLI cache stays minimal; the desktop cache is a separate variant that bakes Xvfb/openbox/xterm/xdotool/scrot so warm desktop boots do not reinstall apt packages. Configure image caching with `defaultImageCache`, for example `defaultImageCache: { dir: "/var/cache/automify-qemu", forcePrepare: true }`. Run `npx automify-qemu-image` to pre-warm the QEMU CLI cache. Run `npx automify-qemu-image --desktop` to pre-warm the QEMU desktop cache. Run `npx automify-qemu-image --image-url https://example.com/path/linux.qcow2 --cache-dir /var/cache/automify-qemu-custom` to pre-warm an alternate cloud qcow2 cache, then use the same URL and cache directory at runtime with `vm.imageUrl` or `qemuImageUrl` plus `defaultImageCache`. Run `npx automify-qemu-image --force-download` to replace the cached base image and rebuild the prepared image. Local disks passed with `image` or `vm.image` are not prepared by the cache command; they must already be bootable with SSH access. On ARM hosts Automify auto-detects common QEMU UEFI firmware paths; set `AUTOMIFY_QEMU_FIRMWARE` if your QEMU install keeps the firmware elsewhere.
|
|
783
|
+
|
|
565
784
|
## License
|
|
566
785
|
|
|
567
786
|
MIT
|
|
787
|
+
|
|
788
|
+
## Disclaimer
|
|
789
|
+
|
|
790
|
+
Automify is distributed "as is", without warranty of any kind. Automation can control browsers, shells, desktops, files, and external services; you are responsible for how you configure and run it, and for any events associated with that use. To the maximum extent permitted by law, the author is not liable for losses, damages, data loss, service disruption, or other consequences arising from use of the software.
|
|
791
|
+
|
|
792
|
+
Created by [Aldo Vincenti](https://aldovincenti.com).
|
|
@@ -22,17 +22,14 @@ await automify.withBrowser(
|
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
async (browser) => {
|
|
25
|
-
return browser.do(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
console.log("Action:", action);
|
|
32
|
-
return true;
|
|
33
|
-
}
|
|
25
|
+
return browser.do("Find the contact page and report the support address", {
|
|
26
|
+
safety: {
|
|
27
|
+
onCheck: async ({ checks, action }) => {
|
|
28
|
+
console.log("Safety checks:", checks);
|
|
29
|
+
console.log("Action:", action);
|
|
30
|
+
return true;
|
|
34
31
|
}
|
|
35
32
|
}
|
|
36
|
-
);
|
|
33
|
+
});
|
|
37
34
|
}
|
|
38
35
|
);
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { initAutomify } from "../src/index.js";
|
|
2
|
+
|
|
3
|
+
const automify = initAutomify({
|
|
4
|
+
provider: {
|
|
5
|
+
type: "openai",
|
|
6
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
7
|
+
model: process.env.OPENAI_MODEL ?? "gpt-5.5"
|
|
8
|
+
}
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
const cli = automify.virtualCli({
|
|
12
|
+
vm: {
|
|
13
|
+
memory: "2g",
|
|
14
|
+
cpus: 2
|
|
15
|
+
},
|
|
16
|
+
additionalAptPackages: ["coreutils"],
|
|
17
|
+
shared: { hostPath: process.cwd(), containerPath: "/workspace" },
|
|
18
|
+
command: {
|
|
19
|
+
allow: ["cat /etc/os-release", "uname -m", "pwd"]
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
const result = await cli.do("Run 'cat /etc/os-release', 'uname -m', and 'pwd', then summarize the VM environment.");
|
|
25
|
+
console.log(result.text);
|
|
26
|
+
} finally {
|
|
27
|
+
await cli.close();
|
|
28
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
|
|
4
|
+
import { createVirtualDesktopComputer, initAutomify } from "../src/index.js";
|
|
5
|
+
|
|
6
|
+
const automify = initAutomify({
|
|
7
|
+
provider: {
|
|
8
|
+
type: "openai",
|
|
9
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
10
|
+
model: process.env.OPENAI_MODEL ?? "gpt-5.5"
|
|
11
|
+
}
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const computer = await createVirtualDesktopComputer({
|
|
15
|
+
vm: {
|
|
16
|
+
memory: "2g",
|
|
17
|
+
cpus: 2
|
|
18
|
+
},
|
|
19
|
+
desktop: {
|
|
20
|
+
startupCommand: "xterm"
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
const desktop = automify.computer({ computer });
|
|
26
|
+
const result = await desktop.do(
|
|
27
|
+
"Use the open terminal to run 'uname -a' and summarize the VM system information shown on screen.",
|
|
28
|
+
{
|
|
29
|
+
screenshots: {
|
|
30
|
+
initial: join(tmpdir(), "automify-qemu-desktop-initial.png"),
|
|
31
|
+
final: join(tmpdir(), "automify-qemu-desktop-final.png")
|
|
32
|
+
},
|
|
33
|
+
limits: { steps: 12 }
|
|
34
|
+
}
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
console.log(result.text);
|
|
38
|
+
console.log(result.finalScreenshot);
|
|
39
|
+
} finally {
|
|
40
|
+
await computer.close();
|
|
41
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "automify",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "AI computer use for browser, CLI, and desktop in Node.js.",
|
|
5
5
|
"homepage": "https://aldovincenti.github.io/automify",
|
|
6
6
|
"bugs": {
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
"main": "src/index.js",
|
|
15
15
|
"types": "src/index.d.ts",
|
|
16
16
|
"bin": {
|
|
17
|
-
"automify-install-desktop": "./scripts/install-desktop.js"
|
|
17
|
+
"automify-install-desktop": "./scripts/install-desktop.js",
|
|
18
|
+
"automify-qemu-image": "./scripts/qemu-image.js"
|
|
18
19
|
},
|
|
19
20
|
"exports": {
|
|
20
21
|
".": {
|
|
@@ -42,6 +43,8 @@
|
|
|
42
43
|
"test": "node --test test/*.test.js",
|
|
43
44
|
"test:e2e": "node --test test/e2e/*.e2e.test.js",
|
|
44
45
|
"test:live": "RUN_OPENAI_E2E=1 node --test test/e2e/live-openai.e2e.test.js",
|
|
46
|
+
"test:live:qemu": "RUN_QEMU_DEBIAN_E2E=1 node --test test/e2e/qemu-runtimes.e2e.test.js",
|
|
47
|
+
"test:live:qemu:desktop": "RUN_QEMU_DESKTOP_E2E=1 node --test --test-name-pattern \"QEMU virtual desktop\" test/e2e/desktop-runtimes.e2e.test.js",
|
|
45
48
|
"format": "prettier --write .",
|
|
46
49
|
"format:check": "prettier --check ."
|
|
47
50
|
},
|
|
@@ -2,7 +2,9 @@ import { writeFile } from "node:fs/promises";
|
|
|
2
2
|
import { argumentReference } from "../src/lib/argument-reference.js";
|
|
3
3
|
|
|
4
4
|
const rows = argumentReference
|
|
5
|
-
.map(
|
|
5
|
+
.map(
|
|
6
|
+
(entry) => `| \`${entry.surface}\` | ${entry.preferred.map((name) => `\`${name}\``).join(", ")} | ${entry.notes} |`
|
|
7
|
+
)
|
|
6
8
|
.join("\n");
|
|
7
9
|
|
|
8
10
|
const markdown = `# Automify Argument Reference
|