automify 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -36
- package/examples/browser-with-safety.js +7 -10
- package/examples/cli-qemu.js +28 -0
- package/examples/desktop-qemu.js +41 -0
- package/package.json +5 -2
- package/scripts/generate-argument-reference.js +3 -1
- package/scripts/qemu-image.js +154 -0
- package/src/index.d.ts +368 -10
- package/src/index.js +18 -38
- package/src/lib/adapter-toolkit.js +8 -4
- package/src/lib/anthropic-model-adapter.js +24 -13
- package/src/lib/argument-reference.js +60 -8
- package/src/lib/automify.js +96 -0
- package/src/lib/cli-automify.js +41 -2
- package/src/lib/computer-automify.js +45 -26
- package/src/lib/docker-cli-automify.js +2 -6
- package/src/lib/docker-desktop-computer.js +7 -13
- package/src/lib/file-data.js +6 -6
- package/src/lib/init.js +14 -3
- package/src/lib/local-desktop-computer.js +2 -1
- package/src/lib/openai-responses-client.js +10 -3
- package/src/lib/presets.js +50 -2
- package/src/lib/qemu-cli-automify.js +568 -0
- package/src/lib/qemu-desktop-computer.js +681 -0
- package/src/lib/qemu-runtime.js +654 -0
- package/src/lib/runtime.js +23 -2
- package/src/lib/screen-recording.js +184 -0
- package/src/lib/task.js +564 -0
- package/src/lib/virtual-shared-folder.js +3 -1
package/src/index.js
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
|
-
export {
|
|
2
|
-
|
|
3
|
-
createAutomify,
|
|
4
|
-
AutomifyError,
|
|
5
|
-
SafetyCheckError,
|
|
6
|
-
MaxStepsExceededError
|
|
7
|
-
} from "./lib/automify.js";
|
|
1
|
+
export { Automify, createAutomify, AutomifyError, SafetyCheckError, MaxStepsExceededError } from "./lib/automify.js";
|
|
2
|
+
export { AutomifyTask, createTask } from "./lib/task.js";
|
|
8
3
|
|
|
9
4
|
export { OpenAIResponsesClient } from "./lib/openai-responses-client.js";
|
|
10
5
|
export { AnthropicModelAdapter, createAnthropicModelAdapter } from "./lib/anthropic-model-adapter.js";
|
|
11
|
-
export {
|
|
12
|
-
createBrowserComputer,
|
|
13
|
-
createPlaywrightComputer,
|
|
14
|
-
executePlaywrightAction
|
|
15
|
-
} from "./lib/playwright-computer.js";
|
|
6
|
+
export { createBrowserComputer, createPlaywrightComputer, executePlaywrightAction } from "./lib/playwright-computer.js";
|
|
16
7
|
export {
|
|
17
8
|
captureLocalDesktopScreenshot,
|
|
18
9
|
createLocalDesktopComputer,
|
|
@@ -20,19 +11,16 @@ export {
|
|
|
20
11
|
} from "./lib/local-desktop-computer.js";
|
|
21
12
|
export {
|
|
22
13
|
DockerDesktopSession,
|
|
23
|
-
DockerVirtualDesktopSession,
|
|
24
14
|
createDockerDesktopComputer,
|
|
25
|
-
createVirtualDesktopComputer,
|
|
26
15
|
defaultDockerDesktopImage,
|
|
27
|
-
|
|
28
|
-
dockerDesktopDockerfile,
|
|
29
|
-
virtualDesktopDockerfile
|
|
16
|
+
dockerDesktopDockerfile
|
|
30
17
|
} from "./lib/docker-desktop-computer.js";
|
|
31
18
|
export {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
} from "./lib/
|
|
19
|
+
QemuDesktopSession,
|
|
20
|
+
createVirtualDesktopComputer,
|
|
21
|
+
defaultVirtualDesktopImage
|
|
22
|
+
} from "./lib/qemu-desktop-computer.js";
|
|
23
|
+
export { BrowserAutomify, createBrowserAutomify, withBrowserAutomify } from "./lib/browser-automify.js";
|
|
36
24
|
export { initAutomify } from "./lib/init.js";
|
|
37
25
|
export { createModelAdapter } from "./lib/model-adapter.js";
|
|
38
26
|
export {
|
|
@@ -57,27 +45,19 @@ export { jsonOutput } from "./lib/output.js";
|
|
|
57
45
|
export {
|
|
58
46
|
DockerComputerAutomify,
|
|
59
47
|
LocalComputerAutomify,
|
|
48
|
+
VirtualComputerAutomify,
|
|
60
49
|
createComputerAutomify,
|
|
61
50
|
createDockerComputerAutomify,
|
|
62
|
-
createLocalComputerAutomify
|
|
51
|
+
createLocalComputerAutomify,
|
|
52
|
+
createVirtualComputerAutomify
|
|
63
53
|
} from "./lib/computer-automify.js";
|
|
54
|
+
export { CliAutomify, createCliAutomify, runShellCommand } from "./lib/cli-automify.js";
|
|
55
|
+
export { DockerCliAutomify, DockerCliSession, createDockerCliAutomify } from "./lib/docker-cli-automify.js";
|
|
64
56
|
export {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
runShellCommand
|
|
68
|
-
} from "./lib/cli-automify.js";
|
|
69
|
-
export {
|
|
70
|
-
DockerCliAutomify,
|
|
71
|
-
DockerCliSession,
|
|
72
|
-
createDockerCliAutomify,
|
|
73
|
-
DockerVirtualCliSession,
|
|
57
|
+
QemuCliSession,
|
|
58
|
+
QemuVirtualCliSession,
|
|
74
59
|
VirtualCliAutomify,
|
|
75
60
|
createVirtualCliAutomify
|
|
76
|
-
} from "./lib/
|
|
77
|
-
export {
|
|
78
|
-
fileToEvaluate,
|
|
79
|
-
fileToData,
|
|
80
|
-
filesToEvaluate,
|
|
81
|
-
filesToData
|
|
82
|
-
} from "./lib/file-data.js";
|
|
61
|
+
} from "./lib/qemu-cli-automify.js";
|
|
62
|
+
export { fileToEvaluate, fileToData, filesToEvaluate, filesToData } from "./lib/file-data.js";
|
|
83
63
|
export { argumentReference } from "./lib/argument-reference.js";
|
|
@@ -34,10 +34,14 @@ export function computerCall(action, options = {}) {
|
|
|
34
34
|
export function runCommandCall(command, options = {}) {
|
|
35
35
|
const callId = options.callId ?? options.call_id ?? `call_${Date.now()}`;
|
|
36
36
|
|
|
37
|
-
return functionCall(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
37
|
+
return functionCall(
|
|
38
|
+
"run_command",
|
|
39
|
+
{ command, cwd: options.cwd, timeoutMs: options.timeoutMs },
|
|
40
|
+
{
|
|
41
|
+
...withoutKeys(options, ["callId", "call_id", "cwd", "timeoutMs"]),
|
|
42
|
+
callId
|
|
43
|
+
}
|
|
44
|
+
);
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
export function functionCall(name, args = {}, options = {}) {
|
|
@@ -75,12 +75,18 @@ export class AnthropicModelAdapter {
|
|
|
75
75
|
? await this.responseTransform(data, { payload, context, request: finalRequest })
|
|
76
76
|
: data;
|
|
77
77
|
const automifyResponse = this.#fromAnthropicResponse(finalData, context, payload);
|
|
78
|
-
const previous = payload.previous_response_id ? this.transcripts.get(payload.previous_response_id) ?? [] : [];
|
|
79
|
-
this.transcripts.set(
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
78
|
+
const previous = payload.previous_response_id ? (this.transcripts.get(payload.previous_response_id) ?? []) : [];
|
|
79
|
+
this.transcripts.set(
|
|
80
|
+
automifyResponse.id,
|
|
81
|
+
compactMessagesForStorage(
|
|
82
|
+
[
|
|
83
|
+
...previous,
|
|
84
|
+
...this.#userMessagesFromPayload(payload, context),
|
|
85
|
+
{ role: "assistant", content: finalData.content ?? [] }
|
|
86
|
+
],
|
|
87
|
+
context
|
|
88
|
+
)
|
|
89
|
+
);
|
|
84
90
|
|
|
85
91
|
return automifyResponse;
|
|
86
92
|
}
|
|
@@ -100,8 +106,11 @@ export class AnthropicModelAdapter {
|
|
|
100
106
|
}
|
|
101
107
|
|
|
102
108
|
async #toAnthropicRequest(payload, context) {
|
|
103
|
-
const previous = payload.previous_response_id ? this.transcripts.get(payload.previous_response_id) ?? [] : [];
|
|
104
|
-
const messages = compactMessagesForRequest(
|
|
109
|
+
const previous = payload.previous_response_id ? (this.transcripts.get(payload.previous_response_id) ?? []) : [];
|
|
110
|
+
const messages = compactMessagesForRequest(
|
|
111
|
+
[...previous, ...this.#userMessagesFromPayload(payload, context)],
|
|
112
|
+
context
|
|
113
|
+
);
|
|
105
114
|
const tools = this.#toolsFromPayload(payload);
|
|
106
115
|
|
|
107
116
|
return removeUndefined({
|
|
@@ -204,11 +213,13 @@ export class AnthropicModelAdapter {
|
|
|
204
213
|
|
|
205
214
|
if (item.type === "tool_use") {
|
|
206
215
|
if (item.name === "run_command") {
|
|
207
|
-
output.push(
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
216
|
+
output.push(
|
|
217
|
+
runCommandCall(item.input?.command ?? "", {
|
|
218
|
+
callId: item.id,
|
|
219
|
+
cwd: item.input?.cwd,
|
|
220
|
+
timeoutMs: item.input?.timeoutMs
|
|
221
|
+
})
|
|
222
|
+
);
|
|
212
223
|
} else if (item.name === "computer") {
|
|
213
224
|
output.push(computerCall(mapAnthropicComputerAction(item.input), { callId: item.id }));
|
|
214
225
|
} else {
|
|
@@ -11,10 +11,11 @@ export const argumentReference = [
|
|
|
11
11
|
"hooks",
|
|
12
12
|
"screenshots",
|
|
13
13
|
"screenshot",
|
|
14
|
+
"recording",
|
|
14
15
|
"command"
|
|
15
16
|
],
|
|
16
17
|
notes:
|
|
17
|
-
"Use data for structured JSON, evaluate for files the model should inspect directly, limits.steps to change the max model-action turns, and command only on CLI surfaces."
|
|
18
|
+
"Use data for structured JSON, evaluate for files the model should inspect directly, limits.steps to change the max model-action turns, recording to save visual runs with ffmpeg, and command only on CLI surfaces."
|
|
18
19
|
},
|
|
19
20
|
{
|
|
20
21
|
surface: "automify.browser()",
|
|
@@ -51,7 +52,24 @@ export const argumentReference = [
|
|
|
51
52
|
"logFile"
|
|
52
53
|
],
|
|
53
54
|
notes:
|
|
54
|
-
'Requires Docker to be installed and running. Use additionalAptPackages to apt-install
|
|
55
|
+
'Requires Docker to be installed and running. Use additionalAptPackages to apt-install packages before commands run. Use preset: "repo" to mount the current workspace at /workspace and allow common repo commands. Use logFile to capture CLI and Docker container events.'
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
surface: "automify.virtualCli()",
|
|
59
|
+
preferred: [
|
|
60
|
+
"preset",
|
|
61
|
+
"vm",
|
|
62
|
+
"ssh",
|
|
63
|
+
"defaultImageCache",
|
|
64
|
+
"additionalAptPackages",
|
|
65
|
+
"workdir",
|
|
66
|
+
"shared",
|
|
67
|
+
"sharedFiles",
|
|
68
|
+
"command",
|
|
69
|
+
"logFile"
|
|
70
|
+
],
|
|
71
|
+
notes:
|
|
72
|
+
'Creates a QEMU-backed CLI runner. Without image or vm.image, Automify uses the prepared Debian image cache by default; use defaultImageCache to configure or refresh that cache. Use preset: "repo" to expose the current workspace at /workspace.'
|
|
55
73
|
},
|
|
56
74
|
{
|
|
57
75
|
surface: "automify.dockerComputer()",
|
|
@@ -66,19 +84,36 @@ export const argumentReference = [
|
|
|
66
84
|
"logFile"
|
|
67
85
|
],
|
|
68
86
|
notes:
|
|
69
|
-
"Creates a Docker-backed Linux desktop runner and requires Docker to be installed and running. Pass startupCommand or desktop.startupCommand to launch the initial app. Use additionalAptPackages to apt-install extra
|
|
87
|
+
"Creates a Docker-backed Linux desktop runner and requires Docker to be installed and running. Pass startupCommand or desktop.startupCommand to launch the initial app. Use additionalAptPackages to apt-install extra packages. Use logFile to capture automation and Docker desktop events. Explicit container names are locked per name until close()."
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
surface: "automify.virtualComputer()",
|
|
91
|
+
preferred: [
|
|
92
|
+
"preset",
|
|
93
|
+
"vm",
|
|
94
|
+
"ssh",
|
|
95
|
+
"viewport",
|
|
96
|
+
"desktop",
|
|
97
|
+
"defaultImageCache",
|
|
98
|
+
"additionalAptPackages",
|
|
99
|
+
"shared",
|
|
100
|
+
"sharedFiles",
|
|
101
|
+
"logFile"
|
|
102
|
+
],
|
|
103
|
+
notes:
|
|
104
|
+
"Creates a QEMU-backed Linux desktop runner. Without image or vm.image, Automify uses the prepared Debian image cache by default. Pass startupCommand or desktop.startupCommand to launch the initial app; the guest runs Xvfb/openbox/xdotool/scrot over SSH."
|
|
70
105
|
},
|
|
71
106
|
{
|
|
72
107
|
surface: "automify.localComputer()",
|
|
73
|
-
preferred: ["viewport", "mouse", "keyboard", "calibration", "virtualDisplay", "logFile"],
|
|
108
|
+
preferred: ["viewport", "mouse", "keyboard", "calibration", "virtualDisplay", "lockResource", "logFile"],
|
|
74
109
|
notes:
|
|
75
|
-
"Creates a local desktop runner and takes an exclusive cross-process lock until close(). Linux local desktop requires X11/Xorg or Xvfb; Wayland is not supported. Use logFile to capture automation and local desktop events."
|
|
110
|
+
"Creates a local desktop runner and takes an exclusive cross-process lock until close(). Linux local desktop requires X11/Xorg or Xvfb; Wayland is not supported. Use lockResource only for independent desktop sessions or isolated tests. Use logFile to capture automation and local desktop events."
|
|
76
111
|
},
|
|
77
112
|
{
|
|
78
113
|
surface: "createLocalDesktopComputer()",
|
|
79
|
-
preferred: ["viewport", "mouse", "keyboard", "calibration", "virtualDisplay", "logFile"],
|
|
114
|
+
preferred: ["viewport", "mouse", "keyboard", "calibration", "virtualDisplay", "lockResource", "logFile"],
|
|
80
115
|
notes:
|
|
81
|
-
"Grouped mouse, keyboard, and calibration options are preferred over the older flat names. Linux local desktop requires X11/Xorg or Xvfb; Wayland is not supported.
|
|
116
|
+
"Grouped mouse, keyboard, and calibration options are preferred over the older flat names. Linux local desktop requires X11/Xorg or Xvfb; Wayland is not supported. Local desktop control takes an exclusive cross-process lock until close(); use lockResource only for independent desktop sessions or isolated tests. Use logFile to capture local desktop events."
|
|
82
117
|
},
|
|
83
118
|
{
|
|
84
119
|
surface: "createDockerDesktopComputer()",
|
|
@@ -93,6 +128,23 @@ export const argumentReference = [
|
|
|
93
128
|
"logFile"
|
|
94
129
|
],
|
|
95
130
|
notes:
|
|
96
|
-
"Requires Docker to be installed and running. container controls Docker and resource limits; startupCommand or desktop.startupCommand is required; shared/sharedFiles control host file access. Use additionalAptPackages to apt-install extra
|
|
131
|
+
"Requires Docker to be installed and running. container controls Docker and resource limits; startupCommand or desktop.startupCommand is required; shared/sharedFiles control host file access. Use additionalAptPackages to apt-install extra packages and logFile to capture Docker desktop events."
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
surface: "createVirtualDesktopComputer()",
|
|
135
|
+
preferred: [
|
|
136
|
+
"preset",
|
|
137
|
+
"vm",
|
|
138
|
+
"ssh",
|
|
139
|
+
"viewport",
|
|
140
|
+
"desktop",
|
|
141
|
+
"defaultImageCache",
|
|
142
|
+
"additionalAptPackages",
|
|
143
|
+
"shared",
|
|
144
|
+
"sharedFiles",
|
|
145
|
+
"logFile"
|
|
146
|
+
],
|
|
147
|
+
notes:
|
|
148
|
+
"Requires QEMU. Without image or vm.image, Automify uses the prepared Debian image cache by default. vm controls QEMU, custom images, resources, acceleration, and cache configuration; ssh controls login for custom images; startupCommand or desktop.startupCommand is required."
|
|
97
149
|
}
|
|
98
150
|
];
|
package/src/lib/automify.js
CHANGED
|
@@ -6,6 +6,8 @@ import { OpenAIResponsesClient } from "./openai-responses-client.js";
|
|
|
6
6
|
import { toDataUrl } from "./adapter-toolkit.js";
|
|
7
7
|
import { filesToEvaluate } from "./file-data.js";
|
|
8
8
|
import { buildRunResult, buildTextConfig } from "./result.js";
|
|
9
|
+
import { startScreenRecording } from "./screen-recording.js";
|
|
10
|
+
import { createTask } from "./task.js";
|
|
9
11
|
import {
|
|
10
12
|
callHook,
|
|
11
13
|
debugLog,
|
|
@@ -58,6 +60,7 @@ export class Automify {
|
|
|
58
60
|
initialScreenshot,
|
|
59
61
|
finalScreenshot,
|
|
60
62
|
actionScreenshots,
|
|
63
|
+
screenRecording,
|
|
61
64
|
trace,
|
|
62
65
|
silent,
|
|
63
66
|
debug,
|
|
@@ -87,6 +90,7 @@ export class Automify {
|
|
|
87
90
|
this.initialScreenshot = initialScreenshot;
|
|
88
91
|
this.finalScreenshot = finalScreenshot;
|
|
89
92
|
this.actionScreenshots = actionScreenshots;
|
|
93
|
+
this.screenRecording = screenRecording;
|
|
90
94
|
this.trace = trace;
|
|
91
95
|
this.silent = silent;
|
|
92
96
|
this.debug = debug;
|
|
@@ -103,6 +107,7 @@ export class Automify {
|
|
|
103
107
|
const { data, options } = normalizeDoArguments(runOptions, maybeOptions);
|
|
104
108
|
const previousSilent = this.silent;
|
|
105
109
|
if ("silent" in options) this.silent = options.silent;
|
|
110
|
+
let screenRecording = null;
|
|
106
111
|
|
|
107
112
|
try {
|
|
108
113
|
const maxSteps = options.maxSteps ?? this.maxSteps;
|
|
@@ -120,12 +125,14 @@ export class Automify {
|
|
|
120
125
|
const initialScreenshotPath = initialScreenshotPathFor(options, this);
|
|
121
126
|
const finalScreenshotPath = finalScreenshotPathFor(options, this);
|
|
122
127
|
const actionScreenshotsPath = actionScreenshotsPathFor(options, this);
|
|
128
|
+
const screenRecordingPath = screenRecordingPathFor(options, this);
|
|
123
129
|
this.#debug("run_start", {
|
|
124
130
|
model,
|
|
125
131
|
maxSteps,
|
|
126
132
|
initialScreenshot: initialScreenshotPath ?? null,
|
|
127
133
|
finalScreenshot: finalScreenshotPath ?? null,
|
|
128
134
|
actionScreenshots: actionScreenshotsPath ?? null,
|
|
135
|
+
screenRecording: screenRecordingPath ?? null,
|
|
129
136
|
screenshotDetail: screenshotDetailFor(options, this),
|
|
130
137
|
screenshotMaxWidth: screenshotMaxWidthFor(options, this),
|
|
131
138
|
screenshotMaxHeight: screenshotMaxHeightFor(options, this)
|
|
@@ -137,10 +144,12 @@ export class Automify {
|
|
|
137
144
|
initialScreenshot: initialScreenshotPath ?? null,
|
|
138
145
|
finalScreenshot: finalScreenshotPath ?? null,
|
|
139
146
|
actionScreenshots: actionScreenshotsPath ?? null,
|
|
147
|
+
screenRecording: screenRecordingPath ?? null,
|
|
140
148
|
screenshotDetail: screenshotDetailFor(options, this)
|
|
141
149
|
});
|
|
142
150
|
|
|
143
151
|
await this.#assertAllowedCurrentUrl(options);
|
|
152
|
+
screenRecording = await this.#startScreenRecording({ instruction, data }, options, trace);
|
|
144
153
|
|
|
145
154
|
const initial = await this.#initialInput(
|
|
146
155
|
instruction,
|
|
@@ -173,6 +182,8 @@ export class Automify {
|
|
|
173
182
|
const result = buildRunResult(response, steps, options.output);
|
|
174
183
|
const finalScreenshot = await this.#saveFinalScreenshot({ response, steps }, options, trace);
|
|
175
184
|
if (finalScreenshot) result.finalScreenshot = finalScreenshot;
|
|
185
|
+
const recording = await this.#stopScreenRecording(screenRecording, { response, steps }, trace);
|
|
186
|
+
if (recording) result.recording = recording;
|
|
176
187
|
if (traceEnabled) result.trace = traceEvents;
|
|
177
188
|
await this.#complete(result, { instruction, data }, options);
|
|
178
189
|
return result;
|
|
@@ -357,10 +368,49 @@ export class Automify {
|
|
|
357
368
|
|
|
358
369
|
throw new MaxStepsExceededError(maxSteps);
|
|
359
370
|
} finally {
|
|
371
|
+
await this.#stopScreenRecording(screenRecording, { discarded: true }).catch((error) => {
|
|
372
|
+
this.#debug("recording_stop_failed", { reason: error?.message });
|
|
373
|
+
});
|
|
360
374
|
this.silent = previousSilent;
|
|
361
375
|
}
|
|
362
376
|
}
|
|
363
377
|
|
|
378
|
+
task(options = {}) {
|
|
379
|
+
return createTask(this, options);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
addStep(instruction, options = {}) {
|
|
383
|
+
return this.task().addStep(instruction, options);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
addAct(instruction, options = {}) {
|
|
387
|
+
return this.task().addAct(instruction, options);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
addWait(conditionOrMs, options = {}) {
|
|
391
|
+
return this.task().addWait(conditionOrMs, options);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
addWaitFor(condition, options = {}) {
|
|
395
|
+
return this.task().addWaitFor(condition, options);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
addPause(ms, options = {}) {
|
|
399
|
+
return this.task().addPause(ms, options);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
addObserve(instruction, options = {}) {
|
|
403
|
+
return this.task().addObserve(instruction, options);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
addExtract(instruction, options = {}) {
|
|
407
|
+
return this.task().addExtract(instruction, options);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
addAssert(instruction, options = {}) {
|
|
411
|
+
return this.task().addAssert(instruction, options);
|
|
412
|
+
}
|
|
413
|
+
|
|
364
414
|
async #initialInput(instruction, data, options, trace) {
|
|
365
415
|
const content = [{ type: "input_text", text: formatInstruction(instruction, data, this.computer, options) }];
|
|
366
416
|
content.push(...(await evaluationContentFor(options.filesToEvaluate)));
|
|
@@ -494,6 +544,41 @@ export class Automify {
|
|
|
494
544
|
return this.#redactScreenshot(screenshot, context, options);
|
|
495
545
|
}
|
|
496
546
|
|
|
547
|
+
async #startScreenRecording(context, options, trace) {
|
|
548
|
+
const screenRecording = screenRecordingFor(options, this);
|
|
549
|
+
if (!screenRecording) return null;
|
|
550
|
+
|
|
551
|
+
const recording = await startScreenRecording(screenRecording, {
|
|
552
|
+
...context,
|
|
553
|
+
captureFrame: (frameContext) => this.#captureRawScreenshot(frameContext, options)
|
|
554
|
+
});
|
|
555
|
+
this.#debug("recording_start", {
|
|
556
|
+
path: recording.path,
|
|
557
|
+
framesDir: recording.framesDir,
|
|
558
|
+
fps: recording.fps
|
|
559
|
+
});
|
|
560
|
+
trace?.({
|
|
561
|
+
type: "recording_start",
|
|
562
|
+
path: recording.path,
|
|
563
|
+
fps: recording.fps
|
|
564
|
+
});
|
|
565
|
+
return recording;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
async #stopScreenRecording(recording, context, trace) {
|
|
569
|
+
if (!recording || recording.stopped) return null;
|
|
570
|
+
recording.stopped = true;
|
|
571
|
+
const result = await recording.stop(context);
|
|
572
|
+
if (!result) return null;
|
|
573
|
+
|
|
574
|
+
this.#debug("recording_stop", result);
|
|
575
|
+
trace?.({
|
|
576
|
+
type: "recording_stop",
|
|
577
|
+
...result
|
|
578
|
+
});
|
|
579
|
+
return result;
|
|
580
|
+
}
|
|
581
|
+
|
|
497
582
|
#computerTool(options) {
|
|
498
583
|
return cleanUndefined({
|
|
499
584
|
type: "computer",
|
|
@@ -813,6 +898,17 @@ function actionScreenshotsPathFor(options, automify) {
|
|
|
813
898
|
return resolveScreenshotPath(options.actionScreenshots ?? automify.actionScreenshots);
|
|
814
899
|
}
|
|
815
900
|
|
|
901
|
+
function screenRecordingFor(options, automify) {
|
|
902
|
+
return options.screenRecording ?? automify.screenRecording;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
function screenRecordingPathFor(options, automify) {
|
|
906
|
+
const recording = screenRecordingFor(options, automify);
|
|
907
|
+
if (typeof recording === "string") return recording;
|
|
908
|
+
if (recording && typeof recording === "object" && typeof recording.path === "string") return recording.path;
|
|
909
|
+
return null;
|
|
910
|
+
}
|
|
911
|
+
|
|
816
912
|
function resolveScreenshotPath(value) {
|
|
817
913
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
818
914
|
}
|
package/src/lib/cli-automify.js
CHANGED
|
@@ -4,6 +4,7 @@ import { OpenAIResponsesClient } from "./openai-responses-client.js";
|
|
|
4
4
|
import { filesToEvaluate } from "./file-data.js";
|
|
5
5
|
import { applyCliPreset } from "./presets.js";
|
|
6
6
|
import { buildRunResult, buildTextConfig } from "./result.js";
|
|
7
|
+
import { createTask } from "./task.js";
|
|
7
8
|
import {
|
|
8
9
|
AUTOMIFY_OPTION_KEYS,
|
|
9
10
|
COMMAND_OPTION_KEYS,
|
|
@@ -188,7 +189,7 @@ export class CliAutomify {
|
|
|
188
189
|
role: "user",
|
|
189
190
|
content: [
|
|
190
191
|
{ type: "input_text", text: formatCliInstruction(instruction, data, options.cwd ?? this.cwd) },
|
|
191
|
-
...await evaluationContentFor(options.filesToEvaluate)
|
|
192
|
+
...(await evaluationContentFor(options.filesToEvaluate))
|
|
192
193
|
]
|
|
193
194
|
}
|
|
194
195
|
],
|
|
@@ -292,6 +293,42 @@ export class CliAutomify {
|
|
|
292
293
|
}
|
|
293
294
|
}
|
|
294
295
|
|
|
296
|
+
task(options = {}) {
|
|
297
|
+
return createTask(this, options);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
addStep(instruction, options = {}) {
|
|
301
|
+
return this.task().addStep(instruction, options);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
addAct(instruction, options = {}) {
|
|
305
|
+
return this.task().addAct(instruction, options);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
addWait(conditionOrMs, options = {}) {
|
|
309
|
+
return this.task().addWait(conditionOrMs, options);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
addWaitFor(condition, options = {}) {
|
|
313
|
+
return this.task().addWaitFor(condition, options);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
addPause(ms, options = {}) {
|
|
317
|
+
return this.task().addPause(ms, options);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
addObserve(instruction, options = {}) {
|
|
321
|
+
return this.task().addObserve(instruction, options);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
addExtract(instruction, options = {}) {
|
|
325
|
+
return this.task().addExtract(instruction, options);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
addAssert(instruction, options = {}) {
|
|
329
|
+
return this.task().addAssert(instruction, options);
|
|
330
|
+
}
|
|
331
|
+
|
|
295
332
|
async #confirm(command, call, response, options) {
|
|
296
333
|
const approval = options.approval ?? this.approval;
|
|
297
334
|
if (approval === "never") return;
|
|
@@ -360,7 +397,9 @@ function cliInstructions(options) {
|
|
|
360
397
|
"After a command changes files, runs tests, or produces the requested result, decide from its output whether another command is necessary. Stop when the task is complete and return a concise summary instead of calling more tools."
|
|
361
398
|
].join("\n"),
|
|
362
399
|
commandPolicyGuidance(options)
|
|
363
|
-
]
|
|
400
|
+
]
|
|
401
|
+
.filter(Boolean)
|
|
402
|
+
.join("\n\n");
|
|
364
403
|
}
|
|
365
404
|
|
|
366
405
|
function assertModel(model) {
|
|
@@ -1,26 +1,14 @@
|
|
|
1
1
|
import { Automify, createAutomify } from "./automify.js";
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
} from "./
|
|
6
|
-
import {
|
|
7
|
-
createLocalDesktopComputer,
|
|
8
|
-
LOCAL_DESKTOP_COMPUTER_OPTION_KEYS
|
|
9
|
-
} from "./local-desktop-computer.js";
|
|
10
|
-
import {
|
|
11
|
-
AUTOMIFY_OPTION_KEYS,
|
|
12
|
-
assertKnownOptions,
|
|
13
|
-
mergeOptionKeys,
|
|
14
|
-
pickKnownOptions
|
|
15
|
-
} from "./runtime.js";
|
|
2
|
+
import { createDockerDesktopComputer, DOCKER_DESKTOP_COMPUTER_OPTION_KEYS } from "./docker-desktop-computer.js";
|
|
3
|
+
import { createVirtualDesktopComputer, VIRTUAL_DESKTOP_COMPUTER_OPTION_KEYS } from "./qemu-desktop-computer.js";
|
|
4
|
+
import { createLocalDesktopComputer, LOCAL_DESKTOP_COMPUTER_OPTION_KEYS } from "./local-desktop-computer.js";
|
|
5
|
+
import { AUTOMIFY_OPTION_KEYS, assertKnownOptions, mergeOptionKeys, pickKnownOptions } from "./runtime.js";
|
|
16
6
|
|
|
17
|
-
const DOCKER_COMPUTER_AUTOMIFY_OPTION_KEYS = mergeOptionKeys(
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
);
|
|
21
|
-
const LOCAL_COMPUTER_AUTOMIFY_OPTION_KEYS = mergeOptionKeys(
|
|
7
|
+
const DOCKER_COMPUTER_AUTOMIFY_OPTION_KEYS = mergeOptionKeys(AUTOMIFY_OPTION_KEYS, DOCKER_DESKTOP_COMPUTER_OPTION_KEYS);
|
|
8
|
+
const LOCAL_COMPUTER_AUTOMIFY_OPTION_KEYS = mergeOptionKeys(AUTOMIFY_OPTION_KEYS, LOCAL_DESKTOP_COMPUTER_OPTION_KEYS);
|
|
9
|
+
const VIRTUAL_COMPUTER_AUTOMIFY_OPTION_KEYS = mergeOptionKeys(
|
|
22
10
|
AUTOMIFY_OPTION_KEYS,
|
|
23
|
-
|
|
11
|
+
VIRTUAL_DESKTOP_COMPUTER_OPTION_KEYS
|
|
24
12
|
);
|
|
25
13
|
|
|
26
14
|
export function createComputerAutomify(options = {}) {
|
|
@@ -33,9 +21,9 @@ export function createComputerAutomify(options = {}) {
|
|
|
33
21
|
export async function createDockerComputerAutomify(options = {}) {
|
|
34
22
|
assertKnownOptions("Docker computer adapter", options, DOCKER_COMPUTER_AUTOMIFY_OPTION_KEYS);
|
|
35
23
|
const usesProvidedComputer = Boolean(options.computer);
|
|
36
|
-
const computer =
|
|
37
|
-
|
|
38
|
-
|
|
24
|
+
const computer =
|
|
25
|
+
options.computer ??
|
|
26
|
+
(await createDockerDesktopComputer(pickKnownOptions(options, DOCKER_DESKTOP_COMPUTER_OPTION_KEYS)));
|
|
39
27
|
const automifyOptions = pickKnownOptions(options, AUTOMIFY_OPTION_KEYS);
|
|
40
28
|
if (!usesProvidedComputer) {
|
|
41
29
|
delete automifyOptions.instructions;
|
|
@@ -47,12 +35,29 @@ export async function createDockerComputerAutomify(options = {}) {
|
|
|
47
35
|
});
|
|
48
36
|
}
|
|
49
37
|
|
|
38
|
+
export async function createVirtualComputerAutomify(options = {}) {
|
|
39
|
+
assertKnownOptions("QEMU virtual computer adapter", options, VIRTUAL_COMPUTER_AUTOMIFY_OPTION_KEYS);
|
|
40
|
+
const usesProvidedComputer = Boolean(options.computer);
|
|
41
|
+
const computer =
|
|
42
|
+
options.computer ??
|
|
43
|
+
(await createVirtualDesktopComputer(pickKnownOptions(options, VIRTUAL_DESKTOP_COMPUTER_OPTION_KEYS)));
|
|
44
|
+
const automifyOptions = pickKnownOptions(options, AUTOMIFY_OPTION_KEYS);
|
|
45
|
+
if (!usesProvidedComputer) {
|
|
46
|
+
delete automifyOptions.instructions;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return new VirtualComputerAutomify({
|
|
50
|
+
...automifyOptions,
|
|
51
|
+
computer
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
50
55
|
export async function createLocalComputerAutomify(options = {}) {
|
|
51
56
|
assertKnownOptions("local computer adapter", options, LOCAL_COMPUTER_AUTOMIFY_OPTION_KEYS);
|
|
52
57
|
const usesProvidedComputer = Boolean(options.computer);
|
|
53
|
-
const computer =
|
|
54
|
-
|
|
55
|
-
|
|
58
|
+
const computer =
|
|
59
|
+
options.computer ??
|
|
60
|
+
(await createLocalDesktopComputer(pickKnownOptions(options, LOCAL_DESKTOP_COMPUTER_OPTION_KEYS)));
|
|
56
61
|
const automifyOptions = pickKnownOptions(options, AUTOMIFY_OPTION_KEYS);
|
|
57
62
|
if (!usesProvidedComputer) {
|
|
58
63
|
delete automifyOptions.instructions;
|
|
@@ -91,6 +96,20 @@ export class DockerComputerAutomify extends Automify {
|
|
|
91
96
|
}
|
|
92
97
|
}
|
|
93
98
|
|
|
99
|
+
export class VirtualComputerAutomify extends Automify {
|
|
100
|
+
constructor(options) {
|
|
101
|
+
super(pickKnownOptions(options, AUTOMIFY_OPTION_KEYS));
|
|
102
|
+
this.session = this.computer.session;
|
|
103
|
+
this.sharedFolder = this.computer.sharedFolder;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async close() {
|
|
107
|
+
if (typeof this.computer.close === "function") {
|
|
108
|
+
await this.computer.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
94
113
|
function defaultComputerEnvironment() {
|
|
95
114
|
switch (process.platform) {
|
|
96
115
|
case "darwin":
|
|
@@ -20,7 +20,7 @@ const execFileAsync = promisify(execFile);
|
|
|
20
20
|
const DEFAULT_IMAGE = "debian:bookworm-slim";
|
|
21
21
|
const DEFAULT_CWD = "/workspace";
|
|
22
22
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
23
|
-
const
|
|
23
|
+
const DOCKER_CLI_OPTION_KEYS = mergeOptionKeys(AUTOMIFY_OPTION_KEYS, [
|
|
24
24
|
"preset",
|
|
25
25
|
"command",
|
|
26
26
|
"commands",
|
|
@@ -297,12 +297,8 @@ export class DockerCliSession {
|
|
|
297
297
|
}
|
|
298
298
|
}
|
|
299
299
|
|
|
300
|
-
export const createVirtualCliAutomify = createDockerCliAutomify;
|
|
301
|
-
export const VirtualCliAutomify = DockerCliAutomify;
|
|
302
|
-
export const DockerVirtualCliSession = DockerCliSession;
|
|
303
|
-
|
|
304
300
|
function normalizeVirtualCliOptions(options = {}) {
|
|
305
|
-
assertKnownOptions("Docker CLI adapter", options,
|
|
301
|
+
assertKnownOptions("Docker CLI adapter", options, DOCKER_CLI_OPTION_KEYS);
|
|
306
302
|
assertKnownOptions("Docker CLI container", options.container, CONTAINER_OPTION_KEYS);
|
|
307
303
|
options = applyDockerCliPreset(options);
|
|
308
304
|
const container = options.container ?? {};
|