codex-computer-use-mcp 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -17
- package/integrations/pi/index.ts +115 -75
- package/package.json +4 -9
- package/dist/code-executor.d.ts +0 -23
- package/dist/code-executor.js +0 -216
- package/dist/code-executor.js.map +0 -1
- package/dist/code-worker.d.ts +0 -1
- package/dist/code-worker.js +0 -120
- package/dist/code-worker.js.map +0 -1
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ OpenAI does not produce or endorse this independent project. It relies on an exp
|
|
|
10
10
|
- Node.js 22 or newer
|
|
11
11
|
- the official ChatGPT macOS app, by default at `/Applications/ChatGPT.app`
|
|
12
12
|
- the Computer Use component installed by ChatGPT, by default under `~/.codex/computer-use/`
|
|
13
|
-
- Pi 0.
|
|
13
|
+
- Pi 0.99.0 or newer when using the Pi integration
|
|
14
14
|
|
|
15
15
|
macOS Screen Recording, Accessibility and TCC controls still apply.
|
|
16
16
|
|
|
@@ -22,25 +22,22 @@ Install from npm:
|
|
|
22
22
|
pi install npm:codex-computer-use-mcp
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
-
The extension registers and
|
|
26
|
-
|
|
27
|
-
```text
|
|
28
|
-
computer_use({ code: string })
|
|
29
|
-
```
|
|
30
|
-
|
|
31
|
-
The code runs with `sky`, `emit`, `emitImage` and a persistent `store` object. `sky` exposes all ten official Computer Use methods, so known sequential actions can run without a model round-trip between each action:
|
|
25
|
+
The extension registers the ten official Computer Use methods as Pi tools named `computer_use_<method>`, such as `computer_use_get_app_state` and `computer_use_click`. They use Pi's `codemode` exposure, so agents call and compose them from Pi's built-in `codemode` tool together with other Pi and MCP tools:
|
|
32
26
|
|
|
33
27
|
```js
|
|
34
|
-
const state = await
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
await
|
|
39
|
-
|
|
40
|
-
|
|
28
|
+
const state = await tools.computer_use_get_app_state({ app: "TextEdit" });
|
|
29
|
+
text(state.text);
|
|
30
|
+
image(state.screenshot);
|
|
31
|
+
|
|
32
|
+
await tools.computer_use_click({ app: "TextEdit", element_index: "7" });
|
|
33
|
+
await tools.computer_use_type_text({ app: "TextEdit", text: "hello" });
|
|
34
|
+
const next = await tools.computer_use_get_app_state({ app: "TextEdit" });
|
|
35
|
+
text(next.text);
|
|
41
36
|
```
|
|
42
37
|
|
|
43
|
-
|
|
38
|
+
The extension does not change which tools are active. Turn on `codemode` with `"defaultTools": ["+codemode"]` in `settings.json` or with `pi --tools +codemode`; Pi also turns it on when an MCP server uses `codemode` exposure. When neither `codemode` nor a Computer Use tool is active, the extension shows one notice at the first prompt.
|
|
39
|
+
|
|
40
|
+
`computer_use_get_app_state` resolves to `{ app, text, screenshot }`, where `screenshot` is an image block for `image()` or `null`. It may return an accessibility-tree diff after the first inspection; pass `disableDiff: true` to request a fresh full tree. `computer_use_list_apps` resolves to the official text inventory produced by the app-server transport; unlike native `@oai/sky`, that transport does not provide the structured app array. Other methods resolve to their official text output. A failed call rejects with the official error text. Calls run one at a time, and calls after `get_app_state` reuse the signed session until the agent settles. The namespace instructions, available from `describeNamespace("computer_use")`, summarise this workflow.
|
|
44
41
|
|
|
45
42
|
Use `/computer-use-status` to inspect the installed component and transport status.
|
|
46
43
|
|
|
@@ -78,7 +75,7 @@ Example Pi MCP configuration:
|
|
|
78
75
|
|
|
79
76
|
## Behaviour
|
|
80
77
|
|
|
81
|
-
The adapter has one mode. Pi
|
|
78
|
+
The adapter has one mode. Pi and MCP both expose the ten official methods as typed tools; Pi gives them `codemode` exposure. Both are available without wrapper permission prompts. It adds no app allowlist, action gate, intent classifier, selector rewrite or focus policy. App-server uses Codex Full access. The adapter forwards any elicitation that the official host still emits.
|
|
82
79
|
|
|
83
80
|
Production calls require verified OpenAI-signed app-server and Computer Use binaries with Team ID `2DC432GLL2`. The adapter uses an isolated, credential-free app-server context. It rejects any model-turn activity. Calls after `get_app_state` reuse the signed session, preserving element identifiers and official app state.
|
|
84
81
|
|
package/integrations/pi/index.ts
CHANGED
|
@@ -8,55 +8,71 @@ import {
|
|
|
8
8
|
getAgentDir,
|
|
9
9
|
truncateHead,
|
|
10
10
|
type ExtensionAPI,
|
|
11
|
+
type ToolNamespace,
|
|
11
12
|
} from "@earendil-works/pi-coding-agent";
|
|
12
|
-
import { Text } from "@earendil-works/pi-tui";
|
|
13
13
|
import { z } from "zod";
|
|
14
14
|
import { getDirectStatus } from "../../dist/direct-service.js";
|
|
15
15
|
import {
|
|
16
16
|
type DirectBrokerElicitationRequest,
|
|
17
17
|
type DirectBrokerElicitationResponse,
|
|
18
18
|
} from "../../dist/direct-broker.js";
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
19
|
+
import { DirectSessionExecutor } from "../../dist/session-executor.js";
|
|
20
|
+
import {
|
|
21
|
+
COMPUTER_USE_METHODS,
|
|
22
|
+
TOOL_INPUT_SCHEMAS,
|
|
23
|
+
TOOL_METADATA,
|
|
24
|
+
type DirectMethod,
|
|
25
|
+
type JsonObject,
|
|
26
|
+
} from "../../dist/tools.js";
|
|
21
27
|
|
|
22
28
|
const jsonObjectSchema = z.record(z.string(), z.json());
|
|
23
|
-
|
|
29
|
+
|
|
30
|
+
export const COMPUTER_USE_NAMESPACE: ToolNamespace = {
|
|
31
|
+
name: "computer_use",
|
|
32
|
+
description: "OpenAI's official signed macOS Computer Use. Call get_app_state for an app before acting on it, run known actions in sequence, then call get_app_state again before deciding the next step.",
|
|
33
|
+
instructions: `Compose these tools in codemode scripts:
|
|
34
|
+
|
|
35
|
+
const state = await tools.computer_use_get_app_state({ app: "TextEdit" });
|
|
36
|
+
text(state.text);
|
|
37
|
+
image(state.screenshot);
|
|
38
|
+
await tools.computer_use_click({ app: "TextEdit", element_index: "7" });
|
|
39
|
+
await tools.computer_use_type_text({ app: "TextEdit", text: "hello" });
|
|
40
|
+
|
|
41
|
+
get_app_state resolves to { app, text, screenshot }. text is the accessibility tree; after the first inspection it may be a diff, so pass disableDiff: true for a fresh full tree. screenshot is an image block for image(), or null.
|
|
42
|
+
Take element_index values from the latest get_app_state text, and inspect again after acting.
|
|
43
|
+
Calls run one at a time, in the order the script makes them.
|
|
44
|
+
list_apps resolves to the official text inventory. A failed call rejects with the official error text.`,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export function computerUseToolName(method: DirectMethod): string {
|
|
48
|
+
return `computer_use_${method}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const appStateOutputSchema = {
|
|
24
52
|
type: "object",
|
|
25
53
|
additionalProperties: false,
|
|
26
|
-
required: ["
|
|
54
|
+
required: ["app", "text", "screenshot"],
|
|
27
55
|
properties: {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
56
|
+
app: { type: "string" },
|
|
57
|
+
text: { type: "string", description: "Accessibility tree, or a diff after the first inspection" },
|
|
58
|
+
screenshot: {
|
|
59
|
+
anyOf: [
|
|
60
|
+
{
|
|
61
|
+
type: "object",
|
|
62
|
+
additionalProperties: false,
|
|
63
|
+
required: ["type", "data", "mimeType"],
|
|
64
|
+
properties: {
|
|
65
|
+
type: { type: "string", const: "image" },
|
|
66
|
+
data: { type: "string" },
|
|
67
|
+
mimeType: { type: "string" },
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
{ type: "null" },
|
|
71
|
+
],
|
|
31
72
|
},
|
|
32
73
|
},
|
|
33
74
|
} as const;
|
|
34
75
|
|
|
35
|
-
const codeDescription = `Run JavaScript that composes OpenAI's official signed macOS Computer Use methods in one call. No nested model is used.
|
|
36
|
-
|
|
37
|
-
Available globals:
|
|
38
|
-
- sky.list_apps() -> text app inventory
|
|
39
|
-
- sky.get_app_state({ app, disableDiff? }) -> { app, text, screenshot }
|
|
40
|
-
- sky.click({ app, element_index?, x?, y?, mouse_button?, click_count? })
|
|
41
|
-
- sky.perform_secondary_action({ app, element_index, action })
|
|
42
|
-
- sky.set_value({ app, element_index, value })
|
|
43
|
-
- sky.select_text({ app, element_index, text, prefix?, suffix?, selection? })
|
|
44
|
-
- sky.scroll({ app, element_index, direction, pages? })
|
|
45
|
-
- sky.drag({ app, from_x, from_y, to_x, to_y })
|
|
46
|
-
- sky.press_key({ app, key })
|
|
47
|
-
- sky.type_text({ app, text })
|
|
48
|
-
- emit(value) returns text or JSON to Pi
|
|
49
|
-
- emitImage(state.screenshot) returns a screenshot to Pi
|
|
50
|
-
- store is a persistent JSON object shared across calls
|
|
51
|
-
|
|
52
|
-
get_app_state may return an accessibility-tree diff after the first inspection. Pass disableDiff: true when you need a fresh full tree.
|
|
53
|
-
|
|
54
|
-
Example:
|
|
55
|
-
const state = await sky.get_app_state({ app: "TextEdit" });
|
|
56
|
-
emit(state.text);
|
|
57
|
-
|
|
58
|
-
Batch known actions sequentially, then inspect again before deciding the next step.`;
|
|
59
|
-
|
|
60
76
|
interface PiContentResult {
|
|
61
77
|
content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }>;
|
|
62
78
|
fullOutputPath?: string;
|
|
@@ -160,9 +176,11 @@ export async function handleOfficialElicitation(
|
|
|
160
176
|
}
|
|
161
177
|
}
|
|
162
178
|
|
|
163
|
-
|
|
179
|
+
type ComputerUseSession = Pick<DirectSessionExecutor, "execute" | "close">;
|
|
180
|
+
|
|
181
|
+
export default function directComputerUse(pi: ExtensionAPI, session: ComputerUseSession = new DirectSessionExecutor()) {
|
|
164
182
|
const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
|
|
165
|
-
const
|
|
183
|
+
const toolNames = COMPUTER_USE_METHODS.map(computerUseToolName);
|
|
166
184
|
|
|
167
185
|
pi.registerCommand("computer-use-status", {
|
|
168
186
|
description: "Show Computer Use status",
|
|
@@ -171,45 +189,67 @@ export default function directComputerUse(pi: ExtensionAPI) {
|
|
|
171
189
|
},
|
|
172
190
|
});
|
|
173
191
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
:
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
192
|
+
for (const method of COMPUTER_USE_METHODS) {
|
|
193
|
+
pi.registerTool({
|
|
194
|
+
name: computerUseToolName(method),
|
|
195
|
+
label: `Computer Use ${method}`,
|
|
196
|
+
description: TOOL_METADATA[method].description,
|
|
197
|
+
// SAFETY: Pi accepts standard JSON Schema objects as custom-tool parameter schemas.
|
|
198
|
+
parameters: TOOL_INPUT_SCHEMAS[method] as any,
|
|
199
|
+
// SAFETY: Pi accepts standard JSON Schema objects as custom-tool output schemas.
|
|
200
|
+
outputSchema: method === "get_app_state" ? appStateOutputSchema as any : undefined,
|
|
201
|
+
exposure: "codemode",
|
|
202
|
+
namespace: COMPUTER_USE_NAMESPACE,
|
|
203
|
+
annotations: TOOL_METADATA[method].annotations,
|
|
204
|
+
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
205
|
+
const args = jsonObjectSchema.parse(params);
|
|
206
|
+
const response = await session.execute(method, args, {
|
|
207
|
+
stateRoot,
|
|
208
|
+
signal,
|
|
209
|
+
supportsOpenAiFormElicitation: true,
|
|
210
|
+
onElicitation: (request) => handleOfficialElicitation(
|
|
211
|
+
request,
|
|
212
|
+
ctx,
|
|
213
|
+
async (url) => (await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 })).code === 0,
|
|
214
|
+
),
|
|
215
|
+
});
|
|
216
|
+
const text = response.content
|
|
217
|
+
.filter((block) => block.type !== "image")
|
|
218
|
+
.map((block) => String(block.text ?? ""))
|
|
219
|
+
.join("\n");
|
|
220
|
+
if (response.isError) throw new Error(text || "Official Computer Use returned an error");
|
|
221
|
+
const rendered = await toPiContent(response.content);
|
|
222
|
+
const details: JsonObject = {};
|
|
223
|
+
if (rendered.fullOutputPath) details.fullOutputPath = rendered.fullOutputPath;
|
|
224
|
+
if (method !== "get_app_state") return { content: rendered.content, details };
|
|
225
|
+
const screenshot = response.content.find((block) => block.type === "image");
|
|
226
|
+
return {
|
|
227
|
+
content: rendered.content,
|
|
228
|
+
details,
|
|
229
|
+
structuredContent: {
|
|
230
|
+
app: String(args.app),
|
|
231
|
+
text,
|
|
232
|
+
screenshot: screenshot
|
|
233
|
+
? { type: "image", data: String(screenshot.data), mimeType: String(screenshot.mimeType) }
|
|
234
|
+
: null,
|
|
235
|
+
},
|
|
236
|
+
};
|
|
237
|
+
},
|
|
238
|
+
});
|
|
239
|
+
}
|
|
211
240
|
|
|
212
|
-
|
|
213
|
-
pi.on("
|
|
214
|
-
pi.on("
|
|
241
|
+
let reachabilityChecked = false;
|
|
242
|
+
pi.on("session_start", () => { reachabilityChecked = false; });
|
|
243
|
+
pi.on("before_agent_start", (_event, ctx) => {
|
|
244
|
+
if (reachabilityChecked) return;
|
|
245
|
+
reachabilityChecked = true;
|
|
246
|
+
const active = pi.getActiveTools();
|
|
247
|
+
if (active.includes("codemode") || toolNames.some((name) => active.includes(name))) return;
|
|
248
|
+
ctx.ui.notify(
|
|
249
|
+
"Computer Use tools are called from codemode, which is not active. Add \"+codemode\" to defaultTools in settings.json, or start Pi with --tools +codemode.",
|
|
250
|
+
"warning",
|
|
251
|
+
);
|
|
252
|
+
});
|
|
253
|
+
pi.on("agent_settled", () => session.close());
|
|
254
|
+
pi.on("session_shutdown", () => session.close());
|
|
215
255
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codex-computer-use-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Expose the official macOS Computer Use tools directly to Pi and MCP clients without a nested model.",
|
|
5
5
|
"author": "Thomas Mustier",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,24 +44,19 @@
|
|
|
44
44
|
"check:pi": "npm run build && npx tsc --ignoreConfig --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --skipLibCheck --types node integrations/pi/index.ts"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@modelcontextprotocol/sdk": "1.
|
|
47
|
+
"@modelcontextprotocol/sdk": "1.32.1",
|
|
48
48
|
"zod": "4.4.3"
|
|
49
49
|
},
|
|
50
50
|
"peerDependencies": {
|
|
51
|
-
"@earendil-works/pi-coding-agent": "
|
|
52
|
-
"@earendil-works/pi-tui": "*"
|
|
51
|
+
"@earendil-works/pi-coding-agent": ">=0.99.0"
|
|
53
52
|
},
|
|
54
53
|
"peerDependenciesMeta": {
|
|
55
54
|
"@earendil-works/pi-coding-agent": {
|
|
56
55
|
"optional": true
|
|
57
|
-
},
|
|
58
|
-
"@earendil-works/pi-tui": {
|
|
59
|
-
"optional": true
|
|
60
56
|
}
|
|
61
57
|
},
|
|
62
58
|
"devDependencies": {
|
|
63
|
-
"@earendil-works/pi-coding-agent": "
|
|
64
|
-
"@earendil-works/pi-tui": "0.80.7",
|
|
59
|
+
"@earendil-works/pi-coding-agent": "1.2.0",
|
|
65
60
|
"@oxlint/plugins": "1.79.0",
|
|
66
61
|
"@types/node": "24.13.3",
|
|
67
62
|
"oxlint": "1.79.0",
|
package/dist/code-executor.d.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { DirectServiceDependencies } from "./direct-service.ts";
|
|
2
|
-
import { DirectSessionExecutor } from "./session-executor.ts";
|
|
3
|
-
import { type DirectMethod, type JsonObject, type JsonValue } from "./tools.ts";
|
|
4
|
-
export interface ComputerUseCodeResult {
|
|
5
|
-
content: JsonObject[];
|
|
6
|
-
calls: DirectMethod[];
|
|
7
|
-
error?: string;
|
|
8
|
-
}
|
|
9
|
-
type CodeSessionExecutor = Pick<DirectSessionExecutor, "execute" | "close">;
|
|
10
|
-
export declare class ComputerUseCodeExecutor {
|
|
11
|
-
private queue;
|
|
12
|
-
private readonly sessionExecutor;
|
|
13
|
-
private readonly codeSliceTimeoutMs;
|
|
14
|
-
private readonly screenshots;
|
|
15
|
-
private nextScreenshotId;
|
|
16
|
-
readonly store: Record<string, JsonValue | undefined>;
|
|
17
|
-
constructor(sessionExecutor?: CodeSessionExecutor, codeSliceTimeoutMs?: number);
|
|
18
|
-
private runExclusive;
|
|
19
|
-
private registerScreenshot;
|
|
20
|
-
execute(code: string, dependencies: DirectServiceDependencies): Promise<ComputerUseCodeResult>;
|
|
21
|
-
close(): Promise<void>;
|
|
22
|
-
}
|
|
23
|
-
export {};
|
package/dist/code-executor.js
DELETED
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
import { Worker } from "node:worker_threads";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
import { DirectSessionExecutor } from "./session-executor.js";
|
|
4
|
-
import { COMPUTER_USE_METHODS, } from "./tools.js";
|
|
5
|
-
const MAX_CODE_BYTES = 20_000;
|
|
6
|
-
const MAX_CALLS = 50;
|
|
7
|
-
const MAX_SCREENSHOT_HANDLES = 50;
|
|
8
|
-
const MAX_EMITTED_IMAGES = 10;
|
|
9
|
-
const MAX_EMITTED_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
10
|
-
const CODE_SLICE_TIMEOUT_MS = 5_000;
|
|
11
|
-
const WORKER_STARTUP_TIMEOUT_MS = 5_000;
|
|
12
|
-
const workerMessageSchema = z.discriminatedUnion("type", [
|
|
13
|
-
z.object({ type: z.literal("ready") }),
|
|
14
|
-
z.object({ type: z.literal("call"), id: z.number().int(), method: z.enum(COMPUTER_USE_METHODS), args: z.string() }),
|
|
15
|
-
z.object({ type: z.literal("emit"), value: z.string() }),
|
|
16
|
-
z.object({ type: z.literal("emit_image"), value: z.string() }),
|
|
17
|
-
z.object({ type: z.literal("done"), store: z.string(), error: z.string().optional() }),
|
|
18
|
-
]);
|
|
19
|
-
const argumentsSchema = z.record(z.string(), z.json());
|
|
20
|
-
const emittedValueSchema = z.json();
|
|
21
|
-
const emittedStringSchema = z.string();
|
|
22
|
-
const screenshotHandleSchema = z.object({
|
|
23
|
-
type: z.literal("computer_use_screenshot"),
|
|
24
|
-
id: z.string(),
|
|
25
|
-
});
|
|
26
|
-
export class ComputerUseCodeExecutor {
|
|
27
|
-
queue = Promise.resolve();
|
|
28
|
-
sessionExecutor;
|
|
29
|
-
codeSliceTimeoutMs;
|
|
30
|
-
screenshots = new Map();
|
|
31
|
-
nextScreenshotId = 1;
|
|
32
|
-
store = {};
|
|
33
|
-
constructor(sessionExecutor = new DirectSessionExecutor(), codeSliceTimeoutMs = CODE_SLICE_TIMEOUT_MS) {
|
|
34
|
-
this.sessionExecutor = sessionExecutor;
|
|
35
|
-
this.codeSliceTimeoutMs = codeSliceTimeoutMs;
|
|
36
|
-
}
|
|
37
|
-
runExclusive(operation) {
|
|
38
|
-
const result = this.queue.then(operation);
|
|
39
|
-
this.queue = result.then(() => undefined, () => undefined);
|
|
40
|
-
return result;
|
|
41
|
-
}
|
|
42
|
-
registerScreenshot(image) {
|
|
43
|
-
const id = String(this.nextScreenshotId++);
|
|
44
|
-
this.screenshots.set(id, image);
|
|
45
|
-
if (this.screenshots.size > MAX_SCREENSHOT_HANDLES) {
|
|
46
|
-
this.screenshots.delete(this.screenshots.keys().next().value);
|
|
47
|
-
}
|
|
48
|
-
return { type: "computer_use_screenshot", id };
|
|
49
|
-
}
|
|
50
|
-
execute(code, dependencies) {
|
|
51
|
-
return this.runExclusive(async () => {
|
|
52
|
-
if (Buffer.byteLength(code, "utf8") > MAX_CODE_BYTES) {
|
|
53
|
-
throw new Error(`Computer Use code exceeds ${MAX_CODE_BYTES} bytes`);
|
|
54
|
-
}
|
|
55
|
-
return new Promise((resolve, reject) => {
|
|
56
|
-
const content = [];
|
|
57
|
-
const calls = [];
|
|
58
|
-
const worker = new Worker(import.meta.url.endsWith(".ts")
|
|
59
|
-
? new URL("../dist/code-worker.js", import.meta.url)
|
|
60
|
-
: new URL("./code-worker.js", import.meta.url), { workerData: { code, store: this.store } });
|
|
61
|
-
let emittedImages = 0;
|
|
62
|
-
let emittedImageBytes = 0;
|
|
63
|
-
let settled = false;
|
|
64
|
-
let timer;
|
|
65
|
-
const clearTimer = () => {
|
|
66
|
-
if (timer)
|
|
67
|
-
clearTimeout(timer);
|
|
68
|
-
timer = undefined;
|
|
69
|
-
};
|
|
70
|
-
const finish = (result) => {
|
|
71
|
-
if (settled)
|
|
72
|
-
return;
|
|
73
|
-
settled = true;
|
|
74
|
-
clearTimer();
|
|
75
|
-
dependencies.signal?.removeEventListener("abort", abort);
|
|
76
|
-
void worker.terminate();
|
|
77
|
-
resolve(result);
|
|
78
|
-
};
|
|
79
|
-
const fail = (error) => {
|
|
80
|
-
if (settled)
|
|
81
|
-
return;
|
|
82
|
-
settled = true;
|
|
83
|
-
clearTimer();
|
|
84
|
-
dependencies.signal?.removeEventListener("abort", abort);
|
|
85
|
-
void worker.terminate();
|
|
86
|
-
reject(error);
|
|
87
|
-
};
|
|
88
|
-
const stopWithError = (message) => {
|
|
89
|
-
content.push({ type: "text", text: `Computer Use code stopped: ${message}` });
|
|
90
|
-
finish({ content, calls, error: message });
|
|
91
|
-
};
|
|
92
|
-
const armTimer = () => {
|
|
93
|
-
clearTimer();
|
|
94
|
-
if (settled)
|
|
95
|
-
return;
|
|
96
|
-
timer = setTimeout(() => stopWithError(`execution exceeded ${this.codeSliceTimeoutMs}ms between Computer Use calls`), this.codeSliceTimeoutMs);
|
|
97
|
-
timer.unref();
|
|
98
|
-
};
|
|
99
|
-
const abort = () => fail(new Error("Computer Use code cancelled"));
|
|
100
|
-
if (dependencies.signal?.aborted) {
|
|
101
|
-
abort();
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
dependencies.signal?.addEventListener("abort", abort, { once: true });
|
|
105
|
-
let messageQueue = Promise.resolve();
|
|
106
|
-
worker.on("message", (rawMessage) => {
|
|
107
|
-
messageQueue = messageQueue.then(async () => {
|
|
108
|
-
if (settled)
|
|
109
|
-
return;
|
|
110
|
-
const message = workerMessageSchema.parse(rawMessage);
|
|
111
|
-
if (message.type === "ready") {
|
|
112
|
-
armTimer();
|
|
113
|
-
return;
|
|
114
|
-
}
|
|
115
|
-
if (message.type === "emit") {
|
|
116
|
-
const parsed = emittedValueSchema.parse(JSON.parse(message.value));
|
|
117
|
-
const stringResult = emittedStringSchema.safeParse(parsed);
|
|
118
|
-
content.push({ type: "text", text: stringResult.success ? stringResult.data : JSON.stringify(parsed, null, 2) });
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
if (message.type === "emit_image") {
|
|
122
|
-
const handle = screenshotHandleSchema.parse(JSON.parse(message.value));
|
|
123
|
-
const image = this.screenshots.get(handle.id);
|
|
124
|
-
if (!image) {
|
|
125
|
-
stopWithError("screenshot is no longer available");
|
|
126
|
-
return;
|
|
127
|
-
}
|
|
128
|
-
const imageBytes = Buffer.byteLength(image.data, "utf8");
|
|
129
|
-
if (emittedImages >= MAX_EMITTED_IMAGES || emittedImageBytes + imageBytes > MAX_EMITTED_IMAGE_BYTES) {
|
|
130
|
-
stopWithError(`emitted images exceed ${MAX_EMITTED_IMAGES} images or ${MAX_EMITTED_IMAGE_BYTES} bytes`);
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
emittedImages += 1;
|
|
134
|
-
emittedImageBytes += imageBytes;
|
|
135
|
-
content.push(image);
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
if (message.type === "done") {
|
|
139
|
-
const nextStore = argumentsSchema.parse(JSON.parse(message.store));
|
|
140
|
-
for (const key of Object.keys(this.store))
|
|
141
|
-
delete this.store[key];
|
|
142
|
-
Object.assign(this.store, nextStore);
|
|
143
|
-
if (message.error)
|
|
144
|
-
stopWithError(message.error);
|
|
145
|
-
else
|
|
146
|
-
finish({ content, calls });
|
|
147
|
-
return;
|
|
148
|
-
}
|
|
149
|
-
if (calls.length >= MAX_CALLS) {
|
|
150
|
-
worker.postMessage({ type: "call_result", id: message.id, error: `Computer Use code exceeded ${MAX_CALLS} calls` });
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
|
-
clearTimer();
|
|
154
|
-
calls.push(message.method);
|
|
155
|
-
try {
|
|
156
|
-
const args = argumentsSchema.parse(JSON.parse(message.args));
|
|
157
|
-
const response = await this.sessionExecutor.execute(message.method, args, dependencies);
|
|
158
|
-
const text = response.content
|
|
159
|
-
.filter((block) => block.type !== "image")
|
|
160
|
-
.map((block) => String(block.text ?? ""))
|
|
161
|
-
.join("\n");
|
|
162
|
-
if (response.isError)
|
|
163
|
-
throw new Error(text.slice(0, 2_000) || "Official Computer Use returned an error");
|
|
164
|
-
let value = response.structuredContent;
|
|
165
|
-
if (message.method === "get_app_state") {
|
|
166
|
-
const block = response.content.find((item) => item.type === "image");
|
|
167
|
-
const screenshot = block ? {
|
|
168
|
-
type: "image",
|
|
169
|
-
data: String(block.data),
|
|
170
|
-
mimeType: String(block.mimeType),
|
|
171
|
-
} : undefined;
|
|
172
|
-
value = {
|
|
173
|
-
app: String(args.app),
|
|
174
|
-
text,
|
|
175
|
-
screenshot: screenshot ? this.registerScreenshot(screenshot) : null,
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
else if (value === undefined) {
|
|
179
|
-
value = text || undefined;
|
|
180
|
-
}
|
|
181
|
-
worker.postMessage({ type: "call_result", id: message.id, value: JSON.stringify(value ?? null) });
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
worker.postMessage({
|
|
185
|
-
type: "call_result",
|
|
186
|
-
id: message.id,
|
|
187
|
-
error: error instanceof Error ? error.message : String(error),
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
finally {
|
|
191
|
-
armTimer();
|
|
192
|
-
}
|
|
193
|
-
}).catch((error) => fail(error instanceof Error ? error : new Error(String(error))));
|
|
194
|
-
});
|
|
195
|
-
worker.once("error", (error) => fail(error));
|
|
196
|
-
worker.once("exit", (code) => {
|
|
197
|
-
if (!settled)
|
|
198
|
-
fail(new Error(`Computer Use code worker exited before completion (${code})`));
|
|
199
|
-
});
|
|
200
|
-
timer = setTimeout(() => fail(new Error("Computer Use code worker failed to start")), WORKER_STARTUP_TIMEOUT_MS);
|
|
201
|
-
timer.unref();
|
|
202
|
-
});
|
|
203
|
-
});
|
|
204
|
-
}
|
|
205
|
-
async close() {
|
|
206
|
-
await this.runExclusive(async () => {
|
|
207
|
-
try {
|
|
208
|
-
await this.sessionExecutor.close();
|
|
209
|
-
}
|
|
210
|
-
finally {
|
|
211
|
-
this.screenshots.clear();
|
|
212
|
-
}
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
//# sourceMappingURL=code-executor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"code-executor.js","sourceRoot":"","sources":["../src/code-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EACN,oBAAoB,GAIpB,MAAM,YAAY,CAAC;AAEpB,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACjD,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAcxC,MAAM,mBAAmB,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACxD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;IACtC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnH,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC9D,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACtF,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACvD,MAAM,kBAAkB,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACpC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;AACvC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,yBAAyB,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACd,CAAC,CAAC;AAIH,MAAM,OAAO,uBAAuB;IAC3B,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACjB,eAAe,CAAsB;IACrC,kBAAkB,CAAS;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IACrD,gBAAgB,GAAG,CAAC,CAAC;IACpB,KAAK,GAA0C,EAAE,CAAC;IAE3D,YAAY,eAAe,GAAwB,IAAI,qBAAqB,EAAE,EAAE,kBAAkB,GAAG,qBAAqB;QACzH,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC9C,CAAC;IAEO,YAAY,CAAI,SAA2B;QAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,kBAAkB,CAAC,KAAiB;QAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,sBAAsB,EAAE,CAAC;YACpD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,yBAAyB,EAAE,EAAE,EAAE,CAAC;IAChD,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,YAAuC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,cAAc,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,cAAc,QAAQ,CAAC,CAAC;YACtE,CAAC;YACD,OAAO,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC7D,MAAM,OAAO,GAAiB,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAmB,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CACxB,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC9B,CAAC,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,kBAAkB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,EAC/C,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAC3C,CAAC;gBACF,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1B,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,IAAI,KAAiC,CAAC;gBACtC,MAAM,UAAU,GAAG,GAAS,EAAE;oBAC7B,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC/B,KAAK,GAAG,SAAS,CAAC;gBACnB,CAAC,CAAC;gBACF,MAAM,MAAM,GAAG,CAAC,MAA6B,EAAQ,EAAE;oBACtD,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,UAAU,EAAE,CAAC;oBACb,YAAY,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACzD,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;oBACxB,OAAO,CAAC,MAAM,CAAC,CAAC;gBACjB,CAAC,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC,KAAY,EAAQ,EAAE;oBACnC,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,UAAU,EAAE,CAAC;oBACb,YAAY,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACzD,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;gBACf,CAAC,CAAC;gBACF,MAAM,aAAa,GAAG,CAAC,OAAe,EAAQ,EAAE;oBAC/C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,8BAA8B,OAAO,EAAE,EAAE,CAAC,CAAC;oBAC9E,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5C,CAAC,CAAC;gBACF,MAAM,QAAQ,GAAG,GAAS,EAAE;oBAC3B,UAAU,EAAE,CAAC;oBACb,IAAI,OAAO;wBAAE,OAAO;oBACpB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,sBAAsB,IAAI,CAAC,kBAAkB,+BAA+B,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC/I,KAAK,CAAC,KAAK,EAAE,CAAC;gBACf,CAAC,CAAC;gBACF,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;gBACzE,IAAI,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBAClC,KAAK,EAAE,CAAC;oBACR,OAAO;gBACR,CAAC;gBACD,YAAY,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtE,IAAI,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,UAA8B,EAAE,EAAE;oBACvD,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;wBAC3C,IAAI,OAAO;4BAAE,OAAO;wBACpB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;wBACtD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;4BAC9B,QAAQ,EAAE,CAAC;4BACX,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACnE,MAAM,YAAY,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BAC3D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BACjH,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;4BACnC,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACvE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;4BAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;gCACZ,aAAa,CAAC,mCAAmC,CAAC,CAAC;gCACnD,OAAO;4BACR,CAAC;4BACD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;4BACzD,IAAI,aAAa,IAAI,kBAAkB,IAAI,iBAAiB,GAAG,UAAU,GAAG,uBAAuB,EAAE,CAAC;gCACrG,aAAa,CAAC,yBAAyB,kBAAkB,cAAc,uBAAuB,QAAQ,CAAC,CAAC;gCACxG,OAAO;4BACR,CAAC;4BACD,aAAa,IAAI,CAAC,CAAC;4BACnB,iBAAiB,IAAI,UAAU,CAAC;4BAChC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BACpB,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BAC7B,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACnE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gCAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;4BAClE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;4BACrC,IAAI,OAAO,CAAC,KAAK;gCAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;;gCAC3C,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;4BAChC,OAAO;wBACR,CAAC;wBAED,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;4BAC/B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,8BAA8B,SAAS,QAAQ,EAAE,CAAC,CAAC;4BACpH,OAAO;wBACR,CAAC;wBACD,UAAU,EAAE,CAAC;wBACb,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC3B,IAAI,CAAC;4BACJ,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;4BACxF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO;iCAC3B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;iCACzC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;iCACxC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACb,IAAI,QAAQ,CAAC,OAAO;gCAAE,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,yCAAyC,CAAC,CAAC;4BACzG,IAAI,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC;4BACvC,IAAI,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;gCACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;gCACrE,MAAM,UAAU,GAA2B,KAAK,CAAC,CAAC,CAAC;oCAClD,IAAI,EAAE,OAAO;oCACb,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;oCACxB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;iCAChC,CAAC,CAAC,CAAC,SAAS,CAAC;gCACd,KAAK,GAAG;oCACP,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;oCACrB,IAAI;oCACJ,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;iCACnE,CAAC;4BACH,CAAC;iCAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gCAChC,KAAK,GAAG,IAAI,IAAI,SAAS,CAAC;4BAC3B,CAAC;4BACD,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;wBACnG,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BAChB,MAAM,CAAC,WAAW,CAAC;gCAClB,IAAI,EAAE,aAAa;gCACnB,EAAE,EAAE,OAAO,CAAC,EAAE;gCACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;6BAC7D,CAAC,CAAC;wBACJ,CAAC;gCAAS,CAAC;4BACV,QAAQ,EAAE,CAAC;wBACZ,CAAC;oBACF,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtF,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;oBAC5B,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,IAAI,KAAK,CAAC,sDAAsD,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC9F,CAAC,CAAC,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;gBACjH,KAAK,CAAC,KAAK,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YACpC,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAC1B,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}
|
package/dist/code-worker.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
package/dist/code-worker.js
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
import vm from "node:vm";
|
|
2
|
-
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
-
import { z } from "zod";
|
|
4
|
-
const MAX_BRIDGE_BYTES = 1_000_000;
|
|
5
|
-
const MAX_EMITS = 100;
|
|
6
|
-
const bridgeStringSchema = z.string();
|
|
7
|
-
// SAFETY: ComputerUseCodeExecutor is the sole worker creator and supplies this exact cloneable shape.
|
|
8
|
-
const input = workerData;
|
|
9
|
-
const port = parentPort;
|
|
10
|
-
if (!port)
|
|
11
|
-
throw new Error("Computer Use code worker requires a parent port");
|
|
12
|
-
let nextCallId = 1;
|
|
13
|
-
const pending = new Map();
|
|
14
|
-
port.on("message", (message) => {
|
|
15
|
-
const waiter = pending.get(message.id);
|
|
16
|
-
if (!waiter)
|
|
17
|
-
return;
|
|
18
|
-
pending.delete(message.id);
|
|
19
|
-
if ("error" in message)
|
|
20
|
-
waiter.reject(new Error(message.error));
|
|
21
|
-
else
|
|
22
|
-
waiter.resolve(message.value);
|
|
23
|
-
});
|
|
24
|
-
let emitCount = 0;
|
|
25
|
-
let emittedBytes = 0;
|
|
26
|
-
const checkBridgeValue = (value, label) => {
|
|
27
|
-
const bytes = Buffer.byteLength(value, "utf8");
|
|
28
|
-
if (bytes > MAX_BRIDGE_BYTES)
|
|
29
|
-
throw new Error(`${label} exceeds ${MAX_BRIDGE_BYTES} bytes`);
|
|
30
|
-
return bytes;
|
|
31
|
-
};
|
|
32
|
-
const callBridge = (method, args) => {
|
|
33
|
-
checkBridgeValue(args, "Computer Use arguments");
|
|
34
|
-
const id = nextCallId++;
|
|
35
|
-
return new Promise((resolve, reject) => {
|
|
36
|
-
pending.set(id, { resolve, reject });
|
|
37
|
-
port.postMessage({ type: "call", id, method, args });
|
|
38
|
-
});
|
|
39
|
-
};
|
|
40
|
-
const emitValue = (type, value) => {
|
|
41
|
-
if (emitCount >= MAX_EMITS)
|
|
42
|
-
throw new Error(`Computer Use code exceeded ${MAX_EMITS} emits`);
|
|
43
|
-
const bytes = checkBridgeValue(value, "Computer Use emitted value");
|
|
44
|
-
if (emittedBytes + bytes > MAX_BRIDGE_BYTES) {
|
|
45
|
-
throw new Error(`Computer Use emitted output exceeds ${MAX_BRIDGE_BYTES} bytes`);
|
|
46
|
-
}
|
|
47
|
-
emitCount += 1;
|
|
48
|
-
emittedBytes += bytes;
|
|
49
|
-
port.postMessage({ type, value });
|
|
50
|
-
};
|
|
51
|
-
const emitBridge = (value) => emitValue("emit", value);
|
|
52
|
-
const emitImageBridge = (value) => emitValue("emit_image", value);
|
|
53
|
-
Object.setPrototypeOf(callBridge, null);
|
|
54
|
-
Object.setPrototypeOf(emitBridge, null);
|
|
55
|
-
Object.setPrototypeOf(emitImageBridge, null);
|
|
56
|
-
const context = vm.createContext({
|
|
57
|
-
__callBridge: callBridge,
|
|
58
|
-
__emitBridge: emitBridge,
|
|
59
|
-
__emitImageBridge: emitImageBridge,
|
|
60
|
-
__storeJson: JSON.stringify(input.store),
|
|
61
|
-
}, {
|
|
62
|
-
codeGeneration: { strings: false, wasm: false },
|
|
63
|
-
name: "computer-use",
|
|
64
|
-
});
|
|
65
|
-
const bootstrap = new vm.Script(`(() => {
|
|
66
|
-
const callBridge = globalThis.__callBridge;
|
|
67
|
-
const emitBridge = globalThis.__emitBridge;
|
|
68
|
-
const emitImageBridge = globalThis.__emitImageBridge;
|
|
69
|
-
const storeJson = globalThis.__storeJson;
|
|
70
|
-
delete globalThis.__callBridge;
|
|
71
|
-
delete globalThis.__emitBridge;
|
|
72
|
-
delete globalThis.__emitImageBridge;
|
|
73
|
-
delete globalThis.__storeJson;
|
|
74
|
-
const call = async (method, args = {}) => JSON.parse(await callBridge(method, JSON.stringify(args)));
|
|
75
|
-
globalThis.sky = Object.freeze({
|
|
76
|
-
list_apps: (args) => call("list_apps", args),
|
|
77
|
-
get_app_state: (args) => call("get_app_state", args),
|
|
78
|
-
click: (args) => call("click", args),
|
|
79
|
-
perform_secondary_action: (args) => call("perform_secondary_action", args),
|
|
80
|
-
set_value: (args) => call("set_value", args),
|
|
81
|
-
select_text: (args) => call("select_text", args),
|
|
82
|
-
scroll: (args) => call("scroll", args),
|
|
83
|
-
drag: (args) => call("drag", args),
|
|
84
|
-
press_key: (args) => call("press_key", args),
|
|
85
|
-
type_text: (args) => call("type_text", args),
|
|
86
|
-
});
|
|
87
|
-
globalThis.emit = (value) => emitBridge(JSON.stringify(value));
|
|
88
|
-
globalThis.emitImage = (value) => {
|
|
89
|
-
if (value === null) throw new Error("get_app_state returned no screenshot");
|
|
90
|
-
emitImageBridge(JSON.stringify(value));
|
|
91
|
-
};
|
|
92
|
-
globalThis.store = JSON.parse(storeJson);
|
|
93
|
-
})()`);
|
|
94
|
-
const serializedStore = () => {
|
|
95
|
-
const parsed = bridgeStringSchema.safeParse(new vm.Script("JSON.stringify(store)").runInContext(context));
|
|
96
|
-
if (!parsed.success)
|
|
97
|
-
throw new Error("Computer Use store must remain a JSON object");
|
|
98
|
-
checkBridgeValue(parsed.data, "Computer Use store");
|
|
99
|
-
return parsed.data;
|
|
100
|
-
};
|
|
101
|
-
try {
|
|
102
|
-
bootstrap.runInContext(context);
|
|
103
|
-
port.postMessage({ type: "ready" });
|
|
104
|
-
const script = new vm.Script(`(async () => {\n${input.code}\n})()`, { filename: "computer-use.js" });
|
|
105
|
-
await script.runInContext(context);
|
|
106
|
-
port.postMessage({ type: "done", store: serializedStore() });
|
|
107
|
-
}
|
|
108
|
-
catch (cause) {
|
|
109
|
-
let error = cause instanceof Error ? cause.message : String(cause);
|
|
110
|
-
let store;
|
|
111
|
-
try {
|
|
112
|
-
store = serializedStore();
|
|
113
|
-
}
|
|
114
|
-
catch (storeError) {
|
|
115
|
-
store = JSON.stringify(input.store);
|
|
116
|
-
error = storeError instanceof Error ? storeError.message : String(storeError);
|
|
117
|
-
}
|
|
118
|
-
port.postMessage({ type: "done", store, error });
|
|
119
|
-
}
|
|
120
|
-
//# sourceMappingURL=code-worker.js.map
|
package/dist/code-worker.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"code-worker.js","sourceRoot":"","sources":["../src/code-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;AAEtC,sGAAsG;AACtG,MAAM,KAAK,GAAG,UAAyB,CAAC;AACxC,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,CAAC,IAAI;IAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;AAE9E,IAAI,UAAU,GAAG,CAAC,CAAC;AACnB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwE,CAAC;AAEhG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA0B,EAAE,EAAE;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,IAAI,OAAO,IAAI,OAAO;QAAE,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;;QAC3D,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC,CAAC,CAAC;AAEH,IAAI,SAAS,GAAG,CAAC,CAAC;AAClB,IAAI,YAAY,GAAG,CAAC,CAAC;AACrB,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE;IACjE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,gBAAgB;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,gBAAgB,QAAQ,CAAC,CAAC;IAC5F,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,IAAY,EAAmB,EAAE;IACpE,gBAAgB,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AACF,MAAM,SAAS,GAAG,CAAC,IAA2B,EAAE,KAAa,EAAQ,EAAE;IACtE,IAAI,SAAS,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,QAAQ,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;IACpE,IAAI,YAAY,GAAG,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,uCAAuC,gBAAgB,QAAQ,CAAC,CAAC;IAClF,CAAC;IACD,SAAS,IAAI,CAAC,CAAC;IACf,YAAY,IAAI,KAAK,CAAC;IACtB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACnC,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAQ,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACrE,MAAM,eAAe,GAAG,CAAC,KAAa,EAAQ,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;AAChF,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACxC,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACxC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;AAE7C,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC;IAChC,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,UAAU;IACxB,iBAAiB,EAAE,eAAe;IAClC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;CACxC,EAAE;IACF,cAAc,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;IAC/C,IAAI,EAAE,cAAc;CACpB,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4B3B,CAAC,CAAC;AAEP,MAAM,eAAe,GAAG,GAAW,EAAE;IACpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1G,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACrF,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC,IAAI,CAAC;AACpB,CAAC,CAAC;AAEF,IAAI,CAAC;IACJ,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,mBAAmB,KAAK,CAAC,IAAI,QAAQ,EAAE,EAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACrG,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;AAC9D,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IAChB,IAAI,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACJ,KAAK,GAAG,eAAe,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,UAAU,EAAE,CAAC;QACrB,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,KAAK,GAAG,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"}
|