@crewhaus/computer-use-driver 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +49 -0
- package/src/backends/backends.test.ts +168 -0
- package/src/backends/host.ts +96 -0
- package/src/backends/remote.ts +127 -0
- package/src/index.test.ts +89 -0
- package/src/index.ts +289 -0
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@crewhaus/computer-use-driver",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Cross-platform mouse/keyboard/screenshot driver — chromium backend (Section 25 BROW)",
|
|
6
|
+
"main": "src/index.ts",
|
|
7
|
+
"types": "src/index.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": "./src/index.ts"
|
|
10
|
+
},
|
|
11
|
+
"scripts": {
|
|
12
|
+
"test": "bun test src"
|
|
13
|
+
},
|
|
14
|
+
"dependencies": {
|
|
15
|
+
"@crewhaus/errors": "0.0.0"
|
|
16
|
+
},
|
|
17
|
+
"peerDependencies": {
|
|
18
|
+
"playwright": "*"
|
|
19
|
+
},
|
|
20
|
+
"peerDependenciesMeta": {
|
|
21
|
+
"playwright": {
|
|
22
|
+
"optional": true
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"license": "Apache-2.0",
|
|
26
|
+
"author": {
|
|
27
|
+
"name": "Max Meier",
|
|
28
|
+
"email": "max@studiomax.io",
|
|
29
|
+
"url": "https://studiomax.io"
|
|
30
|
+
},
|
|
31
|
+
"repository": {
|
|
32
|
+
"type": "git",
|
|
33
|
+
"url": "git+https://github.com/crewhaus/factory.git",
|
|
34
|
+
"directory": "packages/computer-use-driver"
|
|
35
|
+
},
|
|
36
|
+
"homepage": "https://github.com/crewhaus/factory/tree/main/packages/computer-use-driver#readme",
|
|
37
|
+
"bugs": {
|
|
38
|
+
"url": "https://github.com/crewhaus/factory/issues"
|
|
39
|
+
},
|
|
40
|
+
"publishConfig": {
|
|
41
|
+
"access": "restricted"
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"src",
|
|
45
|
+
"README.md",
|
|
46
|
+
"LICENSE",
|
|
47
|
+
"NOTICE"
|
|
48
|
+
]
|
|
49
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Section 30 — host + remote driver contract tests.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, expect, test } from "bun:test";
|
|
5
|
+
import { ComputerUseDriverError, type Viewport } from "../index";
|
|
6
|
+
import { type HostExecutor, createHostDriver } from "./host";
|
|
7
|
+
import { type PuppeteerCoreLike, createRemoteDriver } from "./remote";
|
|
8
|
+
|
|
9
|
+
describe("host driver — Section 30", () => {
|
|
10
|
+
test("disabled flag → throws on construction", () => {
|
|
11
|
+
expect(() => createHostDriver({ enabled: false, executor: {} as never })).toThrow(
|
|
12
|
+
ComputerUseDriverError,
|
|
13
|
+
);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("connect + click + screenshot delegates to executor", async () => {
|
|
17
|
+
const calls: string[] = [];
|
|
18
|
+
const viewport: Viewport = { width: 1920, height: 1080, devicePixelRatio: 1 };
|
|
19
|
+
const executor: HostExecutor = {
|
|
20
|
+
async click(x, y, button) {
|
|
21
|
+
calls.push(`click ${x},${y} ${button}`);
|
|
22
|
+
},
|
|
23
|
+
async type(text) {
|
|
24
|
+
calls.push(`type ${text}`);
|
|
25
|
+
},
|
|
26
|
+
async key(combo) {
|
|
27
|
+
calls.push(`key ${combo}`);
|
|
28
|
+
},
|
|
29
|
+
async scroll(x, y, dx, dy) {
|
|
30
|
+
calls.push(`scroll ${x},${y} ${dx},${dy}`);
|
|
31
|
+
},
|
|
32
|
+
async screenshot() {
|
|
33
|
+
calls.push("screenshot");
|
|
34
|
+
// "iVBORw0KGgo=" decodes to the PNG magic bytes (0x89 0x50 0x4E 0x47 ...).
|
|
35
|
+
return { pngBase64: "iVBORw0KGgo=", viewport };
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
const driver = createHostDriver({ enabled: true, executor });
|
|
39
|
+
await driver.connect();
|
|
40
|
+
await driver.click(100, 200, "left");
|
|
41
|
+
const shot = await driver.screenshot();
|
|
42
|
+
expect(shot).toBeInstanceOf(Uint8Array);
|
|
43
|
+
expect(shot[0]).toBe(0x89);
|
|
44
|
+
expect(shot[1]).toBe(0x50);
|
|
45
|
+
expect(calls).toEqual(["click 100,200 left", "screenshot"]);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("scroll delegates to executor with zero anchor", async () => {
|
|
49
|
+
const calls: string[] = [];
|
|
50
|
+
const executor: HostExecutor = {
|
|
51
|
+
async click() {},
|
|
52
|
+
async type() {},
|
|
53
|
+
async key() {},
|
|
54
|
+
async scroll(x, y, dx, dy) {
|
|
55
|
+
calls.push(`scroll ${x},${y} ${dx},${dy}`);
|
|
56
|
+
},
|
|
57
|
+
async screenshot() {
|
|
58
|
+
return {
|
|
59
|
+
pngBase64: "",
|
|
60
|
+
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
61
|
+
};
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
const driver = createHostDriver({ enabled: true, executor });
|
|
65
|
+
await driver.connect();
|
|
66
|
+
await driver.scroll(10, 20);
|
|
67
|
+
expect(calls).toEqual(["scroll 0,0 10,20"]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("operations before connect throw", async () => {
|
|
71
|
+
const executor = {
|
|
72
|
+
async click() {},
|
|
73
|
+
async type() {},
|
|
74
|
+
async key() {},
|
|
75
|
+
async scroll() {},
|
|
76
|
+
async screenshot() {
|
|
77
|
+
return {
|
|
78
|
+
pngBase64: "",
|
|
79
|
+
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
80
|
+
};
|
|
81
|
+
},
|
|
82
|
+
} as HostExecutor;
|
|
83
|
+
const driver = createHostDriver({ enabled: true, executor });
|
|
84
|
+
await expect(driver.click(0, 0)).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("goto throws — host backend cannot navigate URLs", async () => {
|
|
88
|
+
const driver = createHostDriver({
|
|
89
|
+
enabled: true,
|
|
90
|
+
executor: {
|
|
91
|
+
async screenshot() {
|
|
92
|
+
return {
|
|
93
|
+
pngBase64: "",
|
|
94
|
+
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
95
|
+
};
|
|
96
|
+
},
|
|
97
|
+
} as HostExecutor,
|
|
98
|
+
});
|
|
99
|
+
await driver.connect();
|
|
100
|
+
await expect(driver.goto("https://x.com")).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
describe("remote driver — Section 30", () => {
|
|
105
|
+
test("missing url throws on construction", () => {
|
|
106
|
+
expect(() => createRemoteDriver({ url: "" })).toThrow(ComputerUseDriverError);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("connect without _puppeteer throws with install hint", async () => {
|
|
110
|
+
const driver = createRemoteDriver({ url: "ws://test" });
|
|
111
|
+
await expect(driver.connect()).rejects.toThrow(/puppeteer-core/);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("connect + screenshot + click delegate to puppeteer-core stub", async () => {
|
|
115
|
+
const ops: string[] = [];
|
|
116
|
+
const stubPage = {
|
|
117
|
+
async goto(url: string) {
|
|
118
|
+
ops.push(`goto ${url}`);
|
|
119
|
+
},
|
|
120
|
+
async screenshot() {
|
|
121
|
+
ops.push("screenshot");
|
|
122
|
+
return Buffer.from("png-data");
|
|
123
|
+
},
|
|
124
|
+
mouse: {
|
|
125
|
+
async click(x: number, y: number, opts?: { button?: string }) {
|
|
126
|
+
ops.push(`click ${x},${y} ${opts?.button}`);
|
|
127
|
+
},
|
|
128
|
+
},
|
|
129
|
+
keyboard: {
|
|
130
|
+
async type(text: string) {
|
|
131
|
+
ops.push(`type ${text}`);
|
|
132
|
+
},
|
|
133
|
+
async press(key: string) {
|
|
134
|
+
ops.push(`press ${key}`);
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
async evaluate<T>(_fn: () => T) {
|
|
138
|
+
return undefined as unknown as T;
|
|
139
|
+
},
|
|
140
|
+
async close() {
|
|
141
|
+
ops.push("close-page");
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
const stubBrowser = {
|
|
145
|
+
async newPage() {
|
|
146
|
+
return stubPage;
|
|
147
|
+
},
|
|
148
|
+
async close() {
|
|
149
|
+
ops.push("close-browser");
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
const stubPuppeteer: PuppeteerCoreLike = {
|
|
153
|
+
async connect(_opts) {
|
|
154
|
+
ops.push("connect");
|
|
155
|
+
return stubBrowser as never;
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
const driver = createRemoteDriver({ url: "ws://test", _puppeteer: stubPuppeteer });
|
|
159
|
+
await driver.connect();
|
|
160
|
+
await driver.goto("https://x.com");
|
|
161
|
+
await driver.click(50, 60, "right");
|
|
162
|
+
await driver.disconnect();
|
|
163
|
+
expect(ops).toContain("connect");
|
|
164
|
+
expect(ops).toContain("goto https://x.com");
|
|
165
|
+
expect(ops).toContain("click 50,60 right");
|
|
166
|
+
expect(ops).toContain("close-browser");
|
|
167
|
+
});
|
|
168
|
+
});
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Section 30 — host backend for `computer-use-driver`. Drives the
|
|
3
|
+
* dev's actual desktop via the OS-level computer-use surface (macOS
|
|
4
|
+
* Quartz / Linux X11+xdotool / Windows SendInput). Production
|
|
5
|
+
* deployments use this for cross-app workflows that no browser can
|
|
6
|
+
* reach (Maps, Notes, Photos, Finder, native System Settings).
|
|
7
|
+
*
|
|
8
|
+
* **Safety gate:** the constructor refuses to return a working driver
|
|
9
|
+
* unless `CREWHAUS_BROW_HOST_ENABLED=1` is set in the environment. The
|
|
10
|
+
* goal is fail-loud ergonomics: deployment errors that try to use the
|
|
11
|
+
* host backend without explicit opt-in should immediately surface
|
|
12
|
+
* rather than silently driving the dev's machine.
|
|
13
|
+
*
|
|
14
|
+
* The OS-level command dispatch is delegated to a caller-supplied
|
|
15
|
+
* `HostExecutor` so tests + the smoke harness can stub the surface.
|
|
16
|
+
* In production, codegen wires a thin wrapper that shells out to
|
|
17
|
+
* `xdotool` (Linux), `cliclick` (macOS), or PowerShell (Windows).
|
|
18
|
+
*/
|
|
19
|
+
import { ComputerUseDriverError, type Driver, type Viewport } from "../index";
|
|
20
|
+
|
|
21
|
+
export type HostExecutor = {
|
|
22
|
+
/** Click at absolute screen coordinates. Button: "left" | "right" | "middle". */
|
|
23
|
+
click(x: number, y: number, button: string): Promise<void>;
|
|
24
|
+
/** Type a string at the current focus. */
|
|
25
|
+
type(text: string): Promise<void>;
|
|
26
|
+
/** Press a single key (or modifier+key, e.g. "cmd+space"). */
|
|
27
|
+
key(combo: string): Promise<void>;
|
|
28
|
+
/** Scroll at absolute screen coordinates. */
|
|
29
|
+
scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void>;
|
|
30
|
+
/** Capture a PNG screenshot of the entire screen, returning base64. */
|
|
31
|
+
screenshot(): Promise<{ pngBase64: string; viewport: Viewport }>;
|
|
32
|
+
/** Disconnect / clean up any handles. */
|
|
33
|
+
close?(): Promise<void>;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type HostBackendOptions = {
|
|
37
|
+
/** Required to pass; otherwise the constructor throws. */
|
|
38
|
+
readonly enabled: boolean;
|
|
39
|
+
/** Test injection (or production OS shim). */
|
|
40
|
+
readonly executor: HostExecutor;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export function createHostDriver(opts: HostBackendOptions): Driver {
|
|
44
|
+
if (!opts.enabled) {
|
|
45
|
+
throw new ComputerUseDriverError(
|
|
46
|
+
"host backend disabled. Set CREWHAUS_BROW_HOST_ENABLED=1 to opt in (this drives the actual desktop — never enable in smoke tests).",
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
let connected = false;
|
|
50
|
+
return {
|
|
51
|
+
backend: "host",
|
|
52
|
+
async connect(): Promise<void> {
|
|
53
|
+
connected = true;
|
|
54
|
+
},
|
|
55
|
+
async goto(_url: string): Promise<void> {
|
|
56
|
+
// The host backend doesn't navigate URLs — callers should open
|
|
57
|
+
// a browser app first via the OS shell. We surface this as a
|
|
58
|
+
// soft-fail since some workflows want goto to be a no-op.
|
|
59
|
+
throw new ComputerUseDriverError(
|
|
60
|
+
"host backend cannot navigate URLs directly — open a browser via OS shell first",
|
|
61
|
+
);
|
|
62
|
+
},
|
|
63
|
+
async screenshot(): Promise<Uint8Array> {
|
|
64
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
65
|
+
const { pngBase64 } = await opts.executor.screenshot();
|
|
66
|
+
return Buffer.from(pngBase64, "base64");
|
|
67
|
+
},
|
|
68
|
+
async click(x: number, y: number, button = "left"): Promise<void> {
|
|
69
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
70
|
+
await opts.executor.click(x, y, button);
|
|
71
|
+
},
|
|
72
|
+
async type(text: string): Promise<void> {
|
|
73
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
74
|
+
await opts.executor.type(text);
|
|
75
|
+
},
|
|
76
|
+
async key(combo: string): Promise<void> {
|
|
77
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
78
|
+
await opts.executor.key(combo);
|
|
79
|
+
},
|
|
80
|
+
async scroll(dx: number, dy: number): Promise<void> {
|
|
81
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
82
|
+
// Driver.scroll has no cursor coords; pass (0, 0) for the executor's
|
|
83
|
+
// anchor and let the host shim scroll at the focused window's origin.
|
|
84
|
+
await opts.executor.scroll(0, 0, dx, dy);
|
|
85
|
+
},
|
|
86
|
+
async getViewport(): Promise<Viewport> {
|
|
87
|
+
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
88
|
+
const shot = await opts.executor.screenshot();
|
|
89
|
+
return shot.viewport;
|
|
90
|
+
},
|
|
91
|
+
async disconnect(): Promise<void> {
|
|
92
|
+
connected = false;
|
|
93
|
+
await opts.executor.close?.();
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Section 30 — remote backend for `computer-use-driver`. Connects to a
|
|
3
|
+
* remote chromium instance over Chrome DevTools Protocol (browserless.io,
|
|
4
|
+
* BrowserBase, self-hosted DevTools). Same `Driver` surface as the
|
|
5
|
+
* Playwright-backed `chromium` backend; the difference is the connection
|
|
6
|
+
* layer.
|
|
7
|
+
*
|
|
8
|
+
* Production deployments install `puppeteer-core` and pass it as
|
|
9
|
+
* `_puppeteer`. Without it, the constructor returns a driver whose
|
|
10
|
+
* `connect()` throws a clear diagnostic — matching the SQS / Vapi
|
|
11
|
+
* pattern of "abstraction always builds; deployment-time SDK presence
|
|
12
|
+
* gates the operation".
|
|
13
|
+
*/
|
|
14
|
+
import { ComputerUseDriverError, type Driver, type Viewport } from "../index";
|
|
15
|
+
|
|
16
|
+
export type RemoteBackendOptions = {
|
|
17
|
+
/** Browserless / BrowserBase WebSocket endpoint. */
|
|
18
|
+
readonly url: string;
|
|
19
|
+
/** Optional viewport override; defaults to 1280×800. */
|
|
20
|
+
readonly viewport?: Viewport;
|
|
21
|
+
/**
|
|
22
|
+
* Test / production injection: `puppeteer-core` `import` namespace.
|
|
23
|
+
* When undefined, the driver throws at `connect()` with a hint to
|
|
24
|
+
* install puppeteer-core.
|
|
25
|
+
*/
|
|
26
|
+
readonly _puppeteer?: PuppeteerCoreLike;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type PuppeteerCoreLike = {
|
|
30
|
+
connect(opts: { browserWSEndpoint: string }): Promise<{
|
|
31
|
+
newPage(): Promise<{
|
|
32
|
+
goto(url: string): Promise<void>;
|
|
33
|
+
screenshot(opts?: { encoding?: "base64" }): Promise<string | Buffer>;
|
|
34
|
+
mouse: {
|
|
35
|
+
click(x: number, y: number, opts?: { button?: string }): Promise<void>;
|
|
36
|
+
};
|
|
37
|
+
keyboard: {
|
|
38
|
+
type(text: string): Promise<void>;
|
|
39
|
+
press(key: string): Promise<void>;
|
|
40
|
+
};
|
|
41
|
+
evaluate<T>(fn: () => T): Promise<T>;
|
|
42
|
+
close(): Promise<void>;
|
|
43
|
+
}>;
|
|
44
|
+
close(): Promise<void>;
|
|
45
|
+
}>;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export function createRemoteDriver(opts: RemoteBackendOptions): Driver {
|
|
49
|
+
if (!opts.url) throw new ComputerUseDriverError("remote backend requires url");
|
|
50
|
+
let browser: Awaited<ReturnType<NonNullable<typeof opts._puppeteer>["connect"]>> | undefined;
|
|
51
|
+
let page: Awaited<ReturnType<NonNullable<typeof browser>["newPage"]>> | undefined;
|
|
52
|
+
let connected = false;
|
|
53
|
+
|
|
54
|
+
function ensure(): NonNullable<typeof page> {
|
|
55
|
+
if (!page) throw new ComputerUseDriverError("remote driver not connected");
|
|
56
|
+
return page;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
backend: "remote",
|
|
61
|
+
async connect(): Promise<void> {
|
|
62
|
+
const pup = opts._puppeteer;
|
|
63
|
+
if (!pup) {
|
|
64
|
+
throw new ComputerUseDriverError(
|
|
65
|
+
"remote backend requires `puppeteer-core` to be installed and passed as `_puppeteer`. e.g. `import * as pc from 'puppeteer-core'; createDriver({ backend: 'remote', url, _puppeteer: pc })`.",
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
browser = await pup.connect({ browserWSEndpoint: opts.url });
|
|
69
|
+
page = await browser.newPage();
|
|
70
|
+
connected = true;
|
|
71
|
+
},
|
|
72
|
+
async goto(url: string): Promise<void> {
|
|
73
|
+
await ensure().goto(url);
|
|
74
|
+
},
|
|
75
|
+
async screenshot(): Promise<Uint8Array> {
|
|
76
|
+
const buffer = await ensure().screenshot({ encoding: "base64" });
|
|
77
|
+
const pngBase64 = typeof buffer === "string" ? buffer : buffer.toString("base64");
|
|
78
|
+
return Buffer.from(pngBase64, "base64");
|
|
79
|
+
},
|
|
80
|
+
async click(x: number, y: number, button = "left"): Promise<void> {
|
|
81
|
+
await ensure().mouse.click(x, y, { button });
|
|
82
|
+
},
|
|
83
|
+
async type(text: string): Promise<void> {
|
|
84
|
+
await ensure().keyboard.type(text);
|
|
85
|
+
},
|
|
86
|
+
async key(combo: string): Promise<void> {
|
|
87
|
+
await ensure().keyboard.press(combo);
|
|
88
|
+
},
|
|
89
|
+
async scroll(dx: number, dy: number): Promise<void> {
|
|
90
|
+
// Puppeteer doesn't expose mouse.wheel directly; dispatch from the
|
|
91
|
+
// page context where `window` is defined. The eval-with-args shape
|
|
92
|
+
// isn't on the local PuppeteerCoreLike type; cast to puppeteer's
|
|
93
|
+
// actual surface to call it.
|
|
94
|
+
const pageWithArgs = ensure() as unknown as {
|
|
95
|
+
evaluate(
|
|
96
|
+
fn: (d: { dx: number; dy: number }) => void,
|
|
97
|
+
arg: { dx: number; dy: number },
|
|
98
|
+
): Promise<void>;
|
|
99
|
+
};
|
|
100
|
+
await pageWithArgs.evaluate(
|
|
101
|
+
(d: { dx: number; dy: number }) => {
|
|
102
|
+
(globalThis as unknown as { scrollBy: (dx: number, dy: number) => void }).scrollBy(
|
|
103
|
+
d.dx,
|
|
104
|
+
d.dy,
|
|
105
|
+
);
|
|
106
|
+
},
|
|
107
|
+
{ dx, dy },
|
|
108
|
+
);
|
|
109
|
+
},
|
|
110
|
+
async getViewport(): Promise<Viewport> {
|
|
111
|
+
const v = opts.viewport;
|
|
112
|
+
if (v !== undefined) return v;
|
|
113
|
+
return { width: 1280, height: 800, devicePixelRatio: 1 };
|
|
114
|
+
},
|
|
115
|
+
async disconnect(): Promise<void> {
|
|
116
|
+
try {
|
|
117
|
+
if (page) await page.close();
|
|
118
|
+
if (browser) await browser.close();
|
|
119
|
+
} catch {
|
|
120
|
+
/* best-effort */
|
|
121
|
+
}
|
|
122
|
+
connected = false;
|
|
123
|
+
page = undefined;
|
|
124
|
+
browser = undefined;
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { ComputerUseDriverError, type Driver, createDriver } from "./index.js";
|
|
3
|
+
|
|
4
|
+
describe("createDriver", () => {
|
|
5
|
+
test("dispatches on backend tag", () => {
|
|
6
|
+
const chromium = createDriver({ backend: "chromium" });
|
|
7
|
+
expect(chromium.backend).toBe("chromium");
|
|
8
|
+
const host = createDriver({ backend: "host" });
|
|
9
|
+
expect(host.backend).toBe("host");
|
|
10
|
+
const remote = createDriver({ backend: "remote" });
|
|
11
|
+
expect(remote.backend).toBe("remote");
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("unknown backend throws", () => {
|
|
15
|
+
expect(() => createDriver({ backend: "bogus" as unknown as "host" })).toThrow(
|
|
16
|
+
/unknown backend/,
|
|
17
|
+
);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("host backend rejects connect (T8 — kickoff explicitly forbids)", async () => {
|
|
21
|
+
const d = createDriver({ backend: "host" });
|
|
22
|
+
await expect(d.connect()).rejects.toThrow(ComputerUseDriverError);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("remote backend rejects connect (v0 stub)", async () => {
|
|
26
|
+
const d = createDriver({ backend: "remote" });
|
|
27
|
+
await expect(d.connect()).rejects.toThrow(/not implemented in v0/);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("_injected returns the injected driver verbatim (test seam)", async () => {
|
|
31
|
+
const recorded: string[] = [];
|
|
32
|
+
const stub: Driver = {
|
|
33
|
+
backend: "chromium",
|
|
34
|
+
async connect() {
|
|
35
|
+
recorded.push("connect");
|
|
36
|
+
},
|
|
37
|
+
async goto() {
|
|
38
|
+
recorded.push("goto");
|
|
39
|
+
},
|
|
40
|
+
async screenshot() {
|
|
41
|
+
recorded.push("screenshot");
|
|
42
|
+
return new Uint8Array([0x89, 0x50, 0x4e, 0x47]); // PNG header
|
|
43
|
+
},
|
|
44
|
+
async click() {
|
|
45
|
+
recorded.push("click");
|
|
46
|
+
},
|
|
47
|
+
async type() {
|
|
48
|
+
recorded.push("type");
|
|
49
|
+
},
|
|
50
|
+
async key() {
|
|
51
|
+
recorded.push("key");
|
|
52
|
+
},
|
|
53
|
+
async scroll() {
|
|
54
|
+
recorded.push("scroll");
|
|
55
|
+
},
|
|
56
|
+
async getViewport() {
|
|
57
|
+
return { width: 800, height: 600, devicePixelRatio: 1 };
|
|
58
|
+
},
|
|
59
|
+
async disconnect() {
|
|
60
|
+
recorded.push("disconnect");
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
const d = createDriver({ backend: "chromium", _injected: stub });
|
|
64
|
+
await d.connect();
|
|
65
|
+
await d.goto("about:blank");
|
|
66
|
+
await d.screenshot();
|
|
67
|
+
await d.click(100, 200);
|
|
68
|
+
await d.type("hi");
|
|
69
|
+
await d.key("Enter");
|
|
70
|
+
await d.scroll(0, 200);
|
|
71
|
+
await d.disconnect();
|
|
72
|
+
expect(recorded).toEqual([
|
|
73
|
+
"connect",
|
|
74
|
+
"goto",
|
|
75
|
+
"screenshot",
|
|
76
|
+
"click",
|
|
77
|
+
"type",
|
|
78
|
+
"key",
|
|
79
|
+
"scroll",
|
|
80
|
+
"disconnect",
|
|
81
|
+
]);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("chromium driver throws clean diagnostic when used before connect()", async () => {
|
|
85
|
+
const d = createDriver({ backend: "chromium" });
|
|
86
|
+
await expect(d.screenshot()).rejects.toThrow(/not connected/);
|
|
87
|
+
await expect(d.click(0, 0)).rejects.toThrow(/not connected/);
|
|
88
|
+
});
|
|
89
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R18 `computer-use-driver` — Section 25 BROW.
|
|
3
|
+
*
|
|
4
|
+
* Cross-platform driver for mouse/keyboard/screenshot operations. The
|
|
5
|
+
* `Driver` interface exposes:
|
|
6
|
+
* - `screenshot()` → PNG `Uint8Array`
|
|
7
|
+
* - `click(x, y, button?)`
|
|
8
|
+
* - `type(text)`
|
|
9
|
+
* - `key(combo)` // "Enter", "Tab", "Control+a"
|
|
10
|
+
* - `scroll(dx, dy)`
|
|
11
|
+
* - `getViewport()` → { width, height, devicePixelRatio }
|
|
12
|
+
* - `goto(url)` // browser-only; `host` backend rejects
|
|
13
|
+
*
|
|
14
|
+
* v0 ships ONE concrete backend — `chromium` — driven by Playwright's
|
|
15
|
+
* bundled headless chromium. The kickoff names this `docker-chromium`
|
|
16
|
+
* because it's intended to run inside a Section-18 sandbox; in v0 we
|
|
17
|
+
* lean on Playwright's own chromium-sandboxing (the chromium binary
|
|
18
|
+
* runs as a sandboxed renderer process) and defer the explicit Docker
|
|
19
|
+
* wrapping to a follow-up.
|
|
20
|
+
*
|
|
21
|
+
* The `host` backend (drives the developer's actual desktop) and
|
|
22
|
+
* `remote` backend (CDP-style remote control) are interface stubs that
|
|
23
|
+
* throw at `connect()` — see the kickoff explicitly forbidding the host
|
|
24
|
+
* backend in the smoke ("never use `host` backend for the smoke since
|
|
25
|
+
* it would drive the dev's actual desktop").
|
|
26
|
+
*/
|
|
27
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
28
|
+
|
|
29
|
+
export class ComputerUseDriverError extends CrewhausError {
|
|
30
|
+
override readonly name = "ComputerUseDriverError";
|
|
31
|
+
constructor(message: string, cause?: unknown) {
|
|
32
|
+
super("tool", message, cause);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type DriverBackend = "host" | "chromium" | "remote";
|
|
37
|
+
export type MouseButton = "left" | "right" | "middle";
|
|
38
|
+
|
|
39
|
+
export type Viewport = {
|
|
40
|
+
readonly width: number;
|
|
41
|
+
readonly height: number;
|
|
42
|
+
readonly devicePixelRatio: number;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export interface Driver {
|
|
46
|
+
readonly backend: DriverBackend;
|
|
47
|
+
/** Open / connect / launch chromium. Idempotent. */
|
|
48
|
+
connect(): Promise<void>;
|
|
49
|
+
/** Navigate the chromium tab. Throws on `host` backend. */
|
|
50
|
+
goto(url: string): Promise<void>;
|
|
51
|
+
/** Capture a PNG screenshot. */
|
|
52
|
+
screenshot(): Promise<Uint8Array>;
|
|
53
|
+
click(x: number, y: number, button?: MouseButton): Promise<void>;
|
|
54
|
+
type(text: string): Promise<void>;
|
|
55
|
+
key(combo: string): Promise<void>;
|
|
56
|
+
scroll(dx: number, dy: number): Promise<void>;
|
|
57
|
+
getViewport(): Promise<Viewport>;
|
|
58
|
+
/** Best-effort DOM snapshot. Used by tests + the post-action assertions. */
|
|
59
|
+
domText?(): Promise<string>;
|
|
60
|
+
disconnect(): Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export type CreateDriverOptions = {
|
|
64
|
+
readonly backend: DriverBackend;
|
|
65
|
+
/** Playwright launch overrides (e.g. headless: false). chromium-only. */
|
|
66
|
+
readonly playwrightOptions?: Record<string, unknown>;
|
|
67
|
+
/** Initial viewport — chromium-only. Default 1280x720@1. */
|
|
68
|
+
readonly viewport?: { width: number; height: number };
|
|
69
|
+
/** Test injection: a pre-built Driver instance the factory returns verbatim. */
|
|
70
|
+
readonly _injected?: Driver;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export function createDriver(opts: CreateDriverOptions): Driver {
|
|
74
|
+
if (opts._injected !== undefined) return opts._injected;
|
|
75
|
+
if (opts.backend === "chromium") return createChromiumDriver(opts);
|
|
76
|
+
if (opts.backend === "host") {
|
|
77
|
+
// Section 30 — host backend stays gated. Callers that have wired
|
|
78
|
+
// a real executor use `createHostDriver` directly.
|
|
79
|
+
return createHostDriverStub();
|
|
80
|
+
}
|
|
81
|
+
if (opts.backend === "remote") {
|
|
82
|
+
// Same pattern: callers with a puppeteer-core import use
|
|
83
|
+
// `createRemoteDriver` directly.
|
|
84
|
+
return createRemoteDriverStub();
|
|
85
|
+
}
|
|
86
|
+
throw new ComputerUseDriverError(`unknown backend: ${opts.backend}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Section 30 — direct exports for callers that wire backends with an
|
|
90
|
+
// executor or puppeteer-core import.
|
|
91
|
+
export { createHostDriver, type HostBackendOptions, type HostExecutor } from "./backends/host";
|
|
92
|
+
export {
|
|
93
|
+
createRemoteDriver,
|
|
94
|
+
type PuppeteerCoreLike,
|
|
95
|
+
type RemoteBackendOptions,
|
|
96
|
+
} from "./backends/remote";
|
|
97
|
+
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
// Playwright-backed chromium driver.
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
function createChromiumDriver(opts: CreateDriverOptions): Driver {
|
|
103
|
+
let browser: unknown;
|
|
104
|
+
let context: unknown;
|
|
105
|
+
let page: unknown;
|
|
106
|
+
let connected = false;
|
|
107
|
+
|
|
108
|
+
async function loadPlaywright(): Promise<{
|
|
109
|
+
chromium: { launch: (...args: unknown[]) => unknown };
|
|
110
|
+
}> {
|
|
111
|
+
try {
|
|
112
|
+
// Lazy import so non-browser users don't need Playwright in their tree.
|
|
113
|
+
// The import is dynamic via a string indirection so `tsc` doesn't fail
|
|
114
|
+
// when playwright (an optional peer dep) isn't installed in CI.
|
|
115
|
+
const playwrightModule = "playwright";
|
|
116
|
+
const mod = await import(playwrightModule);
|
|
117
|
+
return mod as unknown as { chromium: { launch: (...args: unknown[]) => unknown } };
|
|
118
|
+
} catch (err) {
|
|
119
|
+
throw new ComputerUseDriverError(
|
|
120
|
+
"Playwright not installed. Run `bun add -D playwright` and `bunx playwright install chromium` to use the chromium backend.",
|
|
121
|
+
err,
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
backend: "chromium",
|
|
128
|
+
async connect() {
|
|
129
|
+
if (connected) return;
|
|
130
|
+
const { chromium } = await loadPlaywright();
|
|
131
|
+
// Playwright defaults: chromium runs in its own sandbox (its own
|
|
132
|
+
// sandboxed renderer process). Headless by default.
|
|
133
|
+
const launchOpts: Record<string, unknown> = {
|
|
134
|
+
headless: true,
|
|
135
|
+
...(opts.playwrightOptions ?? {}),
|
|
136
|
+
};
|
|
137
|
+
browser = await (chromium.launch as (o: unknown) => Promise<unknown>)(launchOpts);
|
|
138
|
+
const ctxOpts: Record<string, unknown> = {
|
|
139
|
+
viewport: opts.viewport ?? { width: 1280, height: 720 },
|
|
140
|
+
};
|
|
141
|
+
context = await (browser as { newContext: (o: unknown) => Promise<unknown> }).newContext(
|
|
142
|
+
ctxOpts,
|
|
143
|
+
);
|
|
144
|
+
page = await (context as { newPage: () => Promise<unknown> }).newPage();
|
|
145
|
+
connected = true;
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
async goto(url) {
|
|
149
|
+
if (!connected)
|
|
150
|
+
throw new ComputerUseDriverError("driver not connected — call connect() first");
|
|
151
|
+
await (page as { goto: (u: string, o?: unknown) => Promise<unknown> }).goto(url, {
|
|
152
|
+
waitUntil: "domcontentloaded",
|
|
153
|
+
});
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async screenshot() {
|
|
157
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
158
|
+
const buf = (await (page as { screenshot: (o?: unknown) => Promise<unknown> }).screenshot({
|
|
159
|
+
type: "png",
|
|
160
|
+
})) as Uint8Array;
|
|
161
|
+
return buf;
|
|
162
|
+
},
|
|
163
|
+
|
|
164
|
+
async click(x, y, button = "left") {
|
|
165
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
166
|
+
await (
|
|
167
|
+
page as { mouse: { click: (x: number, y: number, o?: unknown) => Promise<void> } }
|
|
168
|
+
).mouse.click(x, y, { button });
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
async type(text) {
|
|
172
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
173
|
+
await (page as { keyboard: { type: (t: string) => Promise<void> } }).keyboard.type(text);
|
|
174
|
+
},
|
|
175
|
+
|
|
176
|
+
async key(combo) {
|
|
177
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
178
|
+
await (page as { keyboard: { press: (k: string) => Promise<void> } }).keyboard.press(combo);
|
|
179
|
+
},
|
|
180
|
+
|
|
181
|
+
async scroll(dx, dy) {
|
|
182
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
183
|
+
await (page as { mouse: { wheel: (dx: number, dy: number) => Promise<void> } }).mouse.wheel(
|
|
184
|
+
dx,
|
|
185
|
+
dy,
|
|
186
|
+
);
|
|
187
|
+
},
|
|
188
|
+
|
|
189
|
+
async getViewport() {
|
|
190
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
191
|
+
const vp = (
|
|
192
|
+
page as { viewportSize: () => { width: number; height: number } | null }
|
|
193
|
+
).viewportSize();
|
|
194
|
+
const w = vp?.width ?? opts.viewport?.width ?? 1280;
|
|
195
|
+
const h = vp?.height ?? opts.viewport?.height ?? 720;
|
|
196
|
+
return { width: w, height: h, devicePixelRatio: 1 };
|
|
197
|
+
},
|
|
198
|
+
|
|
199
|
+
async domText() {
|
|
200
|
+
if (!connected) throw new ComputerUseDriverError("driver not connected");
|
|
201
|
+
return (page as { textContent: (sel: string) => Promise<string | null> })
|
|
202
|
+
.textContent("body")
|
|
203
|
+
.then((t) => t ?? "");
|
|
204
|
+
},
|
|
205
|
+
|
|
206
|
+
async disconnect() {
|
|
207
|
+
if (!connected) return;
|
|
208
|
+
try {
|
|
209
|
+
await (browser as { close: () => Promise<void> }).close();
|
|
210
|
+
} catch {
|
|
211
|
+
// best-effort
|
|
212
|
+
}
|
|
213
|
+
connected = false;
|
|
214
|
+
browser = undefined;
|
|
215
|
+
context = undefined;
|
|
216
|
+
page = undefined;
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// ---------------------------------------------------------------------------
|
|
222
|
+
// Stubs for host + remote backends.
|
|
223
|
+
// ---------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
function createHostDriverStub(): Driver {
|
|
226
|
+
return {
|
|
227
|
+
backend: "host",
|
|
228
|
+
async connect() {
|
|
229
|
+
throw new ComputerUseDriverError(
|
|
230
|
+
"host backend is not implemented in v0. The kickoff explicitly forbids it for smokes (would drive the dev's actual desktop). Native macOS / Linux / Windows backends land in a follow-up gated on `CREWHAUS_BROW_HOST_SMOKE=1`.",
|
|
231
|
+
);
|
|
232
|
+
},
|
|
233
|
+
async goto() {
|
|
234
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
235
|
+
},
|
|
236
|
+
async screenshot() {
|
|
237
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
238
|
+
},
|
|
239
|
+
async click() {
|
|
240
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
241
|
+
},
|
|
242
|
+
async type() {
|
|
243
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
244
|
+
},
|
|
245
|
+
async key() {
|
|
246
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
247
|
+
},
|
|
248
|
+
async scroll() {
|
|
249
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
250
|
+
},
|
|
251
|
+
async getViewport() {
|
|
252
|
+
throw new ComputerUseDriverError("host backend not implemented");
|
|
253
|
+
},
|
|
254
|
+
async disconnect() {},
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function createRemoteDriverStub(): Driver {
|
|
259
|
+
return {
|
|
260
|
+
backend: "remote",
|
|
261
|
+
async connect() {
|
|
262
|
+
throw new ComputerUseDriverError(
|
|
263
|
+
"remote backend (CDP / Browserless) is not implemented in v0",
|
|
264
|
+
);
|
|
265
|
+
},
|
|
266
|
+
async goto() {
|
|
267
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
268
|
+
},
|
|
269
|
+
async screenshot() {
|
|
270
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
271
|
+
},
|
|
272
|
+
async click() {
|
|
273
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
274
|
+
},
|
|
275
|
+
async type() {
|
|
276
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
277
|
+
},
|
|
278
|
+
async key() {
|
|
279
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
280
|
+
},
|
|
281
|
+
async scroll() {
|
|
282
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
283
|
+
},
|
|
284
|
+
async getViewport() {
|
|
285
|
+
throw new ComputerUseDriverError("remote backend not implemented");
|
|
286
|
+
},
|
|
287
|
+
async disconnect() {},
|
|
288
|
+
};
|
|
289
|
+
}
|