@crewhaus/computer-use-driver 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/host.d.ts +43 -0
- package/dist/backends/host.js +75 -0
- package/dist/backends/remote.d.ts +51 -0
- package/dist/backends/remote.js +85 -0
- package/dist/errors.d.ts +5 -0
- package/dist/errors.js +7 -0
- package/dist/index.d.ts +89 -0
- package/dist/index.js +257 -0
- package/dist/ssrf-proxy.d.ts +26 -0
- package/dist/ssrf-proxy.js +341 -0
- package/package.json +9 -6
- package/src/backends/backends.test.ts +0 -396
- package/src/backends/host.ts +0 -96
- package/src/backends/remote.ts +0 -127
- package/src/chromium-import-fail.test.ts +0 -39
- package/src/chromium.test.ts +0 -376
- package/src/errors.ts +0 -8
- package/src/index.test.ts +0 -141
- package/src/index.ts +0 -347
- package/src/ssrf-proxy.test.ts +0 -259
- package/src/ssrf-proxy.ts +0 -376
|
@@ -1,396 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Section 30 — host + remote driver contract tests.
|
|
3
|
-
*/
|
|
4
|
-
import { describe, expect, test } from "bun:test";
|
|
5
|
-
import { ComputerUseDriverError, type Viewport } from "../index";
|
|
6
|
-
import { type HostExecutor, createHostDriver } from "./host";
|
|
7
|
-
import { type PuppeteerCoreLike, createRemoteDriver } from "./remote";
|
|
8
|
-
|
|
9
|
-
describe("host driver — Section 30", () => {
|
|
10
|
-
test("disabled flag → throws on construction", () => {
|
|
11
|
-
expect(() => createHostDriver({ enabled: false, executor: {} as never })).toThrow(
|
|
12
|
-
ComputerUseDriverError,
|
|
13
|
-
);
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
test("connect + every operation delegates to the executor", async () => {
|
|
17
|
-
const calls: string[] = [];
|
|
18
|
-
const viewport: Viewport = { width: 1920, height: 1080, devicePixelRatio: 1 };
|
|
19
|
-
let closed = false;
|
|
20
|
-
const executor: HostExecutor = {
|
|
21
|
-
async click(x, y, button) {
|
|
22
|
-
calls.push(`click ${x},${y} ${button}`);
|
|
23
|
-
},
|
|
24
|
-
async type(text) {
|
|
25
|
-
calls.push(`type ${text}`);
|
|
26
|
-
},
|
|
27
|
-
async key(combo) {
|
|
28
|
-
calls.push(`key ${combo}`);
|
|
29
|
-
},
|
|
30
|
-
async scroll(x, y, dx, dy) {
|
|
31
|
-
calls.push(`scroll ${x},${y} ${dx},${dy}`);
|
|
32
|
-
},
|
|
33
|
-
async screenshot() {
|
|
34
|
-
calls.push("screenshot");
|
|
35
|
-
// "iVBORw0KGgo=" decodes to the PNG magic bytes (0x89 0x50 0x4E 0x47 ...).
|
|
36
|
-
return { pngBase64: "iVBORw0KGgo=", viewport };
|
|
37
|
-
},
|
|
38
|
-
async close() {
|
|
39
|
-
closed = true;
|
|
40
|
-
},
|
|
41
|
-
};
|
|
42
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
43
|
-
await driver.connect();
|
|
44
|
-
await driver.click(100, 200, "left");
|
|
45
|
-
const shot = await driver.screenshot();
|
|
46
|
-
expect(shot).toBeInstanceOf(Uint8Array);
|
|
47
|
-
expect(shot[0]).toBe(0x89);
|
|
48
|
-
expect(shot[1]).toBe(0x50);
|
|
49
|
-
await driver.type("hello");
|
|
50
|
-
await driver.key("cmd+space");
|
|
51
|
-
await driver.scroll(5, 6);
|
|
52
|
-
const vp = await driver.getViewport();
|
|
53
|
-
expect(vp).toEqual(viewport);
|
|
54
|
-
await driver.disconnect();
|
|
55
|
-
expect(closed).toBe(true);
|
|
56
|
-
expect(calls).toEqual([
|
|
57
|
-
"click 100,200 left",
|
|
58
|
-
"screenshot",
|
|
59
|
-
"type hello",
|
|
60
|
-
"key cmd+space",
|
|
61
|
-
"scroll 0,0 5,6",
|
|
62
|
-
"screenshot", // getViewport screenshots to read the viewport
|
|
63
|
-
]);
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
test("click defaults to the left button when none is given", async () => {
|
|
67
|
-
const calls: string[] = [];
|
|
68
|
-
const executor: HostExecutor = {
|
|
69
|
-
async click(x, y, button) {
|
|
70
|
-
calls.push(`click ${x},${y} ${button}`);
|
|
71
|
-
},
|
|
72
|
-
async type() {},
|
|
73
|
-
async key() {},
|
|
74
|
-
async scroll() {},
|
|
75
|
-
async screenshot() {
|
|
76
|
-
return { pngBase64: "", viewport: { width: 0, height: 0, devicePixelRatio: 1 } };
|
|
77
|
-
},
|
|
78
|
-
};
|
|
79
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
80
|
-
await driver.connect();
|
|
81
|
-
await driver.click(7, 8);
|
|
82
|
-
expect(calls).toEqual(["click 7,8 left"]);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
test("disconnect tolerates an executor without a close() method", async () => {
|
|
86
|
-
const executor: HostExecutor = {
|
|
87
|
-
async click() {},
|
|
88
|
-
async type() {},
|
|
89
|
-
async key() {},
|
|
90
|
-
async scroll() {},
|
|
91
|
-
async screenshot() {
|
|
92
|
-
return { pngBase64: "", viewport: { width: 0, height: 0, devicePixelRatio: 1 } };
|
|
93
|
-
},
|
|
94
|
-
// no close — optional-chaining must not throw
|
|
95
|
-
};
|
|
96
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
97
|
-
await driver.connect();
|
|
98
|
-
await expect(driver.disconnect()).resolves.toBeUndefined();
|
|
99
|
-
// After disconnect, guarded operations throw again.
|
|
100
|
-
await expect(driver.screenshot()).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
test("type / key / getViewport reject before connect", async () => {
|
|
104
|
-
const executor: HostExecutor = {
|
|
105
|
-
async click() {},
|
|
106
|
-
async type() {},
|
|
107
|
-
async key() {},
|
|
108
|
-
async scroll() {},
|
|
109
|
-
async screenshot() {
|
|
110
|
-
return { pngBase64: "", viewport: { width: 0, height: 0, devicePixelRatio: 1 } };
|
|
111
|
-
},
|
|
112
|
-
};
|
|
113
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
114
|
-
await expect(driver.type("x")).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
115
|
-
await expect(driver.key("Enter")).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
116
|
-
await expect(driver.scroll(0, 0)).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
117
|
-
await expect(driver.getViewport()).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
118
|
-
await expect(driver.screenshot()).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
test("scroll delegates to executor with zero anchor", async () => {
|
|
122
|
-
const calls: string[] = [];
|
|
123
|
-
const executor: HostExecutor = {
|
|
124
|
-
async click() {},
|
|
125
|
-
async type() {},
|
|
126
|
-
async key() {},
|
|
127
|
-
async scroll(x, y, dx, dy) {
|
|
128
|
-
calls.push(`scroll ${x},${y} ${dx},${dy}`);
|
|
129
|
-
},
|
|
130
|
-
async screenshot() {
|
|
131
|
-
return {
|
|
132
|
-
pngBase64: "",
|
|
133
|
-
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
134
|
-
};
|
|
135
|
-
},
|
|
136
|
-
};
|
|
137
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
138
|
-
await driver.connect();
|
|
139
|
-
await driver.scroll(10, 20);
|
|
140
|
-
expect(calls).toEqual(["scroll 0,0 10,20"]);
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
test("operations before connect throw", async () => {
|
|
144
|
-
const executor = {
|
|
145
|
-
async click() {},
|
|
146
|
-
async type() {},
|
|
147
|
-
async key() {},
|
|
148
|
-
async scroll() {},
|
|
149
|
-
async screenshot() {
|
|
150
|
-
return {
|
|
151
|
-
pngBase64: "",
|
|
152
|
-
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
153
|
-
};
|
|
154
|
-
},
|
|
155
|
-
} as HostExecutor;
|
|
156
|
-
const driver = createHostDriver({ enabled: true, executor });
|
|
157
|
-
await expect(driver.click(0, 0)).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
158
|
-
});
|
|
159
|
-
|
|
160
|
-
test("goto throws — host backend cannot navigate URLs", async () => {
|
|
161
|
-
const driver = createHostDriver({
|
|
162
|
-
enabled: true,
|
|
163
|
-
executor: {
|
|
164
|
-
async screenshot() {
|
|
165
|
-
return {
|
|
166
|
-
pngBase64: "",
|
|
167
|
-
viewport: { width: 0, height: 0, devicePixelRatio: 1 },
|
|
168
|
-
};
|
|
169
|
-
},
|
|
170
|
-
} as HostExecutor,
|
|
171
|
-
});
|
|
172
|
-
await driver.connect();
|
|
173
|
-
await expect(driver.goto("https://x.com")).rejects.toBeInstanceOf(ComputerUseDriverError);
|
|
174
|
-
});
|
|
175
|
-
});
|
|
176
|
-
|
|
177
|
-
describe("remote driver — Section 30", () => {
|
|
178
|
-
test("missing url throws on construction", () => {
|
|
179
|
-
expect(() => createRemoteDriver({ url: "" })).toThrow(ComputerUseDriverError);
|
|
180
|
-
});
|
|
181
|
-
|
|
182
|
-
test("connect without _puppeteer throws with install hint", async () => {
|
|
183
|
-
const driver = createRemoteDriver({ url: "ws://test" });
|
|
184
|
-
await expect(driver.connect()).rejects.toThrow(/puppeteer-core/);
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
test("connect + screenshot + click delegate to puppeteer-core stub", async () => {
|
|
188
|
-
const ops: string[] = [];
|
|
189
|
-
const stubPage = {
|
|
190
|
-
async goto(url: string) {
|
|
191
|
-
ops.push(`goto ${url}`);
|
|
192
|
-
},
|
|
193
|
-
async screenshot() {
|
|
194
|
-
ops.push("screenshot");
|
|
195
|
-
return Buffer.from("png-data");
|
|
196
|
-
},
|
|
197
|
-
mouse: {
|
|
198
|
-
async click(x: number, y: number, opts?: { button?: string }) {
|
|
199
|
-
ops.push(`click ${x},${y} ${opts?.button}`);
|
|
200
|
-
},
|
|
201
|
-
},
|
|
202
|
-
keyboard: {
|
|
203
|
-
async type(text: string) {
|
|
204
|
-
ops.push(`type ${text}`);
|
|
205
|
-
},
|
|
206
|
-
async press(key: string) {
|
|
207
|
-
ops.push(`press ${key}`);
|
|
208
|
-
},
|
|
209
|
-
},
|
|
210
|
-
async evaluate<T>(_fn: () => T) {
|
|
211
|
-
return undefined as unknown as T;
|
|
212
|
-
},
|
|
213
|
-
async close() {
|
|
214
|
-
ops.push("close-page");
|
|
215
|
-
},
|
|
216
|
-
};
|
|
217
|
-
const stubBrowser = {
|
|
218
|
-
async newPage() {
|
|
219
|
-
return stubPage;
|
|
220
|
-
},
|
|
221
|
-
async close() {
|
|
222
|
-
ops.push("close-browser");
|
|
223
|
-
},
|
|
224
|
-
};
|
|
225
|
-
const stubPuppeteer: PuppeteerCoreLike = {
|
|
226
|
-
async connect(_opts) {
|
|
227
|
-
ops.push("connect");
|
|
228
|
-
return stubBrowser as never;
|
|
229
|
-
},
|
|
230
|
-
};
|
|
231
|
-
const driver = createRemoteDriver({ url: "ws://test", _puppeteer: stubPuppeteer });
|
|
232
|
-
await driver.connect();
|
|
233
|
-
await driver.goto("https://x.com");
|
|
234
|
-
await driver.click(50, 60, "right");
|
|
235
|
-
await driver.disconnect();
|
|
236
|
-
expect(ops).toContain("connect");
|
|
237
|
-
expect(ops).toContain("goto https://x.com");
|
|
238
|
-
expect(ops).toContain("click 50,60 right");
|
|
239
|
-
expect(ops).toContain("close-browser");
|
|
240
|
-
});
|
|
241
|
-
|
|
242
|
-
/** Build a fully-stubbed puppeteer-core whose page records into `ops`. */
|
|
243
|
-
function makeStub(ops: string[], opts?: { screenshotAsString?: boolean }) {
|
|
244
|
-
const page = {
|
|
245
|
-
async goto(url: string) {
|
|
246
|
-
ops.push(`goto ${url}`);
|
|
247
|
-
},
|
|
248
|
-
async screenshot(_o?: { encoding?: "base64" }) {
|
|
249
|
-
ops.push("screenshot");
|
|
250
|
-
// "aGk=" is base64 for "hi"; cover both the string and Buffer arms.
|
|
251
|
-
return opts?.screenshotAsString ? "aGk=" : Buffer.from("hi");
|
|
252
|
-
},
|
|
253
|
-
mouse: {
|
|
254
|
-
async click(x: number, y: number, o?: { button?: string }) {
|
|
255
|
-
ops.push(`click ${x},${y} ${o?.button}`);
|
|
256
|
-
},
|
|
257
|
-
},
|
|
258
|
-
keyboard: {
|
|
259
|
-
async type(text: string) {
|
|
260
|
-
ops.push(`type ${text}`);
|
|
261
|
-
},
|
|
262
|
-
async press(key: string) {
|
|
263
|
-
ops.push(`press ${key}`);
|
|
264
|
-
},
|
|
265
|
-
},
|
|
266
|
-
// The real driver casts the page and calls evaluate(fn, arg). Run the fn
|
|
267
|
-
// against a stub global exposing scrollBy so the closure body executes.
|
|
268
|
-
async evaluate(fn: (d: { dx: number; dy: number }) => void, arg: { dx: number; dy: number }) {
|
|
269
|
-
const g = globalThis as Record<string, unknown>;
|
|
270
|
-
const prev = g["scrollBy"];
|
|
271
|
-
g["scrollBy"] = (dx: number, dy: number) => {
|
|
272
|
-
ops.push(`scrollBy ${dx},${dy}`);
|
|
273
|
-
};
|
|
274
|
-
try {
|
|
275
|
-
fn(arg);
|
|
276
|
-
} finally {
|
|
277
|
-
g["scrollBy"] = prev;
|
|
278
|
-
}
|
|
279
|
-
},
|
|
280
|
-
async close() {
|
|
281
|
-
ops.push("close-page");
|
|
282
|
-
},
|
|
283
|
-
};
|
|
284
|
-
const browser = {
|
|
285
|
-
async newPage() {
|
|
286
|
-
return page;
|
|
287
|
-
},
|
|
288
|
-
async close() {
|
|
289
|
-
ops.push("close-browser");
|
|
290
|
-
},
|
|
291
|
-
};
|
|
292
|
-
const puppeteer: PuppeteerCoreLike = {
|
|
293
|
-
async connect(_o) {
|
|
294
|
-
return browser as never;
|
|
295
|
-
},
|
|
296
|
-
};
|
|
297
|
-
return { puppeteer, page, browser };
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
test("screenshot decodes a base64 string from puppeteer", async () => {
|
|
301
|
-
const ops: string[] = [];
|
|
302
|
-
const { puppeteer } = makeStub(ops, { screenshotAsString: true });
|
|
303
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
304
|
-
await driver.connect();
|
|
305
|
-
const png = await driver.screenshot();
|
|
306
|
-
expect(png).toBeInstanceOf(Uint8Array);
|
|
307
|
-
expect(Buffer.from(png).toString()).toBe("hi");
|
|
308
|
-
});
|
|
309
|
-
|
|
310
|
-
test("screenshot re-encodes a Buffer from puppeteer", async () => {
|
|
311
|
-
const ops: string[] = [];
|
|
312
|
-
const { puppeteer } = makeStub(ops, { screenshotAsString: false });
|
|
313
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
314
|
-
await driver.connect();
|
|
315
|
-
const png = await driver.screenshot();
|
|
316
|
-
expect(Buffer.from(png).toString()).toBe("hi");
|
|
317
|
-
});
|
|
318
|
-
|
|
319
|
-
test("type / key / scroll delegate through puppeteer", async () => {
|
|
320
|
-
const ops: string[] = [];
|
|
321
|
-
const { puppeteer } = makeStub(ops);
|
|
322
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
323
|
-
await driver.connect();
|
|
324
|
-
await driver.type("hello");
|
|
325
|
-
await driver.key("Enter");
|
|
326
|
-
await driver.scroll(11, 22);
|
|
327
|
-
expect(ops).toContain("type hello");
|
|
328
|
-
expect(ops).toContain("press Enter");
|
|
329
|
-
// The scroll closure runs in the (stubbed) page context and calls scrollBy.
|
|
330
|
-
expect(ops).toContain("scrollBy 11,22");
|
|
331
|
-
});
|
|
332
|
-
|
|
333
|
-
test("click defaults to the left button when none is given", async () => {
|
|
334
|
-
const ops: string[] = [];
|
|
335
|
-
const { puppeteer } = makeStub(ops);
|
|
336
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
337
|
-
await driver.connect();
|
|
338
|
-
await driver.click(1, 2);
|
|
339
|
-
expect(ops).toContain("click 1,2 left");
|
|
340
|
-
});
|
|
341
|
-
|
|
342
|
-
test("getViewport returns the override when provided", async () => {
|
|
343
|
-
const ops: string[] = [];
|
|
344
|
-
const { puppeteer } = makeStub(ops);
|
|
345
|
-
const viewport: Viewport = { width: 640, height: 480, devicePixelRatio: 2 };
|
|
346
|
-
const driver = createRemoteDriver({ url: "ws://t", viewport, _puppeteer: puppeteer });
|
|
347
|
-
await driver.connect();
|
|
348
|
-
expect(await driver.getViewport()).toEqual(viewport);
|
|
349
|
-
});
|
|
350
|
-
|
|
351
|
-
test("getViewport falls back to 1280x800 when no override", async () => {
|
|
352
|
-
const ops: string[] = [];
|
|
353
|
-
const { puppeteer } = makeStub(ops);
|
|
354
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
355
|
-
await driver.connect();
|
|
356
|
-
expect(await driver.getViewport()).toEqual({ width: 1280, height: 800, devicePixelRatio: 1 });
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
test("operations before connect throw 'not connected'", async () => {
|
|
360
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: makeStub([]).puppeteer });
|
|
361
|
-
await expect(driver.goto("https://x.com")).rejects.toThrow(/not connected/);
|
|
362
|
-
await expect(driver.screenshot()).rejects.toThrow(/not connected/);
|
|
363
|
-
await expect(driver.click(0, 0)).rejects.toThrow(/not connected/);
|
|
364
|
-
await expect(driver.type("x")).rejects.toThrow(/not connected/);
|
|
365
|
-
await expect(driver.key("Enter")).rejects.toThrow(/not connected/);
|
|
366
|
-
await expect(driver.scroll(0, 0)).rejects.toThrow(/not connected/);
|
|
367
|
-
});
|
|
368
|
-
|
|
369
|
-
test("disconnect closes page then browser and resets state", async () => {
|
|
370
|
-
const ops: string[] = [];
|
|
371
|
-
const { puppeteer } = makeStub(ops);
|
|
372
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
373
|
-
await driver.connect();
|
|
374
|
-
await driver.disconnect();
|
|
375
|
-
expect(ops).toContain("close-page");
|
|
376
|
-
expect(ops).toContain("close-browser");
|
|
377
|
-
// After disconnect the page handle is cleared → operations throw again.
|
|
378
|
-
await expect(driver.click(0, 0)).rejects.toThrow(/not connected/);
|
|
379
|
-
});
|
|
380
|
-
|
|
381
|
-
test("disconnect swallows close() failures (best-effort)", async () => {
|
|
382
|
-
const ops: string[] = [];
|
|
383
|
-
const { puppeteer, page } = makeStub(ops);
|
|
384
|
-
page.close = async () => {
|
|
385
|
-
throw new Error("page close failed");
|
|
386
|
-
};
|
|
387
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: puppeteer });
|
|
388
|
-
await driver.connect();
|
|
389
|
-
await expect(driver.disconnect()).resolves.toBeUndefined();
|
|
390
|
-
});
|
|
391
|
-
|
|
392
|
-
test("disconnect before connect is a no-op (no page / browser handles)", async () => {
|
|
393
|
-
const driver = createRemoteDriver({ url: "ws://t", _puppeteer: makeStub([]).puppeteer });
|
|
394
|
-
await expect(driver.disconnect()).resolves.toBeUndefined();
|
|
395
|
-
});
|
|
396
|
-
});
|
package/src/backends/host.ts
DELETED
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Section 30 — host backend for `computer-use-driver`. Drives the
|
|
3
|
-
* dev's actual desktop via the OS-level computer-use surface (macOS
|
|
4
|
-
* Quartz / Linux X11+xdotool / Windows SendInput). Production
|
|
5
|
-
* deployments use this for cross-app workflows that no browser can
|
|
6
|
-
* reach (Maps, Notes, Photos, Finder, native System Settings).
|
|
7
|
-
*
|
|
8
|
-
* **Safety gate:** the constructor refuses to return a working driver
|
|
9
|
-
* unless `CREWHAUS_BROW_HOST_ENABLED=1` is set in the environment. The
|
|
10
|
-
* goal is fail-loud ergonomics: deployment errors that try to use the
|
|
11
|
-
* host backend without explicit opt-in should immediately surface
|
|
12
|
-
* rather than silently driving the dev's machine.
|
|
13
|
-
*
|
|
14
|
-
* The OS-level command dispatch is delegated to a caller-supplied
|
|
15
|
-
* `HostExecutor` so tests + the smoke harness can stub the surface.
|
|
16
|
-
* In production, codegen wires a thin wrapper that shells out to
|
|
17
|
-
* `xdotool` (Linux), `cliclick` (macOS), or PowerShell (Windows).
|
|
18
|
-
*/
|
|
19
|
-
import { ComputerUseDriverError, type Driver, type Viewport } from "../index";
|
|
20
|
-
|
|
21
|
-
export type HostExecutor = {
|
|
22
|
-
/** Click at absolute screen coordinates. Button: "left" | "right" | "middle". */
|
|
23
|
-
click(x: number, y: number, button: string): Promise<void>;
|
|
24
|
-
/** Type a string at the current focus. */
|
|
25
|
-
type(text: string): Promise<void>;
|
|
26
|
-
/** Press a single key (or modifier+key, e.g. "cmd+space"). */
|
|
27
|
-
key(combo: string): Promise<void>;
|
|
28
|
-
/** Scroll at absolute screen coordinates. */
|
|
29
|
-
scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void>;
|
|
30
|
-
/** Capture a PNG screenshot of the entire screen, returning base64. */
|
|
31
|
-
screenshot(): Promise<{ pngBase64: string; viewport: Viewport }>;
|
|
32
|
-
/** Disconnect / clean up any handles. */
|
|
33
|
-
close?(): Promise<void>;
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
export type HostBackendOptions = {
|
|
37
|
-
/** Required to pass; otherwise the constructor throws. */
|
|
38
|
-
readonly enabled: boolean;
|
|
39
|
-
/** Test injection (or production OS shim). */
|
|
40
|
-
readonly executor: HostExecutor;
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
export function createHostDriver(opts: HostBackendOptions): Driver {
|
|
44
|
-
if (!opts.enabled) {
|
|
45
|
-
throw new ComputerUseDriverError(
|
|
46
|
-
"host backend disabled. Set CREWHAUS_BROW_HOST_ENABLED=1 to opt in (this drives the actual desktop — never enable in smoke tests).",
|
|
47
|
-
);
|
|
48
|
-
}
|
|
49
|
-
let connected = false;
|
|
50
|
-
return {
|
|
51
|
-
backend: "host",
|
|
52
|
-
async connect(): Promise<void> {
|
|
53
|
-
connected = true;
|
|
54
|
-
},
|
|
55
|
-
async goto(_url: string): Promise<void> {
|
|
56
|
-
// The host backend doesn't navigate URLs — callers should open
|
|
57
|
-
// a browser app first via the OS shell. We surface this as a
|
|
58
|
-
// soft-fail since some workflows want goto to be a no-op.
|
|
59
|
-
throw new ComputerUseDriverError(
|
|
60
|
-
"host backend cannot navigate URLs directly — open a browser via OS shell first",
|
|
61
|
-
);
|
|
62
|
-
},
|
|
63
|
-
async screenshot(): Promise<Uint8Array> {
|
|
64
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
65
|
-
const { pngBase64 } = await opts.executor.screenshot();
|
|
66
|
-
return Buffer.from(pngBase64, "base64");
|
|
67
|
-
},
|
|
68
|
-
async click(x: number, y: number, button = "left"): Promise<void> {
|
|
69
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
70
|
-
await opts.executor.click(x, y, button);
|
|
71
|
-
},
|
|
72
|
-
async type(text: string): Promise<void> {
|
|
73
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
74
|
-
await opts.executor.type(text);
|
|
75
|
-
},
|
|
76
|
-
async key(combo: string): Promise<void> {
|
|
77
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
78
|
-
await opts.executor.key(combo);
|
|
79
|
-
},
|
|
80
|
-
async scroll(dx: number, dy: number): Promise<void> {
|
|
81
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
82
|
-
// Driver.scroll has no cursor coords; pass (0, 0) for the executor's
|
|
83
|
-
// anchor and let the host shim scroll at the focused window's origin.
|
|
84
|
-
await opts.executor.scroll(0, 0, dx, dy);
|
|
85
|
-
},
|
|
86
|
-
async getViewport(): Promise<Viewport> {
|
|
87
|
-
if (!connected) throw new ComputerUseDriverError("host driver not connected");
|
|
88
|
-
const shot = await opts.executor.screenshot();
|
|
89
|
-
return shot.viewport;
|
|
90
|
-
},
|
|
91
|
-
async disconnect(): Promise<void> {
|
|
92
|
-
connected = false;
|
|
93
|
-
await opts.executor.close?.();
|
|
94
|
-
},
|
|
95
|
-
};
|
|
96
|
-
}
|
package/src/backends/remote.ts
DELETED
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Section 30 — remote backend for `computer-use-driver`. Connects to a
|
|
3
|
-
* remote chromium instance over Chrome DevTools Protocol (browserless.io,
|
|
4
|
-
* BrowserBase, self-hosted DevTools). Same `Driver` surface as the
|
|
5
|
-
* Playwright-backed `chromium` backend; the difference is the connection
|
|
6
|
-
* layer.
|
|
7
|
-
*
|
|
8
|
-
* Production deployments install `puppeteer-core` and pass it as
|
|
9
|
-
* `_puppeteer`. Without it, the constructor returns a driver whose
|
|
10
|
-
* `connect()` throws a clear diagnostic — matching the SQS / Vapi
|
|
11
|
-
* pattern of "abstraction always builds; deployment-time SDK presence
|
|
12
|
-
* gates the operation".
|
|
13
|
-
*/
|
|
14
|
-
import { ComputerUseDriverError, type Driver, type Viewport } from "../index";
|
|
15
|
-
|
|
16
|
-
export type RemoteBackendOptions = {
|
|
17
|
-
/** Browserless / BrowserBase WebSocket endpoint. */
|
|
18
|
-
readonly url: string;
|
|
19
|
-
/** Optional viewport override; defaults to 1280×800. */
|
|
20
|
-
readonly viewport?: Viewport;
|
|
21
|
-
/**
|
|
22
|
-
* Test / production injection: `puppeteer-core` `import` namespace.
|
|
23
|
-
* When undefined, the driver throws at `connect()` with a hint to
|
|
24
|
-
* install puppeteer-core.
|
|
25
|
-
*/
|
|
26
|
-
readonly _puppeteer?: PuppeteerCoreLike;
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
export type PuppeteerCoreLike = {
|
|
30
|
-
connect(opts: { browserWSEndpoint: string }): Promise<{
|
|
31
|
-
newPage(): Promise<{
|
|
32
|
-
goto(url: string): Promise<void>;
|
|
33
|
-
screenshot(opts?: { encoding?: "base64" }): Promise<string | Buffer>;
|
|
34
|
-
mouse: {
|
|
35
|
-
click(x: number, y: number, opts?: { button?: string }): Promise<void>;
|
|
36
|
-
};
|
|
37
|
-
keyboard: {
|
|
38
|
-
type(text: string): Promise<void>;
|
|
39
|
-
press(key: string): Promise<void>;
|
|
40
|
-
};
|
|
41
|
-
evaluate<T>(fn: () => T): Promise<T>;
|
|
42
|
-
close(): Promise<void>;
|
|
43
|
-
}>;
|
|
44
|
-
close(): Promise<void>;
|
|
45
|
-
}>;
|
|
46
|
-
};
|
|
47
|
-
|
|
48
|
-
export function createRemoteDriver(opts: RemoteBackendOptions): Driver {
|
|
49
|
-
if (!opts.url) throw new ComputerUseDriverError("remote backend requires url");
|
|
50
|
-
let browser: Awaited<ReturnType<NonNullable<typeof opts._puppeteer>["connect"]>> | undefined;
|
|
51
|
-
let page: Awaited<ReturnType<NonNullable<typeof browser>["newPage"]>> | undefined;
|
|
52
|
-
let connected = false;
|
|
53
|
-
|
|
54
|
-
function ensure(): NonNullable<typeof page> {
|
|
55
|
-
if (!page) throw new ComputerUseDriverError("remote driver not connected");
|
|
56
|
-
return page;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
return {
|
|
60
|
-
backend: "remote",
|
|
61
|
-
async connect(): Promise<void> {
|
|
62
|
-
const pup = opts._puppeteer;
|
|
63
|
-
if (!pup) {
|
|
64
|
-
throw new ComputerUseDriverError(
|
|
65
|
-
"remote backend requires `puppeteer-core` to be installed and passed as `_puppeteer`. e.g. `import * as pc from 'puppeteer-core'; createDriver({ backend: 'remote', url, _puppeteer: pc })`.",
|
|
66
|
-
);
|
|
67
|
-
}
|
|
68
|
-
browser = await pup.connect({ browserWSEndpoint: opts.url });
|
|
69
|
-
page = await browser.newPage();
|
|
70
|
-
connected = true;
|
|
71
|
-
},
|
|
72
|
-
async goto(url: string): Promise<void> {
|
|
73
|
-
await ensure().goto(url);
|
|
74
|
-
},
|
|
75
|
-
async screenshot(): Promise<Uint8Array> {
|
|
76
|
-
const buffer = await ensure().screenshot({ encoding: "base64" });
|
|
77
|
-
const pngBase64 = typeof buffer === "string" ? buffer : buffer.toString("base64");
|
|
78
|
-
return Buffer.from(pngBase64, "base64");
|
|
79
|
-
},
|
|
80
|
-
async click(x: number, y: number, button = "left"): Promise<void> {
|
|
81
|
-
await ensure().mouse.click(x, y, { button });
|
|
82
|
-
},
|
|
83
|
-
async type(text: string): Promise<void> {
|
|
84
|
-
await ensure().keyboard.type(text);
|
|
85
|
-
},
|
|
86
|
-
async key(combo: string): Promise<void> {
|
|
87
|
-
await ensure().keyboard.press(combo);
|
|
88
|
-
},
|
|
89
|
-
async scroll(dx: number, dy: number): Promise<void> {
|
|
90
|
-
// Puppeteer doesn't expose mouse.wheel directly; dispatch from the
|
|
91
|
-
// page context where `window` is defined. The eval-with-args shape
|
|
92
|
-
// isn't on the local PuppeteerCoreLike type; cast to puppeteer's
|
|
93
|
-
// actual surface to call it.
|
|
94
|
-
const pageWithArgs = ensure() as unknown as {
|
|
95
|
-
evaluate(
|
|
96
|
-
fn: (d: { dx: number; dy: number }) => void,
|
|
97
|
-
arg: { dx: number; dy: number },
|
|
98
|
-
): Promise<void>;
|
|
99
|
-
};
|
|
100
|
-
await pageWithArgs.evaluate(
|
|
101
|
-
(d: { dx: number; dy: number }) => {
|
|
102
|
-
(globalThis as unknown as { scrollBy: (dx: number, dy: number) => void }).scrollBy(
|
|
103
|
-
d.dx,
|
|
104
|
-
d.dy,
|
|
105
|
-
);
|
|
106
|
-
},
|
|
107
|
-
{ dx, dy },
|
|
108
|
-
);
|
|
109
|
-
},
|
|
110
|
-
async getViewport(): Promise<Viewport> {
|
|
111
|
-
const v = opts.viewport;
|
|
112
|
-
if (v !== undefined) return v;
|
|
113
|
-
return { width: 1280, height: 800, devicePixelRatio: 1 };
|
|
114
|
-
},
|
|
115
|
-
async disconnect(): Promise<void> {
|
|
116
|
-
try {
|
|
117
|
-
if (page) await page.close();
|
|
118
|
-
if (browser) await browser.close();
|
|
119
|
-
} catch {
|
|
120
|
-
/* best-effort */
|
|
121
|
-
}
|
|
122
|
-
connected = false;
|
|
123
|
-
page = undefined;
|
|
124
|
-
browser = undefined;
|
|
125
|
-
},
|
|
126
|
-
};
|
|
127
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Catalog R18 — covers the chromium driver's `loadPlaywright` failure path
|
|
3
|
-
* (the catch arm that rethrows a `ComputerUseDriverError` with an install
|
|
4
|
-
* hint when `import("playwright")` rejects).
|
|
5
|
-
*
|
|
6
|
-
* The failing import is injected via the `_importPlaywright` seam instead of
|
|
7
|
-
* `mock.module("playwright", throwingFactory)`: bun's `mock.module` is
|
|
8
|
-
* process-global, and once "playwright" has been successfully imported
|
|
9
|
-
* anywhere in the test process (chromium.test.ts imports it as a fake), a
|
|
10
|
-
* *throwing* factory is evaluated eagerly and blows up at registration time.
|
|
11
|
-
* Because `bun test` runs every file in one process in nondeterministic
|
|
12
|
-
* order, the module-mock approach worked or exploded depending on which file
|
|
13
|
-
* ran first; the seam is order-independent.
|
|
14
|
-
*/
|
|
15
|
-
import { expect, test } from "bun:test";
|
|
16
|
-
import { ComputerUseDriverError, createDriver } from "./index.js";
|
|
17
|
-
|
|
18
|
-
// Rejects the way a missing optional peer dep does, exercising the same
|
|
19
|
-
// catch arm in `loadPlaywright` as a genuine import failure.
|
|
20
|
-
const failingImport = () => Promise.reject(new Error("Cannot find module 'playwright'"));
|
|
21
|
-
|
|
22
|
-
test("connect surfaces a clean diagnostic when playwright import fails", async () => {
|
|
23
|
-
const d = createDriver({ backend: "chromium", _importPlaywright: failingImport });
|
|
24
|
-
await expect(d.connect()).rejects.toThrow(/Playwright not installed/);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
test("the import error is preserved as the error cause", async () => {
|
|
28
|
-
const d = createDriver({ backend: "chromium", _importPlaywright: failingImport });
|
|
29
|
-
try {
|
|
30
|
-
await d.connect();
|
|
31
|
-
throw new Error("should have thrown");
|
|
32
|
-
} catch (err) {
|
|
33
|
-
expect(err).toBeInstanceOf(ComputerUseDriverError);
|
|
34
|
-
// `cause` is the standard Error.cause carrying the original import error.
|
|
35
|
-
const cause = (err as { cause?: unknown }).cause;
|
|
36
|
-
expect(cause).toBeInstanceOf(Error);
|
|
37
|
-
expect((cause as Error).message).toContain("Cannot find module");
|
|
38
|
-
}
|
|
39
|
-
});
|