@omercnet/paseo-shared-browser 1.2.1 → 1.2.2-next.237.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +312 -25
- package/REMOTE_BROWSER_RESEARCH.md +395 -0
- package/STREAMING_DESIGN.md +191 -0
- package/client/browser-auto-capture-density.test.ts +102 -0
- package/client/browser-auto-capture-density.ts +30 -0
- package/client/browser-canvas-input.test.ts +198 -0
- package/client/browser-canvas-input.ts +202 -0
- package/client/browser-canvas-keyboard-web.test.ts +113 -0
- package/client/browser-canvas-keyboard-web.ts +230 -0
- package/client/browser-canvas-keyboard.test.ts +123 -0
- package/client/browser-canvas-keyboard.ts +126 -0
- package/client/browser-canvas-layout.test.ts +109 -0
- package/client/browser-canvas-layout.ts +51 -0
- package/client/browser-canvas-viewport-web.test.ts +77 -0
- package/client/browser-canvas-viewport-web.ts +63 -0
- package/client/browser-canvas-viewport.tsx +152 -0
- package/client/browser-capture-cadence.test.ts +9 -0
- package/client/browser-capture-cadence.ts +6 -0
- package/client/browser-capture-density-controls.tsx +75 -0
- package/client/browser-chrome.tsx +234 -0
- package/client/browser-client-presence.ts +184 -0
- package/client/browser-compose-controls.test.tsx +275 -0
- package/client/browser-compose-controls.tsx +174 -0
- package/client/browser-control-button.tsx +109 -0
- package/client/browser-control-dialog.tsx +71 -0
- package/client/browser-control-request.test.ts +27 -0
- package/client/browser-control-request.ts +57 -0
- package/client/browser-control-tooltip-web.ts +10 -0
- package/client/browser-dialog-body.tsx +22 -0
- package/client/browser-emulation-mode.test.ts +75 -0
- package/client/browser-emulation-mode.ts +98 -0
- package/client/browser-fill-viewport.test.ts +45 -0
- package/client/browser-fill-viewport.ts +28 -0
- package/client/browser-frame-buffer.test.ts +157 -0
- package/client/browser-frame-buffer.ts +148 -0
- package/client/browser-frame-image.tsx +62 -0
- package/client/browser-image-load.test.ts +23 -0
- package/client/browser-image-load.ts +26 -0
- package/client/browser-input-queue.test.ts +1201 -0
- package/client/browser-input-queue.ts +505 -0
- package/client/browser-menu-placement.test.ts +90 -0
- package/client/browser-menu-placement.ts +78 -0
- package/client/browser-panel-styles.ts +384 -0
- package/client/browser-quality-controls.test.ts +26 -0
- package/client/browser-quality-controls.tsx +139 -0
- package/client/browser-resolution-picker.tsx +189 -0
- package/client/browser-tabs.tsx +117 -0
- package/client/browser-toolbar-menu-web.test.ts +185 -0
- package/client/browser-toolbar-menu-web.ts +159 -0
- package/client/browser-toolbar-menu.tsx +255 -0
- package/client/browser-toolbar-submenu.tsx +93 -0
- package/client/browser-video-bytes.test.ts +44 -0
- package/client/browser-video-decoder.test.ts +473 -0
- package/client/browser-video-decoder.ts +311 -0
- package/client/browser-video-surface.tsx +21 -0
- package/client/browser-video-visibility.test.ts +45 -0
- package/client/browser-viewer-recovery.test.ts +111 -0
- package/client/browser-viewer-recovery.ts +78 -0
- package/client/browser.tsx +1642 -1569
- package/client/favorite-star.tsx +23 -0
- package/client/frame-lifecycle.ts +62 -59
- package/client/use-browser-auto-capture-density.test.ts +104 -0
- package/client/use-browser-auto-capture-density.ts +43 -0
- package/client/use-browser-canvas-input.ts +370 -0
- package/client/use-browser-capture-density.ts +87 -0
- package/client/use-browser-emulation-mode.ts +56 -0
- package/client/use-browser-fill-viewport.test.ts +108 -0
- package/client/use-browser-fill-viewport.ts +62 -0
- package/client/use-browser-frame-buffer.ts +66 -0
- package/client/use-browser-image-capture.ts +139 -0
- package/client/use-browser-scoped-mutation.test.ts +118 -0
- package/client/use-browser-scoped-mutation.ts +59 -0
- package/client/use-browser-video.test.ts +991 -0
- package/client/use-browser-video.ts +474 -0
- package/client/use-browser-viewer-recovery.ts +26 -0
- package/client/use-resolution-favorites.ts +90 -0
- package/client/web.test.ts +125 -0
- package/client/web.ts +405 -0
- package/index.client.tsx +0 -10
- package/index.server.ts +32 -0
- package/npm-shrinkwrap.json +1834 -0
- package/package.json +13 -8
- package/paseo-plugin.json +3 -1
- package/scripts/prepare-dependencies.mjs +1 -1
- package/scripts/prepare-runtime.mjs +1 -1
- package/server/agent-browser-runtime.ts +1598 -314
- package/server/browser-cursor.test.ts +64 -0
- package/server/browser-cursor.ts +69 -0
- package/server/browser-frame-context.ts +22 -0
- package/server/browser-frame-epoch.test.ts +95 -0
- package/server/browser-frame-rejection.ts +17 -0
- package/server/browser-gesture.ts +120 -0
- package/server/browser-operation-diagnostics.test.ts +79 -0
- package/server/browser-operation-diagnostics.ts +103 -0
- package/server/browser-policy.ts +1313 -151
- package/server/browser-video-input-authority.test.ts +208 -0
- package/server/browser-video-policy.test.ts +564 -0
- package/server/browser-video-socket.test.ts +368 -0
- package/server/browser.ts +151 -16
- package/server/capture-transport-policy.test.ts +105 -0
- package/server/capture-transport-policy.ts +154 -0
- package/server/capture-transport-runtime.test.ts +213 -0
- package/server/daemon-launch-chain.test.ts +70 -0
- package/server/daemon-shutdown.test.ts +289 -0
- package/server/discrete-input-freshness.test.ts +364 -0
- package/server/discrete-input-native-lifetime.test.ts +150 -0
- package/server/display-loss-recovery.test.ts +294 -0
- package/server/emulation-restoration.test.ts +262 -0
- package/server/input-cleanup.ts +23 -0
- package/server/input-generation.test.ts +31 -0
- package/server/input-generation.ts +36 -0
- package/server/jpeg-capture-demand.test.ts +76 -0
- package/server/jpeg-capture-demand.ts +98 -0
- package/server/jpeg-frame.ts +29 -0
- package/server/keyboard-input.test.ts +61 -0
- package/server/keyboard-input.ts +104 -0
- package/server/live-input-policy.test.ts +958 -0
- package/server/live-input-runtime.test.ts +732 -0
- package/server/live-input-supervisor.test.ts +147 -0
- package/server/mcp-entry.ts +44 -2
- package/server/native-video-backpressure.test.ts +228 -0
- package/server/native-video-buffer.test.ts +141 -0
- package/server/native-video-buffer.ts +114 -0
- package/server/native-video-capture.test.ts +599 -0
- package/server/native-video-capture.ts +497 -0
- package/server/native-video-extension-source.test.ts +201 -0
- package/server/native-video-extension-source.ts +364 -0
- package/server/native-video-extension.test.ts +41 -0
- package/server/native-video-extension.ts +83 -0
- package/server/native-video-packet.ts +9 -0
- package/server/native-video-settings.test.ts +245 -0
- package/server/navigation-budget.ts +8 -0
- package/server/navigation-commit.ts +125 -0
- package/server/navigation-policy.test.ts +153 -0
- package/server/process-identity.ts +58 -0
- package/server/reattach-held-input.test.ts +143 -0
- package/server/runtime-frame-reuse.test.ts +295 -0
- package/server/runtime-jpeg-demand.test.ts +200 -0
- package/server/runtime-jpeg-shutdown.test.ts +210 -0
- package/server/runtime-jpeg-source-order.test.ts +201 -0
- package/server/runtime-launch-environment.test.ts +111 -0
- package/server/runtime-navigation.test.ts +327 -0
- package/server/runtime-owner-display.test.ts +152 -0
- package/server/runtime-owner.ts +305 -31
- package/server/runtime-page-state.test.ts +249 -0
- package/server/runtime-protocol.ts +31 -1
- package/server/runtime-target-retention.test.ts +181 -0
- package/server/runtime-video-opt-in.test.ts +82 -0
- package/server/runtime-video.test.ts +444 -0
- package/server/screencast-source-time.ts +46 -0
- package/server/stale-wheel-admission.test.ts +136 -0
- package/server/supervisor-client.ts +28 -4
- package/server/supervisor-heartbeat.test.ts +80 -0
- package/server/supervisor.ts +553 -55
- package/server/video-source-recovery.test.ts +20 -0
- package/server/video-source-recovery.ts +64 -0
- package/server/viewer-capture-lifetime.ts +51 -0
- package/server/virtual-display.test.ts +184 -0
- package/server/virtual-display.ts +236 -0
- package/shared/browser-display-preferences.ts +46 -0
- package/shared/browser-video.ts +80 -0
- package/shared/browser.ts +286 -65
- package/shared/capture-density.ts +16 -0
- package/shared/capture-settings.ts +17 -0
- package/shared/device-presets.ts +90 -0
- package/shared/resolution-menu.test.ts +120 -0
- package/shared/resolution-menu.ts +72 -0
- package/shared/video-settings.test.ts +38 -0
- package/shared/video-settings.ts +34 -0
- package/shared/viewport-limits.ts +5 -0
package/server/runtime-owner.ts
CHANGED
|
@@ -1,26 +1,86 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
-
import {
|
|
4
|
+
import { browserGestureKeySchema, MAX_BROWSER_TABS } from "../shared/browser";
|
|
5
|
+
import { videoBitrateSchema, videoFrameRateSchema } from "../shared/video-settings";
|
|
6
|
+
import {
|
|
7
|
+
AgentBrowserRuntime,
|
|
8
|
+
type AgentBrowserRuntimeOptions,
|
|
9
|
+
type BrowserViewport,
|
|
10
|
+
} from "./agent-browser-runtime";
|
|
5
11
|
import { resolveBrowserRuntimeRoot } from "./runtime-path";
|
|
6
12
|
import type { JsonValue } from "./runtime-protocol";
|
|
7
13
|
import type { RuntimeOwner } from "./supervisor";
|
|
14
|
+
import {
|
|
15
|
+
createPrivateVirtualDisplay,
|
|
16
|
+
PrivateDisplayError,
|
|
17
|
+
type PrivateVirtualDisplay,
|
|
18
|
+
} from "./virtual-display";
|
|
8
19
|
|
|
9
20
|
const DEFAULT_BROWSER_URL = "https://example.com/";
|
|
10
21
|
|
|
11
22
|
interface OwnedRuntime {
|
|
12
23
|
runtimeId: string;
|
|
13
24
|
runtime: AgentBrowserRuntime;
|
|
25
|
+
rootTargetId: string;
|
|
26
|
+
runtimeOptions: AgentBrowserRuntimeOptions;
|
|
27
|
+
tabs: Map<string, Promise<AgentBrowserRuntime>>;
|
|
28
|
+
display: PrivateVirtualDisplay | null;
|
|
29
|
+
/** Bounded, secret-free reason an opted-in private display is not in use. */
|
|
30
|
+
displayNotice: string | null;
|
|
14
31
|
}
|
|
15
32
|
interface RuntimeOwnerOptions {
|
|
16
33
|
initialUrl?: string;
|
|
17
34
|
headed?: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* Trusted opt-in for a private Linux Xvfb display. When omitted, each runtime creation reads
|
|
37
|
+
* PASEO_SHARED_BROWSER_XVFB from this process's environment and enables it only for "1".
|
|
38
|
+
* Explicit false overrides the environment.
|
|
39
|
+
*/
|
|
40
|
+
virtualDisplay?: boolean;
|
|
41
|
+
/** Encoded video is opt-in; defaults to PASEO_SHARED_BROWSER_VIDEO=1 in the daemon environment. */
|
|
42
|
+
nativeVideo?: boolean;
|
|
18
43
|
}
|
|
19
44
|
|
|
20
45
|
function paseoHome(): string {
|
|
21
46
|
return process.env.PASEO_HOME ?? join(homedir(), ".paseo");
|
|
22
47
|
}
|
|
23
48
|
|
|
49
|
+
/** Retain omission for legacy/native held-key inference; reject malformed explicit masks. */
|
|
50
|
+
function optionalMouseModifiers(data: Record<string, JsonValue>): [] | [number] {
|
|
51
|
+
return data.modifiers === undefined
|
|
52
|
+
? []
|
|
53
|
+
: [browserGestureKeySchema.shape.modifiers.parse(data.modifiers)];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Give each page its own CDP attachment and media state within one Chromium profile. */
|
|
57
|
+
async function tabRuntime(owned: OwnedRuntime, targetId: string): Promise<AgentBrowserRuntime> {
|
|
58
|
+
const existing = owned.tabs.get(targetId);
|
|
59
|
+
if (existing) return existing;
|
|
60
|
+
|
|
61
|
+
const attaching = (async () => {
|
|
62
|
+
const targets = await owned.runtime.targets();
|
|
63
|
+
if (!targets.some((target) => target.targetId === targetId)) {
|
|
64
|
+
throw new Error("Browser tab is no longer available");
|
|
65
|
+
}
|
|
66
|
+
const runtime = new AgentBrowserRuntime(owned.runtimeOptions);
|
|
67
|
+
try {
|
|
68
|
+
await runtime.attachTarget(targetId);
|
|
69
|
+
return runtime;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
await runtime.shutdown().catch(() => undefined);
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
})();
|
|
75
|
+
owned.tabs.set(targetId, attaching);
|
|
76
|
+
try {
|
|
77
|
+
return await attaching;
|
|
78
|
+
} catch (error) {
|
|
79
|
+
if (owned.tabs.get(targetId) === attaching) owned.tabs.delete(targetId);
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
24
84
|
export async function createRuntimeOwner(
|
|
25
85
|
options: RuntimeOwnerOptions = {},
|
|
26
86
|
): Promise<RuntimeOwner<OwnedRuntime>> {
|
|
@@ -48,97 +108,311 @@ export async function createRuntimeOwner(
|
|
|
48
108
|
return {
|
|
49
109
|
async create(workspaceId) {
|
|
50
110
|
const hash = createHash("sha256").update(workspaceId).digest("hex");
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
111
|
+
// Explicit headed mode uses the caller's real display. An opted-in hidden Linux mode
|
|
112
|
+
// gets a private virtual mouse so native pointer media survive emulation.
|
|
113
|
+
let display: PrivateVirtualDisplay | null = null;
|
|
114
|
+
let displayNotice: string | null = null;
|
|
115
|
+
const wantsDisplay = options.virtualDisplay ?? process.env.PASEO_SHARED_BROWSER_XVFB === "1";
|
|
116
|
+
if (!options.headed && wantsDisplay) {
|
|
117
|
+
try {
|
|
118
|
+
display = await createPrivateVirtualDisplay();
|
|
119
|
+
if (!display) displayNotice = "Private display unavailable: Xvfb was not found.";
|
|
120
|
+
} catch (error) {
|
|
121
|
+
// The helper has finished cleanup and no Chromium was constructed yet.
|
|
122
|
+
displayNotice = `Private display unavailable: ${
|
|
123
|
+
error instanceof PrivateDisplayError ? error.message : "unexpected startup failure"
|
|
124
|
+
}.`;
|
|
125
|
+
}
|
|
126
|
+
if (displayNotice) {
|
|
127
|
+
// The detached supervisor discards stdio; the notice also reaches the viewer state.
|
|
128
|
+
console.warn(`[shared-browser] ${displayNotice} Using headless mode.`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
let runtime: AgentBrowserRuntime | null = null;
|
|
132
|
+
try {
|
|
133
|
+
const runtimeOptions: AgentBrowserRuntimeOptions = {
|
|
134
|
+
binaryPath,
|
|
135
|
+
executablePath,
|
|
136
|
+
profilePath: join(root, "profiles", hash),
|
|
137
|
+
ipcDirectory,
|
|
138
|
+
session: `ws-${hash.slice(0, 16)}`,
|
|
139
|
+
initialUrl: options.initialUrl ?? DEFAULT_BROWSER_URL,
|
|
140
|
+
headed: display ? true : (options.headed ?? false),
|
|
141
|
+
nativeVideo: options.nativeVideo ?? process.env.PASEO_SHARED_BROWSER_VIDEO === "1",
|
|
142
|
+
...(display ? { launchEnvironment: display.launchEnvironment } : {}),
|
|
143
|
+
};
|
|
144
|
+
runtime = new AgentBrowserRuntime(runtimeOptions);
|
|
145
|
+
await runtime.launch();
|
|
146
|
+
display?.assertAvailable();
|
|
147
|
+
const identity = await runtime.identity();
|
|
148
|
+
return {
|
|
149
|
+
runtimeId: randomUUID(),
|
|
150
|
+
runtime,
|
|
151
|
+
rootTargetId: identity.targetId,
|
|
152
|
+
runtimeOptions,
|
|
153
|
+
tabs: new Map(),
|
|
154
|
+
display,
|
|
155
|
+
displayNotice,
|
|
156
|
+
};
|
|
157
|
+
} catch (error) {
|
|
158
|
+
let cleanupFailure: unknown = null;
|
|
159
|
+
try {
|
|
160
|
+
await runtime?.shutdown();
|
|
161
|
+
} catch (failure) {
|
|
162
|
+
cleanupFailure = failure;
|
|
163
|
+
} finally {
|
|
164
|
+
await display?.stop();
|
|
165
|
+
}
|
|
166
|
+
// An unconfirmed cleanup must not hide why creation failed.
|
|
167
|
+
if (cleanupFailure)
|
|
168
|
+
throw new AggregateError(
|
|
169
|
+
[error, cleanupFailure],
|
|
170
|
+
`Runtime creation failed (${error instanceof Error ? error.message : String(error)}) and its cleanup was not confirmed`,
|
|
171
|
+
);
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
62
174
|
},
|
|
63
175
|
async request(owned, operation, input) {
|
|
176
|
+
owned.display?.assertAvailable();
|
|
64
177
|
const data =
|
|
65
178
|
input && typeof input === "object" && !Array.isArray(input)
|
|
66
179
|
? (input as Record<string, JsonValue>)
|
|
67
180
|
: {};
|
|
181
|
+
if (operation === "tabs.list") {
|
|
182
|
+
return (await owned.runtime.targets()).map(({ targetId, title, url }) => ({
|
|
183
|
+
targetId,
|
|
184
|
+
title,
|
|
185
|
+
url,
|
|
186
|
+
}));
|
|
187
|
+
}
|
|
188
|
+
if (operation === "tabs.create") {
|
|
189
|
+
if ((await owned.runtime.targets()).length >= MAX_BROWSER_TABS)
|
|
190
|
+
throw new Error("Browser tab limit reached");
|
|
191
|
+
return { targetId: await owned.runtime.createTarget() };
|
|
192
|
+
}
|
|
193
|
+
if (operation === "tabs.close") {
|
|
194
|
+
const targetId = String(data.targetId);
|
|
195
|
+
const targets = await owned.runtime.targets();
|
|
196
|
+
if (!targets.some((target) => target.targetId === targetId)) {
|
|
197
|
+
throw new Error("Browser tab is no longer available");
|
|
198
|
+
}
|
|
199
|
+
if (targets.length <= 1) {
|
|
200
|
+
throw new Error("The last browser tab cannot be closed");
|
|
201
|
+
}
|
|
202
|
+
const controller = owned.tabs.get(targetId);
|
|
203
|
+
if (controller) await (await controller).shutdown();
|
|
204
|
+
if (owned.rootTargetId === targetId) {
|
|
205
|
+
const successor = targets.find((target) => target.targetId !== targetId)!;
|
|
206
|
+
await owned.runtime.selectTarget(successor.targetId, false);
|
|
207
|
+
owned.rootTargetId = successor.targetId;
|
|
208
|
+
}
|
|
209
|
+
await owned.runtime.closeTarget(targetId);
|
|
210
|
+
owned.tabs.delete(targetId);
|
|
211
|
+
return null;
|
|
212
|
+
}
|
|
213
|
+
const targetId = typeof data.targetId === "string" ? data.targetId : null;
|
|
214
|
+
// Keep the launch page on its original CDP session. A second session on
|
|
215
|
+
// the same target can reset emulation and interrupt its video source.
|
|
216
|
+
const runtime =
|
|
217
|
+
targetId && (targetId !== owned.rootTargetId || owned.tabs.has(targetId))
|
|
218
|
+
? await tabRuntime(owned, targetId)
|
|
219
|
+
: owned.runtime;
|
|
220
|
+
const gestureId = typeof data.gestureId === "string" ? data.gestureId : undefined;
|
|
68
221
|
switch (operation) {
|
|
69
|
-
case "identity":
|
|
70
|
-
|
|
222
|
+
case "identity": {
|
|
223
|
+
const identity = await runtime.identity();
|
|
224
|
+
return {
|
|
225
|
+
...identity,
|
|
226
|
+
...(owned.displayNotice ? { displayNotice: owned.displayNotice } : {}),
|
|
227
|
+
} as unknown as JsonValue;
|
|
228
|
+
}
|
|
71
229
|
case "state":
|
|
72
|
-
return
|
|
230
|
+
if (data.timeoutMs === undefined) return runtime.state() as unknown as JsonValue;
|
|
231
|
+
if (typeof data.timeoutMs !== "number")
|
|
232
|
+
throw new Error("Invalid browser metadata timeout");
|
|
233
|
+
return runtime.state({ timeoutMs: data.timeoutMs }) as unknown as JsonValue;
|
|
73
234
|
case "navigate":
|
|
74
|
-
await
|
|
235
|
+
await runtime.navigate(String(data.url));
|
|
75
236
|
return null;
|
|
76
237
|
case "back":
|
|
77
|
-
await
|
|
238
|
+
await runtime.back();
|
|
78
239
|
return null;
|
|
79
240
|
case "forward":
|
|
80
|
-
await
|
|
241
|
+
await runtime.forward();
|
|
81
242
|
return null;
|
|
82
243
|
case "reload":
|
|
83
|
-
await
|
|
244
|
+
await runtime.reload();
|
|
84
245
|
return null;
|
|
85
246
|
case "emulate":
|
|
86
|
-
await
|
|
247
|
+
await runtime.emulate(data as unknown as BrowserViewport);
|
|
248
|
+
return null;
|
|
249
|
+
case "capture.density":
|
|
250
|
+
await runtime.setCaptureDensity(Number(data.density), Number(data.deviceScaleFactor));
|
|
87
251
|
return null;
|
|
88
252
|
case "screencast.start":
|
|
89
|
-
await
|
|
253
|
+
await runtime.startScreencast(Number(data.quality));
|
|
90
254
|
return null;
|
|
91
255
|
case "screencast.stop":
|
|
92
|
-
await
|
|
256
|
+
await runtime.stopScreencast();
|
|
257
|
+
return null;
|
|
258
|
+
case "video.read":
|
|
259
|
+
return (await runtime.readVideo({
|
|
260
|
+
quality: data.quality === "low" || data.quality === "medium" ? data.quality : "high",
|
|
261
|
+
streamId: typeof data.streamId === "string" ? data.streamId : null,
|
|
262
|
+
requestKeyFrame: data.requestKeyFrame === true,
|
|
263
|
+
...(data.bitrate === undefined
|
|
264
|
+
? {}
|
|
265
|
+
: { bitrate: videoBitrateSchema.parse(data.bitrate) }),
|
|
266
|
+
...(data.fps === undefined ? {} : { fps: videoFrameRateSchema.parse(data.fps) }),
|
|
267
|
+
afterSequence: Number(data.afterSequence ?? 0),
|
|
268
|
+
waitMs: Number(data.waitMs ?? 250),
|
|
269
|
+
})) as unknown as JsonValue;
|
|
270
|
+
case "video.stop":
|
|
271
|
+
await runtime.stopVideo();
|
|
272
|
+
return null;
|
|
273
|
+
case "video.invalidate":
|
|
274
|
+
runtime.invalidateQueuedVideoFrames();
|
|
93
275
|
return null;
|
|
94
276
|
case "frame":
|
|
95
|
-
return (await
|
|
277
|
+
return (await runtime.frame(
|
|
96
278
|
Number(data.maxBytes),
|
|
97
279
|
Number(data.quality),
|
|
98
280
|
Number(data.waitMs),
|
|
99
281
|
)) as unknown as JsonValue;
|
|
100
282
|
case "mouse.move":
|
|
101
|
-
await
|
|
283
|
+
await runtime.mouseMove(
|
|
284
|
+
Number(data.x),
|
|
285
|
+
Number(data.y),
|
|
286
|
+
gestureId,
|
|
287
|
+
...optionalMouseModifiers(data),
|
|
288
|
+
);
|
|
102
289
|
return null;
|
|
103
290
|
case "mouse.down":
|
|
104
|
-
await
|
|
291
|
+
await runtime.mouseDown(
|
|
105
292
|
Number(data.x),
|
|
106
293
|
Number(data.y),
|
|
107
294
|
String(data.button) as "left" | "middle" | "right",
|
|
108
295
|
Number(data.clickCount),
|
|
296
|
+
gestureId,
|
|
297
|
+
...optionalMouseModifiers(data),
|
|
109
298
|
);
|
|
110
299
|
return null;
|
|
111
300
|
case "mouse.up":
|
|
112
|
-
await
|
|
301
|
+
await runtime.mouseUp(
|
|
113
302
|
Number(data.x),
|
|
114
303
|
Number(data.y),
|
|
115
304
|
String(data.button) as "left" | "middle" | "right",
|
|
116
305
|
Number(data.clickCount),
|
|
306
|
+
gestureId,
|
|
307
|
+
...optionalMouseModifiers(data),
|
|
117
308
|
);
|
|
118
309
|
return null;
|
|
119
310
|
case "mouse.wheel":
|
|
120
|
-
await
|
|
311
|
+
await runtime.wheel(
|
|
121
312
|
Number(data.x),
|
|
122
313
|
Number(data.y),
|
|
123
314
|
Number(data.deltaX),
|
|
124
315
|
Number(data.deltaY),
|
|
316
|
+
gestureId,
|
|
317
|
+
...optionalMouseModifiers(data),
|
|
125
318
|
);
|
|
126
319
|
return null;
|
|
320
|
+
case "mouse.leave":
|
|
321
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
322
|
+
await runtime.mouseLeave(gestureId);
|
|
323
|
+
return null;
|
|
324
|
+
case "input.begin":
|
|
325
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
326
|
+
if (typeof data.expectedInputGeneration !== "string") {
|
|
327
|
+
throw new Error("Expected native input generation is required");
|
|
328
|
+
}
|
|
329
|
+
await runtime.beginLiveInput(gestureId, data.expectedInputGeneration);
|
|
330
|
+
return null;
|
|
331
|
+
case "input.check":
|
|
332
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
333
|
+
await runtime.assertLiveInput(gestureId);
|
|
334
|
+
return null;
|
|
335
|
+
case "input.end":
|
|
336
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
337
|
+
await runtime.endLiveInput(gestureId);
|
|
338
|
+
return null;
|
|
339
|
+
case "cursor":
|
|
340
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
341
|
+
return runtime.cursorAt(Number(data.x), Number(data.y), gestureId);
|
|
342
|
+
case "touch": {
|
|
343
|
+
if (!gestureId || !Array.isArray(data.points) || data.points.length > 5)
|
|
344
|
+
throw new Error("Invalid touch input");
|
|
345
|
+
const points = data.points.map((value) => {
|
|
346
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
347
|
+
throw new Error("Invalid touch point");
|
|
348
|
+
const id = value.id;
|
|
349
|
+
if (
|
|
350
|
+
typeof id !== "number" ||
|
|
351
|
+
typeof value.x !== "number" ||
|
|
352
|
+
typeof value.y !== "number"
|
|
353
|
+
)
|
|
354
|
+
throw new Error("Invalid touch point numbers");
|
|
355
|
+
if (!Number.isSafeInteger(id) || id < 0 || id > 2_147_483_647)
|
|
356
|
+
throw new Error("Invalid touch identifier");
|
|
357
|
+
return { x: Number(value.x), y: Number(value.y), id };
|
|
358
|
+
});
|
|
359
|
+
if (new Set(points.map((point) => point.id)).size !== points.length)
|
|
360
|
+
throw new Error("Duplicate touch identifiers");
|
|
361
|
+
const type = data.type;
|
|
362
|
+
if (type !== "start" && type !== "move" && type !== "end" && type !== "cancel")
|
|
363
|
+
throw new Error("Invalid touch phase");
|
|
364
|
+
if ((type === "end" || type === "cancel") !== (points.length === 0))
|
|
365
|
+
throw new Error("Invalid touch phase points");
|
|
366
|
+
const nativeType =
|
|
367
|
+
type === "start"
|
|
368
|
+
? "touchStart"
|
|
369
|
+
: type === "move"
|
|
370
|
+
? "touchMove"
|
|
371
|
+
: type === "end"
|
|
372
|
+
? "touchEnd"
|
|
373
|
+
: "touchCancel";
|
|
374
|
+
await runtime.touch(nativeType, points, gestureId);
|
|
375
|
+
return null;
|
|
376
|
+
}
|
|
377
|
+
case "input.key":
|
|
378
|
+
if (!gestureId) throw new Error("Gesture identity is required");
|
|
379
|
+
await runtime.dispatchKey(browserGestureKeySchema.parse(data.event), gestureId);
|
|
380
|
+
return null;
|
|
381
|
+
case "input.text":
|
|
382
|
+
if (
|
|
383
|
+
!gestureId ||
|
|
384
|
+
typeof data.text !== "string" ||
|
|
385
|
+
data.text.length < 1 ||
|
|
386
|
+
data.text.length > 16_000
|
|
387
|
+
) {
|
|
388
|
+
throw new Error("Invalid committed text input");
|
|
389
|
+
}
|
|
390
|
+
await runtime.insertText(data.text, gestureId);
|
|
391
|
+
return null;
|
|
127
392
|
case "text.insert":
|
|
128
|
-
await
|
|
393
|
+
await runtime.insertText(String(data.text), gestureId);
|
|
129
394
|
return null;
|
|
130
395
|
case "key.down":
|
|
131
|
-
await
|
|
396
|
+
await runtime.keyDown(String(data.key), String(data.key), gestureId);
|
|
132
397
|
return null;
|
|
133
398
|
case "key.up":
|
|
134
|
-
await
|
|
399
|
+
await runtime.keyUp(String(data.key), String(data.key), gestureId);
|
|
135
400
|
return null;
|
|
136
401
|
default:
|
|
137
402
|
throw new Error(`Unknown browser runtime operation: ${operation}`);
|
|
138
403
|
}
|
|
139
404
|
},
|
|
140
405
|
async stop(owned) {
|
|
141
|
-
|
|
406
|
+
try {
|
|
407
|
+
await Promise.allSettled(
|
|
408
|
+
[...owned.tabs.values()].map(async (controller) =>
|
|
409
|
+
(await controller).shutdown().catch(() => undefined),
|
|
410
|
+
),
|
|
411
|
+
);
|
|
412
|
+
await owned.runtime.shutdown();
|
|
413
|
+
} finally {
|
|
414
|
+
await owned.display?.stop();
|
|
415
|
+
}
|
|
142
416
|
},
|
|
143
417
|
};
|
|
144
418
|
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/** Fresh metadata reads overlap CDP latency but never accept mixed native identities. */
|
|
2
|
+
import { describe, expect, it, vi } from "vitest";
|
|
3
|
+
import { AgentBrowserRuntime } from "./agent-browser-runtime";
|
|
4
|
+
|
|
5
|
+
function fixture() {
|
|
6
|
+
const runtime = new AgentBrowserRuntime({
|
|
7
|
+
binaryPath: "/tmp/unlaunched-browser",
|
|
8
|
+
executablePath: "/tmp/unlaunched-chromium",
|
|
9
|
+
profilePath: "/tmp/uncreated-state-profile",
|
|
10
|
+
ipcDirectory: "/tmp/uncreated-state-ipc",
|
|
11
|
+
session: "state-test",
|
|
12
|
+
});
|
|
13
|
+
const calls: string[] = [];
|
|
14
|
+
const history = Promise.withResolvers<{
|
|
15
|
+
currentIndex: number;
|
|
16
|
+
entries: { url: string; title: string }[];
|
|
17
|
+
}>();
|
|
18
|
+
const metadata = Promise.withResolvers<{
|
|
19
|
+
result: { value?: { url?: string; title?: string } };
|
|
20
|
+
}>();
|
|
21
|
+
const retryHistory = Promise.withResolvers<Awaited<typeof history.promise>>();
|
|
22
|
+
const retryMetadata = Promise.withResolvers<Awaited<typeof metadata.promise>>();
|
|
23
|
+
const timeouts: number[] = [];
|
|
24
|
+
let onRead: (method: string, attempt: number) => void = () => {};
|
|
25
|
+
const page = {
|
|
26
|
+
send: (method: string, _input: unknown, options: { timeoutMs: number }) => {
|
|
27
|
+
const attempt = calls.filter((call) => call === method).length;
|
|
28
|
+
calls.push(method);
|
|
29
|
+
timeouts.push(options.timeoutMs);
|
|
30
|
+
onRead(method, attempt);
|
|
31
|
+
if (method === "Page.getNavigationHistory") {
|
|
32
|
+
return attempt === 0 ? history.promise : retryHistory.promise;
|
|
33
|
+
}
|
|
34
|
+
return attempt === 0 ? metadata.promise : retryMetadata.promise;
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
const control = runtime as unknown as {
|
|
38
|
+
page: typeof page;
|
|
39
|
+
attachmentGeneration: number;
|
|
40
|
+
documentGeneration: number;
|
|
41
|
+
stopping: boolean;
|
|
42
|
+
requirePage(): Promise<typeof page>;
|
|
43
|
+
};
|
|
44
|
+
control.page = page;
|
|
45
|
+
control.requirePage = async () => page;
|
|
46
|
+
return {
|
|
47
|
+
runtime,
|
|
48
|
+
control,
|
|
49
|
+
page,
|
|
50
|
+
calls,
|
|
51
|
+
history,
|
|
52
|
+
metadata,
|
|
53
|
+
retryHistory,
|
|
54
|
+
retryMetadata,
|
|
55
|
+
timeouts,
|
|
56
|
+
intercept: (callback: typeof onRead) => {
|
|
57
|
+
onRead = callback;
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const entries = [
|
|
62
|
+
{ url: "https://example.test/previous", title: "Previous" },
|
|
63
|
+
{ url: "https://example.test/current", title: "History title" },
|
|
64
|
+
];
|
|
65
|
+
|
|
66
|
+
describe("fresh parallel native page metadata", () => {
|
|
67
|
+
it("discards both old reads and returns one fresh pair after a same-page document change", async () => {
|
|
68
|
+
const f = fixture();
|
|
69
|
+
const pending = f.runtime.state();
|
|
70
|
+
await Promise.resolve();
|
|
71
|
+
f.control.documentGeneration++;
|
|
72
|
+
f.retryHistory.resolve({
|
|
73
|
+
currentIndex: 0,
|
|
74
|
+
entries: [{ url: "https://example.test/new", title: "New history" }],
|
|
75
|
+
});
|
|
76
|
+
f.retryMetadata.resolve({
|
|
77
|
+
result: { value: { url: "https://example.test/new#ready", title: "New DOM" } },
|
|
78
|
+
});
|
|
79
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
80
|
+
f.metadata.resolve({
|
|
81
|
+
result: { value: { url: "https://example.test/old", title: "Old DOM" } },
|
|
82
|
+
});
|
|
83
|
+
expect(await pending).toEqual({
|
|
84
|
+
url: "https://example.test/new#ready",
|
|
85
|
+
title: "New DOM",
|
|
86
|
+
canGoBack: false,
|
|
87
|
+
canGoForward: false,
|
|
88
|
+
inputGeneration: "0:1",
|
|
89
|
+
});
|
|
90
|
+
expect(f.calls).toEqual([
|
|
91
|
+
"Page.getNavigationHistory",
|
|
92
|
+
"Runtime.evaluate",
|
|
93
|
+
"Page.getNavigationHistory",
|
|
94
|
+
"Runtime.evaluate",
|
|
95
|
+
]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("starts both independent reads before either resolves and retains native history facts", async () => {
|
|
99
|
+
const f = fixture();
|
|
100
|
+
const pending = f.runtime.state();
|
|
101
|
+
await Promise.resolve();
|
|
102
|
+
expect(f.calls).toEqual(["Page.getNavigationHistory", "Runtime.evaluate"]);
|
|
103
|
+
f.metadata.resolve({
|
|
104
|
+
result: { value: { url: "https://example.test/current#hash", title: "Fresh title" } },
|
|
105
|
+
});
|
|
106
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
107
|
+
expect(await pending).toMatchObject({
|
|
108
|
+
url: "https://example.test/current#hash",
|
|
109
|
+
title: "Fresh title",
|
|
110
|
+
canGoBack: true,
|
|
111
|
+
canGoForward: false,
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
it("preserves existing native-history fallback when DOM values are absent", async () => {
|
|
115
|
+
const f = fixture();
|
|
116
|
+
const pending = f.runtime.state();
|
|
117
|
+
f.metadata.resolve({ result: {} });
|
|
118
|
+
f.history.resolve({ currentIndex: 0, entries });
|
|
119
|
+
expect(await pending).toMatchObject({
|
|
120
|
+
url: entries[0]!.url,
|
|
121
|
+
title: "Previous",
|
|
122
|
+
canGoBack: false,
|
|
123
|
+
canGoForward: true,
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
it("refuses attachment, page replacement or shutdown during either pending observation", async () => {
|
|
127
|
+
for (const kind of ["attachment", "page", "shutdown"] as const) {
|
|
128
|
+
const f = fixture();
|
|
129
|
+
const pending = f.runtime.state();
|
|
130
|
+
await Promise.resolve();
|
|
131
|
+
if (kind === "shutdown") f.control.stopping = true;
|
|
132
|
+
if (kind === "attachment") f.control.attachmentGeneration++;
|
|
133
|
+
if (kind === "page")
|
|
134
|
+
f.control.page = { send: () => Promise.resolve({}) } as unknown as typeof f.page;
|
|
135
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
136
|
+
f.metadata.resolve({ result: { value: { title: "Obsolete document" } } });
|
|
137
|
+
await expect(pending).rejects.toThrow(
|
|
138
|
+
kind === "shutdown" ? "shutting down" : "metadata changed",
|
|
139
|
+
);
|
|
140
|
+
expect(f.calls).toHaveLength(2);
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
it("refuses repeated document changes after exactly two read pairs", async () => {
|
|
144
|
+
const f = fixture();
|
|
145
|
+
f.intercept((method, attempt) => {
|
|
146
|
+
if (method === "Runtime.evaluate" && attempt === 1) f.control.documentGeneration++;
|
|
147
|
+
});
|
|
148
|
+
const pending = f.runtime.state();
|
|
149
|
+
await Promise.resolve();
|
|
150
|
+
f.control.documentGeneration++;
|
|
151
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
152
|
+
f.metadata.resolve({ result: {} });
|
|
153
|
+
f.retryHistory.resolve({ currentIndex: 0, entries });
|
|
154
|
+
f.retryMetadata.resolve({ result: {} });
|
|
155
|
+
await expect(pending).rejects.toThrow("metadata changed");
|
|
156
|
+
expect(f.calls).toHaveLength(4);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("shares the original timeout budget across both pairs without restarting attachment", async () => {
|
|
160
|
+
const f = fixture();
|
|
161
|
+
let clock = 1_000;
|
|
162
|
+
const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
|
|
163
|
+
try {
|
|
164
|
+
const pending = f.runtime.state({ timeoutMs: 1_000 });
|
|
165
|
+
await Promise.resolve();
|
|
166
|
+
clock += 700;
|
|
167
|
+
f.control.documentGeneration++;
|
|
168
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
169
|
+
f.metadata.resolve({ result: {} });
|
|
170
|
+
f.retryHistory.resolve({ currentIndex: 0, entries });
|
|
171
|
+
f.retryMetadata.resolve({ result: {} });
|
|
172
|
+
await pending;
|
|
173
|
+
expect(f.timeouts).toHaveLength(4);
|
|
174
|
+
expect(f.timeouts.slice(0, 2).every((timeout) => timeout <= 1_000)).toBe(true);
|
|
175
|
+
expect(f.timeouts.slice(2).every((timeout) => timeout <= 300)).toBe(true);
|
|
176
|
+
} finally {
|
|
177
|
+
monotonic.mockRestore();
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it("does not begin a second pair after the original budget expires", async () => {
|
|
182
|
+
const f = fixture();
|
|
183
|
+
let clock = 1_000;
|
|
184
|
+
const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
|
|
185
|
+
try {
|
|
186
|
+
const pending = f.runtime.state({ timeoutMs: 1_000 });
|
|
187
|
+
await Promise.resolve();
|
|
188
|
+
clock += 1_000;
|
|
189
|
+
f.control.documentGeneration++;
|
|
190
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
191
|
+
f.metadata.resolve({ result: {} });
|
|
192
|
+
await expect(pending).rejects.toThrow("metadata observation timed out");
|
|
193
|
+
expect(f.calls).toHaveLength(2);
|
|
194
|
+
} finally {
|
|
195
|
+
monotonic.mockRestore();
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
for (const phase of ["first", "second"] as const) {
|
|
200
|
+
it(`refuses successful held ${phase} replies at or after the original deadline`, async () => {
|
|
201
|
+
const f = fixture();
|
|
202
|
+
let clock = 1_000;
|
|
203
|
+
const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
|
|
204
|
+
try {
|
|
205
|
+
const secondStarted = Promise.withResolvers<void>();
|
|
206
|
+
f.intercept((method, attempt) => {
|
|
207
|
+
if (method === "Runtime.evaluate" && attempt === 1) secondStarted.resolve();
|
|
208
|
+
});
|
|
209
|
+
const pending = f.runtime.state({ timeoutMs: 1_000 });
|
|
210
|
+
await Promise.resolve();
|
|
211
|
+
if (phase === "second") {
|
|
212
|
+
clock += 700;
|
|
213
|
+
f.control.documentGeneration++;
|
|
214
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
215
|
+
f.metadata.resolve({ result: {} });
|
|
216
|
+
await secondStarted.promise;
|
|
217
|
+
expect(f.calls).toHaveLength(4);
|
|
218
|
+
clock += 301;
|
|
219
|
+
f.retryHistory.resolve({ currentIndex: 0, entries });
|
|
220
|
+
f.retryMetadata.resolve({ result: { value: { title: "Late fresh metadata" } } });
|
|
221
|
+
} else {
|
|
222
|
+
clock += 1_000;
|
|
223
|
+
f.history.resolve({ currentIndex: 1, entries });
|
|
224
|
+
f.metadata.resolve({ result: { value: { title: "Late metadata" } } });
|
|
225
|
+
}
|
|
226
|
+
await expect(pending).rejects.toThrow("metadata observation timed out");
|
|
227
|
+
expect(f.calls).toHaveLength(phase === "second" ? 4 : 2);
|
|
228
|
+
} finally {
|
|
229
|
+
monotonic.mockRestore();
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
it("propagates failed native reads rather than reusing earlier metadata", async () => {
|
|
235
|
+
for (const failed of ["history", "metadata"]) {
|
|
236
|
+
const f = fixture();
|
|
237
|
+
const pending = f.runtime.state();
|
|
238
|
+
const refusal = expect(pending).rejects.toThrow("Read unavailable");
|
|
239
|
+
await Promise.resolve();
|
|
240
|
+
f.control.documentGeneration++;
|
|
241
|
+
if (failed === "history") f.history.reject(new Error("Read unavailable"));
|
|
242
|
+
else f.history.resolve({ currentIndex: 1, entries });
|
|
243
|
+
if (failed === "metadata") f.metadata.reject(new Error("Read unavailable"));
|
|
244
|
+
else f.metadata.resolve({ result: {} });
|
|
245
|
+
await refusal;
|
|
246
|
+
expect(f.calls).toHaveLength(2);
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
});
|