@omercnet/paseo-shared-browser 1.2.2-next.232.1 → 1.2.2-next.237.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +312 -25
- package/REMOTE_BROWSER_RESEARCH.md +395 -0
- package/STREAMING_DESIGN.md +191 -0
- package/client/browser-auto-capture-density.test.ts +102 -0
- package/client/browser-auto-capture-density.ts +30 -0
- package/client/browser-canvas-input.test.ts +198 -0
- package/client/browser-canvas-input.ts +202 -0
- package/client/browser-canvas-keyboard-web.test.ts +113 -0
- package/client/browser-canvas-keyboard-web.ts +230 -0
- package/client/browser-canvas-keyboard.test.ts +123 -0
- package/client/browser-canvas-keyboard.ts +126 -0
- package/client/browser-canvas-layout.test.ts +109 -0
- package/client/browser-canvas-layout.ts +51 -0
- package/client/browser-canvas-viewport-web.test.ts +77 -0
- package/client/browser-canvas-viewport-web.ts +63 -0
- package/client/browser-canvas-viewport.tsx +152 -0
- package/client/browser-capture-cadence.test.ts +9 -0
- package/client/browser-capture-cadence.ts +6 -0
- package/client/browser-capture-density-controls.tsx +75 -0
- package/client/browser-chrome.tsx +234 -0
- package/client/browser-client-presence.ts +184 -0
- package/client/browser-compose-controls.test.tsx +275 -0
- package/client/browser-compose-controls.tsx +174 -0
- package/client/browser-control-button.tsx +109 -0
- package/client/browser-control-dialog.tsx +71 -0
- package/client/browser-control-request.test.ts +27 -0
- package/client/browser-control-request.ts +57 -0
- package/client/browser-control-tooltip-web.ts +10 -0
- package/client/browser-dialog-body.tsx +22 -0
- package/client/browser-emulation-mode.test.ts +75 -0
- package/client/browser-emulation-mode.ts +98 -0
- package/client/browser-fill-viewport.test.ts +45 -0
- package/client/browser-fill-viewport.ts +28 -0
- package/client/browser-frame-buffer.test.ts +157 -0
- package/client/browser-frame-buffer.ts +148 -0
- package/client/browser-frame-image.tsx +62 -0
- package/client/browser-image-load.test.ts +23 -0
- package/client/browser-image-load.ts +26 -0
- package/client/browser-input-queue.test.ts +1201 -0
- package/client/browser-input-queue.ts +505 -0
- package/client/browser-menu-placement.test.ts +90 -0
- package/client/browser-menu-placement.ts +78 -0
- package/client/browser-panel-styles.ts +384 -0
- package/client/browser-quality-controls.test.ts +26 -0
- package/client/browser-quality-controls.tsx +139 -0
- package/client/browser-resolution-picker.tsx +189 -0
- package/client/browser-tabs.tsx +117 -0
- package/client/browser-toolbar-menu-web.test.ts +185 -0
- package/client/browser-toolbar-menu-web.ts +159 -0
- package/client/browser-toolbar-menu.tsx +255 -0
- package/client/browser-toolbar-submenu.tsx +93 -0
- package/client/browser-video-bytes.test.ts +44 -0
- package/client/browser-video-decoder.test.ts +473 -0
- package/client/browser-video-decoder.ts +311 -0
- package/client/browser-video-surface.tsx +21 -0
- package/client/browser-video-visibility.test.ts +45 -0
- package/client/browser-viewer-recovery.test.ts +111 -0
- package/client/browser-viewer-recovery.ts +78 -0
- package/client/browser.tsx +1642 -1569
- package/client/favorite-star.tsx +23 -0
- package/client/frame-lifecycle.ts +62 -59
- package/client/use-browser-auto-capture-density.test.ts +104 -0
- package/client/use-browser-auto-capture-density.ts +43 -0
- package/client/use-browser-canvas-input.ts +370 -0
- package/client/use-browser-capture-density.ts +87 -0
- package/client/use-browser-emulation-mode.ts +56 -0
- package/client/use-browser-fill-viewport.test.ts +108 -0
- package/client/use-browser-fill-viewport.ts +62 -0
- package/client/use-browser-frame-buffer.ts +66 -0
- package/client/use-browser-image-capture.ts +139 -0
- package/client/use-browser-scoped-mutation.test.ts +118 -0
- package/client/use-browser-scoped-mutation.ts +59 -0
- package/client/use-browser-video.test.ts +991 -0
- package/client/use-browser-video.ts +474 -0
- package/client/use-browser-viewer-recovery.ts +26 -0
- package/client/use-resolution-favorites.ts +90 -0
- package/client/web.test.ts +125 -0
- package/client/web.ts +405 -0
- package/index.client.tsx +0 -10
- package/index.server.ts +32 -0
- package/package.json +9 -5
- package/paseo-plugin.json +3 -1
- package/scripts/prepare-runtime.mjs +1 -1
- package/server/agent-browser-runtime.ts +1598 -314
- package/server/browser-cursor.test.ts +64 -0
- package/server/browser-cursor.ts +69 -0
- package/server/browser-frame-context.ts +22 -0
- package/server/browser-frame-epoch.test.ts +95 -0
- package/server/browser-frame-rejection.ts +17 -0
- package/server/browser-gesture.ts +120 -0
- package/server/browser-operation-diagnostics.test.ts +79 -0
- package/server/browser-operation-diagnostics.ts +103 -0
- package/server/browser-policy.ts +1313 -151
- package/server/browser-video-input-authority.test.ts +208 -0
- package/server/browser-video-policy.test.ts +564 -0
- package/server/browser-video-socket.test.ts +368 -0
- package/server/browser.ts +151 -16
- package/server/capture-transport-policy.test.ts +105 -0
- package/server/capture-transport-policy.ts +154 -0
- package/server/capture-transport-runtime.test.ts +213 -0
- package/server/daemon-launch-chain.test.ts +70 -0
- package/server/daemon-shutdown.test.ts +289 -0
- package/server/discrete-input-freshness.test.ts +364 -0
- package/server/discrete-input-native-lifetime.test.ts +150 -0
- package/server/display-loss-recovery.test.ts +294 -0
- package/server/emulation-restoration.test.ts +262 -0
- package/server/input-cleanup.ts +23 -0
- package/server/input-generation.test.ts +31 -0
- package/server/input-generation.ts +36 -0
- package/server/jpeg-capture-demand.test.ts +76 -0
- package/server/jpeg-capture-demand.ts +98 -0
- package/server/jpeg-frame.ts +29 -0
- package/server/keyboard-input.test.ts +61 -0
- package/server/keyboard-input.ts +104 -0
- package/server/live-input-policy.test.ts +958 -0
- package/server/live-input-runtime.test.ts +732 -0
- package/server/live-input-supervisor.test.ts +147 -0
- package/server/mcp-entry.ts +44 -2
- package/server/native-video-backpressure.test.ts +228 -0
- package/server/native-video-buffer.test.ts +141 -0
- package/server/native-video-buffer.ts +114 -0
- package/server/native-video-capture.test.ts +599 -0
- package/server/native-video-capture.ts +497 -0
- package/server/native-video-extension-source.test.ts +201 -0
- package/server/native-video-extension-source.ts +364 -0
- package/server/native-video-extension.test.ts +41 -0
- package/server/native-video-extension.ts +83 -0
- package/server/native-video-packet.ts +9 -0
- package/server/native-video-settings.test.ts +245 -0
- package/server/navigation-budget.ts +8 -0
- package/server/navigation-commit.ts +125 -0
- package/server/navigation-policy.test.ts +153 -0
- package/server/process-identity.ts +58 -0
- package/server/reattach-held-input.test.ts +143 -0
- package/server/runtime-frame-reuse.test.ts +295 -0
- package/server/runtime-jpeg-demand.test.ts +200 -0
- package/server/runtime-jpeg-shutdown.test.ts +210 -0
- package/server/runtime-jpeg-source-order.test.ts +201 -0
- package/server/runtime-launch-environment.test.ts +111 -0
- package/server/runtime-navigation.test.ts +327 -0
- package/server/runtime-owner-display.test.ts +152 -0
- package/server/runtime-owner.ts +305 -31
- package/server/runtime-page-state.test.ts +249 -0
- package/server/runtime-protocol.ts +31 -1
- package/server/runtime-target-retention.test.ts +181 -0
- package/server/runtime-video-opt-in.test.ts +82 -0
- package/server/runtime-video.test.ts +444 -0
- package/server/screencast-source-time.ts +46 -0
- package/server/stale-wheel-admission.test.ts +136 -0
- package/server/supervisor-client.ts +28 -4
- package/server/supervisor-heartbeat.test.ts +80 -0
- package/server/supervisor.ts +553 -55
- package/server/video-source-recovery.test.ts +20 -0
- package/server/video-source-recovery.ts +64 -0
- package/server/viewer-capture-lifetime.ts +51 -0
- package/server/virtual-display.test.ts +184 -0
- package/server/virtual-display.ts +236 -0
- package/shared/browser-display-preferences.ts +46 -0
- package/shared/browser-video.ts +80 -0
- package/shared/browser.ts +286 -65
- package/shared/capture-density.ts +16 -0
- package/shared/capture-settings.ts +17 -0
- package/shared/device-presets.ts +90 -0
- package/shared/resolution-menu.test.ts +120 -0
- package/shared/resolution-menu.ts +72 -0
- package/shared/video-settings.test.ts +38 -0
- package/shared/video-settings.ts +34 -0
- package/shared/viewport-limits.ts +5 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/** Actual plugin bridge socket: a held media reply must not delay a later input command. */
|
|
2
|
+
|
|
3
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { performance } from "node:perf_hooks";
|
|
7
|
+
import { expect, it } from "vitest";
|
|
8
|
+
import type { BrowserFrame, BrowserState } from "../shared/browser";
|
|
9
|
+
import type { BrowserVideoReadReply } from "../shared/browser-video";
|
|
10
|
+
import { RuntimeProtocolError } from "./runtime-protocol";
|
|
11
|
+
import { resolveSupervisorPaths, startSupervisorServer } from "./supervisor";
|
|
12
|
+
import { SupervisorClient } from "./supervisor-client";
|
|
13
|
+
|
|
14
|
+
it("keeps video past unrelated teardown and input past video on the authenticated socket", async () => {
|
|
15
|
+
const home = await mkdtemp(join(tmpdir(), "owned-video-socket-"));
|
|
16
|
+
let release = () => {};
|
|
17
|
+
let started = () => {};
|
|
18
|
+
let block = false;
|
|
19
|
+
const gate = new Promise<void>((resolve) => {
|
|
20
|
+
release = resolve;
|
|
21
|
+
});
|
|
22
|
+
const arrived = new Promise<void>((resolve) => {
|
|
23
|
+
started = resolve;
|
|
24
|
+
});
|
|
25
|
+
const retirementGate = Promise.withResolvers<void>();
|
|
26
|
+
const retirementStarted = Promise.withResolvers<void>();
|
|
27
|
+
let blockRetirement = false;
|
|
28
|
+
let retirement: Promise<unknown> | null = null;
|
|
29
|
+
const stateGate = Promise.withResolvers<void>();
|
|
30
|
+
const inputPublished = Promise.withResolvers<void>();
|
|
31
|
+
let blockState = false;
|
|
32
|
+
let jpegQuality: unknown;
|
|
33
|
+
let videoSequence = 0;
|
|
34
|
+
const calls: string[] = [];
|
|
35
|
+
const running = await startSupervisorServer(
|
|
36
|
+
{
|
|
37
|
+
create: async () => ({ runtimeId: "r".repeat(32) }),
|
|
38
|
+
stop: async () => {},
|
|
39
|
+
request: async (_runtime, operation, parameters) => {
|
|
40
|
+
calls.push(operation);
|
|
41
|
+
if (operation === "screencast.stop" && blockRetirement) {
|
|
42
|
+
retirementStarted.resolve();
|
|
43
|
+
await retirementGate.promise;
|
|
44
|
+
}
|
|
45
|
+
if (operation === "identity") return { userAgent: "Owned fixture" };
|
|
46
|
+
if (operation === "frame") {
|
|
47
|
+
jpegQuality = (parameters as { quality?: unknown }).quality;
|
|
48
|
+
return {
|
|
49
|
+
dataBase64: "AA==",
|
|
50
|
+
byteLength: 1,
|
|
51
|
+
width: 1280,
|
|
52
|
+
height: 800,
|
|
53
|
+
capturedAt: new Date().toISOString(),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
if (operation === "mouse.move") inputPublished.resolve();
|
|
57
|
+
if (operation === "state") {
|
|
58
|
+
if (blockState) await stateGate.promise;
|
|
59
|
+
return {
|
|
60
|
+
url: "https://fixture.invalid/",
|
|
61
|
+
title: "Owned",
|
|
62
|
+
inputGeneration: "1:1",
|
|
63
|
+
canGoBack: false,
|
|
64
|
+
canGoForward: false,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
if (operation === "video.read") {
|
|
68
|
+
if (block) {
|
|
69
|
+
started();
|
|
70
|
+
await gate;
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
status: "ready",
|
|
74
|
+
streamId: "s".repeat(32),
|
|
75
|
+
inputGeneration: "1:1",
|
|
76
|
+
packets: [
|
|
77
|
+
{
|
|
78
|
+
streamId: "s".repeat(32),
|
|
79
|
+
captureGeneration: 1,
|
|
80
|
+
sequence: ++videoSequence,
|
|
81
|
+
timestampUs: 1000,
|
|
82
|
+
type: "key",
|
|
83
|
+
codec: "vp8",
|
|
84
|
+
width: 1280,
|
|
85
|
+
height: 800,
|
|
86
|
+
dataBase64: "AA==",
|
|
87
|
+
capturedAt: new Date().toISOString(),
|
|
88
|
+
capturedAtMonotonicMs: performance.now(),
|
|
89
|
+
},
|
|
90
|
+
],
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return null;
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
resolveSupervisorPaths(home),
|
|
97
|
+
);
|
|
98
|
+
const client = new SupervisorClient({
|
|
99
|
+
bridgeId: "owned-video-socket",
|
|
100
|
+
paths: resolveSupervisorPaths(home),
|
|
101
|
+
});
|
|
102
|
+
type MediaReply = {
|
|
103
|
+
state: BrowserState;
|
|
104
|
+
frame?: BrowserFrame | null;
|
|
105
|
+
packets?: BrowserVideoReadReply["packets"];
|
|
106
|
+
};
|
|
107
|
+
let pending: Promise<MediaReply> | null = null;
|
|
108
|
+
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
109
|
+
try {
|
|
110
|
+
await client.connect();
|
|
111
|
+
const attached = await client.requestBrowser<{ viewerToken: string }>("attach", {
|
|
112
|
+
workspaceId: "owned",
|
|
113
|
+
viewerLabel: "Owned fixture",
|
|
114
|
+
});
|
|
115
|
+
const control = await client.requestBrowser<{ controlToken: string; state: BrowserState }>(
|
|
116
|
+
"acquire-control",
|
|
117
|
+
{ viewerToken: attached.viewerToken },
|
|
118
|
+
);
|
|
119
|
+
const input = {
|
|
120
|
+
viewerToken: attached.viewerToken,
|
|
121
|
+
quality: "high",
|
|
122
|
+
streamId: null,
|
|
123
|
+
afterSequence: 0,
|
|
124
|
+
waitMs: 250,
|
|
125
|
+
requestKeyFrame: false,
|
|
126
|
+
};
|
|
127
|
+
const maximum = await client.requestBrowser<MediaReply>("capture", {
|
|
128
|
+
viewerToken: attached.viewerToken,
|
|
129
|
+
quality: "maximum",
|
|
130
|
+
});
|
|
131
|
+
expect(maximum.frame?.mimeType).toBe("image/jpeg");
|
|
132
|
+
expect(jpegQuality).toBe(100);
|
|
133
|
+
const initial = await client.requestBrowser<MediaReply>("video.read", input);
|
|
134
|
+
// Switching panels can retire one workspace while another keeps playing.
|
|
135
|
+
// Its native teardown must not occupy the shared socket's video lane.
|
|
136
|
+
const retiring = await client.requestBrowser<{ viewerToken: string }>("attach", {
|
|
137
|
+
workspaceId: "retiring",
|
|
138
|
+
viewerLabel: "Retiring fixture",
|
|
139
|
+
});
|
|
140
|
+
blockRetirement = true;
|
|
141
|
+
retirement = client.requestBrowser("detach", { viewerToken: retiring.viewerToken });
|
|
142
|
+
await retirementStarted.promise;
|
|
143
|
+
const independentRead = client.requestBrowser<MediaReply>("video.read", input);
|
|
144
|
+
const progressed = await Promise.race([
|
|
145
|
+
independentRead.then(() => true),
|
|
146
|
+
new Promise<boolean>((resolve) => {
|
|
147
|
+
timeout = setTimeout(() => resolve(false), 250);
|
|
148
|
+
}),
|
|
149
|
+
]);
|
|
150
|
+
retirementGate.resolve();
|
|
151
|
+
await retirement;
|
|
152
|
+
await independentRead;
|
|
153
|
+
expect(progressed).toBe(true);
|
|
154
|
+
blockRetirement = false;
|
|
155
|
+
const frame = initial.packets![0]!.frame;
|
|
156
|
+
const stateReads = calls.filter((call) => call === "state").length;
|
|
157
|
+
block = true;
|
|
158
|
+
blockState = true;
|
|
159
|
+
pending = client.requestBrowser<MediaReply>("video.read", input);
|
|
160
|
+
await arrived;
|
|
161
|
+
expect(calls.filter((call) => call === "state")).toHaveLength(stateReads);
|
|
162
|
+
const mutation = client.requestBrowser("input", {
|
|
163
|
+
viewerToken: attached.viewerToken,
|
|
164
|
+
controlToken: control.controlToken,
|
|
165
|
+
expected: {
|
|
166
|
+
sessionId: initial.state.sessionId,
|
|
167
|
+
navigationGeneration: initial.state.navigationGeneration,
|
|
168
|
+
viewportGeneration: initial.state.viewportGeneration,
|
|
169
|
+
},
|
|
170
|
+
target: {
|
|
171
|
+
frameId: frame.frameId,
|
|
172
|
+
navigationGeneration: frame.navigationGeneration,
|
|
173
|
+
viewportGeneration: frame.viewportGeneration,
|
|
174
|
+
},
|
|
175
|
+
event: { kind: "move", point: { x: 10, y: 20, width: 1280, height: 800 } },
|
|
176
|
+
});
|
|
177
|
+
// Discrete input first reconciles the captured document, so a stalled metadata
|
|
178
|
+
// read must hold back every native primitive rather than let one reach an
|
|
179
|
+
// unobserved page. The pending video read must not be what delays it.
|
|
180
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
181
|
+
expect(calls).not.toContain("mouse.move");
|
|
182
|
+
expect(calls).not.toContain("input.begin");
|
|
183
|
+
stateGate.resolve();
|
|
184
|
+
const accepted = await Promise.race([
|
|
185
|
+
inputPublished.promise.then(() => true),
|
|
186
|
+
new Promise<boolean>((resolve) => {
|
|
187
|
+
timeout = setTimeout(() => resolve(false), 1000);
|
|
188
|
+
}),
|
|
189
|
+
]);
|
|
190
|
+
// Published through a pinned channel while the video read is still pending.
|
|
191
|
+
expect(accepted).toBe(true);
|
|
192
|
+
expect(calls.indexOf("input.begin")).toBeGreaterThan(-1);
|
|
193
|
+
expect(calls.indexOf("input.begin")).toBeLessThan(calls.indexOf("mouse.move"));
|
|
194
|
+
await mutation;
|
|
195
|
+
expect(calls).toContain("input.end");
|
|
196
|
+
const finalStateReads = calls.filter((call) => call === "state").length;
|
|
197
|
+
release();
|
|
198
|
+
expect((await pending).packets).toEqual([]);
|
|
199
|
+
expect(calls.filter((call) => call === "state")).toHaveLength(finalStateReads);
|
|
200
|
+
} finally {
|
|
201
|
+
if (timeout) clearTimeout(timeout);
|
|
202
|
+
retirementGate.resolve();
|
|
203
|
+
await retirement?.catch(() => undefined);
|
|
204
|
+
stateGate.resolve();
|
|
205
|
+
release();
|
|
206
|
+
await pending?.catch(() => undefined);
|
|
207
|
+
client.disconnect();
|
|
208
|
+
await running.close();
|
|
209
|
+
await rm(home, { recursive: true, force: true });
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("bounds sixteen viewers' media while retaining input, cancel and heartbeat capacity", async () => {
|
|
214
|
+
const home = await mkdtemp(join(tmpdir(), "owned-heartbeat-cap-review-"));
|
|
215
|
+
let release!: () => void;
|
|
216
|
+
let allStarted!: () => void;
|
|
217
|
+
let reads = 0;
|
|
218
|
+
const calls: string[] = [];
|
|
219
|
+
const barrier = new Promise<void>((resolve) => {
|
|
220
|
+
release = resolve;
|
|
221
|
+
});
|
|
222
|
+
const started = new Promise<void>((resolve) => {
|
|
223
|
+
allStarted = resolve;
|
|
224
|
+
});
|
|
225
|
+
const paths = resolveSupervisorPaths(home);
|
|
226
|
+
const running = await startSupervisorServer(
|
|
227
|
+
{
|
|
228
|
+
create: async () => ({ runtimeId: "r".repeat(32) }),
|
|
229
|
+
stop: async () => {},
|
|
230
|
+
async request(_runtime, operation) {
|
|
231
|
+
calls.push(operation);
|
|
232
|
+
if (operation === "identity") return { userAgent: "Fixture" };
|
|
233
|
+
if (operation === "state")
|
|
234
|
+
return {
|
|
235
|
+
url: "https://fixture.invalid/",
|
|
236
|
+
title: "Fixture",
|
|
237
|
+
inputGeneration: "1:1",
|
|
238
|
+
canGoBack: false,
|
|
239
|
+
canGoForward: false,
|
|
240
|
+
};
|
|
241
|
+
if (operation === "frame")
|
|
242
|
+
return {
|
|
243
|
+
dataBase64: "AA==",
|
|
244
|
+
byteLength: 1,
|
|
245
|
+
width: 1280,
|
|
246
|
+
height: 800,
|
|
247
|
+
capturedAt: new Date().toISOString(),
|
|
248
|
+
};
|
|
249
|
+
if (operation === "video.read") {
|
|
250
|
+
if (++reads === 12) allStarted();
|
|
251
|
+
await barrier;
|
|
252
|
+
return {
|
|
253
|
+
status: "waiting",
|
|
254
|
+
streamId: "s".repeat(32),
|
|
255
|
+
inputGeneration: "1:1",
|
|
256
|
+
packets: [],
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
return null;
|
|
260
|
+
},
|
|
261
|
+
},
|
|
262
|
+
paths,
|
|
263
|
+
);
|
|
264
|
+
const client = new SupervisorClient({ bridgeId: "owned-cap-review", paths });
|
|
265
|
+
const pending: Promise<unknown>[] = [];
|
|
266
|
+
try {
|
|
267
|
+
const lease = await client.connect();
|
|
268
|
+
const viewers: string[] = [];
|
|
269
|
+
for (let i = 0; i < 16; i++) {
|
|
270
|
+
const attached = await client.requestBrowser<{ viewerToken: string }>("attach", {
|
|
271
|
+
workspaceId: "owned",
|
|
272
|
+
viewerLabel: `Fixture ${i}`,
|
|
273
|
+
});
|
|
274
|
+
viewers.push(attached.viewerToken);
|
|
275
|
+
}
|
|
276
|
+
const viewerToken = viewers[0]!;
|
|
277
|
+
const { controlToken } = await client.requestBrowser<{ controlToken: string }>(
|
|
278
|
+
"acquire-control",
|
|
279
|
+
{ viewerToken },
|
|
280
|
+
);
|
|
281
|
+
const captured = await client.requestBrowser<{ state: BrowserState; frame: BrowserFrame }>(
|
|
282
|
+
"capture",
|
|
283
|
+
{ viewerToken },
|
|
284
|
+
);
|
|
285
|
+
const expected = {
|
|
286
|
+
sessionId: captured.state.sessionId,
|
|
287
|
+
runtimeId: captured.state.runtimeId!,
|
|
288
|
+
bridgeEpoch: captured.state.bridgeEpoch!,
|
|
289
|
+
navigationGeneration: captured.state.navigationGeneration,
|
|
290
|
+
viewportGeneration: captured.state.viewportGeneration,
|
|
291
|
+
};
|
|
292
|
+
const target = {
|
|
293
|
+
frameId: captured.frame.frameId,
|
|
294
|
+
navigationGeneration: captured.frame.navigationGeneration,
|
|
295
|
+
viewportGeneration: captured.frame.viewportGeneration,
|
|
296
|
+
};
|
|
297
|
+
const context = { viewerToken, controlToken, expected };
|
|
298
|
+
const gesture = await client.requestBrowser<{ gestureId: string; nextSequence: number }>(
|
|
299
|
+
"gesture.begin",
|
|
300
|
+
{
|
|
301
|
+
...context,
|
|
302
|
+
target,
|
|
303
|
+
pointerKind: "mouse",
|
|
304
|
+
},
|
|
305
|
+
);
|
|
306
|
+
for (const viewerToken of viewers)
|
|
307
|
+
pending.push(
|
|
308
|
+
client
|
|
309
|
+
.requestBrowser("video.read", {
|
|
310
|
+
viewerToken,
|
|
311
|
+
quality: "high",
|
|
312
|
+
streamId: null,
|
|
313
|
+
afterSequence: 0,
|
|
314
|
+
waitMs: 250,
|
|
315
|
+
requestKeyFrame: false,
|
|
316
|
+
})
|
|
317
|
+
.catch((error) => error),
|
|
318
|
+
);
|
|
319
|
+
await started;
|
|
320
|
+
const internal = client as unknown as { send(request: unknown): Promise<unknown> };
|
|
321
|
+
await expect(
|
|
322
|
+
internal.send({
|
|
323
|
+
method: "bridge.heartbeat",
|
|
324
|
+
bridgeId: "owned-cap-review",
|
|
325
|
+
epoch: lease.epoch,
|
|
326
|
+
}),
|
|
327
|
+
).resolves.toMatchObject({ epoch: lease.epoch });
|
|
328
|
+
// JPEG fallback uses the same media cap, so excess readers cannot consume
|
|
329
|
+
// the normal headroom reserved for an ordered press and its cleanup.
|
|
330
|
+
await expect(client.requestBrowser("capture", { viewerToken })).rejects.toMatchObject({
|
|
331
|
+
code: "RUNTIME_BUSY",
|
|
332
|
+
});
|
|
333
|
+
const pressed = await client.requestBrowser<{ nextSequence: number }>("gesture.update", {
|
|
334
|
+
...context,
|
|
335
|
+
gestureId: gesture.gestureId,
|
|
336
|
+
sequence: gesture.nextSequence,
|
|
337
|
+
target,
|
|
338
|
+
event: {
|
|
339
|
+
kind: "down",
|
|
340
|
+
point: { x: 10, y: 20, width: 1280, height: 800 },
|
|
341
|
+
button: "left",
|
|
342
|
+
clickCount: 1,
|
|
343
|
+
},
|
|
344
|
+
});
|
|
345
|
+
await client.requestBrowser("gesture.end", {
|
|
346
|
+
...context,
|
|
347
|
+
gestureId: gesture.gestureId,
|
|
348
|
+
sequence: pressed.nextSequence,
|
|
349
|
+
cancel: true,
|
|
350
|
+
});
|
|
351
|
+
expect(calls).toContain("mouse.down");
|
|
352
|
+
expect(calls).toContain("input.end");
|
|
353
|
+
expect(reads).toBe(12);
|
|
354
|
+
release();
|
|
355
|
+
const results = await Promise.all(pending);
|
|
356
|
+
expect(
|
|
357
|
+
results.filter(
|
|
358
|
+
(value) => value instanceof RuntimeProtocolError && value.code === "RUNTIME_BUSY",
|
|
359
|
+
),
|
|
360
|
+
).toHaveLength(4);
|
|
361
|
+
} finally {
|
|
362
|
+
release();
|
|
363
|
+
await Promise.allSettled(pending);
|
|
364
|
+
client.disconnect();
|
|
365
|
+
await running.close();
|
|
366
|
+
await rm(home, { recursive: true, force: true });
|
|
367
|
+
}
|
|
368
|
+
});
|
package/server/browser.ts
CHANGED
|
@@ -9,14 +9,25 @@ import type {
|
|
|
9
9
|
acquireControlRpc,
|
|
10
10
|
applyDevicePresetRpc,
|
|
11
11
|
attachBrowserRpc,
|
|
12
|
+
beginBrowserGestureRpc,
|
|
12
13
|
captureBrowserRpc,
|
|
14
|
+
closeBrowserRpc,
|
|
15
|
+
closeBrowserTabRpc,
|
|
16
|
+
createBrowserTabRpc,
|
|
13
17
|
detachBrowserRpc,
|
|
18
|
+
endBrowserGestureRpc,
|
|
19
|
+
listBrowserTabsRpc,
|
|
14
20
|
listOpenBrowserWorkspacesRpc,
|
|
15
21
|
navigateBrowserRpc,
|
|
16
22
|
releaseControlRpc,
|
|
23
|
+
reopenBrowserRpc,
|
|
17
24
|
resizeBrowserRpc,
|
|
18
25
|
sendBrowserInputRpc,
|
|
26
|
+
setCaptureDensityRpc,
|
|
27
|
+
updateBrowserGestureRpc,
|
|
19
28
|
} from "../shared/browser";
|
|
29
|
+
import type { BrowserVideoReadInput, BrowserVideoReadReply } from "../shared/browser-video";
|
|
30
|
+
import { createBrowserOperationDiagnostics } from "./browser-operation-diagnostics";
|
|
20
31
|
import { resolveBrowserRuntimeRoot } from "./runtime-path";
|
|
21
32
|
import type { JsonValue } from "./runtime-protocol";
|
|
22
33
|
import { resolveSupervisorPaths } from "./supervisor";
|
|
@@ -33,8 +44,18 @@ type AttachInput = RpcInput<typeof attachBrowserRpc>;
|
|
|
33
44
|
type AttachOutput = RpcOutput<typeof attachBrowserRpc>;
|
|
34
45
|
type DetachInput = RpcInput<typeof detachBrowserRpc>;
|
|
35
46
|
type DetachOutput = RpcOutput<typeof detachBrowserRpc>;
|
|
47
|
+
type DensityInput = RpcInput<typeof setCaptureDensityRpc>;
|
|
48
|
+
type DensityOutput = RpcOutput<typeof setCaptureDensityRpc>;
|
|
36
49
|
type CaptureInput = RpcInput<typeof captureBrowserRpc>;
|
|
37
50
|
type CaptureOutput = RpcOutput<typeof captureBrowserRpc>;
|
|
51
|
+
type CloseInput = RpcInput<typeof closeBrowserRpc>;
|
|
52
|
+
type CloseOutput = RpcOutput<typeof closeBrowserRpc>;
|
|
53
|
+
type CloseTabInput = RpcInput<typeof closeBrowserTabRpc>;
|
|
54
|
+
type CloseTabOutput = RpcOutput<typeof closeBrowserTabRpc>;
|
|
55
|
+
type CreateTabOutput = RpcOutput<typeof createBrowserTabRpc>;
|
|
56
|
+
type ListTabsOutput = RpcOutput<typeof listBrowserTabsRpc>;
|
|
57
|
+
type ReopenInput = RpcInput<typeof reopenBrowserRpc>;
|
|
58
|
+
type ReopenOutput = RpcOutput<typeof reopenBrowserRpc>;
|
|
38
59
|
type AcquireControlInput = RpcInput<typeof acquireControlRpc>;
|
|
39
60
|
type AcquireControlOutput = RpcOutput<typeof acquireControlRpc>;
|
|
40
61
|
type ReleaseControlInput = RpcInput<typeof releaseControlRpc>;
|
|
@@ -48,24 +69,60 @@ type ApplyDevicePresetInput = RpcInput<typeof applyDevicePresetRpc>;
|
|
|
48
69
|
type ApplyDevicePresetOutput = RpcOutput<typeof applyDevicePresetRpc>;
|
|
49
70
|
type SendInput = RpcInput<typeof sendBrowserInputRpc>;
|
|
50
71
|
type SendOutput = RpcOutput<typeof sendBrowserInputRpc>;
|
|
72
|
+
type BeginGestureInput = RpcInput<typeof beginBrowserGestureRpc>;
|
|
73
|
+
type BeginGestureOutput = RpcOutput<typeof beginBrowserGestureRpc>;
|
|
74
|
+
type UpdateGestureInput = RpcInput<typeof updateBrowserGestureRpc>;
|
|
75
|
+
type UpdateGestureOutput = RpcOutput<typeof updateBrowserGestureRpc>;
|
|
76
|
+
type EndGestureInput = RpcInput<typeof endBrowserGestureRpc>;
|
|
77
|
+
type EndGestureOutput = RpcOutput<typeof endBrowserGestureRpc>;
|
|
51
78
|
|
|
52
79
|
function paseoHome(): string {
|
|
53
80
|
return process.env.PASEO_HOME ?? join(homedir(), ".paseo");
|
|
54
81
|
}
|
|
55
82
|
|
|
56
83
|
class RemoteBrowserManager {
|
|
84
|
+
private readonly diagnose = createBrowserOperationDiagnostics();
|
|
85
|
+
|
|
86
|
+
private requestBrowser<Result = JsonValue>(operation: string, input: JsonValue): Promise<Result> {
|
|
87
|
+
return this.diagnose(operation, () => this.client.requestBrowser<Result>(operation, input));
|
|
88
|
+
}
|
|
89
|
+
|
|
57
90
|
constructor(private readonly client: SupervisorClient) {}
|
|
58
91
|
|
|
59
92
|
async connect(): Promise<void> {
|
|
60
93
|
await this.client.connect();
|
|
61
94
|
}
|
|
62
95
|
|
|
63
|
-
attach(workspaceId: string, viewerLabel: string): Promise<AttachOutput> {
|
|
64
|
-
return this.
|
|
96
|
+
attach(workspaceId: string, viewerLabel: string, tabId?: string): Promise<AttachOutput> {
|
|
97
|
+
return this.requestBrowser<AttachOutput>("attach", {
|
|
98
|
+
workspaceId,
|
|
99
|
+
viewerLabel,
|
|
100
|
+
...(tabId ? { tabId } : {}),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
listTabs(viewerToken: string): Promise<ListTabsOutput> {
|
|
105
|
+
return this.requestBrowser<ListTabsOutput>("tabs.list", { viewerToken });
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
createTab(viewerToken: string): Promise<CreateTabOutput> {
|
|
109
|
+
return this.requestBrowser<CreateTabOutput>("tabs.create", { viewerToken });
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
closeTab(input: CloseTabInput): Promise<CloseTabOutput> {
|
|
113
|
+
return this.requestBrowser<CloseTabOutput>("tabs.close", input);
|
|
65
114
|
}
|
|
66
115
|
|
|
67
116
|
detach(viewerToken: string): Promise<DetachOutput> {
|
|
68
|
-
return this.
|
|
117
|
+
return this.requestBrowser<DetachOutput>("detach", { viewerToken });
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
close(input: CloseInput): Promise<CloseOutput> {
|
|
121
|
+
return this.requestBrowser<CloseOutput>("close", input);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
reopen(workspaceId: string): Promise<ReopenOutput> {
|
|
125
|
+
return this.requestBrowser<ReopenOutput>("reopen", { workspaceId });
|
|
69
126
|
}
|
|
70
127
|
|
|
71
128
|
capture(
|
|
@@ -73,48 +130,65 @@ class RemoteBrowserManager {
|
|
|
73
130
|
quality: CaptureInput["quality"],
|
|
74
131
|
knownFrameId: string | null,
|
|
75
132
|
): Promise<CaptureOutput> {
|
|
76
|
-
return this.
|
|
133
|
+
return this.requestBrowser<CaptureOutput>("capture", {
|
|
77
134
|
viewerToken,
|
|
78
135
|
quality,
|
|
79
136
|
knownFrameId,
|
|
80
137
|
});
|
|
81
138
|
}
|
|
82
139
|
|
|
140
|
+
readVideo(input: BrowserVideoReadInput): Promise<BrowserVideoReadReply> {
|
|
141
|
+
return this.requestBrowser<BrowserVideoReadReply>("video.read", input as unknown as JsonValue);
|
|
142
|
+
}
|
|
143
|
+
|
|
83
144
|
acquireControl(viewerToken: string, takeover: boolean): Promise<AcquireControlOutput> {
|
|
84
|
-
return this.
|
|
145
|
+
return this.requestBrowser<AcquireControlOutput>("acquire-control", {
|
|
85
146
|
viewerToken,
|
|
86
147
|
takeover,
|
|
87
148
|
});
|
|
88
149
|
}
|
|
89
150
|
|
|
90
151
|
releaseControl(viewerToken: string, controlToken: string): Promise<ReleaseControlOutput> {
|
|
91
|
-
return this.
|
|
152
|
+
return this.requestBrowser<ReleaseControlOutput>("release-control", {
|
|
92
153
|
viewerToken,
|
|
93
154
|
controlToken,
|
|
94
155
|
});
|
|
95
156
|
}
|
|
96
157
|
|
|
97
158
|
navigate(input: NavigateInput): Promise<NavigateOutput> {
|
|
98
|
-
return this.
|
|
159
|
+
return this.requestBrowser<NavigateOutput>("navigate", input as unknown as JsonValue);
|
|
99
160
|
}
|
|
100
161
|
|
|
101
162
|
resize(input: ResizeInput): Promise<ResizeOutput> {
|
|
102
|
-
return this.
|
|
163
|
+
return this.requestBrowser<ResizeOutput>("viewport", input as unknown as JsonValue);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
setCaptureDensity(input: DensityInput): Promise<DensityOutput> {
|
|
167
|
+
return this.requestBrowser<DensityOutput>("capture.density", input as unknown as JsonValue);
|
|
103
168
|
}
|
|
104
169
|
|
|
105
170
|
applyDevicePreset(input: ApplyDevicePresetInput): Promise<ApplyDevicePresetOutput> {
|
|
106
|
-
return this.
|
|
107
|
-
"device",
|
|
108
|
-
input as unknown as JsonValue,
|
|
109
|
-
);
|
|
171
|
+
return this.requestBrowser<ApplyDevicePresetOutput>("device", input as unknown as JsonValue);
|
|
110
172
|
}
|
|
111
173
|
|
|
112
174
|
sendInput(input: SendInput): Promise<SendOutput> {
|
|
113
|
-
return this.
|
|
175
|
+
return this.requestBrowser<SendOutput>("input", input as unknown as JsonValue);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
beginGesture(input: BeginGestureInput): Promise<BeginGestureOutput> {
|
|
179
|
+
return this.requestBrowser("gesture.begin", input as unknown as JsonValue);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
updateGesture(input: UpdateGestureInput): Promise<UpdateGestureOutput> {
|
|
183
|
+
return this.requestBrowser("gesture.update", input as unknown as JsonValue);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
endGesture(input: EndGestureInput): Promise<EndGestureOutput> {
|
|
187
|
+
return this.requestBrowser("gesture.end", input as unknown as JsonValue);
|
|
114
188
|
}
|
|
115
189
|
|
|
116
190
|
async listOpenWorkspaceIds(): Promise<string[]> {
|
|
117
|
-
const result = await this.
|
|
191
|
+
const result = await this.requestBrowser("list", {});
|
|
118
192
|
if (
|
|
119
193
|
!result ||
|
|
120
194
|
typeof result !== "object" ||
|
|
@@ -132,7 +206,7 @@ class RemoteBrowserManager {
|
|
|
132
206
|
}
|
|
133
207
|
|
|
134
208
|
async archiveWorkspace(workspaceId: string): Promise<void> {
|
|
135
|
-
await this.
|
|
209
|
+
await this.requestBrowser("archive", { workspaceId });
|
|
136
210
|
}
|
|
137
211
|
|
|
138
212
|
issueAgentTicket(ticket: string): Promise<void> {
|
|
@@ -237,13 +311,46 @@ export async function handleAttachBrowser(
|
|
|
237
311
|
): Promise<AttachOutput> {
|
|
238
312
|
const workspace = await context.paseo.workspaces.ref(input.workspaceId).refresh();
|
|
239
313
|
if (!workspace) throw new Error("Workspace not found");
|
|
240
|
-
return (await getProductionManager()).attach(input.workspaceId, input.viewerLabel);
|
|
314
|
+
return (await getProductionManager()).attach(input.workspaceId, input.viewerLabel, input.tabId);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export async function handleListBrowserTabs({
|
|
318
|
+
viewerToken,
|
|
319
|
+
}: {
|
|
320
|
+
viewerToken: string;
|
|
321
|
+
}): Promise<ListTabsOutput> {
|
|
322
|
+
return (await getProductionManager()).listTabs(viewerToken);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export async function handleCreateBrowserTab({
|
|
326
|
+
viewerToken,
|
|
327
|
+
}: {
|
|
328
|
+
viewerToken: string;
|
|
329
|
+
}): Promise<CreateTabOutput> {
|
|
330
|
+
return (await getProductionManager()).createTab(viewerToken);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export async function handleCloseBrowserTab(input: CloseTabInput): Promise<CloseTabOutput> {
|
|
334
|
+
return (await getProductionManager()).closeTab(input);
|
|
241
335
|
}
|
|
242
336
|
|
|
243
337
|
export async function handleDetachBrowser({ viewerToken }: DetachInput): Promise<DetachOutput> {
|
|
244
338
|
return (await getProductionManager()).detach(viewerToken);
|
|
245
339
|
}
|
|
246
340
|
|
|
341
|
+
export async function handleCloseBrowser(input: CloseInput): Promise<CloseOutput> {
|
|
342
|
+
return (await getProductionManager()).close(input);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export async function handleReopenBrowser(
|
|
346
|
+
input: ReopenInput,
|
|
347
|
+
context: PluginHandlerContext,
|
|
348
|
+
): Promise<ReopenOutput> {
|
|
349
|
+
const workspace = await context.paseo.workspaces.ref(input.workspaceId).refresh();
|
|
350
|
+
if (!workspace) throw new Error("Workspace not found");
|
|
351
|
+
return (await getProductionManager()).reopen(input.workspaceId);
|
|
352
|
+
}
|
|
353
|
+
|
|
247
354
|
export async function handleWorkspaceArchived(workspaceId: string): Promise<void> {
|
|
248
355
|
if (productionStopped) return;
|
|
249
356
|
await (await getProductionManager()).archiveWorkspace(workspaceId);
|
|
@@ -283,6 +390,10 @@ export async function handleResizeBrowser(input: ResizeInput): Promise<ResizeOut
|
|
|
283
390
|
return (await getProductionManager()).resize(input);
|
|
284
391
|
}
|
|
285
392
|
|
|
393
|
+
export async function handleSetCaptureDensity(input: DensityInput): Promise<DensityOutput> {
|
|
394
|
+
return (await getProductionManager()).setCaptureDensity(input);
|
|
395
|
+
}
|
|
396
|
+
|
|
286
397
|
export async function handleApplyDevicePreset(
|
|
287
398
|
input: ApplyDevicePresetInput,
|
|
288
399
|
): Promise<ApplyDevicePresetOutput> {
|
|
@@ -293,9 +404,33 @@ export async function handleSendBrowserInput(input: SendInput): Promise<SendOutp
|
|
|
293
404
|
return (await getProductionManager()).sendInput(input);
|
|
294
405
|
}
|
|
295
406
|
|
|
407
|
+
/** Human-viewer live gestures use the existing authenticated supervisor, never an agent ticket. */
|
|
408
|
+
export async function handleBeginBrowserGesture(
|
|
409
|
+
input: BeginGestureInput,
|
|
410
|
+
): Promise<BeginGestureOutput> {
|
|
411
|
+
return (await getProductionManager()).beginGesture(input);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
export async function handleUpdateBrowserGesture(
|
|
415
|
+
input: UpdateGestureInput,
|
|
416
|
+
): Promise<UpdateGestureOutput> {
|
|
417
|
+
return (await getProductionManager()).updateGesture(input);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
export async function handleEndBrowserGesture(input: EndGestureInput): Promise<EndGestureOutput> {
|
|
421
|
+
return (await getProductionManager()).endGesture(input);
|
|
422
|
+
}
|
|
423
|
+
|
|
296
424
|
export async function cleanupBrowserServer(): Promise<void> {
|
|
297
425
|
productionStopped = true;
|
|
298
426
|
if (productionStart) await productionStart.catch(() => undefined);
|
|
299
427
|
productionManager?.disconnect();
|
|
300
428
|
productionManager = null;
|
|
301
429
|
}
|
|
430
|
+
|
|
431
|
+
/** Read bounded native video without acquiring control or exposing another transport. */
|
|
432
|
+
export async function handleReadBrowserVideo(
|
|
433
|
+
input: BrowserVideoReadInput,
|
|
434
|
+
): Promise<BrowserVideoReadReply> {
|
|
435
|
+
return (await getProductionManager()).readVideo(input);
|
|
436
|
+
}
|