@push.rocks/smartbrowser 2.0.11 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.smartconfig.json +49 -0
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/index.d.ts +1 -1
- package/dist_ts/index.js +7 -3
- package/dist_ts_web/00_commitinfo_data.d.ts +8 -0
- package/dist_ts_web/00_commitinfo_data.js +9 -0
- package/dist_ts_web/classes.livebrowsercanvasrenderer.d.ts +134 -0
- package/dist_ts_web/classes.livebrowsercanvasrenderer.js +1408 -0
- package/dist_ts_web/index.d.ts +3 -0
- package/dist_ts_web/index.js +2 -0
- package/dist_ts_web/interfaces.livebrowsercanvas.d.ts +41 -0
- package/dist_ts_web/interfaces.livebrowsercanvas.js +2 -0
- package/license.md +19 -0
- package/package.json +32 -20
- package/readme.hints.md +26 -12
- package/readme.md +96 -10
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/index.ts +7 -6
- package/ts_web/00_commitinfo_data.ts +8 -0
- package/ts_web/classes.livebrowsercanvasrenderer.ts +1722 -0
- package/ts_web/index.ts +27 -0
- package/ts_web/interfaces.livebrowsercanvas.ts +83 -0
- package/npmextra.json +0 -33
|
@@ -0,0 +1,1722 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ILiveBrowserFrame,
|
|
3
|
+
ILiveBrowserInputBase,
|
|
4
|
+
ILiveBrowserKeyInput,
|
|
5
|
+
ILiveBrowserModifierState,
|
|
6
|
+
ILiveBrowserMouseInput,
|
|
7
|
+
ILiveBrowserState,
|
|
8
|
+
ILiveBrowserViewport,
|
|
9
|
+
TLiveBrowserEvent,
|
|
10
|
+
} from '@push.rocks/smartpuppeteer';
|
|
11
|
+
|
|
12
|
+
import type {
|
|
13
|
+
ILiveBrowserCanvasError,
|
|
14
|
+
ILiveBrowserCanvasOperationOptions,
|
|
15
|
+
ILiveBrowserCanvasRendererOptions,
|
|
16
|
+
} from './interfaces.livebrowsercanvas.js';
|
|
17
|
+
|
|
18
|
+
const maxInputQueueLength = 128;
|
|
19
|
+
const maxPendingAcknowledgements = 16;
|
|
20
|
+
const maxInputReleaseAttempts = 2;
|
|
21
|
+
const maxFrameDimension = 12288;
|
|
22
|
+
const maxFramePixelArea = 8294400;
|
|
23
|
+
const maxFrameByteLength = (maxFramePixelArea * 4) + 1048576;
|
|
24
|
+
const maxViewportWidth = 4096;
|
|
25
|
+
const maxViewportHeight = 4096;
|
|
26
|
+
const maxDeviceScaleFactor = 3;
|
|
27
|
+
const coordinateEdgeInset = 0.001;
|
|
28
|
+
const defaultOperationTimeoutMs = 10000;
|
|
29
|
+
|
|
30
|
+
class LiveBrowserOperationTimeoutError extends Error {}
|
|
31
|
+
class LiveBrowserRunAbortedError extends Error {}
|
|
32
|
+
class LiveBrowserFrameProtocolError extends Error {}
|
|
33
|
+
class LiveBrowserFrameIntegrityError extends Error {}
|
|
34
|
+
|
|
35
|
+
interface IFrameWork {
|
|
36
|
+
frame: ILiveBrowserFrame;
|
|
37
|
+
runEpoch: number;
|
|
38
|
+
acknowledgementAttempted: boolean;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
interface IDisplayedFrame extends ILiveBrowserInputBase {
|
|
42
|
+
sequence: number;
|
|
43
|
+
viewport: ILiveBrowserViewport;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
type TInputCommandKind = 'key' | 'mouse' | 'mouseMove' | 'text' | 'wheel';
|
|
47
|
+
|
|
48
|
+
interface IInputCommand {
|
|
49
|
+
kind: TInputCommandKind;
|
|
50
|
+
runEpoch: number;
|
|
51
|
+
execute: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<void>;
|
|
52
|
+
onAttempt?: () => void;
|
|
53
|
+
onSuccess?: () => void;
|
|
54
|
+
resolve: () => void;
|
|
55
|
+
reject: (errorArg: unknown) => void;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
interface IResizeFence {
|
|
59
|
+
minimumViewportRevision: number;
|
|
60
|
+
target: ILiveBrowserViewport;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const viewportEquals = (
|
|
64
|
+
firstArg: ILiveBrowserViewport,
|
|
65
|
+
secondArg: ILiveBrowserViewport,
|
|
66
|
+
): boolean => (
|
|
67
|
+
firstArg.width === secondArg.width
|
|
68
|
+
&& firstArg.height === secondArg.height
|
|
69
|
+
&& firstArg.deviceScaleFactor === secondArg.deviceScaleFactor
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
const createModifiers = (
|
|
73
|
+
eventArg: MouseEvent | KeyboardEvent,
|
|
74
|
+
): ILiveBrowserModifierState => ({
|
|
75
|
+
alt: eventArg.altKey,
|
|
76
|
+
control: eventArg.ctrlKey,
|
|
77
|
+
meta: eventArg.metaKey,
|
|
78
|
+
shift: eventArg.shiftKey,
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
const createFrameAcknowledgementRequest = (frameArg: ILiveBrowserFrame) => ({
|
|
82
|
+
tabId: frameArg.tabId,
|
|
83
|
+
sequence: frameArg.sequence,
|
|
84
|
+
generation: frameArg.generation,
|
|
85
|
+
viewportRevision: frameArg.viewportRevision,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
const createInputIdentity = (displayedFrameArg: IDisplayedFrame): ILiveBrowserInputBase => ({
|
|
89
|
+
tabId: displayedFrameArg.tabId,
|
|
90
|
+
generation: displayedFrameArg.generation,
|
|
91
|
+
viewportRevision: displayedFrameArg.viewportRevision,
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const getErrorMessage = (errorArg: unknown): string => (
|
|
95
|
+
errorArg instanceof Error ? errorArg.message : String(errorArg)
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Renders transport-neutral live browser frames and maps DOM input back to the
|
|
100
|
+
* currently displayed SmartPuppeteer frame identity.
|
|
101
|
+
*/
|
|
102
|
+
export class LiveBrowserCanvasRenderer {
|
|
103
|
+
private readonly canvas: HTMLCanvasElement;
|
|
104
|
+
private readonly canvasContext: CanvasRenderingContext2D;
|
|
105
|
+
private readonly client: ILiveBrowserCanvasRendererOptions['client'];
|
|
106
|
+
private readonly focusTarget: HTMLElement;
|
|
107
|
+
private readonly resizeTarget?: Element;
|
|
108
|
+
private readonly getDeviceScaleFactor: () => number;
|
|
109
|
+
private readonly operationTimeoutMs: number;
|
|
110
|
+
private readonly frameDecodeTimeoutMs: number;
|
|
111
|
+
private readonly onError?: ILiveBrowserCanvasRendererOptions['onError'];
|
|
112
|
+
private readonly onFrameRendered?: ILiveBrowserCanvasRendererOptions['onFrameRendered'];
|
|
113
|
+
private readonly initialCanvasWidth: number;
|
|
114
|
+
private readonly initialCanvasHeight: number;
|
|
115
|
+
|
|
116
|
+
private lifecycleTail: Promise<void> = Promise.resolve();
|
|
117
|
+
private runEpoch = 0;
|
|
118
|
+
private running = false;
|
|
119
|
+
private suspended = false;
|
|
120
|
+
private stopping = false;
|
|
121
|
+
private interruptionRequested = false;
|
|
122
|
+
private runController?: AbortController;
|
|
123
|
+
private operationControllers = new Set<AbortController>();
|
|
124
|
+
private acceptingFrames = false;
|
|
125
|
+
private state?: ILiveBrowserState;
|
|
126
|
+
private unsubscribe?: () => void;
|
|
127
|
+
private listenerController?: AbortController;
|
|
128
|
+
private resizeObserver?: ResizeObserver;
|
|
129
|
+
private focusWasAdjusted = false;
|
|
130
|
+
private focusHadTabIndex = false;
|
|
131
|
+
private focusTabIndexValue: string | null = null;
|
|
132
|
+
|
|
133
|
+
private displayedFrame?: IDisplayedFrame;
|
|
134
|
+
private highestFrameSequence = -1;
|
|
135
|
+
private queuedFrame?: IFrameWork;
|
|
136
|
+
private frameProcessingPromise?: Promise<void>;
|
|
137
|
+
private acknowledgementPromises = new Set<Promise<void>>();
|
|
138
|
+
private rawDecodeSettlementPromises = new Set<Promise<void>>();
|
|
139
|
+
|
|
140
|
+
private inputBlocked = true;
|
|
141
|
+
private inputCommands: IInputCommand[] = [];
|
|
142
|
+
private inputProcessing = false;
|
|
143
|
+
private inputIdleResolvers: Array<() => void> = [];
|
|
144
|
+
private inputResetTail: Promise<void> = Promise.resolve();
|
|
145
|
+
private inputResetPromise?: Promise<void>;
|
|
146
|
+
private inputResetCount = 0;
|
|
147
|
+
private inputOverflowRecoveryPending = false;
|
|
148
|
+
private pressedKeys = new Map<string, ILiveBrowserKeyInput>();
|
|
149
|
+
private pressedMouseButtons = new Map<string, ILiveBrowserMouseInput>();
|
|
150
|
+
private capturedPointerIds = new Set<number>();
|
|
151
|
+
|
|
152
|
+
private pendingViewport?: ILiveBrowserViewport;
|
|
153
|
+
private inFlightViewport?: ILiveBrowserViewport;
|
|
154
|
+
private viewportProcessing = false;
|
|
155
|
+
private viewportProcessingPromise?: Promise<void>;
|
|
156
|
+
private resizeFence?: IResizeFence;
|
|
157
|
+
|
|
158
|
+
constructor(optionsArg: ILiveBrowserCanvasRendererOptions) {
|
|
159
|
+
this.canvas = optionsArg.canvas;
|
|
160
|
+
this.client = optionsArg.client;
|
|
161
|
+
this.focusTarget = optionsArg.focusTarget ?? this.canvas;
|
|
162
|
+
this.resizeTarget = optionsArg.resizeTarget;
|
|
163
|
+
this.getDeviceScaleFactor = optionsArg.getDeviceScaleFactor
|
|
164
|
+
?? (() => window.devicePixelRatio || 1);
|
|
165
|
+
this.operationTimeoutMs = this.validateTimeout(
|
|
166
|
+
optionsArg.operationTimeoutMs ?? defaultOperationTimeoutMs,
|
|
167
|
+
'operationTimeoutMs',
|
|
168
|
+
);
|
|
169
|
+
this.frameDecodeTimeoutMs = this.validateTimeout(
|
|
170
|
+
optionsArg.frameDecodeTimeoutMs ?? this.operationTimeoutMs,
|
|
171
|
+
'frameDecodeTimeoutMs',
|
|
172
|
+
);
|
|
173
|
+
this.onError = optionsArg.onError;
|
|
174
|
+
this.onFrameRendered = optionsArg.onFrameRendered;
|
|
175
|
+
this.initialCanvasWidth = this.canvas.width;
|
|
176
|
+
this.initialCanvasHeight = this.canvas.height;
|
|
177
|
+
|
|
178
|
+
const canvasContext = this.canvas.getContext('2d');
|
|
179
|
+
if (!canvasContext) {
|
|
180
|
+
throw new Error('LiveBrowserCanvasRenderer requires a 2D canvas context');
|
|
181
|
+
}
|
|
182
|
+
this.canvasContext = canvasContext;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
public get isRunning(): boolean {
|
|
186
|
+
return this.running;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
public get isSuspended(): boolean {
|
|
190
|
+
return this.suspended;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
public start(): Promise<void> {
|
|
194
|
+
return this.enqueueLifecycle(async () => {
|
|
195
|
+
if (this.running) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (this.suspended) {
|
|
199
|
+
throw new Error('Live browser renderer is suspended; call resume() to start a fresh run');
|
|
200
|
+
}
|
|
201
|
+
await this.startRun(false);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
public stop(): Promise<void> {
|
|
206
|
+
this.interruptRun(this.runEpoch);
|
|
207
|
+
return this.enqueueLifecycle(async () => {
|
|
208
|
+
const runEpoch = this.runEpoch;
|
|
209
|
+
this.interruptRun(runEpoch);
|
|
210
|
+
if (this.stopping) {
|
|
211
|
+
await this.finishRun(runEpoch, false);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (!this.suspended) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
this.suspended = false;
|
|
218
|
+
this.restoreCanvasAndFocus();
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Aborts the current transport generation without replaying input recovery.
|
|
224
|
+
*/
|
|
225
|
+
public suspend(): Promise<void> {
|
|
226
|
+
this.interruptRun(this.runEpoch);
|
|
227
|
+
return this.enqueueLifecycle(async () => {
|
|
228
|
+
const runEpoch = this.runEpoch;
|
|
229
|
+
this.interruptRun(runEpoch);
|
|
230
|
+
if (!this.stopping) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
await this.finishRun(runEpoch, true);
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Starts a new transport generation that requires new state and a new frame.
|
|
239
|
+
*/
|
|
240
|
+
public resume(): Promise<void> {
|
|
241
|
+
return this.enqueueLifecycle(async () => {
|
|
242
|
+
if (this.running) {
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (!this.suspended) {
|
|
246
|
+
throw new Error('Live browser renderer is not suspended');
|
|
247
|
+
}
|
|
248
|
+
await this.startRun(true);
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Sends arbitrary committed text, including text produced by an external IME.
|
|
254
|
+
*/
|
|
255
|
+
public async insertText(textArg: string): Promise<void> {
|
|
256
|
+
const displayedFrame = this.requireInputFrame();
|
|
257
|
+
const inputIdentity = createInputIdentity(displayedFrame);
|
|
258
|
+
await this.enqueueInputCommand({
|
|
259
|
+
kind: 'text',
|
|
260
|
+
runEpoch: this.runEpoch,
|
|
261
|
+
execute: async (optionsArg) => {
|
|
262
|
+
await this.client.insertText({
|
|
263
|
+
...inputIdentity,
|
|
264
|
+
text: textArg,
|
|
265
|
+
}, optionsArg);
|
|
266
|
+
},
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Reads the configured stable resize target and requests the corresponding
|
|
272
|
+
* remote CSS viewport. Intrinsic canvas dimensions are never used here.
|
|
273
|
+
*/
|
|
274
|
+
public syncViewport(): void {
|
|
275
|
+
if (!this.running || this.stopping || this.interruptionRequested || !this.resizeTarget) {
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
const width = Math.floor(this.resizeTarget.clientWidth);
|
|
279
|
+
const height = Math.floor(this.resizeTarget.clientHeight);
|
|
280
|
+
this.queueViewport(width, height, this.runEpoch);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
private enqueueLifecycle(operationArg: () => Promise<void>): Promise<void> {
|
|
284
|
+
const operation = this.lifecycleTail.then(operationArg, operationArg);
|
|
285
|
+
this.lifecycleTail = operation.then(() => undefined, () => undefined);
|
|
286
|
+
return operation;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private async startRun(resumingArg: boolean): Promise<void> {
|
|
290
|
+
await Promise.all([...this.rawDecodeSettlementPromises]);
|
|
291
|
+
const runEpoch = ++this.runEpoch;
|
|
292
|
+
this.runController = new AbortController();
|
|
293
|
+
this.running = true;
|
|
294
|
+
this.suspended = false;
|
|
295
|
+
this.stopping = false;
|
|
296
|
+
this.interruptionRequested = false;
|
|
297
|
+
this.acceptingFrames = true;
|
|
298
|
+
this.inputBlocked = true;
|
|
299
|
+
this.state = undefined;
|
|
300
|
+
this.displayedFrame = undefined;
|
|
301
|
+
this.highestFrameSequence = -1;
|
|
302
|
+
this.clearCanvas();
|
|
303
|
+
|
|
304
|
+
try {
|
|
305
|
+
this.makeFocusTargetFocusable();
|
|
306
|
+
this.installDomListeners(runEpoch);
|
|
307
|
+
this.unsubscribe = this.client.onEvent((eventArg) => {
|
|
308
|
+
this.handleEvent(eventArg, runEpoch);
|
|
309
|
+
});
|
|
310
|
+
this.applyState(this.client.getState(), runEpoch);
|
|
311
|
+
this.installResizeObserver(runEpoch);
|
|
312
|
+
} catch (error) {
|
|
313
|
+
this.interruptRun(runEpoch);
|
|
314
|
+
await this.finishRun(runEpoch, resumingArg);
|
|
315
|
+
throw error;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
private interruptRun(runEpochArg: number): void {
|
|
320
|
+
if (
|
|
321
|
+
runEpochArg !== this.runEpoch
|
|
322
|
+
|| (!this.running && !this.stopping)
|
|
323
|
+
) {
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
this.running = false;
|
|
327
|
+
this.stopping = true;
|
|
328
|
+
this.interruptionRequested = true;
|
|
329
|
+
this.acceptingFrames = false;
|
|
330
|
+
this.inputBlocked = true;
|
|
331
|
+
this.pendingViewport = undefined;
|
|
332
|
+
this.queuedFrame = undefined;
|
|
333
|
+
this.displayedFrame = undefined;
|
|
334
|
+
this.resizeFence = undefined;
|
|
335
|
+
this.releasePointerCaptures();
|
|
336
|
+
this.clearCanvas();
|
|
337
|
+
|
|
338
|
+
const interruptionError = new LiveBrowserRunAbortedError(
|
|
339
|
+
`Live browser renderer run ${runEpochArg} was interrupted`,
|
|
340
|
+
);
|
|
341
|
+
this.runController?.abort(interruptionError);
|
|
342
|
+
for (const controller of this.operationControllers) {
|
|
343
|
+
controller.abort(interruptionError);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
347
|
+
for (const command of abandonedCommands) {
|
|
348
|
+
command.reject(interruptionError);
|
|
349
|
+
}
|
|
350
|
+
this.pressedKeys.clear();
|
|
351
|
+
this.pressedMouseButtons.clear();
|
|
352
|
+
this.inputOverflowRecoveryPending = false;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
private async finishRun(runEpochArg: number, suspendArg: boolean): Promise<void> {
|
|
356
|
+
if (runEpochArg !== this.runEpoch) {
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
try {
|
|
361
|
+
this.resizeObserver?.disconnect();
|
|
362
|
+
this.resizeObserver = undefined;
|
|
363
|
+
this.listenerController?.abort();
|
|
364
|
+
this.listenerController = undefined;
|
|
365
|
+
const unsubscribe = this.unsubscribe;
|
|
366
|
+
this.unsubscribe = undefined;
|
|
367
|
+
if (unsubscribe) {
|
|
368
|
+
try {
|
|
369
|
+
unsubscribe();
|
|
370
|
+
} catch (error) {
|
|
371
|
+
this.reportError({
|
|
372
|
+
code: 'renderer_cleanup_failed',
|
|
373
|
+
message: `Could not unsubscribe the live browser renderer: ${getErrorMessage(error)}`,
|
|
374
|
+
cause: error,
|
|
375
|
+
}, runEpochArg);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
await Promise.allSettled([
|
|
380
|
+
this.viewportProcessingPromise,
|
|
381
|
+
this.inputResetTail,
|
|
382
|
+
this.waitForInputIdle(),
|
|
383
|
+
this.frameProcessingPromise,
|
|
384
|
+
...this.acknowledgementPromises,
|
|
385
|
+
]);
|
|
386
|
+
} finally {
|
|
387
|
+
for (const controller of this.operationControllers) {
|
|
388
|
+
controller.abort(new LiveBrowserRunAbortedError(
|
|
389
|
+
`Live browser renderer run ${runEpochArg} finished`,
|
|
390
|
+
));
|
|
391
|
+
}
|
|
392
|
+
this.operationControllers.clear();
|
|
393
|
+
this.runController = undefined;
|
|
394
|
+
this.running = false;
|
|
395
|
+
this.suspended = suspendArg;
|
|
396
|
+
this.acceptingFrames = false;
|
|
397
|
+
this.state = undefined;
|
|
398
|
+
this.displayedFrame = undefined;
|
|
399
|
+
this.highestFrameSequence = -1;
|
|
400
|
+
this.queuedFrame = undefined;
|
|
401
|
+
this.frameProcessingPromise = undefined;
|
|
402
|
+
this.acknowledgementPromises.clear();
|
|
403
|
+
this.resizeFence = undefined;
|
|
404
|
+
this.pendingViewport = undefined;
|
|
405
|
+
this.inFlightViewport = undefined;
|
|
406
|
+
this.viewportProcessing = false;
|
|
407
|
+
this.viewportProcessingPromise = undefined;
|
|
408
|
+
this.inputResetPromise = undefined;
|
|
409
|
+
this.inputResetTail = Promise.resolve();
|
|
410
|
+
this.inputResetCount = 0;
|
|
411
|
+
this.inputBlocked = true;
|
|
412
|
+
this.pressedKeys.clear();
|
|
413
|
+
this.pressedMouseButtons.clear();
|
|
414
|
+
this.releasePointerCaptures();
|
|
415
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
416
|
+
for (const command of abandonedCommands) {
|
|
417
|
+
command.reject(new Error('Live browser renderer stopped before dispatching input'));
|
|
418
|
+
}
|
|
419
|
+
const inputIdleResolvers = this.inputIdleResolvers;
|
|
420
|
+
this.inputIdleResolvers = [];
|
|
421
|
+
for (const resolver of inputIdleResolvers) {
|
|
422
|
+
resolver();
|
|
423
|
+
}
|
|
424
|
+
this.restoreCanvasAndFocus();
|
|
425
|
+
this.interruptionRequested = false;
|
|
426
|
+
this.stopping = false;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
private restoreCanvasAndFocus(): void {
|
|
431
|
+
this.canvas.width = this.initialCanvasWidth;
|
|
432
|
+
this.canvas.height = this.initialCanvasHeight;
|
|
433
|
+
this.restoreFocusTarget();
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
private installDomListeners(runEpochArg: number): void {
|
|
437
|
+
const listenerController = new AbortController();
|
|
438
|
+
const signal = listenerController.signal;
|
|
439
|
+
this.listenerController = listenerController;
|
|
440
|
+
|
|
441
|
+
this.canvas.addEventListener('pointerdown', (eventArg) => {
|
|
442
|
+
this.handlePointerDown(eventArg, runEpochArg);
|
|
443
|
+
}, { signal });
|
|
444
|
+
this.canvas.addEventListener('pointermove', (eventArg) => {
|
|
445
|
+
this.handlePointerMove(eventArg, runEpochArg);
|
|
446
|
+
}, { signal });
|
|
447
|
+
this.canvas.addEventListener('pointerup', (eventArg) => {
|
|
448
|
+
this.handlePointerUp(eventArg, runEpochArg);
|
|
449
|
+
}, { signal });
|
|
450
|
+
this.canvas.addEventListener('pointercancel', (eventArg) => {
|
|
451
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
452
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
453
|
+
}, { signal });
|
|
454
|
+
this.canvas.addEventListener('lostpointercapture', (eventArg) => {
|
|
455
|
+
this.capturedPointerIds.delete(eventArg.pointerId);
|
|
456
|
+
if (this.pressedMouseButtons.size > 0) {
|
|
457
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
458
|
+
}
|
|
459
|
+
}, { signal });
|
|
460
|
+
this.canvas.addEventListener('contextmenu', (eventArg) => {
|
|
461
|
+
eventArg.preventDefault();
|
|
462
|
+
}, { signal });
|
|
463
|
+
this.canvas.addEventListener('wheel', (eventArg) => {
|
|
464
|
+
this.handleWheel(eventArg, runEpochArg);
|
|
465
|
+
}, { passive: false, signal });
|
|
466
|
+
|
|
467
|
+
this.focusTarget.addEventListener('keydown', (eventArg) => {
|
|
468
|
+
this.handleKey(eventArg, 'down', runEpochArg);
|
|
469
|
+
}, { signal });
|
|
470
|
+
this.focusTarget.addEventListener('keyup', (eventArg) => {
|
|
471
|
+
this.handleKey(eventArg, 'up', runEpochArg);
|
|
472
|
+
}, { signal });
|
|
473
|
+
this.focusTarget.addEventListener('compositionend', (eventArg) => {
|
|
474
|
+
if (!this.inputBlocked && eventArg.data) {
|
|
475
|
+
void this.insertText(eventArg.data).catch(() => undefined);
|
|
476
|
+
}
|
|
477
|
+
}, { signal });
|
|
478
|
+
|
|
479
|
+
window.addEventListener('blur', () => {
|
|
480
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
481
|
+
}, { signal });
|
|
482
|
+
window.addEventListener('resize', () => {
|
|
483
|
+
this.syncViewport();
|
|
484
|
+
}, { signal });
|
|
485
|
+
document.addEventListener('visibilitychange', () => {
|
|
486
|
+
if (document.visibilityState === 'hidden') {
|
|
487
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
488
|
+
}
|
|
489
|
+
}, { signal });
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
private installResizeObserver(runEpochArg: number): void {
|
|
493
|
+
if (!this.resizeTarget) {
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
this.resizeObserver = new ResizeObserver((entriesArg) => {
|
|
497
|
+
const matchingEntry = entriesArg.find((entryArg) => entryArg.target === this.resizeTarget);
|
|
498
|
+
if (!matchingEntry) {
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
this.queueViewport(
|
|
502
|
+
Math.floor(matchingEntry.contentRect.width),
|
|
503
|
+
Math.floor(matchingEntry.contentRect.height),
|
|
504
|
+
runEpochArg,
|
|
505
|
+
);
|
|
506
|
+
});
|
|
507
|
+
this.resizeObserver.observe(this.resizeTarget);
|
|
508
|
+
this.syncViewport();
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
private handleEvent(eventArg: TLiveBrowserEvent, runEpochArg: number): void {
|
|
512
|
+
if (eventArg.type === 'frame') {
|
|
513
|
+
this.queueFrame(eventArg.frame, runEpochArg);
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
if (!this.running || runEpochArg !== this.runEpoch) {
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
if (eventArg.type === 'state') {
|
|
520
|
+
this.applyState(eventArg.state, runEpochArg);
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
this.reportError({
|
|
524
|
+
code: 'remote_browser_error',
|
|
525
|
+
message: `${eventArg.error.code}: ${eventArg.error.message}`,
|
|
526
|
+
cause: eventArg.error,
|
|
527
|
+
}, runEpochArg);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
private applyState(stateArg: ILiveBrowserState, runEpochArg: number): void {
|
|
531
|
+
if (!this.running || runEpochArg !== this.runEpoch) {
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
this.state = stateArg;
|
|
535
|
+
|
|
536
|
+
if (this.displayedFrame && this.isStateAheadOfDisplayedFrame(stateArg, this.displayedFrame)) {
|
|
537
|
+
this.displayedFrame = undefined;
|
|
538
|
+
this.clearCanvas();
|
|
539
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
this.updateInputAvailability();
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
private isStateAheadOfDisplayedFrame(
|
|
546
|
+
stateArg: ILiveBrowserState,
|
|
547
|
+
displayedFrameArg: IDisplayedFrame,
|
|
548
|
+
): boolean {
|
|
549
|
+
if (stateArg.status !== 'running' || stateArg.activeTabId !== displayedFrameArg.tabId) {
|
|
550
|
+
return true;
|
|
551
|
+
}
|
|
552
|
+
const activeTab = stateArg.tabs.find((tabArg) => tabArg.id === displayedFrameArg.tabId);
|
|
553
|
+
if (!activeTab || activeTab.status !== 'open') {
|
|
554
|
+
return true;
|
|
555
|
+
}
|
|
556
|
+
return stateArg.viewportRevision > displayedFrameArg.viewportRevision
|
|
557
|
+
|| activeTab.generation > displayedFrameArg.generation;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
private queueFrame(frameArg: ILiveBrowserFrame, runEpochArg: number): void {
|
|
561
|
+
if (!this.running || !this.acceptingFrames || runEpochArg !== this.runEpoch) {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
try {
|
|
566
|
+
this.validateFrame(frameArg);
|
|
567
|
+
if (frameArg.sequence <= this.highestFrameSequence) {
|
|
568
|
+
throw new LiveBrowserFrameProtocolError(
|
|
569
|
+
`frame.sequence ${frameArg.sequence} must be strictly greater than current run high-water ${this.highestFrameSequence}`,
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
} catch (error) {
|
|
573
|
+
this.requestSuspend(runEpochArg);
|
|
574
|
+
this.reportError({
|
|
575
|
+
code: 'frame_render_failed',
|
|
576
|
+
message: `Rejected invalid live browser frame: ${getErrorMessage(error)}`,
|
|
577
|
+
cause: error,
|
|
578
|
+
frame: frameArg,
|
|
579
|
+
}, runEpochArg);
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
this.highestFrameSequence = frameArg.sequence;
|
|
584
|
+
const frameWork: IFrameWork = {
|
|
585
|
+
frame: frameArg,
|
|
586
|
+
runEpoch: runEpochArg,
|
|
587
|
+
acknowledgementAttempted: false,
|
|
588
|
+
};
|
|
589
|
+
if (this.queuedFrame) {
|
|
590
|
+
const supersededFrame = this.queuedFrame;
|
|
591
|
+
this.queuedFrame = undefined;
|
|
592
|
+
void this.acknowledgeFrame(supersededFrame);
|
|
593
|
+
}
|
|
594
|
+
this.queuedFrame = frameWork;
|
|
595
|
+
|
|
596
|
+
this.ensureFrameProcessor();
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
private ensureFrameProcessor(): void {
|
|
600
|
+
if (this.frameProcessingPromise || !this.queuedFrame) {
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
const processingPromise = this.processFrameQueue().finally(() => {
|
|
604
|
+
if (this.frameProcessingPromise === processingPromise) {
|
|
605
|
+
this.frameProcessingPromise = undefined;
|
|
606
|
+
if (this.queuedFrame) {
|
|
607
|
+
this.ensureFrameProcessor();
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
});
|
|
611
|
+
this.frameProcessingPromise = processingPromise;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
private async processFrameQueue(): Promise<void> {
|
|
615
|
+
while (this.queuedFrame) {
|
|
616
|
+
const frameWork = this.queuedFrame;
|
|
617
|
+
this.queuedFrame = undefined;
|
|
618
|
+
await this.processFrame(frameWork);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
private async processFrame(frameWorkArg: IFrameWork): Promise<void> {
|
|
623
|
+
const { frame, runEpoch } = frameWorkArg;
|
|
624
|
+
let imageBitmap: ImageBitmap | undefined;
|
|
625
|
+
let terminalFailure = false;
|
|
626
|
+
let terminalError: unknown;
|
|
627
|
+
try {
|
|
628
|
+
if (!this.canRenderFrame(frame, runEpoch)) {
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
const frameBytes = new Uint8Array(frame.data.byteLength);
|
|
633
|
+
frameBytes.set(frame.data);
|
|
634
|
+
imageBitmap = await this.decodeFrame(new Blob([frameBytes.buffer], {
|
|
635
|
+
type: frame.mimeType,
|
|
636
|
+
}), runEpoch);
|
|
637
|
+
|
|
638
|
+
if (imageBitmap.width !== frame.width || imageBitmap.height !== frame.height) {
|
|
639
|
+
throw new LiveBrowserFrameIntegrityError(
|
|
640
|
+
`decoded frame dimensions ${imageBitmap.width}x${imageBitmap.height} do not match ${frame.width}x${frame.height}`,
|
|
641
|
+
);
|
|
642
|
+
}
|
|
643
|
+
if (!this.canRenderFrame(frame, runEpoch) || frame.sequence !== this.highestFrameSequence) {
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
this.canvas.width = frame.width;
|
|
648
|
+
this.canvas.height = frame.height;
|
|
649
|
+
this.canvasContext.clearRect(0, 0, frame.width, frame.height);
|
|
650
|
+
this.canvasContext.drawImage(imageBitmap, 0, 0, frame.width, frame.height);
|
|
651
|
+
this.displayedFrame = {
|
|
652
|
+
tabId: frame.tabId,
|
|
653
|
+
sequence: frame.sequence,
|
|
654
|
+
generation: frame.generation,
|
|
655
|
+
viewportRevision: frame.viewportRevision,
|
|
656
|
+
viewport: { ...frame.viewport },
|
|
657
|
+
};
|
|
658
|
+
|
|
659
|
+
if (
|
|
660
|
+
this.resizeFence
|
|
661
|
+
&& !this.viewportProcessing
|
|
662
|
+
&& frame.viewportRevision >= this.resizeFence.minimumViewportRevision
|
|
663
|
+
&& viewportEquals(frame.viewport, this.resizeFence.target)
|
|
664
|
+
) {
|
|
665
|
+
this.resizeFence = undefined;
|
|
666
|
+
}
|
|
667
|
+
this.updateInputAvailability();
|
|
668
|
+
this.callFrameRendered(frame, runEpoch);
|
|
669
|
+
} catch (error) {
|
|
670
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
terminalFailure = true;
|
|
674
|
+
terminalError = error;
|
|
675
|
+
} finally {
|
|
676
|
+
imageBitmap?.close();
|
|
677
|
+
await this.acknowledgeFrame(frameWorkArg);
|
|
678
|
+
if (terminalFailure) {
|
|
679
|
+
this.requestSuspend(runEpoch);
|
|
680
|
+
this.reportError({
|
|
681
|
+
code: 'frame_render_failed',
|
|
682
|
+
message: `Could not render live browser frame: ${getErrorMessage(terminalError)}`,
|
|
683
|
+
cause: terminalError,
|
|
684
|
+
frame,
|
|
685
|
+
}, runEpoch);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
private canRenderFrame(frameArg: ILiveBrowserFrame, runEpochArg: number): boolean {
|
|
691
|
+
if (
|
|
692
|
+
!this.running
|
|
693
|
+
|| !this.acceptingFrames
|
|
694
|
+
|| runEpochArg !== this.runEpoch
|
|
695
|
+
|| this.viewportProcessing
|
|
696
|
+
) {
|
|
697
|
+
return false;
|
|
698
|
+
}
|
|
699
|
+
const state = this.state;
|
|
700
|
+
if (!state || state.status !== 'running' || state.activeTabId !== frameArg.tabId) {
|
|
701
|
+
return false;
|
|
702
|
+
}
|
|
703
|
+
const activeTab = state.tabs.find((tabArg) => tabArg.id === frameArg.tabId);
|
|
704
|
+
if (!activeTab || activeTab.status !== 'open') {
|
|
705
|
+
return false;
|
|
706
|
+
}
|
|
707
|
+
if (
|
|
708
|
+
state.viewportRevision > frameArg.viewportRevision
|
|
709
|
+
|| activeTab.generation > frameArg.generation
|
|
710
|
+
) {
|
|
711
|
+
return false;
|
|
712
|
+
}
|
|
713
|
+
if (this.resizeFence) {
|
|
714
|
+
return frameArg.viewportRevision >= this.resizeFence.minimumViewportRevision
|
|
715
|
+
&& viewportEquals(frameArg.viewport, this.resizeFence.target);
|
|
716
|
+
}
|
|
717
|
+
return true;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
private acknowledgeFrame(frameWorkArg: IFrameWork): Promise<void> {
|
|
721
|
+
if (frameWorkArg.acknowledgementAttempted) {
|
|
722
|
+
return Promise.resolve();
|
|
723
|
+
}
|
|
724
|
+
frameWorkArg.acknowledgementAttempted = true;
|
|
725
|
+
|
|
726
|
+
if (!this.isRunActive(frameWorkArg.runEpoch)) {
|
|
727
|
+
return Promise.resolve();
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
if (this.acknowledgementPromises.size >= maxPendingAcknowledgements) {
|
|
731
|
+
this.requestSuspend(frameWorkArg.runEpoch);
|
|
732
|
+
this.reportError({
|
|
733
|
+
code: 'frame_acknowledgement_failed',
|
|
734
|
+
message: `Live browser frame acknowledgement capacity of ${maxPendingAcknowledgements} was reached`,
|
|
735
|
+
}, frameWorkArg.runEpoch);
|
|
736
|
+
return Promise.resolve();
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const acknowledgement = this.runClientOperation(
|
|
740
|
+
(optionsArg) => this.client.acknowledgeFrame(
|
|
741
|
+
createFrameAcknowledgementRequest(frameWorkArg.frame),
|
|
742
|
+
optionsArg,
|
|
743
|
+
),
|
|
744
|
+
'frame acknowledgement',
|
|
745
|
+
frameWorkArg.runEpoch,
|
|
746
|
+
)
|
|
747
|
+
.then(() => undefined)
|
|
748
|
+
.catch((error) => {
|
|
749
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
this.requestSuspend(frameWorkArg.runEpoch);
|
|
753
|
+
this.reportError({
|
|
754
|
+
code: 'frame_acknowledgement_failed',
|
|
755
|
+
message: `Could not acknowledge live browser frame: ${getErrorMessage(error)}`,
|
|
756
|
+
cause: error,
|
|
757
|
+
frame: frameWorkArg.frame,
|
|
758
|
+
}, frameWorkArg.runEpoch);
|
|
759
|
+
});
|
|
760
|
+
let trackedAcknowledgement: Promise<void>;
|
|
761
|
+
trackedAcknowledgement = acknowledgement.finally(() => {
|
|
762
|
+
this.acknowledgementPromises.delete(trackedAcknowledgement);
|
|
763
|
+
});
|
|
764
|
+
this.acknowledgementPromises.add(trackedAcknowledgement);
|
|
765
|
+
return trackedAcknowledgement;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
private handlePointerDown(eventArg: PointerEvent, runEpochArg: number): void {
|
|
769
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
const button = this.getMouseButton(eventArg.button);
|
|
773
|
+
if (!button) {
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
const displayedFrame = this.getInputFrame();
|
|
777
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
eventArg.preventDefault();
|
|
781
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
782
|
+
if (!coordinates) {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
this.focusTarget.focus({ preventScroll: true });
|
|
786
|
+
try {
|
|
787
|
+
this.canvas.setPointerCapture(eventArg.pointerId);
|
|
788
|
+
this.capturedPointerIds.add(eventArg.pointerId);
|
|
789
|
+
} catch {
|
|
790
|
+
// Pointer capture may already have been released by the browser.
|
|
791
|
+
}
|
|
792
|
+
const input: ILiveBrowserMouseInput = {
|
|
793
|
+
...createInputIdentity(displayedFrame),
|
|
794
|
+
type: 'down',
|
|
795
|
+
...coordinates,
|
|
796
|
+
button,
|
|
797
|
+
buttons: eventArg.buttons,
|
|
798
|
+
clickCount: Math.min(3, Math.max(1, eventArg.detail || 1)),
|
|
799
|
+
modifiers: createModifiers(eventArg),
|
|
800
|
+
};
|
|
801
|
+
void this.enqueueInputCommand({
|
|
802
|
+
kind: 'mouse',
|
|
803
|
+
runEpoch: runEpochArg,
|
|
804
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
805
|
+
onAttempt: () => this.pressedMouseButtons.set(button, input),
|
|
806
|
+
}).catch(() => undefined);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
private handlePointerMove(eventArg: PointerEvent, runEpochArg: number): void {
|
|
810
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
const displayedFrame = this.getInputFrame();
|
|
814
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
eventArg.preventDefault();
|
|
818
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
819
|
+
if (!coordinates) {
|
|
820
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
821
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
822
|
+
return;
|
|
823
|
+
}
|
|
824
|
+
const input: ILiveBrowserMouseInput = {
|
|
825
|
+
...createInputIdentity(displayedFrame),
|
|
826
|
+
type: 'move',
|
|
827
|
+
...coordinates,
|
|
828
|
+
button: 'none',
|
|
829
|
+
buttons: eventArg.buttons,
|
|
830
|
+
modifiers: createModifiers(eventArg),
|
|
831
|
+
};
|
|
832
|
+
void this.enqueueInputCommand({
|
|
833
|
+
kind: 'mouseMove',
|
|
834
|
+
runEpoch: runEpochArg,
|
|
835
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
836
|
+
}).catch(() => undefined);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
private handlePointerUp(eventArg: PointerEvent, runEpochArg: number): void {
|
|
840
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
const button = this.getMouseButton(eventArg.button);
|
|
844
|
+
const displayedFrame = this.getInputFrame();
|
|
845
|
+
if (!button || !displayedFrame || runEpochArg !== this.runEpoch) {
|
|
846
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
eventArg.preventDefault();
|
|
850
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
851
|
+
if (!coordinates) {
|
|
852
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
853
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
const input: ILiveBrowserMouseInput = {
|
|
857
|
+
...createInputIdentity(displayedFrame),
|
|
858
|
+
type: 'up',
|
|
859
|
+
...coordinates,
|
|
860
|
+
button,
|
|
861
|
+
buttons: eventArg.buttons,
|
|
862
|
+
clickCount: Math.min(3, Math.max(1, eventArg.detail || 1)),
|
|
863
|
+
modifiers: createModifiers(eventArg),
|
|
864
|
+
};
|
|
865
|
+
void this.enqueueInputCommand({
|
|
866
|
+
kind: 'mouse',
|
|
867
|
+
runEpoch: runEpochArg,
|
|
868
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
869
|
+
onSuccess: () => this.pressedMouseButtons.delete(button),
|
|
870
|
+
}).catch(() => undefined);
|
|
871
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
private handleWheel(eventArg: WheelEvent, runEpochArg: number): void {
|
|
875
|
+
const displayedFrame = this.getInputFrame();
|
|
876
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
877
|
+
return;
|
|
878
|
+
}
|
|
879
|
+
eventArg.preventDefault();
|
|
880
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
881
|
+
if (!coordinates) {
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
const multiplier = eventArg.deltaMode === WheelEvent.DOM_DELTA_LINE
|
|
885
|
+
? 16
|
|
886
|
+
: eventArg.deltaMode === WheelEvent.DOM_DELTA_PAGE
|
|
887
|
+
? displayedFrame.viewport.height
|
|
888
|
+
: 1;
|
|
889
|
+
const clampDelta = (valueArg: number) => Math.max(-1000000, Math.min(1000000, valueArg));
|
|
890
|
+
void this.enqueueInputCommand({
|
|
891
|
+
kind: 'wheel',
|
|
892
|
+
runEpoch: runEpochArg,
|
|
893
|
+
execute: async (optionsArg) => this.client.dispatchWheel({
|
|
894
|
+
...createInputIdentity(displayedFrame),
|
|
895
|
+
...coordinates,
|
|
896
|
+
deltaX: clampDelta(eventArg.deltaX * multiplier),
|
|
897
|
+
deltaY: clampDelta(eventArg.deltaY * multiplier),
|
|
898
|
+
modifiers: createModifiers(eventArg),
|
|
899
|
+
}, optionsArg),
|
|
900
|
+
}).catch(() => undefined);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
private handleKey(
|
|
904
|
+
eventArg: KeyboardEvent,
|
|
905
|
+
typeArg: 'down' | 'up',
|
|
906
|
+
runEpochArg: number,
|
|
907
|
+
): void {
|
|
908
|
+
const displayedFrame = this.getInputFrame();
|
|
909
|
+
if (
|
|
910
|
+
!displayedFrame
|
|
911
|
+
|| runEpochArg !== this.runEpoch
|
|
912
|
+
|| eventArg.isComposing
|
|
913
|
+
|| eventArg.key === 'Process'
|
|
914
|
+
) {
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
eventArg.preventDefault();
|
|
918
|
+
const text = typeArg === 'down'
|
|
919
|
+
&& eventArg.key.length === 1
|
|
920
|
+
&& !eventArg.metaKey
|
|
921
|
+
&& (!eventArg.ctrlKey || eventArg.altKey)
|
|
922
|
+
? eventArg.key
|
|
923
|
+
: undefined;
|
|
924
|
+
const input: ILiveBrowserKeyInput = {
|
|
925
|
+
...createInputIdentity(displayedFrame),
|
|
926
|
+
type: typeArg,
|
|
927
|
+
key: eventArg.key,
|
|
928
|
+
code: eventArg.code || undefined,
|
|
929
|
+
text,
|
|
930
|
+
windowsVirtualKeyCode: eventArg.keyCode || undefined,
|
|
931
|
+
autoRepeat: eventArg.repeat,
|
|
932
|
+
isKeypad: eventArg.location === KeyboardEvent.DOM_KEY_LOCATION_NUMPAD,
|
|
933
|
+
location: eventArg.location,
|
|
934
|
+
modifiers: createModifiers(eventArg),
|
|
935
|
+
};
|
|
936
|
+
const keyIdentity = eventArg.code || eventArg.key;
|
|
937
|
+
void this.enqueueInputCommand({
|
|
938
|
+
kind: 'key',
|
|
939
|
+
runEpoch: runEpochArg,
|
|
940
|
+
execute: async (optionsArg) => this.client.dispatchKey(input, optionsArg),
|
|
941
|
+
onAttempt: typeArg === 'down'
|
|
942
|
+
? () => this.pressedKeys.set(keyIdentity, input)
|
|
943
|
+
: undefined,
|
|
944
|
+
onSuccess: () => {
|
|
945
|
+
if (typeArg === 'up') {
|
|
946
|
+
this.pressedKeys.delete(keyIdentity);
|
|
947
|
+
}
|
|
948
|
+
},
|
|
949
|
+
}).catch(() => undefined);
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
private mapCoordinates(
|
|
953
|
+
eventArg: MouseEvent,
|
|
954
|
+
viewportArg: ILiveBrowserViewport,
|
|
955
|
+
): { x: number; y: number } | undefined {
|
|
956
|
+
const canvasBounds = this.canvas.getBoundingClientRect();
|
|
957
|
+
if (canvasBounds.width <= 0 || canvasBounds.height <= 0) {
|
|
958
|
+
return undefined;
|
|
959
|
+
}
|
|
960
|
+
const x = ((eventArg.clientX - canvasBounds.left) / canvasBounds.width) * viewportArg.width;
|
|
961
|
+
const y = ((eventArg.clientY - canvasBounds.top) / canvasBounds.height) * viewportArg.height;
|
|
962
|
+
return {
|
|
963
|
+
x: Math.max(0, Math.min(viewportArg.width - coordinateEdgeInset, x)),
|
|
964
|
+
y: Math.max(0, Math.min(viewportArg.height - coordinateEdgeInset, y)),
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
private getMouseButton(buttonArg: number): ILiveBrowserMouseInput['button'] | undefined {
|
|
969
|
+
return buttonArg === 0
|
|
970
|
+
? 'left'
|
|
971
|
+
: buttonArg === 1
|
|
972
|
+
? 'middle'
|
|
973
|
+
: buttonArg === 2
|
|
974
|
+
? 'right'
|
|
975
|
+
: buttonArg === 3
|
|
976
|
+
? 'back'
|
|
977
|
+
: buttonArg === 4
|
|
978
|
+
? 'forward'
|
|
979
|
+
: undefined;
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
private getInputFrame(): IDisplayedFrame | undefined {
|
|
983
|
+
return this.inputBlocked || !this.isDisplayedFrameCurrent()
|
|
984
|
+
? undefined
|
|
985
|
+
: this.displayedFrame;
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
private requireInputFrame(): IDisplayedFrame {
|
|
989
|
+
const displayedFrame = this.getInputFrame();
|
|
990
|
+
if (!displayedFrame) {
|
|
991
|
+
throw new Error('Live browser input is unavailable until a current frame is displayed');
|
|
992
|
+
}
|
|
993
|
+
return displayedFrame;
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
private isDisplayedFrameCurrent(): boolean {
|
|
997
|
+
const state = this.state;
|
|
998
|
+
const displayedFrame = this.displayedFrame;
|
|
999
|
+
if (!state || !displayedFrame || state.status !== 'running') {
|
|
1000
|
+
return false;
|
|
1001
|
+
}
|
|
1002
|
+
if (
|
|
1003
|
+
state.activeTabId !== displayedFrame.tabId
|
|
1004
|
+
|| state.viewportRevision > displayedFrame.viewportRevision
|
|
1005
|
+
) {
|
|
1006
|
+
return false;
|
|
1007
|
+
}
|
|
1008
|
+
const activeTab = state.tabs.find((tabArg) => tabArg.id === displayedFrame.tabId);
|
|
1009
|
+
return Boolean(
|
|
1010
|
+
activeTab
|
|
1011
|
+
&& activeTab.status === 'open'
|
|
1012
|
+
&& activeTab.generation <= displayedFrame.generation,
|
|
1013
|
+
);
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
private enqueueInputCommand(commandArg: Omit<IInputCommand, 'reject' | 'resolve'>): Promise<void> {
|
|
1017
|
+
if (!this.running || this.inputBlocked || commandArg.runEpoch !== this.runEpoch) {
|
|
1018
|
+
return Promise.reject(new Error('Live browser input is currently blocked'));
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
return new Promise<void>((resolve, reject) => {
|
|
1022
|
+
const command: IInputCommand = {
|
|
1023
|
+
...commandArg,
|
|
1024
|
+
resolve,
|
|
1025
|
+
reject,
|
|
1026
|
+
};
|
|
1027
|
+
const lastCommand = this.inputCommands.at(-1);
|
|
1028
|
+
if (command.kind === 'mouseMove' && lastCommand?.kind === 'mouseMove') {
|
|
1029
|
+
lastCommand.resolve();
|
|
1030
|
+
this.inputCommands[this.inputCommands.length - 1] = command;
|
|
1031
|
+
} else if (this.inputCommands.length >= maxInputQueueLength) {
|
|
1032
|
+
if (command.kind === 'mouseMove') {
|
|
1033
|
+
resolve();
|
|
1034
|
+
return;
|
|
1035
|
+
}
|
|
1036
|
+
const error = new Error('Live browser input queue reached its capacity');
|
|
1037
|
+
this.reportError({
|
|
1038
|
+
code: 'input_queue_capacity_exceeded',
|
|
1039
|
+
message: error.message,
|
|
1040
|
+
cause: error,
|
|
1041
|
+
}, command.runEpoch);
|
|
1042
|
+
reject(error);
|
|
1043
|
+
this.recoverFromInputOverflow(command.runEpoch);
|
|
1044
|
+
return;
|
|
1045
|
+
} else {
|
|
1046
|
+
this.inputCommands.push(command);
|
|
1047
|
+
}
|
|
1048
|
+
this.processInputCommands();
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
private processInputCommands(): void {
|
|
1053
|
+
if (this.inputProcessing) {
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
this.inputProcessing = true;
|
|
1057
|
+
void (async () => {
|
|
1058
|
+
try {
|
|
1059
|
+
while (this.inputCommands.length > 0) {
|
|
1060
|
+
const command = this.inputCommands.shift()!;
|
|
1061
|
+
if (!this.isRunActive(command.runEpoch)) {
|
|
1062
|
+
command.reject(new LiveBrowserRunAbortedError(
|
|
1063
|
+
`Live browser input belongs to inactive run ${command.runEpoch}`,
|
|
1064
|
+
));
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
try {
|
|
1068
|
+
command.onAttempt?.();
|
|
1069
|
+
await this.runClientOperation(command.execute, 'input dispatch', command.runEpoch);
|
|
1070
|
+
if (!this.isRunActive(command.runEpoch)) {
|
|
1071
|
+
throw new LiveBrowserRunAbortedError(
|
|
1072
|
+
`Live browser input belongs to inactive run ${command.runEpoch}`,
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
command.onSuccess?.();
|
|
1076
|
+
command.resolve();
|
|
1077
|
+
} catch (error) {
|
|
1078
|
+
command.reject(error);
|
|
1079
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
1080
|
+
continue;
|
|
1081
|
+
}
|
|
1082
|
+
if (error instanceof LiveBrowserOperationTimeoutError) {
|
|
1083
|
+
this.requestSuspend(command.runEpoch);
|
|
1084
|
+
}
|
|
1085
|
+
this.reportError({
|
|
1086
|
+
code: 'input_dispatch_failed',
|
|
1087
|
+
message: `Could not dispatch live browser input: ${getErrorMessage(error)}`,
|
|
1088
|
+
cause: error,
|
|
1089
|
+
}, command.runEpoch);
|
|
1090
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
1091
|
+
for (const abandonedCommand of abandonedCommands) {
|
|
1092
|
+
abandonedCommand.reject(
|
|
1093
|
+
new Error('Live browser input was abandoned after a dispatch failure'),
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
if (!(error instanceof LiveBrowserOperationTimeoutError) && !this.stopping) {
|
|
1097
|
+
void this.scheduleInputReset(false, command.runEpoch);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
} finally {
|
|
1102
|
+
this.inputProcessing = false;
|
|
1103
|
+
if (this.inputCommands.length > 0) {
|
|
1104
|
+
this.processInputCommands();
|
|
1105
|
+
} else {
|
|
1106
|
+
const resolvers = this.inputIdleResolvers;
|
|
1107
|
+
this.inputIdleResolvers = [];
|
|
1108
|
+
for (const resolver of resolvers) {
|
|
1109
|
+
resolver();
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
})();
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
private waitForInputIdle(): Promise<void> {
|
|
1117
|
+
if (!this.inputProcessing && this.inputCommands.length === 0) {
|
|
1118
|
+
return Promise.resolve();
|
|
1119
|
+
}
|
|
1120
|
+
return new Promise<void>((resolve) => {
|
|
1121
|
+
this.inputIdleResolvers.push(resolve);
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
private scheduleInputReset(
|
|
1126
|
+
clearDisplayArg: boolean,
|
|
1127
|
+
runEpochArg: number,
|
|
1128
|
+
): Promise<void> {
|
|
1129
|
+
if (runEpochArg !== this.runEpoch || this.stopping) {
|
|
1130
|
+
return Promise.resolve();
|
|
1131
|
+
}
|
|
1132
|
+
this.inputBlocked = true;
|
|
1133
|
+
this.releasePointerCaptures();
|
|
1134
|
+
if (clearDisplayArg) {
|
|
1135
|
+
this.displayedFrame = undefined;
|
|
1136
|
+
this.clearCanvas();
|
|
1137
|
+
}
|
|
1138
|
+
if (this.inputResetPromise) {
|
|
1139
|
+
return this.inputResetPromise;
|
|
1140
|
+
}
|
|
1141
|
+
this.inputResetCount++;
|
|
1142
|
+
let resetOperation: Promise<void>;
|
|
1143
|
+
resetOperation = this.inputResetTail.then(async () => {
|
|
1144
|
+
await this.waitForInputIdle();
|
|
1145
|
+
await this.releasePressedInput(runEpochArg);
|
|
1146
|
+
}).finally(() => {
|
|
1147
|
+
this.inputResetCount--;
|
|
1148
|
+
if (this.inputResetPromise === resetOperation) {
|
|
1149
|
+
this.inputResetPromise = undefined;
|
|
1150
|
+
}
|
|
1151
|
+
this.updateInputAvailability();
|
|
1152
|
+
});
|
|
1153
|
+
this.inputResetPromise = resetOperation;
|
|
1154
|
+
this.inputResetTail = resetOperation.then(() => undefined, () => undefined);
|
|
1155
|
+
return resetOperation;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
private recoverFromInputOverflow(runEpochArg: number): void {
|
|
1159
|
+
if (this.inputOverflowRecoveryPending) {
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
this.inputOverflowRecoveryPending = true;
|
|
1163
|
+
this.inputBlocked = true;
|
|
1164
|
+
void this.scheduleInputReset(false, runEpochArg).finally(() => {
|
|
1165
|
+
this.inputOverflowRecoveryPending = false;
|
|
1166
|
+
});
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
private async releasePressedInput(runEpochArg: number): Promise<void> {
|
|
1170
|
+
for (const [keyIdentity, pressedKey] of [...this.pressedKeys.entries()]) {
|
|
1171
|
+
const released = await this.tryInputRelease(
|
|
1172
|
+
(optionsArg) => this.client.dispatchKey({
|
|
1173
|
+
...pressedKey,
|
|
1174
|
+
type: 'up',
|
|
1175
|
+
text: undefined,
|
|
1176
|
+
autoRepeat: false,
|
|
1177
|
+
modifiers: {},
|
|
1178
|
+
}, optionsArg),
|
|
1179
|
+
runEpochArg,
|
|
1180
|
+
);
|
|
1181
|
+
if (released && this.pressedKeys.get(keyIdentity) === pressedKey) {
|
|
1182
|
+
this.pressedKeys.delete(keyIdentity);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
for (const [button, pressedButton] of [...this.pressedMouseButtons.entries()]) {
|
|
1186
|
+
const released = await this.tryInputRelease(
|
|
1187
|
+
(optionsArg) => this.client.dispatchMouse({
|
|
1188
|
+
...pressedButton,
|
|
1189
|
+
type: 'up',
|
|
1190
|
+
buttons: 0,
|
|
1191
|
+
clickCount: 1,
|
|
1192
|
+
modifiers: {},
|
|
1193
|
+
}, optionsArg),
|
|
1194
|
+
runEpochArg,
|
|
1195
|
+
);
|
|
1196
|
+
if (released && this.pressedMouseButtons.get(button) === pressedButton) {
|
|
1197
|
+
this.pressedMouseButtons.delete(button);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
private async tryInputRelease(
|
|
1203
|
+
operationArg: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<void>,
|
|
1204
|
+
runEpochArg: number,
|
|
1205
|
+
): Promise<boolean> {
|
|
1206
|
+
let lastError: unknown;
|
|
1207
|
+
for (let attempt = 0; attempt < maxInputReleaseAttempts; attempt++) {
|
|
1208
|
+
try {
|
|
1209
|
+
await this.runClientOperation(
|
|
1210
|
+
operationArg,
|
|
1211
|
+
'input release',
|
|
1212
|
+
runEpochArg,
|
|
1213
|
+
);
|
|
1214
|
+
return true;
|
|
1215
|
+
} catch (error) {
|
|
1216
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
1217
|
+
return false;
|
|
1218
|
+
}
|
|
1219
|
+
lastError = error;
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
this.requestSuspend(runEpochArg);
|
|
1223
|
+
this.reportInputReleaseError(lastError, runEpochArg);
|
|
1224
|
+
return false;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
private reportInputReleaseError(errorArg: unknown, runEpochArg: number): void {
|
|
1228
|
+
this.reportError({
|
|
1229
|
+
code: 'input_dispatch_failed',
|
|
1230
|
+
message: `Could not release live browser input: ${getErrorMessage(errorArg)}`,
|
|
1231
|
+
cause: errorArg,
|
|
1232
|
+
}, runEpochArg);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
private queueViewport(widthArg: number, heightArg: number, runEpochArg: number): void {
|
|
1236
|
+
if (
|
|
1237
|
+
!this.running
|
|
1238
|
+
|| this.stopping
|
|
1239
|
+
|| this.interruptionRequested
|
|
1240
|
+
|| runEpochArg !== this.runEpoch
|
|
1241
|
+
|| widthArg <= 0
|
|
1242
|
+
|| heightArg <= 0
|
|
1243
|
+
) {
|
|
1244
|
+
return;
|
|
1245
|
+
}
|
|
1246
|
+
const deviceScaleFactor = this.getDeviceScaleFactor();
|
|
1247
|
+
if (!Number.isFinite(deviceScaleFactor) || deviceScaleFactor <= 0) {
|
|
1248
|
+
this.reportError({
|
|
1249
|
+
code: 'viewport_update_failed',
|
|
1250
|
+
message: 'Live browser device scale factor must be a positive finite number',
|
|
1251
|
+
}, runEpochArg);
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
const viewport: ILiveBrowserViewport = {
|
|
1255
|
+
width: widthArg,
|
|
1256
|
+
height: heightArg,
|
|
1257
|
+
deviceScaleFactor,
|
|
1258
|
+
};
|
|
1259
|
+
if (this.pendingViewport && viewportEquals(viewport, this.pendingViewport)) {
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
if (this.inFlightViewport && viewportEquals(viewport, this.inFlightViewport)) {
|
|
1263
|
+
this.pendingViewport = undefined;
|
|
1264
|
+
return;
|
|
1265
|
+
}
|
|
1266
|
+
if (
|
|
1267
|
+
!this.inFlightViewport
|
|
1268
|
+
&& this.resizeFence
|
|
1269
|
+
&& viewportEquals(viewport, this.resizeFence.target)
|
|
1270
|
+
) {
|
|
1271
|
+
this.pendingViewport = undefined;
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
if (
|
|
1275
|
+
!this.inFlightViewport
|
|
1276
|
+
&& !this.resizeFence
|
|
1277
|
+
&& this.state
|
|
1278
|
+
&& viewportEquals(viewport, this.state.viewport)
|
|
1279
|
+
) {
|
|
1280
|
+
this.pendingViewport = undefined;
|
|
1281
|
+
this.updateInputAvailability();
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
this.pendingViewport = viewport;
|
|
1285
|
+
this.inputBlocked = true;
|
|
1286
|
+
this.ensureViewportProcessor(runEpochArg);
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
private ensureViewportProcessor(runEpochArg: number): void {
|
|
1290
|
+
if (this.viewportProcessingPromise || !this.pendingViewport) {
|
|
1291
|
+
return;
|
|
1292
|
+
}
|
|
1293
|
+
const processingPromise = this.processViewportQueue(runEpochArg).finally(() => {
|
|
1294
|
+
if (this.viewportProcessingPromise === processingPromise) {
|
|
1295
|
+
this.viewportProcessingPromise = undefined;
|
|
1296
|
+
if (
|
|
1297
|
+
this.pendingViewport
|
|
1298
|
+
&& this.running
|
|
1299
|
+
&& !this.stopping
|
|
1300
|
+
&& runEpochArg === this.runEpoch
|
|
1301
|
+
) {
|
|
1302
|
+
this.ensureViewportProcessor(runEpochArg);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
});
|
|
1306
|
+
this.viewportProcessingPromise = processingPromise;
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
private async processViewportQueue(runEpochArg: number): Promise<void> {
|
|
1310
|
+
this.viewportProcessing = true;
|
|
1311
|
+
try {
|
|
1312
|
+
while (
|
|
1313
|
+
this.pendingViewport
|
|
1314
|
+
&& this.running
|
|
1315
|
+
&& !this.stopping
|
|
1316
|
+
&& runEpochArg === this.runEpoch
|
|
1317
|
+
) {
|
|
1318
|
+
const viewport = this.pendingViewport;
|
|
1319
|
+
this.pendingViewport = undefined;
|
|
1320
|
+
this.inFlightViewport = viewport;
|
|
1321
|
+
if (!this.resizeFence && this.state && viewportEquals(viewport, this.state.viewport)) {
|
|
1322
|
+
this.inFlightViewport = undefined;
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
await this.scheduleInputReset(true, runEpochArg);
|
|
1327
|
+
if (!this.running || this.stopping || runEpochArg !== this.runEpoch) {
|
|
1328
|
+
return;
|
|
1329
|
+
}
|
|
1330
|
+
const currentRevision = this.state?.viewportRevision ?? 0;
|
|
1331
|
+
const previousMinimumRevision = this.resizeFence?.minimumViewportRevision ?? currentRevision;
|
|
1332
|
+
try {
|
|
1333
|
+
await this.runClientOperation(
|
|
1334
|
+
(optionsArg) => this.client.setViewport(viewport, optionsArg),
|
|
1335
|
+
'viewport update',
|
|
1336
|
+
runEpochArg,
|
|
1337
|
+
);
|
|
1338
|
+
if (!this.running || this.stopping || runEpochArg !== this.runEpoch) {
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
this.resizeFence = {
|
|
1342
|
+
target: viewport,
|
|
1343
|
+
minimumViewportRevision: Math.max(currentRevision, previousMinimumRevision) + 1,
|
|
1344
|
+
};
|
|
1345
|
+
} catch (error) {
|
|
1346
|
+
if (runEpochArg === this.runEpoch && !(error instanceof LiveBrowserRunAbortedError)) {
|
|
1347
|
+
this.resizeFence = undefined;
|
|
1348
|
+
if (error instanceof LiveBrowserOperationTimeoutError) {
|
|
1349
|
+
this.requestSuspend(runEpochArg);
|
|
1350
|
+
}
|
|
1351
|
+
this.reportError({
|
|
1352
|
+
code: 'viewport_update_failed',
|
|
1353
|
+
message: `Could not update live browser viewport: ${getErrorMessage(error)}`,
|
|
1354
|
+
cause: error,
|
|
1355
|
+
}, runEpochArg);
|
|
1356
|
+
}
|
|
1357
|
+
} finally {
|
|
1358
|
+
if (this.inFlightViewport && viewportEquals(this.inFlightViewport, viewport)) {
|
|
1359
|
+
this.inFlightViewport = undefined;
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
} finally {
|
|
1364
|
+
this.viewportProcessing = false;
|
|
1365
|
+
this.updateInputAvailability();
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
private updateInputAvailability(): void {
|
|
1370
|
+
this.inputBlocked = !(
|
|
1371
|
+
this.running
|
|
1372
|
+
&& this.acceptingFrames
|
|
1373
|
+
&& !this.viewportProcessing
|
|
1374
|
+
&& !this.resizeFence
|
|
1375
|
+
&& this.inputResetCount === 0
|
|
1376
|
+
&& this.isDisplayedFrameCurrent()
|
|
1377
|
+
);
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
private makeFocusTargetFocusable(): void {
|
|
1381
|
+
this.focusWasAdjusted = false;
|
|
1382
|
+
if (this.focusTarget.tabIndex >= 0) {
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
this.focusHadTabIndex = this.focusTarget.hasAttribute('tabindex');
|
|
1386
|
+
this.focusTabIndexValue = this.focusTarget.getAttribute('tabindex');
|
|
1387
|
+
this.focusTarget.setAttribute('tabindex', '0');
|
|
1388
|
+
this.focusWasAdjusted = true;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
private restoreFocusTarget(): void {
|
|
1392
|
+
if (!this.focusWasAdjusted) {
|
|
1393
|
+
return;
|
|
1394
|
+
}
|
|
1395
|
+
if (this.focusHadTabIndex && this.focusTabIndexValue !== null) {
|
|
1396
|
+
this.focusTarget.setAttribute('tabindex', this.focusTabIndexValue);
|
|
1397
|
+
} else {
|
|
1398
|
+
this.focusTarget.removeAttribute('tabindex');
|
|
1399
|
+
}
|
|
1400
|
+
this.focusWasAdjusted = false;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
private clearCanvas(): void {
|
|
1404
|
+
this.canvasContext.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
private callFrameRendered(frameArg: ILiveBrowserFrame, runEpochArg: number): void {
|
|
1408
|
+
if (!this.onFrameRendered || runEpochArg !== this.runEpoch || !this.running) {
|
|
1409
|
+
return;
|
|
1410
|
+
}
|
|
1411
|
+
try {
|
|
1412
|
+
this.onFrameRendered(frameArg);
|
|
1413
|
+
} catch (error) {
|
|
1414
|
+
this.reportError({
|
|
1415
|
+
code: 'renderer_callback_failed',
|
|
1416
|
+
message: `Live browser frame callback failed: ${getErrorMessage(error)}`,
|
|
1417
|
+
cause: error,
|
|
1418
|
+
frame: frameArg,
|
|
1419
|
+
}, runEpochArg);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
private reportError(errorArg: ILiveBrowserCanvasError, runEpochArg: number): void {
|
|
1424
|
+
if (!this.onError || runEpochArg !== this.runEpoch) {
|
|
1425
|
+
return;
|
|
1426
|
+
}
|
|
1427
|
+
try {
|
|
1428
|
+
this.onError(errorArg);
|
|
1429
|
+
} catch {
|
|
1430
|
+
// Consumer error reporting must not break renderer cleanup.
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
private requestSuspend(runEpochArg: number): void {
|
|
1435
|
+
if (!this.isRunActive(runEpochArg) || this.interruptionRequested) {
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
this.interruptRun(runEpochArg);
|
|
1439
|
+
void this.suspend();
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
private validateTimeout(timeoutArg: number, nameArg: string): number {
|
|
1443
|
+
if (!Number.isFinite(timeoutArg) || timeoutArg <= 0) {
|
|
1444
|
+
throw new Error(`${nameArg} must be a positive finite number`);
|
|
1445
|
+
}
|
|
1446
|
+
return timeoutArg;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
private runClientOperation<T>(
|
|
1450
|
+
operationArg: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<T>,
|
|
1451
|
+
operationNameArg: string,
|
|
1452
|
+
runEpochArg: number,
|
|
1453
|
+
): Promise<T> {
|
|
1454
|
+
return this.runAbortableOperation(
|
|
1455
|
+
(signalArg) => operationArg({ signal: signalArg }),
|
|
1456
|
+
this.operationTimeoutMs,
|
|
1457
|
+
operationNameArg,
|
|
1458
|
+
runEpochArg,
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
private runAbortableOperation<T>(
|
|
1463
|
+
operationArg: (signalArg: AbortSignal) => Promise<T>,
|
|
1464
|
+
timeoutMsArg: number,
|
|
1465
|
+
operationNameArg: string,
|
|
1466
|
+
runEpochArg: number,
|
|
1467
|
+
): Promise<T> {
|
|
1468
|
+
const runController = this.runController;
|
|
1469
|
+
if (
|
|
1470
|
+
!runController
|
|
1471
|
+
|| !this.isRunActive(runEpochArg)
|
|
1472
|
+
|| runController.signal.aborted
|
|
1473
|
+
) {
|
|
1474
|
+
return Promise.reject(new LiveBrowserRunAbortedError(
|
|
1475
|
+
`Cannot start ${operationNameArg} for inactive run ${runEpochArg}`,
|
|
1476
|
+
));
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
const operationController = new AbortController();
|
|
1480
|
+
this.operationControllers.add(operationController);
|
|
1481
|
+
const abortFromRun = () => {
|
|
1482
|
+
const reason = runController.signal.reason instanceof Error
|
|
1483
|
+
? runController.signal.reason
|
|
1484
|
+
: new LiveBrowserRunAbortedError(
|
|
1485
|
+
`Live browser renderer run ${runEpochArg} was interrupted`,
|
|
1486
|
+
);
|
|
1487
|
+
operationController.abort(reason);
|
|
1488
|
+
};
|
|
1489
|
+
runController.signal.addEventListener('abort', abortFromRun, { once: true });
|
|
1490
|
+
const timeout = globalThis.setTimeout(() => {
|
|
1491
|
+
operationController.abort(new LiveBrowserOperationTimeoutError(
|
|
1492
|
+
`${operationNameArg} timed out after ${timeoutMsArg}ms`,
|
|
1493
|
+
));
|
|
1494
|
+
}, timeoutMsArg);
|
|
1495
|
+
|
|
1496
|
+
let abortOperation = () => undefined;
|
|
1497
|
+
const operation = new Promise<T>((resolve, reject) => {
|
|
1498
|
+
let settled = false;
|
|
1499
|
+
const settle = (callbackArg: () => void) => {
|
|
1500
|
+
if (!settled) {
|
|
1501
|
+
settled = true;
|
|
1502
|
+
callbackArg();
|
|
1503
|
+
}
|
|
1504
|
+
};
|
|
1505
|
+
abortOperation = () => {
|
|
1506
|
+
const reason = operationController.signal.reason instanceof Error
|
|
1507
|
+
? operationController.signal.reason
|
|
1508
|
+
: new LiveBrowserRunAbortedError(`${operationNameArg} was aborted`);
|
|
1509
|
+
settle(() => reject(reason));
|
|
1510
|
+
};
|
|
1511
|
+
operationController.signal.addEventListener('abort', abortOperation, { once: true });
|
|
1512
|
+
|
|
1513
|
+
let operationResult: Promise<T>;
|
|
1514
|
+
try {
|
|
1515
|
+
operationResult = operationArg(operationController.signal);
|
|
1516
|
+
} catch (error) {
|
|
1517
|
+
settle(() => reject(error));
|
|
1518
|
+
return;
|
|
1519
|
+
}
|
|
1520
|
+
operationResult.then(
|
|
1521
|
+
(valueArg) => {
|
|
1522
|
+
settle(() => resolve(valueArg));
|
|
1523
|
+
},
|
|
1524
|
+
(errorArg) => {
|
|
1525
|
+
settle(() => reject(errorArg));
|
|
1526
|
+
},
|
|
1527
|
+
);
|
|
1528
|
+
}).finally(() => {
|
|
1529
|
+
globalThis.clearTimeout(timeout);
|
|
1530
|
+
runController.signal.removeEventListener('abort', abortFromRun);
|
|
1531
|
+
operationController.signal.removeEventListener('abort', abortOperation);
|
|
1532
|
+
this.operationControllers.delete(operationController);
|
|
1533
|
+
});
|
|
1534
|
+
return operation;
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
private async decodeFrame(blobArg: Blob, runEpochArg: number): Promise<ImageBitmap> {
|
|
1538
|
+
let decodeOperation: Promise<ImageBitmap>;
|
|
1539
|
+
try {
|
|
1540
|
+
decodeOperation = createImageBitmap(blobArg);
|
|
1541
|
+
} catch (error) {
|
|
1542
|
+
throw error;
|
|
1543
|
+
}
|
|
1544
|
+
let rawDecodeSettlementPromise: Promise<void>;
|
|
1545
|
+
rawDecodeSettlementPromise = decodeOperation.then(
|
|
1546
|
+
() => undefined,
|
|
1547
|
+
() => undefined,
|
|
1548
|
+
).finally(() => {
|
|
1549
|
+
this.rawDecodeSettlementPromises.delete(rawDecodeSettlementPromise);
|
|
1550
|
+
});
|
|
1551
|
+
this.rawDecodeSettlementPromises.add(rawDecodeSettlementPromise);
|
|
1552
|
+
try {
|
|
1553
|
+
return await this.runAbortableOperation(
|
|
1554
|
+
async () => decodeOperation,
|
|
1555
|
+
this.frameDecodeTimeoutMs,
|
|
1556
|
+
'frame decode',
|
|
1557
|
+
runEpochArg,
|
|
1558
|
+
);
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
if (
|
|
1561
|
+
error instanceof LiveBrowserOperationTimeoutError
|
|
1562
|
+
|| error instanceof LiveBrowserRunAbortedError
|
|
1563
|
+
) {
|
|
1564
|
+
void decodeOperation.then((lateBitmapArg) => lateBitmapArg.close(), () => undefined);
|
|
1565
|
+
throw error;
|
|
1566
|
+
}
|
|
1567
|
+
throw new LiveBrowserFrameIntegrityError(
|
|
1568
|
+
`frame bytes could not be decoded: ${getErrorMessage(error)}`,
|
|
1569
|
+
{ cause: error },
|
|
1570
|
+
);
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
private isRunActive(runEpochArg: number): boolean {
|
|
1575
|
+
return Boolean(
|
|
1576
|
+
this.running
|
|
1577
|
+
&& !this.stopping
|
|
1578
|
+
&& runEpochArg === this.runEpoch
|
|
1579
|
+
&& this.runController
|
|
1580
|
+
&& !this.runController.signal.aborted,
|
|
1581
|
+
);
|
|
1582
|
+
}
|
|
1583
|
+
|
|
1584
|
+
private validateFrame(frameArg: ILiveBrowserFrame): void {
|
|
1585
|
+
if (!frameArg || typeof frameArg !== 'object') {
|
|
1586
|
+
throw new LiveBrowserFrameProtocolError('frame must be an object');
|
|
1587
|
+
}
|
|
1588
|
+
if (
|
|
1589
|
+
typeof frameArg.tabId !== 'string'
|
|
1590
|
+
|| frameArg.tabId.length === 0
|
|
1591
|
+
|| frameArg.tabId.length > 4096
|
|
1592
|
+
) {
|
|
1593
|
+
throw new LiveBrowserFrameProtocolError('frame.tabId must be a non-empty bounded string');
|
|
1594
|
+
}
|
|
1595
|
+
this.validateProtocolInteger(frameArg.sequence, 'frame.sequence', 1);
|
|
1596
|
+
this.validateProtocolInteger(frameArg.generation, 'frame.generation', 1);
|
|
1597
|
+
this.validateProtocolInteger(frameArg.viewportRevision, 'frame.viewportRevision', 1);
|
|
1598
|
+
this.validateViewport(frameArg.viewport, 'frame.viewport');
|
|
1599
|
+
if (frameArg.format !== 'jpeg' && frameArg.format !== 'png') {
|
|
1600
|
+
throw new LiveBrowserFrameProtocolError('frame.format must be jpeg or png');
|
|
1601
|
+
}
|
|
1602
|
+
const expectedMimeType = frameArg.format === 'jpeg' ? 'image/jpeg' : 'image/png';
|
|
1603
|
+
if (frameArg.mimeType !== expectedMimeType) {
|
|
1604
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1605
|
+
`frame.mimeType must be ${expectedMimeType} for ${frameArg.format}`,
|
|
1606
|
+
);
|
|
1607
|
+
}
|
|
1608
|
+
this.validateProtocolInteger(frameArg.width, 'frame.width', 1, maxFrameDimension);
|
|
1609
|
+
this.validateProtocolInteger(frameArg.height, 'frame.height', 1, maxFrameDimension);
|
|
1610
|
+
if (frameArg.width * frameArg.height > maxFramePixelArea) {
|
|
1611
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1612
|
+
`frame pixel area must not exceed ${maxFramePixelArea}`,
|
|
1613
|
+
);
|
|
1614
|
+
}
|
|
1615
|
+
if (!(frameArg.data instanceof Uint8Array)) {
|
|
1616
|
+
throw new LiveBrowserFrameProtocolError('frame.data must be a Uint8Array');
|
|
1617
|
+
}
|
|
1618
|
+
if (frameArg.data.byteLength === 0 || frameArg.data.byteLength > maxFrameByteLength) {
|
|
1619
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1620
|
+
`frame.data byte length must be between 1 and ${maxFrameByteLength}`,
|
|
1621
|
+
);
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
const metadata = frameArg.metadata;
|
|
1625
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
1626
|
+
throw new LiveBrowserFrameProtocolError('frame.metadata must be an object');
|
|
1627
|
+
}
|
|
1628
|
+
this.validateProtocolNumber(metadata.offsetTop, 'frame.metadata.offsetTop');
|
|
1629
|
+
this.validateProtocolNumber(
|
|
1630
|
+
metadata.pageScaleFactor,
|
|
1631
|
+
'frame.metadata.pageScaleFactor',
|
|
1632
|
+
true,
|
|
1633
|
+
);
|
|
1634
|
+
this.validateProtocolNumber(metadata.deviceWidth, 'frame.metadata.deviceWidth', true);
|
|
1635
|
+
this.validateProtocolNumber(metadata.deviceHeight, 'frame.metadata.deviceHeight', true);
|
|
1636
|
+
if (
|
|
1637
|
+
metadata.deviceWidth > maxFrameDimension
|
|
1638
|
+
|| metadata.deviceHeight > maxFrameDimension
|
|
1639
|
+
|| metadata.deviceWidth * metadata.deviceHeight > maxFramePixelArea
|
|
1640
|
+
) {
|
|
1641
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1642
|
+
`frame metadata dimensions must fit within ${maxFramePixelArea} pixels`,
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
this.validateProtocolNumber(metadata.scrollOffsetX, 'frame.metadata.scrollOffsetX');
|
|
1646
|
+
this.validateProtocolNumber(metadata.scrollOffsetY, 'frame.metadata.scrollOffsetY');
|
|
1647
|
+
if (metadata.timestamp !== undefined) {
|
|
1648
|
+
this.validateProtocolNumber(metadata.timestamp, 'frame.metadata.timestamp');
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
private validateViewport(viewportArg: ILiveBrowserViewport, nameArg: string): void {
|
|
1653
|
+
if (!viewportArg || typeof viewportArg !== 'object') {
|
|
1654
|
+
throw new LiveBrowserFrameProtocolError(`${nameArg} must be an object`);
|
|
1655
|
+
}
|
|
1656
|
+
this.validateProtocolInteger(viewportArg.width, `${nameArg}.width`, 1, maxViewportWidth);
|
|
1657
|
+
this.validateProtocolInteger(viewportArg.height, `${nameArg}.height`, 1, maxViewportHeight);
|
|
1658
|
+
this.validateProtocolNumber(viewportArg.deviceScaleFactor, `${nameArg}.deviceScaleFactor`, true);
|
|
1659
|
+
if (viewportArg.deviceScaleFactor > maxDeviceScaleFactor) {
|
|
1660
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1661
|
+
`${nameArg}.deviceScaleFactor must not exceed ${maxDeviceScaleFactor}`,
|
|
1662
|
+
);
|
|
1663
|
+
}
|
|
1664
|
+
const physicalArea = viewportArg.width
|
|
1665
|
+
* viewportArg.height
|
|
1666
|
+
* viewportArg.deviceScaleFactor
|
|
1667
|
+
* viewportArg.deviceScaleFactor;
|
|
1668
|
+
if (physicalArea > maxFramePixelArea) {
|
|
1669
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1670
|
+
`${nameArg} physical pixel area must not exceed ${maxFramePixelArea}`,
|
|
1671
|
+
);
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
private validateProtocolInteger(
|
|
1676
|
+
valueArg: number,
|
|
1677
|
+
nameArg: string,
|
|
1678
|
+
minimumArg: number,
|
|
1679
|
+
maximumArg = Number.MAX_SAFE_INTEGER,
|
|
1680
|
+
): void {
|
|
1681
|
+
if (
|
|
1682
|
+
!Number.isSafeInteger(valueArg)
|
|
1683
|
+
|| valueArg < minimumArg
|
|
1684
|
+
|| valueArg > maximumArg
|
|
1685
|
+
) {
|
|
1686
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1687
|
+
`${nameArg} must be an integer between ${minimumArg} and ${maximumArg}`,
|
|
1688
|
+
);
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
private validateProtocolNumber(
|
|
1693
|
+
valueArg: number,
|
|
1694
|
+
nameArg: string,
|
|
1695
|
+
positiveArg = false,
|
|
1696
|
+
): void {
|
|
1697
|
+
if (!Number.isFinite(valueArg) || (positiveArg && valueArg <= 0)) {
|
|
1698
|
+
throw new LiveBrowserFrameProtocolError(
|
|
1699
|
+
`${nameArg} must be a ${positiveArg ? 'positive ' : ''}finite number`,
|
|
1700
|
+
);
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
private releasePointerCapture(pointerIdArg: number): void {
|
|
1705
|
+
this.capturedPointerIds.delete(pointerIdArg);
|
|
1706
|
+
try {
|
|
1707
|
+
if (this.canvas.hasPointerCapture(pointerIdArg)) {
|
|
1708
|
+
this.canvas.releasePointerCapture(pointerIdArg);
|
|
1709
|
+
}
|
|
1710
|
+
} catch {
|
|
1711
|
+
// Pointer capture may already have been released by the browser.
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
private releasePointerCaptures(): void {
|
|
1716
|
+
const pointerIds = [...this.capturedPointerIds];
|
|
1717
|
+
this.capturedPointerIds.clear();
|
|
1718
|
+
for (const pointerId of pointerIds) {
|
|
1719
|
+
this.releasePointerCapture(pointerId);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
}
|