@push.rocks/smartbrowser 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/dist_ts_web/classes.livebrowsercanvasrenderer.d.ts +4 -187
- package/dist_ts_web/classes.livebrowsercanvasrenderer.js +5 -1750
- package/dist_ts_web/classes.livebrowserrenderer.d.ts +194 -0
- package/dist_ts_web/classes.livebrowserrenderer.js +1830 -0
- package/dist_ts_web/classes.livebrowservideoreceiver.d.ts +46 -0
- package/dist_ts_web/classes.livebrowservideoreceiver.js +243 -0
- package/dist_ts_web/classes.livebrowservideorenderer.d.ts +6 -0
- package/dist_ts_web/classes.livebrowservideorenderer.js +8 -0
- package/dist_ts_web/index.d.ts +3 -0
- package/dist_ts_web/index.js +2 -1
- package/dist_ts_web/interfaces.livebrowsercanvas.d.ts +8 -2
- package/dist_ts_web/interfaces.livebrowservideo.d.ts +31 -0
- package/dist_ts_web/interfaces.livebrowservideo.js +2 -0
- package/package.json +4 -4
- package/readme.md +25 -1
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/classes.livebrowsercanvasrenderer.ts +5 -2151
- package/ts_web/classes.livebrowserrenderer.ts +2232 -0
- package/ts_web/classes.livebrowservideoreceiver.ts +250 -0
- package/ts_web/classes.livebrowservideorenderer.ts +9 -0
- package/ts_web/index.ts +4 -0
- package/ts_web/interfaces.livebrowsercanvas.ts +11 -1
- package/ts_web/interfaces.livebrowservideo.ts +46 -0
- package/readme.hints.md +0 -34
|
@@ -0,0 +1,2232 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ILiveBrowserFrame,
|
|
3
|
+
ILiveBrowserInputBase,
|
|
4
|
+
ILiveBrowserKeyInput,
|
|
5
|
+
ILiveBrowserModifierState,
|
|
6
|
+
ILiveBrowserMouseInput,
|
|
7
|
+
ILiveBrowserState,
|
|
8
|
+
ILiveBrowserViewport,
|
|
9
|
+
ILiveBrowserWheelInput,
|
|
10
|
+
ILiveVideoSource,
|
|
11
|
+
TLiveBrowserEvent,
|
|
12
|
+
} from '@push.rocks/smartpuppeteer';
|
|
13
|
+
|
|
14
|
+
import type {
|
|
15
|
+
ILiveBrowserCanvasError,
|
|
16
|
+
ILiveBrowserCanvasOperationOptions,
|
|
17
|
+
ILiveBrowserCanvasRendererOptions,
|
|
18
|
+
ILiveBrowserCanvasRendererStatistics,
|
|
19
|
+
} from './interfaces.livebrowsercanvas.js';
|
|
20
|
+
|
|
21
|
+
import { LiveBrowserVideoReceiver } from './classes.livebrowservideoreceiver.js';
|
|
22
|
+
import type { ILiveBrowserVideoRendererOptions, ILiveBrowserVideoRendererStatistics } from './interfaces.livebrowservideo.js';
|
|
23
|
+
|
|
24
|
+
const maxInputQueueLength = 128;
|
|
25
|
+
const maxInFlightInputCommands = 4;
|
|
26
|
+
const maxQueuedCoalescableCommands = 32;
|
|
27
|
+
const maxWheelDelta = 1000000;
|
|
28
|
+
const animationFrameFallbackMs = 16;
|
|
29
|
+
const resizeDebounceMs = 100;
|
|
30
|
+
const maxPendingAcknowledgements = 16;
|
|
31
|
+
const maxInputReleaseAttempts = 2;
|
|
32
|
+
const maxFrameDimension = 12288;
|
|
33
|
+
const maxFramePixelArea = 8294400;
|
|
34
|
+
const maxFrameByteLength = (maxFramePixelArea * 4) + 1048576;
|
|
35
|
+
const maxViewportWidth = 4096;
|
|
36
|
+
const maxViewportHeight = 4096;
|
|
37
|
+
const minDeviceScaleFactor = 0.25;
|
|
38
|
+
const maxDeviceScaleFactor = 3;
|
|
39
|
+
const coordinateEdgeInset = 0.001;
|
|
40
|
+
const defaultOperationTimeoutMs = 10000;
|
|
41
|
+
|
|
42
|
+
class LiveBrowserOperationTimeoutError extends Error {}
|
|
43
|
+
class LiveBrowserRunAbortedError extends Error {}
|
|
44
|
+
class LiveBrowserFrameProtocolError extends Error {}
|
|
45
|
+
class LiveBrowserFrameIntegrityError extends Error {}
|
|
46
|
+
|
|
47
|
+
interface IFrameWork {
|
|
48
|
+
frame: ILiveBrowserFrame;
|
|
49
|
+
runEpoch: number;
|
|
50
|
+
acknowledgementAttempted: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface IDisplayedFrame extends ILiveBrowserInputBase {
|
|
54
|
+
sequence: number;
|
|
55
|
+
viewport: ILiveBrowserViewport;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
type TInputCommandKind = 'key' | 'mouse' | 'mouseMove' | 'text' | 'wheel';
|
|
59
|
+
type TCoalescableInputCommandKind = 'mouseMove' | 'wheel';
|
|
60
|
+
|
|
61
|
+
interface IInputCommand {
|
|
62
|
+
kind: TInputCommandKind;
|
|
63
|
+
runEpoch: number;
|
|
64
|
+
execute: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<void>;
|
|
65
|
+
/** Mutable merged wheel payload that execute() reads at dispatch time. */
|
|
66
|
+
wheelInput?: ILiveBrowserWheelInput;
|
|
67
|
+
onAttempt?: () => void;
|
|
68
|
+
onSuccess?: () => void;
|
|
69
|
+
resolve: () => void;
|
|
70
|
+
reject: (errorArg: unknown) => void;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface IPendingWheel extends ILiveBrowserWheelInput {
|
|
74
|
+
runEpoch: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
interface IPendingDraw {
|
|
78
|
+
bitmap: ImageBitmap;
|
|
79
|
+
frame: ILiveBrowserFrame;
|
|
80
|
+
runEpoch: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface IScheduledFrame {
|
|
84
|
+
animationFrameId?: number;
|
|
85
|
+
timeoutId?: ReturnType<typeof setTimeout>;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const isCoalescableInputCommand = (
|
|
89
|
+
kindArg: TInputCommandKind,
|
|
90
|
+
): kindArg is TCoalescableInputCommandKind => kindArg === 'mouseMove' || kindArg === 'wheel';
|
|
91
|
+
|
|
92
|
+
const clampWheelDelta = (valueArg: number): number => (
|
|
93
|
+
Math.max(-maxWheelDelta, Math.min(maxWheelDelta, valueArg))
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
const mergeWheelInput = (
|
|
97
|
+
targetArg: ILiveBrowserWheelInput,
|
|
98
|
+
sourceArg: ILiveBrowserWheelInput,
|
|
99
|
+
): void => {
|
|
100
|
+
targetArg.x = sourceArg.x;
|
|
101
|
+
targetArg.y = sourceArg.y;
|
|
102
|
+
targetArg.deltaX = clampWheelDelta(targetArg.deltaX + sourceArg.deltaX);
|
|
103
|
+
targetArg.deltaY = clampWheelDelta(targetArg.deltaY + sourceArg.deltaY);
|
|
104
|
+
targetArg.modifiers = sourceArg.modifiers;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const inputIdentityEquals = (
|
|
108
|
+
firstArg: ILiveBrowserInputBase,
|
|
109
|
+
secondArg: ILiveBrowserInputBase,
|
|
110
|
+
): boolean => (
|
|
111
|
+
firstArg.tabId === secondArg.tabId
|
|
112
|
+
&& firstArg.generation === secondArg.generation
|
|
113
|
+
&& firstArg.viewportRevision === secondArg.viewportRevision
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
interface IResizeFence {
|
|
117
|
+
minimumViewportRevision: number;
|
|
118
|
+
target: ILiveBrowserViewport;
|
|
119
|
+
negotiated?: boolean;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const viewportEquals = (
|
|
123
|
+
firstArg: ILiveBrowserViewport,
|
|
124
|
+
secondArg: ILiveBrowserViewport,
|
|
125
|
+
): boolean => (
|
|
126
|
+
firstArg.width === secondArg.width
|
|
127
|
+
&& firstArg.height === secondArg.height
|
|
128
|
+
&& firstArg.deviceScaleFactor === secondArg.deviceScaleFactor
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
const createModifiers = (
|
|
132
|
+
eventArg: MouseEvent | KeyboardEvent,
|
|
133
|
+
): ILiveBrowserModifierState => ({
|
|
134
|
+
alt: eventArg.altKey,
|
|
135
|
+
control: eventArg.ctrlKey,
|
|
136
|
+
meta: eventArg.metaKey,
|
|
137
|
+
shift: eventArg.shiftKey,
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const createFrameAcknowledgementRequest = (frameArg: ILiveBrowserFrame) => ({
|
|
141
|
+
tabId: frameArg.tabId,
|
|
142
|
+
sequence: frameArg.sequence,
|
|
143
|
+
generation: frameArg.generation,
|
|
144
|
+
viewportRevision: frameArg.viewportRevision,
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
const createInputIdentity = (displayedFrameArg: IDisplayedFrame): ILiveBrowserInputBase => ({
|
|
148
|
+
tabId: displayedFrameArg.tabId,
|
|
149
|
+
generation: displayedFrameArg.generation,
|
|
150
|
+
viewportRevision: displayedFrameArg.viewportRevision,
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
const getErrorMessage = (errorArg: unknown): string => (
|
|
154
|
+
errorArg instanceof Error ? errorArg.message : String(errorArg)
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Renders transport-neutral live browser frames and maps DOM input back to the
|
|
159
|
+
* currently displayed SmartPuppeteer frame identity.
|
|
160
|
+
*/
|
|
161
|
+
export class LiveBrowserRenderer {
|
|
162
|
+
private readonly canvas: HTMLElement & { width: number; height: number };
|
|
163
|
+
private readonly videoReceiver?: LiveBrowserVideoReceiver;
|
|
164
|
+
private readonly canvasContext?: CanvasRenderingContext2D;
|
|
165
|
+
private readonly bitmapContext?: ImageBitmapRenderingContext;
|
|
166
|
+
private readonly client: ILiveBrowserCanvasRendererOptions['client'] | ILiveBrowserVideoRendererOptions['client'];
|
|
167
|
+
private readonly focusTarget: HTMLElement;
|
|
168
|
+
private readonly resizeTarget?: Element;
|
|
169
|
+
private readonly getDeviceScaleFactor: () => number;
|
|
170
|
+
private readonly operationTimeoutMs: number;
|
|
171
|
+
private readonly frameDecodeTimeoutMs: number;
|
|
172
|
+
private readonly onError?: ILiveBrowserCanvasRendererOptions['onError'];
|
|
173
|
+
private readonly onFrameRendered?: ILiveBrowserCanvasRendererOptions['onFrameRendered'];
|
|
174
|
+
private readonly initialCanvasWidth: number;
|
|
175
|
+
private readonly initialCanvasHeight: number;
|
|
176
|
+
|
|
177
|
+
private lifecycleTail: Promise<void> = Promise.resolve();
|
|
178
|
+
private runEpoch = 0;
|
|
179
|
+
private running = false;
|
|
180
|
+
private suspended = false;
|
|
181
|
+
private stopping = false;
|
|
182
|
+
private interruptionRequested = false;
|
|
183
|
+
private runController?: AbortController;
|
|
184
|
+
private operationControllers = new Set<AbortController>();
|
|
185
|
+
private acceptingFrames = false;
|
|
186
|
+
private state?: ILiveBrowserState;
|
|
187
|
+
private unsubscribe?: () => void;
|
|
188
|
+
private listenerController?: AbortController;
|
|
189
|
+
private resizeObserver?: ResizeObserver;
|
|
190
|
+
private focusWasAdjusted = false;
|
|
191
|
+
private focusHadTabIndex = false;
|
|
192
|
+
private focusTabIndexValue: string | null = null;
|
|
193
|
+
|
|
194
|
+
private displayedFrame?: IDisplayedFrame;
|
|
195
|
+
private highestFrameSequence = -1;
|
|
196
|
+
private queuedFrame?: IFrameWork;
|
|
197
|
+
private frameProcessingPromise?: Promise<void>;
|
|
198
|
+
private pendingDraw?: IPendingDraw;
|
|
199
|
+
private drawSchedule?: IScheduledFrame;
|
|
200
|
+
private acknowledgementPromises = new Set<Promise<void>>();
|
|
201
|
+
private rawDecodeSettlementPromises = new Set<Promise<void>>();
|
|
202
|
+
|
|
203
|
+
private inputBlocked = true;
|
|
204
|
+
private inputCommands: IInputCommand[] = [];
|
|
205
|
+
private inFlightInputCommands = 0;
|
|
206
|
+
private inFlightWheelCommands = 0;
|
|
207
|
+
private inputIdleResolvers: Array<() => void> = [];
|
|
208
|
+
private inputResetTail: Promise<void> = Promise.resolve();
|
|
209
|
+
private inputResetPromise?: Promise<void>;
|
|
210
|
+
private inputResetCount = 0;
|
|
211
|
+
private pendingWheel?: IPendingWheel;
|
|
212
|
+
private wheelFlushSchedule?: IScheduledFrame;
|
|
213
|
+
private wheelFrameElapsed = true;
|
|
214
|
+
private pressedKeys = new Map<string, ILiveBrowserKeyInput>();
|
|
215
|
+
private pressedMouseButtons = new Map<string, ILiveBrowserMouseInput>();
|
|
216
|
+
private capturedPointerIds = new Set<number>();
|
|
217
|
+
|
|
218
|
+
private pendingViewport?: ILiveBrowserViewport;
|
|
219
|
+
private inFlightViewport?: ILiveBrowserViewport;
|
|
220
|
+
private viewportProcessing = false;
|
|
221
|
+
private viewportProcessingPromise?: Promise<void>;
|
|
222
|
+
private resizeFence?: IResizeFence;
|
|
223
|
+
private lastRequestedViewport?: ILiveBrowserViewport;
|
|
224
|
+
private resizeDebounceTimer?: ReturnType<typeof setTimeout>;
|
|
225
|
+
private pendingResizeSize?: { width: number; height: number };
|
|
226
|
+
|
|
227
|
+
private readonly statistics: Omit<ILiveBrowserCanvasRendererStatistics, 'inputCommandsInFlight'> = {
|
|
228
|
+
framesReceived: 0,
|
|
229
|
+
framesDecoded: 0,
|
|
230
|
+
framesSkipped: 0,
|
|
231
|
+
inputCommandsEnqueued: 0,
|
|
232
|
+
inputCommandsCoalesced: 0,
|
|
233
|
+
lastInputRoundTripMs: 0,
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
constructor(optionsArg: ILiveBrowserCanvasRendererOptions | ILiveBrowserVideoRendererOptions) {
|
|
237
|
+
this.canvas = 'video' in optionsArg ? optionsArg.video : optionsArg.canvas;
|
|
238
|
+
this.client = optionsArg.client;
|
|
239
|
+
this.focusTarget = optionsArg.focusTarget ?? this.canvas;
|
|
240
|
+
this.resizeTarget = optionsArg.resizeTarget;
|
|
241
|
+
this.getDeviceScaleFactor = optionsArg.getDeviceScaleFactor
|
|
242
|
+
?? (() => window.devicePixelRatio || 1);
|
|
243
|
+
this.operationTimeoutMs = this.validateTimeout(
|
|
244
|
+
optionsArg.operationTimeoutMs ?? defaultOperationTimeoutMs,
|
|
245
|
+
'operationTimeoutMs',
|
|
246
|
+
);
|
|
247
|
+
this.frameDecodeTimeoutMs = this.validateTimeout(
|
|
248
|
+
('frameDecodeTimeoutMs' in optionsArg ? optionsArg.frameDecodeTimeoutMs : undefined) ?? this.operationTimeoutMs,
|
|
249
|
+
'frameDecodeTimeoutMs',
|
|
250
|
+
);
|
|
251
|
+
this.onError = optionsArg.onError;
|
|
252
|
+
this.onFrameRendered = 'onFrameRendered' in optionsArg ? optionsArg.onFrameRendered : undefined;
|
|
253
|
+
this.initialCanvasWidth = this.canvas.width;
|
|
254
|
+
this.initialCanvasHeight = this.canvas.height;
|
|
255
|
+
|
|
256
|
+
if ('video' in optionsArg) {
|
|
257
|
+
this.videoReceiver = new LiveBrowserVideoReceiver({
|
|
258
|
+
video: optionsArg.video, client: optionsArg.client, operationTimeoutMs: this.operationTimeoutMs,
|
|
259
|
+
onUnavailable: () => {
|
|
260
|
+
this.displayedFrame = undefined;
|
|
261
|
+
this.inputBlocked = true;
|
|
262
|
+
if (this.running) void this.scheduleInputReset(false, this.runEpoch);
|
|
263
|
+
},
|
|
264
|
+
onPresented: (sourceArg) => this.presentVideo(sourceArg),
|
|
265
|
+
onError: (errorArg) => this.reportError(errorArg, this.runEpoch),
|
|
266
|
+
});
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const bitmapContext = optionsArg.canvas.getContext('bitmaprenderer');
|
|
270
|
+
if (bitmapContext) {
|
|
271
|
+
this.bitmapContext = bitmapContext;
|
|
272
|
+
} else {
|
|
273
|
+
// A canvas that already owns a 2D context keeps the 2D drawing path.
|
|
274
|
+
const canvasContext = optionsArg.canvas.getContext('2d');
|
|
275
|
+
if (!canvasContext) {
|
|
276
|
+
throw new Error('LiveBrowserCanvasRenderer requires a bitmaprenderer or 2D canvas context');
|
|
277
|
+
}
|
|
278
|
+
this.canvasContext = canvasContext;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
public get isRunning(): boolean {
|
|
283
|
+
return this.running;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
public get isSuspended(): boolean {
|
|
287
|
+
return this.suspended;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Cumulative frame and input counters for this renderer instance across runs.
|
|
292
|
+
*/
|
|
293
|
+
public getStatistics(): ILiveBrowserVideoRendererStatistics {
|
|
294
|
+
return {
|
|
295
|
+
...this.statistics,
|
|
296
|
+
...this.videoReceiver?.getCounters(),
|
|
297
|
+
...(this.videoReceiver ? { video: this.videoReceiver.getStatistics() } : {}),
|
|
298
|
+
inputCommandsInFlight: this.inFlightInputCommands,
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
public start(): Promise<void> {
|
|
303
|
+
return this.enqueueLifecycle(async () => {
|
|
304
|
+
if (this.running) {
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
if (this.suspended) {
|
|
308
|
+
throw new Error('Live browser renderer is suspended; call resume() to start a fresh run');
|
|
309
|
+
}
|
|
310
|
+
await this.startRun(false);
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
public stop(): Promise<void> {
|
|
315
|
+
this.interruptRun(this.runEpoch);
|
|
316
|
+
return this.enqueueLifecycle(async () => {
|
|
317
|
+
const runEpoch = this.runEpoch;
|
|
318
|
+
this.interruptRun(runEpoch);
|
|
319
|
+
if (this.stopping) {
|
|
320
|
+
await this.finishRun(runEpoch, false);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
if (!this.suspended) {
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
this.suspended = false;
|
|
327
|
+
this.restoreCanvasAndFocus();
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Aborts the current transport generation without replaying input recovery.
|
|
333
|
+
*/
|
|
334
|
+
public suspend(): Promise<void> {
|
|
335
|
+
this.interruptRun(this.runEpoch);
|
|
336
|
+
return this.enqueueLifecycle(async () => {
|
|
337
|
+
const runEpoch = this.runEpoch;
|
|
338
|
+
this.interruptRun(runEpoch);
|
|
339
|
+
if (!this.stopping) {
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
await this.finishRun(runEpoch, true);
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Starts a new transport generation that requires new state and a new frame.
|
|
348
|
+
*/
|
|
349
|
+
public resume(): Promise<void> {
|
|
350
|
+
return this.enqueueLifecycle(async () => {
|
|
351
|
+
if (this.running) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (!this.suspended) {
|
|
355
|
+
throw new Error('Live browser renderer is not suspended');
|
|
356
|
+
}
|
|
357
|
+
await this.startRun(true);
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Sends arbitrary committed text, including text produced by an external IME.
|
|
363
|
+
*/
|
|
364
|
+
public async insertText(textArg: string): Promise<void> {
|
|
365
|
+
const displayedFrame = this.requireInputFrame();
|
|
366
|
+
const inputIdentity = createInputIdentity(displayedFrame);
|
|
367
|
+
await this.enqueueInputCommand({
|
|
368
|
+
kind: 'text',
|
|
369
|
+
runEpoch: this.runEpoch,
|
|
370
|
+
execute: async (optionsArg) => {
|
|
371
|
+
await this.client.insertText({
|
|
372
|
+
...inputIdentity,
|
|
373
|
+
text: textArg,
|
|
374
|
+
}, optionsArg);
|
|
375
|
+
},
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Reads the configured stable resize target and requests the corresponding
|
|
381
|
+
* remote CSS viewport. Intrinsic canvas dimensions are never used here.
|
|
382
|
+
*/
|
|
383
|
+
public syncViewport(): void {
|
|
384
|
+
if (!this.running || this.stopping || this.interruptionRequested || !this.resizeTarget) {
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
this.queueViewport(
|
|
388
|
+
this.resizeTarget.clientWidth,
|
|
389
|
+
this.resizeTarget.clientHeight,
|
|
390
|
+
this.runEpoch,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
private enqueueLifecycle(operationArg: () => Promise<void>): Promise<void> {
|
|
395
|
+
const operation = this.lifecycleTail.then(operationArg, operationArg);
|
|
396
|
+
this.lifecycleTail = operation.then(() => undefined, () => undefined);
|
|
397
|
+
return operation;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
private async startRun(resumingArg: boolean): Promise<void> {
|
|
401
|
+
await Promise.all([...this.rawDecodeSettlementPromises]);
|
|
402
|
+
const runEpoch = ++this.runEpoch;
|
|
403
|
+
this.runController = new AbortController();
|
|
404
|
+
this.running = true;
|
|
405
|
+
this.suspended = false;
|
|
406
|
+
this.stopping = false;
|
|
407
|
+
this.interruptionRequested = false;
|
|
408
|
+
this.acceptingFrames = true;
|
|
409
|
+
this.inputBlocked = true;
|
|
410
|
+
this.wheelFrameElapsed = true;
|
|
411
|
+
this.state = undefined;
|
|
412
|
+
this.displayedFrame = undefined;
|
|
413
|
+
this.highestFrameSequence = -1;
|
|
414
|
+
this.clearCanvas();
|
|
415
|
+
|
|
416
|
+
try {
|
|
417
|
+
this.makeFocusTargetFocusable();
|
|
418
|
+
this.installDomListeners(runEpoch);
|
|
419
|
+
this.unsubscribe = this.client.onEvent((eventArg) => {
|
|
420
|
+
this.handleEvent(eventArg, runEpoch);
|
|
421
|
+
});
|
|
422
|
+
this.applyState(this.client.getState(), runEpoch);
|
|
423
|
+
this.installResizeObserver(runEpoch);
|
|
424
|
+
} catch (error) {
|
|
425
|
+
this.interruptRun(runEpoch);
|
|
426
|
+
await this.finishRun(runEpoch, resumingArg);
|
|
427
|
+
throw error;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
private interruptRun(runEpochArg: number): void {
|
|
432
|
+
if (
|
|
433
|
+
runEpochArg !== this.runEpoch
|
|
434
|
+
|| (!this.running && !this.stopping)
|
|
435
|
+
) {
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
this.running = false;
|
|
439
|
+
this.stopping = true;
|
|
440
|
+
this.interruptionRequested = true;
|
|
441
|
+
this.acceptingFrames = false;
|
|
442
|
+
this.inputBlocked = true;
|
|
443
|
+
this.pendingViewport = undefined;
|
|
444
|
+
this.queuedFrame = undefined;
|
|
445
|
+
this.displayedFrame = undefined;
|
|
446
|
+
this.resizeFence = undefined;
|
|
447
|
+
this.discardPendingDraw();
|
|
448
|
+
this.discardPendingWheel();
|
|
449
|
+
this.releasePointerCaptures();
|
|
450
|
+
this.clearCanvas();
|
|
451
|
+
this.videoReceiver?.interrupt();
|
|
452
|
+
|
|
453
|
+
const interruptionError = new LiveBrowserRunAbortedError(
|
|
454
|
+
`Live browser renderer run ${runEpochArg} was interrupted`,
|
|
455
|
+
);
|
|
456
|
+
this.runController?.abort(interruptionError);
|
|
457
|
+
for (const controller of this.operationControllers) {
|
|
458
|
+
controller.abort(interruptionError);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
462
|
+
for (const command of abandonedCommands) {
|
|
463
|
+
command.reject(interruptionError);
|
|
464
|
+
}
|
|
465
|
+
this.pressedKeys.clear();
|
|
466
|
+
this.pressedMouseButtons.clear();
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
private async finishRun(runEpochArg: number, suspendArg: boolean): Promise<void> {
|
|
470
|
+
if (runEpochArg !== this.runEpoch) {
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
try {
|
|
475
|
+
this.resizeObserver?.disconnect();
|
|
476
|
+
this.resizeObserver = undefined;
|
|
477
|
+
this.clearResizeDebounce();
|
|
478
|
+
this.listenerController?.abort();
|
|
479
|
+
this.listenerController = undefined;
|
|
480
|
+
const unsubscribe = this.unsubscribe;
|
|
481
|
+
this.unsubscribe = undefined;
|
|
482
|
+
if (unsubscribe) {
|
|
483
|
+
try {
|
|
484
|
+
unsubscribe();
|
|
485
|
+
} catch (error) {
|
|
486
|
+
this.reportError({
|
|
487
|
+
code: 'renderer_cleanup_failed',
|
|
488
|
+
message: `Could not unsubscribe the live browser renderer: ${getErrorMessage(error)}`,
|
|
489
|
+
cause: error,
|
|
490
|
+
}, runEpochArg);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
await Promise.allSettled([
|
|
495
|
+
this.videoReceiver?.stop(),
|
|
496
|
+
this.viewportProcessingPromise,
|
|
497
|
+
this.inputResetTail,
|
|
498
|
+
this.waitForInputIdle(),
|
|
499
|
+
this.frameProcessingPromise,
|
|
500
|
+
...this.acknowledgementPromises,
|
|
501
|
+
]);
|
|
502
|
+
} finally {
|
|
503
|
+
for (const controller of this.operationControllers) {
|
|
504
|
+
controller.abort(new LiveBrowserRunAbortedError(
|
|
505
|
+
`Live browser renderer run ${runEpochArg} finished`,
|
|
506
|
+
));
|
|
507
|
+
}
|
|
508
|
+
this.operationControllers.clear();
|
|
509
|
+
this.runController = undefined;
|
|
510
|
+
this.running = false;
|
|
511
|
+
this.suspended = suspendArg;
|
|
512
|
+
this.acceptingFrames = false;
|
|
513
|
+
this.state = undefined;
|
|
514
|
+
this.displayedFrame = undefined;
|
|
515
|
+
this.highestFrameSequence = -1;
|
|
516
|
+
this.queuedFrame = undefined;
|
|
517
|
+
this.frameProcessingPromise = undefined;
|
|
518
|
+
this.discardPendingDraw();
|
|
519
|
+
this.discardPendingWheel();
|
|
520
|
+
this.clearResizeDebounce();
|
|
521
|
+
this.acknowledgementPromises.clear();
|
|
522
|
+
this.resizeFence = undefined;
|
|
523
|
+
this.lastRequestedViewport = undefined;
|
|
524
|
+
this.pendingViewport = undefined;
|
|
525
|
+
this.inFlightViewport = undefined;
|
|
526
|
+
this.viewportProcessing = false;
|
|
527
|
+
this.viewportProcessingPromise = undefined;
|
|
528
|
+
this.inputResetPromise = undefined;
|
|
529
|
+
this.inputResetTail = Promise.resolve();
|
|
530
|
+
this.inputResetCount = 0;
|
|
531
|
+
this.inputBlocked = true;
|
|
532
|
+
this.pressedKeys.clear();
|
|
533
|
+
this.pressedMouseButtons.clear();
|
|
534
|
+
this.releasePointerCaptures();
|
|
535
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
536
|
+
for (const command of abandonedCommands) {
|
|
537
|
+
command.reject(new Error('Live browser renderer stopped before dispatching input'));
|
|
538
|
+
}
|
|
539
|
+
const inputIdleResolvers = this.inputIdleResolvers;
|
|
540
|
+
this.inputIdleResolvers = [];
|
|
541
|
+
for (const resolver of inputIdleResolvers) {
|
|
542
|
+
resolver();
|
|
543
|
+
}
|
|
544
|
+
this.restoreCanvasAndFocus();
|
|
545
|
+
this.interruptionRequested = false;
|
|
546
|
+
this.stopping = false;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
private restoreCanvasAndFocus(): void {
|
|
551
|
+
this.canvas.width = this.initialCanvasWidth;
|
|
552
|
+
this.canvas.height = this.initialCanvasHeight;
|
|
553
|
+
this.restoreFocusTarget();
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
private installDomListeners(runEpochArg: number): void {
|
|
557
|
+
const listenerController = new AbortController();
|
|
558
|
+
const signal = listenerController.signal;
|
|
559
|
+
this.listenerController = listenerController;
|
|
560
|
+
|
|
561
|
+
this.canvas.addEventListener('pointerdown', (eventArg) => {
|
|
562
|
+
this.handlePointerDown(eventArg, runEpochArg);
|
|
563
|
+
}, { signal });
|
|
564
|
+
this.canvas.addEventListener('pointermove', (eventArg) => {
|
|
565
|
+
this.handlePointerMove(eventArg, runEpochArg);
|
|
566
|
+
}, { signal });
|
|
567
|
+
this.canvas.addEventListener('pointerup', (eventArg) => {
|
|
568
|
+
this.handlePointerUp(eventArg, runEpochArg);
|
|
569
|
+
}, { signal });
|
|
570
|
+
this.canvas.addEventListener('pointercancel', (eventArg) => {
|
|
571
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
572
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
573
|
+
}, { signal });
|
|
574
|
+
this.canvas.addEventListener('lostpointercapture', (eventArg) => {
|
|
575
|
+
this.capturedPointerIds.delete(eventArg.pointerId);
|
|
576
|
+
if (this.pressedMouseButtons.size > 0) {
|
|
577
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
578
|
+
}
|
|
579
|
+
}, { signal });
|
|
580
|
+
this.canvas.addEventListener('contextmenu', (eventArg) => {
|
|
581
|
+
eventArg.preventDefault();
|
|
582
|
+
}, { signal });
|
|
583
|
+
this.canvas.addEventListener('wheel', (eventArg) => {
|
|
584
|
+
this.handleWheel(eventArg, runEpochArg);
|
|
585
|
+
}, { passive: false, signal });
|
|
586
|
+
|
|
587
|
+
this.focusTarget.addEventListener('keydown', (eventArg) => {
|
|
588
|
+
this.handleKey(eventArg, 'down', runEpochArg);
|
|
589
|
+
}, { signal });
|
|
590
|
+
this.focusTarget.addEventListener('keyup', (eventArg) => {
|
|
591
|
+
this.handleKey(eventArg, 'up', runEpochArg);
|
|
592
|
+
}, { signal });
|
|
593
|
+
this.focusTarget.addEventListener('compositionend', (eventArg) => {
|
|
594
|
+
if (!this.inputBlocked && eventArg.data) {
|
|
595
|
+
void this.insertText(eventArg.data).catch(() => undefined);
|
|
596
|
+
}
|
|
597
|
+
}, { signal });
|
|
598
|
+
|
|
599
|
+
window.addEventListener('blur', () => {
|
|
600
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
601
|
+
}, { signal });
|
|
602
|
+
window.addEventListener('resize', () => {
|
|
603
|
+
this.syncViewport();
|
|
604
|
+
}, { signal });
|
|
605
|
+
document.addEventListener('visibilitychange', () => {
|
|
606
|
+
if (this.state && this.running) this.videoReceiver?.update(this.state);
|
|
607
|
+
if (document.visibilityState === 'hidden') {
|
|
608
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
609
|
+
}
|
|
610
|
+
}, { signal });
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
private installResizeObserver(runEpochArg: number): void {
|
|
614
|
+
if (!this.resizeTarget) {
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
this.resizeObserver = new ResizeObserver((entriesArg) => {
|
|
618
|
+
const matchingEntry = entriesArg.find((entryArg) => entryArg.target === this.resizeTarget);
|
|
619
|
+
if (!matchingEntry || runEpochArg !== this.runEpoch) {
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
this.pendingResizeSize = {
|
|
623
|
+
width: matchingEntry.contentRect.width,
|
|
624
|
+
height: matchingEntry.contentRect.height,
|
|
625
|
+
};
|
|
626
|
+
if (this.resizeDebounceTimer !== undefined) {
|
|
627
|
+
globalThis.clearTimeout(this.resizeDebounceTimer);
|
|
628
|
+
}
|
|
629
|
+
this.resizeDebounceTimer = globalThis.setTimeout(() => {
|
|
630
|
+
this.resizeDebounceTimer = undefined;
|
|
631
|
+
const pendingResizeSize = this.pendingResizeSize;
|
|
632
|
+
this.pendingResizeSize = undefined;
|
|
633
|
+
if (pendingResizeSize) {
|
|
634
|
+
this.queueViewport(pendingResizeSize.width, pendingResizeSize.height, runEpochArg);
|
|
635
|
+
}
|
|
636
|
+
}, resizeDebounceMs);
|
|
637
|
+
});
|
|
638
|
+
this.resizeObserver.observe(this.resizeTarget);
|
|
639
|
+
this.syncViewport();
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
private clearResizeDebounce(): void {
|
|
643
|
+
if (this.resizeDebounceTimer !== undefined) {
|
|
644
|
+
globalThis.clearTimeout(this.resizeDebounceTimer);
|
|
645
|
+
this.resizeDebounceTimer = undefined;
|
|
646
|
+
}
|
|
647
|
+
this.pendingResizeSize = undefined;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
private handleEvent(eventArg: TLiveBrowserEvent, runEpochArg: number): void {
|
|
651
|
+
if (eventArg.type === 'frame') {
|
|
652
|
+
if (this.videoReceiver) return;
|
|
653
|
+
this.queueFrame(eventArg.frame, runEpochArg);
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
if (!this.running || runEpochArg !== this.runEpoch) {
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
if (eventArg.type === 'state') {
|
|
660
|
+
this.applyState(eventArg.state, runEpochArg);
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
this.reportError({
|
|
664
|
+
code: 'remote_browser_error',
|
|
665
|
+
message: `${eventArg.error.code}: ${eventArg.error.message}`,
|
|
666
|
+
cause: eventArg.error,
|
|
667
|
+
}, runEpochArg);
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
private applyState(stateArg: ILiveBrowserState, runEpochArg: number): void {
|
|
671
|
+
if (!this.running || runEpochArg !== this.runEpoch) {
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
this.state = stateArg;
|
|
675
|
+
this.followNegotiatedViewport();
|
|
676
|
+
this.videoReceiver?.update(stateArg);
|
|
677
|
+
|
|
678
|
+
if (this.displayedFrame && this.isStateAheadOfDisplayedFrame(stateArg, this.displayedFrame)) {
|
|
679
|
+
this.displayedFrame = undefined;
|
|
680
|
+
this.clearCanvas();
|
|
681
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
this.updateInputAvailability();
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
private presentVideo(sourceArg: ILiveVideoSource): void {
|
|
688
|
+
if (!this.running || !this.state) return;
|
|
689
|
+
const tab = this.state.tabs.find((tabArg) => tabArg.id === sourceArg.tabId);
|
|
690
|
+
if (this.state.status !== 'running' || this.state.activeTabId !== sourceArg.tabId
|
|
691
|
+
|| this.state.viewportRevision !== sourceArg.viewportRevision || !tab?.streaming
|
|
692
|
+
|| tab.status !== 'open' || tab.generation !== sourceArg.generation) return;
|
|
693
|
+
this.displayedFrame = { ...sourceArg, viewport: { ...sourceArg.viewport }, sequence: 0 };
|
|
694
|
+
this.updateInputAvailability();
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
private isStateAheadOfDisplayedFrame(
|
|
698
|
+
stateArg: ILiveBrowserState,
|
|
699
|
+
displayedFrameArg: IDisplayedFrame,
|
|
700
|
+
): boolean {
|
|
701
|
+
if (stateArg.status !== 'running' || stateArg.activeTabId !== displayedFrameArg.tabId) {
|
|
702
|
+
return true;
|
|
703
|
+
}
|
|
704
|
+
const activeTab = stateArg.tabs.find((tabArg) => tabArg.id === displayedFrameArg.tabId);
|
|
705
|
+
if (!activeTab || activeTab.status !== 'open' || !activeTab.streaming) {
|
|
706
|
+
return true;
|
|
707
|
+
}
|
|
708
|
+
return stateArg.viewportRevision > displayedFrameArg.viewportRevision
|
|
709
|
+
|| activeTab.generation > displayedFrameArg.generation;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
private queueFrame(frameArg: ILiveBrowserFrame, runEpochArg: number): void {
|
|
713
|
+
if (!this.running || !this.acceptingFrames || runEpochArg !== this.runEpoch) {
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
this.statistics.framesReceived++;
|
|
717
|
+
|
|
718
|
+
try {
|
|
719
|
+
this.validateFrame(frameArg);
|
|
720
|
+
if (frameArg.sequence <= this.highestFrameSequence) {
|
|
721
|
+
throw new LiveBrowserFrameProtocolError(
|
|
722
|
+
`frame.sequence ${frameArg.sequence} must be strictly greater than current run high-water ${this.highestFrameSequence}`,
|
|
723
|
+
);
|
|
724
|
+
}
|
|
725
|
+
} catch (error) {
|
|
726
|
+
this.requestSuspend(runEpochArg);
|
|
727
|
+
this.reportError({
|
|
728
|
+
code: 'frame_render_failed',
|
|
729
|
+
message: `Rejected invalid live browser frame: ${getErrorMessage(error)}`,
|
|
730
|
+
cause: error,
|
|
731
|
+
frame: frameArg,
|
|
732
|
+
}, runEpochArg);
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
this.highestFrameSequence = frameArg.sequence;
|
|
737
|
+
const frameWork: IFrameWork = {
|
|
738
|
+
frame: frameArg,
|
|
739
|
+
runEpoch: runEpochArg,
|
|
740
|
+
acknowledgementAttempted: false,
|
|
741
|
+
};
|
|
742
|
+
if (this.queuedFrame) {
|
|
743
|
+
// The superseded frame was already acknowledged at receipt.
|
|
744
|
+
this.queuedFrame = undefined;
|
|
745
|
+
this.statistics.framesSkipped++;
|
|
746
|
+
}
|
|
747
|
+
this.queuedFrame = frameWork;
|
|
748
|
+
void this.acknowledgeFrame(frameWork);
|
|
749
|
+
|
|
750
|
+
this.ensureFrameProcessor();
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
private ensureFrameProcessor(): void {
|
|
754
|
+
if (this.frameProcessingPromise || !this.queuedFrame) {
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
const processingPromise = this.processFrameQueue().finally(() => {
|
|
758
|
+
if (this.frameProcessingPromise === processingPromise) {
|
|
759
|
+
this.frameProcessingPromise = undefined;
|
|
760
|
+
if (this.queuedFrame) {
|
|
761
|
+
this.ensureFrameProcessor();
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
});
|
|
765
|
+
this.frameProcessingPromise = processingPromise;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
private async processFrameQueue(): Promise<void> {
|
|
769
|
+
while (this.queuedFrame) {
|
|
770
|
+
const frameWork = this.queuedFrame;
|
|
771
|
+
this.queuedFrame = undefined;
|
|
772
|
+
await this.processFrame(frameWork);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
private async processFrame(frameWorkArg: IFrameWork): Promise<void> {
|
|
777
|
+
const { frame, runEpoch } = frameWorkArg;
|
|
778
|
+
let imageBitmap: ImageBitmap | undefined;
|
|
779
|
+
let terminalFailure = false;
|
|
780
|
+
let terminalError: unknown;
|
|
781
|
+
try {
|
|
782
|
+
if (!this.canRenderFrame(frame, runEpoch)) {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const frameBytes = new Uint8Array(frame.data.byteLength);
|
|
787
|
+
frameBytes.set(frame.data);
|
|
788
|
+
imageBitmap = await this.decodeFrame(new Blob([frameBytes.buffer], {
|
|
789
|
+
type: frame.mimeType,
|
|
790
|
+
}), runEpoch);
|
|
791
|
+
|
|
792
|
+
if (imageBitmap.width !== frame.width || imageBitmap.height !== frame.height) {
|
|
793
|
+
throw new LiveBrowserFrameIntegrityError(
|
|
794
|
+
`decoded frame dimensions ${imageBitmap.width}x${imageBitmap.height} do not match ${frame.width}x${frame.height}`,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
this.statistics.framesDecoded++;
|
|
798
|
+
if (!this.canRenderFrame(frame, runEpoch)) {
|
|
799
|
+
this.statistics.framesSkipped++;
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
this.setPendingDraw({ bitmap: imageBitmap, frame, runEpoch });
|
|
804
|
+
imageBitmap = undefined;
|
|
805
|
+
} catch (error) {
|
|
806
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
terminalFailure = true;
|
|
810
|
+
terminalError = error;
|
|
811
|
+
} finally {
|
|
812
|
+
imageBitmap?.close();
|
|
813
|
+
if (terminalFailure) {
|
|
814
|
+
this.requestSuspend(runEpoch);
|
|
815
|
+
this.reportError({
|
|
816
|
+
code: 'frame_render_failed',
|
|
817
|
+
message: `Could not render live browser frame: ${getErrorMessage(terminalError)}`,
|
|
818
|
+
cause: terminalError,
|
|
819
|
+
frame,
|
|
820
|
+
}, runEpoch);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* Keeps only the newest decoded bitmap and presents it on the next animation frame.
|
|
827
|
+
*/
|
|
828
|
+
private setPendingDraw(pendingDrawArg: IPendingDraw): void {
|
|
829
|
+
if (this.pendingDraw) {
|
|
830
|
+
this.pendingDraw.bitmap.close();
|
|
831
|
+
this.statistics.framesSkipped++;
|
|
832
|
+
}
|
|
833
|
+
this.pendingDraw = pendingDrawArg;
|
|
834
|
+
if (!this.drawSchedule) {
|
|
835
|
+
this.drawSchedule = this.scheduleAnimationFrame(() => {
|
|
836
|
+
this.drawSchedule = undefined;
|
|
837
|
+
this.drawPendingFrame();
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
private discardPendingDraw(): void {
|
|
843
|
+
if (this.drawSchedule) {
|
|
844
|
+
this.cancelScheduledFrame(this.drawSchedule);
|
|
845
|
+
this.drawSchedule = undefined;
|
|
846
|
+
}
|
|
847
|
+
const pendingDraw = this.pendingDraw;
|
|
848
|
+
this.pendingDraw = undefined;
|
|
849
|
+
if (pendingDraw) {
|
|
850
|
+
pendingDraw.bitmap.close();
|
|
851
|
+
this.statistics.framesSkipped++;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
private drawPendingFrame(): void {
|
|
856
|
+
const pendingDraw = this.pendingDraw;
|
|
857
|
+
this.pendingDraw = undefined;
|
|
858
|
+
if (!pendingDraw) {
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
const { bitmap, frame, runEpoch } = pendingDraw;
|
|
862
|
+
if (!this.canRenderFrame(frame, runEpoch)) {
|
|
863
|
+
bitmap.close();
|
|
864
|
+
this.statistics.framesSkipped++;
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
try {
|
|
868
|
+
this.paintBitmap(bitmap, frame.width, frame.height);
|
|
869
|
+
} catch (error) {
|
|
870
|
+
bitmap.close();
|
|
871
|
+
this.requestSuspend(runEpoch);
|
|
872
|
+
this.reportError({
|
|
873
|
+
code: 'frame_render_failed',
|
|
874
|
+
message: `Could not render live browser frame: ${getErrorMessage(error)}`,
|
|
875
|
+
cause: error,
|
|
876
|
+
frame,
|
|
877
|
+
}, runEpoch);
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
this.displayedFrame = {
|
|
881
|
+
tabId: frame.tabId,
|
|
882
|
+
sequence: frame.sequence,
|
|
883
|
+
generation: frame.generation,
|
|
884
|
+
viewportRevision: frame.viewportRevision,
|
|
885
|
+
viewport: { ...frame.viewport },
|
|
886
|
+
};
|
|
887
|
+
|
|
888
|
+
if (
|
|
889
|
+
this.resizeFence
|
|
890
|
+
&& !this.viewportProcessing
|
|
891
|
+
&& frame.viewportRevision >= this.resizeFence.minimumViewportRevision
|
|
892
|
+
&& viewportEquals(frame.viewport, this.resizeFence.target)
|
|
893
|
+
) {
|
|
894
|
+
this.resizeFence = undefined;
|
|
895
|
+
}
|
|
896
|
+
this.updateInputAvailability();
|
|
897
|
+
this.callFrameRendered(frame, runEpoch);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* Presents one decoded bitmap. The canvas backing store is reallocated only
|
|
902
|
+
* when the frame dimensions differ from the current intrinsic dimensions.
|
|
903
|
+
*/
|
|
904
|
+
private paintBitmap(bitmapArg: ImageBitmap, widthArg: number, heightArg: number): void {
|
|
905
|
+
if (this.canvas.width !== widthArg) {
|
|
906
|
+
this.canvas.width = widthArg;
|
|
907
|
+
}
|
|
908
|
+
if (this.canvas.height !== heightArg) {
|
|
909
|
+
this.canvas.height = heightArg;
|
|
910
|
+
}
|
|
911
|
+
if (this.bitmapContext) {
|
|
912
|
+
this.bitmapContext.transferFromImageBitmap(bitmapArg);
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
if (!this.canvasContext) {
|
|
916
|
+
throw new Error('LiveBrowserCanvasRenderer has no rendering context');
|
|
917
|
+
}
|
|
918
|
+
this.canvasContext.clearRect(0, 0, widthArg, heightArg);
|
|
919
|
+
this.canvasContext.drawImage(bitmapArg, 0, 0, widthArg, heightArg);
|
|
920
|
+
bitmapArg.close();
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
private canRenderFrame(frameArg: ILiveBrowserFrame, runEpochArg: number): boolean {
|
|
924
|
+
if (
|
|
925
|
+
!this.running
|
|
926
|
+
|| !this.acceptingFrames
|
|
927
|
+
|| runEpochArg !== this.runEpoch
|
|
928
|
+
) {
|
|
929
|
+
return false;
|
|
930
|
+
}
|
|
931
|
+
const state = this.state;
|
|
932
|
+
if (!state || state.status !== 'running' || state.activeTabId !== frameArg.tabId) {
|
|
933
|
+
return false;
|
|
934
|
+
}
|
|
935
|
+
const activeTab = state.tabs.find((tabArg) => tabArg.id === frameArg.tabId);
|
|
936
|
+
if (!activeTab || activeTab.status !== 'open' || !activeTab.streaming) {
|
|
937
|
+
return false;
|
|
938
|
+
}
|
|
939
|
+
if (
|
|
940
|
+
state.viewportRevision > frameArg.viewportRevision
|
|
941
|
+
|| activeTab.generation > frameArg.generation
|
|
942
|
+
) {
|
|
943
|
+
return false;
|
|
944
|
+
}
|
|
945
|
+
if (this.resizeFence && !this.viewportProcessing) {
|
|
946
|
+
return frameArg.viewportRevision >= this.resizeFence.minimumViewportRevision
|
|
947
|
+
&& viewportEquals(frameArg.viewport, this.resizeFence.target);
|
|
948
|
+
}
|
|
949
|
+
return true;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
private acknowledgeFrame(frameWorkArg: IFrameWork): Promise<void> {
|
|
953
|
+
if (frameWorkArg.acknowledgementAttempted) {
|
|
954
|
+
return Promise.resolve();
|
|
955
|
+
}
|
|
956
|
+
frameWorkArg.acknowledgementAttempted = true;
|
|
957
|
+
|
|
958
|
+
if (!this.isRunActive(frameWorkArg.runEpoch)) {
|
|
959
|
+
return Promise.resolve();
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
if (this.acknowledgementPromises.size >= maxPendingAcknowledgements) {
|
|
963
|
+
this.requestSuspend(frameWorkArg.runEpoch);
|
|
964
|
+
this.reportError({
|
|
965
|
+
code: 'frame_acknowledgement_failed',
|
|
966
|
+
message: `Live browser frame acknowledgement capacity of ${maxPendingAcknowledgements} was reached`,
|
|
967
|
+
}, frameWorkArg.runEpoch);
|
|
968
|
+
return Promise.resolve();
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
const acknowledgement = this.runClientOperation(
|
|
972
|
+
(optionsArg) => {
|
|
973
|
+
if (!('acknowledgeFrame' in this.client)) throw new Error('Video clients do not acknowledge image frames');
|
|
974
|
+
return this.client.acknowledgeFrame(
|
|
975
|
+
createFrameAcknowledgementRequest(frameWorkArg.frame),
|
|
976
|
+
optionsArg,
|
|
977
|
+
);
|
|
978
|
+
},
|
|
979
|
+
'frame acknowledgement',
|
|
980
|
+
frameWorkArg.runEpoch,
|
|
981
|
+
)
|
|
982
|
+
.then(() => undefined)
|
|
983
|
+
.catch((error) => {
|
|
984
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
985
|
+
return;
|
|
986
|
+
}
|
|
987
|
+
this.requestSuspend(frameWorkArg.runEpoch);
|
|
988
|
+
this.reportError({
|
|
989
|
+
code: 'frame_acknowledgement_failed',
|
|
990
|
+
message: `Could not acknowledge live browser frame: ${getErrorMessage(error)}`,
|
|
991
|
+
cause: error,
|
|
992
|
+
frame: frameWorkArg.frame,
|
|
993
|
+
}, frameWorkArg.runEpoch);
|
|
994
|
+
});
|
|
995
|
+
let trackedAcknowledgement: Promise<void>;
|
|
996
|
+
trackedAcknowledgement = acknowledgement.finally(() => {
|
|
997
|
+
this.acknowledgementPromises.delete(trackedAcknowledgement);
|
|
998
|
+
});
|
|
999
|
+
this.acknowledgementPromises.add(trackedAcknowledgement);
|
|
1000
|
+
return trackedAcknowledgement;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
private handlePointerDown(eventArg: PointerEvent, runEpochArg: number): void {
|
|
1004
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
const button = this.getMouseButton(eventArg.button);
|
|
1008
|
+
if (!button) {
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const displayedFrame = this.getInputFrame();
|
|
1012
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
eventArg.preventDefault();
|
|
1016
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
1017
|
+
if (!coordinates) {
|
|
1018
|
+
return;
|
|
1019
|
+
}
|
|
1020
|
+
this.focusTarget.focus({ preventScroll: true });
|
|
1021
|
+
try {
|
|
1022
|
+
this.canvas.setPointerCapture(eventArg.pointerId);
|
|
1023
|
+
this.capturedPointerIds.add(eventArg.pointerId);
|
|
1024
|
+
} catch {
|
|
1025
|
+
// Pointer capture may already have been released by the browser.
|
|
1026
|
+
}
|
|
1027
|
+
const input: ILiveBrowserMouseInput = {
|
|
1028
|
+
...createInputIdentity(displayedFrame),
|
|
1029
|
+
type: 'down',
|
|
1030
|
+
...coordinates,
|
|
1031
|
+
button,
|
|
1032
|
+
buttons: eventArg.buttons,
|
|
1033
|
+
clickCount: Math.min(3, Math.max(1, eventArg.detail || 1)),
|
|
1034
|
+
modifiers: createModifiers(eventArg),
|
|
1035
|
+
};
|
|
1036
|
+
void this.enqueueInputCommand({
|
|
1037
|
+
kind: 'mouse',
|
|
1038
|
+
runEpoch: runEpochArg,
|
|
1039
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
1040
|
+
onAttempt: () => this.pressedMouseButtons.set(button, input),
|
|
1041
|
+
}).catch(() => undefined);
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
private handlePointerMove(eventArg: PointerEvent, runEpochArg: number): void {
|
|
1045
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
const displayedFrame = this.getInputFrame();
|
|
1049
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
1050
|
+
return;
|
|
1051
|
+
}
|
|
1052
|
+
eventArg.preventDefault();
|
|
1053
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
1054
|
+
if (!coordinates) {
|
|
1055
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
1056
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
const input: ILiveBrowserMouseInput = {
|
|
1060
|
+
...createInputIdentity(displayedFrame),
|
|
1061
|
+
type: 'move',
|
|
1062
|
+
...coordinates,
|
|
1063
|
+
button: 'none',
|
|
1064
|
+
buttons: eventArg.buttons,
|
|
1065
|
+
modifiers: createModifiers(eventArg),
|
|
1066
|
+
};
|
|
1067
|
+
void this.enqueueInputCommand({
|
|
1068
|
+
kind: 'mouseMove',
|
|
1069
|
+
runEpoch: runEpochArg,
|
|
1070
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
1071
|
+
}).catch(() => undefined);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
private handlePointerUp(eventArg: PointerEvent, runEpochArg: number): void {
|
|
1075
|
+
if (eventArg.pointerType && eventArg.pointerType !== 'mouse') {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
const button = this.getMouseButton(eventArg.button);
|
|
1079
|
+
const displayedFrame = this.getInputFrame();
|
|
1080
|
+
if (!button || !displayedFrame || runEpochArg !== this.runEpoch) {
|
|
1081
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
eventArg.preventDefault();
|
|
1085
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
1086
|
+
if (!coordinates) {
|
|
1087
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
1088
|
+
void this.scheduleInputReset(false, runEpochArg);
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
const input: ILiveBrowserMouseInput = {
|
|
1092
|
+
...createInputIdentity(displayedFrame),
|
|
1093
|
+
type: 'up',
|
|
1094
|
+
...coordinates,
|
|
1095
|
+
button,
|
|
1096
|
+
buttons: eventArg.buttons,
|
|
1097
|
+
clickCount: Math.min(3, Math.max(1, eventArg.detail || 1)),
|
|
1098
|
+
modifiers: createModifiers(eventArg),
|
|
1099
|
+
};
|
|
1100
|
+
void this.enqueueInputCommand({
|
|
1101
|
+
kind: 'mouse',
|
|
1102
|
+
runEpoch: runEpochArg,
|
|
1103
|
+
execute: async (optionsArg) => this.client.dispatchMouse(input, optionsArg),
|
|
1104
|
+
onSuccess: () => this.pressedMouseButtons.delete(button),
|
|
1105
|
+
}).catch(() => undefined);
|
|
1106
|
+
this.releasePointerCapture(eventArg.pointerId);
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
private handleWheel(eventArg: WheelEvent, runEpochArg: number): void {
|
|
1110
|
+
const displayedFrame = this.getInputFrame();
|
|
1111
|
+
if (!displayedFrame || runEpochArg !== this.runEpoch) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
eventArg.preventDefault();
|
|
1115
|
+
const coordinates = this.mapCoordinates(eventArg, displayedFrame.viewport);
|
|
1116
|
+
if (!coordinates) {
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
const multiplier = eventArg.deltaMode === WheelEvent.DOM_DELTA_LINE
|
|
1120
|
+
? 16
|
|
1121
|
+
: eventArg.deltaMode === WheelEvent.DOM_DELTA_PAGE
|
|
1122
|
+
? displayedFrame.viewport.height
|
|
1123
|
+
: 1;
|
|
1124
|
+
const inputIdentity = createInputIdentity(displayedFrame);
|
|
1125
|
+
const modifiers = createModifiers(eventArg);
|
|
1126
|
+
this.statistics.inputCommandsEnqueued++;
|
|
1127
|
+
const pendingWheel = this.pendingWheel;
|
|
1128
|
+
if (
|
|
1129
|
+
pendingWheel
|
|
1130
|
+
&& pendingWheel.runEpoch === runEpochArg
|
|
1131
|
+
&& inputIdentityEquals(pendingWheel, inputIdentity)
|
|
1132
|
+
) {
|
|
1133
|
+
mergeWheelInput(pendingWheel, {
|
|
1134
|
+
...inputIdentity,
|
|
1135
|
+
...coordinates,
|
|
1136
|
+
deltaX: eventArg.deltaX * multiplier,
|
|
1137
|
+
deltaY: eventArg.deltaY * multiplier,
|
|
1138
|
+
modifiers,
|
|
1139
|
+
});
|
|
1140
|
+
this.statistics.inputCommandsCoalesced++;
|
|
1141
|
+
} else {
|
|
1142
|
+
if (pendingWheel) {
|
|
1143
|
+
this.flushPendingWheel();
|
|
1144
|
+
}
|
|
1145
|
+
this.pendingWheel = {
|
|
1146
|
+
...inputIdentity,
|
|
1147
|
+
...coordinates,
|
|
1148
|
+
deltaX: clampWheelDelta(eventArg.deltaX * multiplier),
|
|
1149
|
+
deltaY: clampWheelDelta(eventArg.deltaY * multiplier),
|
|
1150
|
+
modifiers,
|
|
1151
|
+
runEpoch: runEpochArg,
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
this.tryFlushPendingWheel();
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/**
|
|
1158
|
+
* Flushes accumulated wheel input at most once per animation frame and only
|
|
1159
|
+
* while no wheel dispatch is in flight; the settling dispatch flushes the rest.
|
|
1160
|
+
*/
|
|
1161
|
+
private tryFlushPendingWheel(): void {
|
|
1162
|
+
if (!this.pendingWheel) {
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
if (!this.wheelFrameElapsed) {
|
|
1166
|
+
this.scheduleWheelFlush();
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
if (this.inFlightWheelCommands > 0) {
|
|
1170
|
+
return;
|
|
1171
|
+
}
|
|
1172
|
+
this.flushPendingWheel();
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
private scheduleWheelFlush(): void {
|
|
1176
|
+
if (this.wheelFlushSchedule) {
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
this.wheelFlushSchedule = this.scheduleAnimationFrame(() => {
|
|
1180
|
+
this.wheelFlushSchedule = undefined;
|
|
1181
|
+
this.wheelFrameElapsed = true;
|
|
1182
|
+
this.tryFlushPendingWheel();
|
|
1183
|
+
});
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* Moves the accumulated wheel input into the command queue regardless of the
|
|
1188
|
+
* frame gate so that later discrete commands stay behind it.
|
|
1189
|
+
*/
|
|
1190
|
+
private flushPendingWheel(): void {
|
|
1191
|
+
const pendingWheel = this.pendingWheel;
|
|
1192
|
+
if (!pendingWheel) {
|
|
1193
|
+
return;
|
|
1194
|
+
}
|
|
1195
|
+
this.pendingWheel = undefined;
|
|
1196
|
+
this.wheelFrameElapsed = false;
|
|
1197
|
+
this.scheduleWheelFlush();
|
|
1198
|
+
const displayedFrame = this.getInputFrame();
|
|
1199
|
+
if (
|
|
1200
|
+
!displayedFrame
|
|
1201
|
+
|| pendingWheel.runEpoch !== this.runEpoch
|
|
1202
|
+
|| !inputIdentityEquals(pendingWheel, createInputIdentity(displayedFrame))
|
|
1203
|
+
) {
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
const { runEpoch, ...wheelInput } = pendingWheel;
|
|
1207
|
+
void this.enqueueInputCommand({
|
|
1208
|
+
kind: 'wheel',
|
|
1209
|
+
runEpoch,
|
|
1210
|
+
wheelInput,
|
|
1211
|
+
execute: async (optionsArg) => this.client.dispatchWheel({
|
|
1212
|
+
...wheelInput,
|
|
1213
|
+
deltaX: clampWheelDelta(wheelInput.deltaX),
|
|
1214
|
+
deltaY: clampWheelDelta(wheelInput.deltaY),
|
|
1215
|
+
}, optionsArg),
|
|
1216
|
+
}).catch(() => undefined);
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
private discardPendingWheel(): void {
|
|
1220
|
+
this.pendingWheel = undefined;
|
|
1221
|
+
if (this.wheelFlushSchedule) {
|
|
1222
|
+
this.cancelScheduledFrame(this.wheelFlushSchedule);
|
|
1223
|
+
this.wheelFlushSchedule = undefined;
|
|
1224
|
+
}
|
|
1225
|
+
this.wheelFrameElapsed = true;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
private handleKey(
|
|
1229
|
+
eventArg: KeyboardEvent,
|
|
1230
|
+
typeArg: 'down' | 'up',
|
|
1231
|
+
runEpochArg: number,
|
|
1232
|
+
): void {
|
|
1233
|
+
const displayedFrame = this.getInputFrame();
|
|
1234
|
+
if (
|
|
1235
|
+
!displayedFrame
|
|
1236
|
+
|| runEpochArg !== this.runEpoch
|
|
1237
|
+
|| eventArg.isComposing
|
|
1238
|
+
|| eventArg.key === 'Process'
|
|
1239
|
+
) {
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
eventArg.preventDefault();
|
|
1243
|
+
const text = typeArg === 'down'
|
|
1244
|
+
&& eventArg.key.length === 1
|
|
1245
|
+
&& !eventArg.metaKey
|
|
1246
|
+
&& (!eventArg.ctrlKey || eventArg.altKey)
|
|
1247
|
+
? eventArg.key
|
|
1248
|
+
: undefined;
|
|
1249
|
+
const input: ILiveBrowserKeyInput = {
|
|
1250
|
+
...createInputIdentity(displayedFrame),
|
|
1251
|
+
type: typeArg,
|
|
1252
|
+
key: eventArg.key,
|
|
1253
|
+
code: eventArg.code || undefined,
|
|
1254
|
+
text,
|
|
1255
|
+
windowsVirtualKeyCode: eventArg.keyCode || undefined,
|
|
1256
|
+
autoRepeat: eventArg.repeat,
|
|
1257
|
+
isKeypad: eventArg.location === KeyboardEvent.DOM_KEY_LOCATION_NUMPAD,
|
|
1258
|
+
location: eventArg.location,
|
|
1259
|
+
modifiers: createModifiers(eventArg),
|
|
1260
|
+
};
|
|
1261
|
+
const keyIdentity = eventArg.code || eventArg.key;
|
|
1262
|
+
void this.enqueueInputCommand({
|
|
1263
|
+
kind: 'key',
|
|
1264
|
+
runEpoch: runEpochArg,
|
|
1265
|
+
execute: async (optionsArg) => this.client.dispatchKey(input, optionsArg),
|
|
1266
|
+
onAttempt: typeArg === 'down'
|
|
1267
|
+
? () => this.pressedKeys.set(keyIdentity, input)
|
|
1268
|
+
: undefined,
|
|
1269
|
+
onSuccess: () => {
|
|
1270
|
+
if (typeArg === 'up') {
|
|
1271
|
+
this.pressedKeys.delete(keyIdentity);
|
|
1272
|
+
}
|
|
1273
|
+
},
|
|
1274
|
+
}).catch(() => undefined);
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
private mapCoordinates(
|
|
1278
|
+
eventArg: MouseEvent,
|
|
1279
|
+
viewportArg: ILiveBrowserViewport,
|
|
1280
|
+
): { x: number; y: number } | undefined {
|
|
1281
|
+
const bounds = this.canvas.getBoundingClientRect();
|
|
1282
|
+
let canvasBounds = { left: bounds.left, top: bounds.top, width: bounds.width, height: bounds.height };
|
|
1283
|
+
if (this.videoReceiver && bounds.width > 0 && bounds.height > 0) {
|
|
1284
|
+
const scale = Math.min(bounds.width / viewportArg.width, bounds.height / viewportArg.height);
|
|
1285
|
+
canvasBounds = {
|
|
1286
|
+
left: bounds.left + (bounds.width - viewportArg.width * scale) / 2,
|
|
1287
|
+
top: bounds.top + (bounds.height - viewportArg.height * scale) / 2,
|
|
1288
|
+
width: viewportArg.width * scale, height: viewportArg.height * scale,
|
|
1289
|
+
};
|
|
1290
|
+
if (this.pressedMouseButtons.size === 0 && (eventArg.clientX < canvasBounds.left
|
|
1291
|
+
|| eventArg.clientX >= canvasBounds.left + canvasBounds.width
|
|
1292
|
+
|| eventArg.clientY < canvasBounds.top || eventArg.clientY >= canvasBounds.top + canvasBounds.height)) return undefined;
|
|
1293
|
+
}
|
|
1294
|
+
if (canvasBounds.width <= 0 || canvasBounds.height <= 0) {
|
|
1295
|
+
return undefined;
|
|
1296
|
+
}
|
|
1297
|
+
const x = ((eventArg.clientX - canvasBounds.left) / canvasBounds.width) * viewportArg.width;
|
|
1298
|
+
const y = ((eventArg.clientY - canvasBounds.top) / canvasBounds.height) * viewportArg.height;
|
|
1299
|
+
return {
|
|
1300
|
+
x: Math.max(0, Math.min(viewportArg.width - coordinateEdgeInset, x)),
|
|
1301
|
+
y: Math.max(0, Math.min(viewportArg.height - coordinateEdgeInset, y)),
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
private getMouseButton(buttonArg: number): ILiveBrowserMouseInput['button'] | undefined {
|
|
1306
|
+
return buttonArg === 0
|
|
1307
|
+
? 'left'
|
|
1308
|
+
: buttonArg === 1
|
|
1309
|
+
? 'middle'
|
|
1310
|
+
: buttonArg === 2
|
|
1311
|
+
? 'right'
|
|
1312
|
+
: buttonArg === 3
|
|
1313
|
+
? 'back'
|
|
1314
|
+
: buttonArg === 4
|
|
1315
|
+
? 'forward'
|
|
1316
|
+
: undefined;
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
private getInputFrame(): IDisplayedFrame | undefined {
|
|
1320
|
+
return this.inputBlocked || !this.isDisplayedFrameCurrent()
|
|
1321
|
+
? undefined
|
|
1322
|
+
: this.displayedFrame;
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
private requireInputFrame(): IDisplayedFrame {
|
|
1326
|
+
const displayedFrame = this.getInputFrame();
|
|
1327
|
+
if (!displayedFrame) {
|
|
1328
|
+
throw new Error('Live browser input is unavailable until a current frame is displayed');
|
|
1329
|
+
}
|
|
1330
|
+
return displayedFrame;
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
private isDisplayedFrameCurrent(): boolean {
|
|
1334
|
+
const state = this.state;
|
|
1335
|
+
const displayedFrame = this.displayedFrame;
|
|
1336
|
+
if (!state || !displayedFrame || state.status !== 'running') {
|
|
1337
|
+
return false;
|
|
1338
|
+
}
|
|
1339
|
+
if (
|
|
1340
|
+
state.activeTabId !== displayedFrame.tabId
|
|
1341
|
+
|| state.viewportRevision > displayedFrame.viewportRevision
|
|
1342
|
+
) {
|
|
1343
|
+
return false;
|
|
1344
|
+
}
|
|
1345
|
+
const activeTab = state.tabs.find((tabArg) => tabArg.id === displayedFrame.tabId);
|
|
1346
|
+
return Boolean(
|
|
1347
|
+
activeTab
|
|
1348
|
+
&& activeTab.status === 'open'
|
|
1349
|
+
&& activeTab.streaming
|
|
1350
|
+
&& activeTab.generation <= displayedFrame.generation,
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
private enqueueInputCommand(commandArg: Omit<IInputCommand, 'reject' | 'resolve'>): Promise<void> {
|
|
1355
|
+
if (!this.running || this.inputBlocked || commandArg.runEpoch !== this.runEpoch) {
|
|
1356
|
+
return Promise.reject(new Error('Live browser input is currently blocked'));
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
return new Promise<void>((resolve, reject) => {
|
|
1360
|
+
const command: IInputCommand = {
|
|
1361
|
+
...commandArg,
|
|
1362
|
+
resolve,
|
|
1363
|
+
reject,
|
|
1364
|
+
};
|
|
1365
|
+
if (isCoalescableInputCommand(command.kind)) {
|
|
1366
|
+
this.enqueueCoalescableCommand(command);
|
|
1367
|
+
} else {
|
|
1368
|
+
const queuedDiscreteCommands = this.inputCommands.filter((queuedCommand) => (
|
|
1369
|
+
!isCoalescableInputCommand(queuedCommand.kind)
|
|
1370
|
+
)).length;
|
|
1371
|
+
if (queuedDiscreteCommands >= maxInputQueueLength) {
|
|
1372
|
+
const error = new Error('Live browser input queue reached its capacity');
|
|
1373
|
+
this.reportError({
|
|
1374
|
+
code: 'input_queue_capacity_exceeded',
|
|
1375
|
+
message: error.message,
|
|
1376
|
+
cause: error,
|
|
1377
|
+
}, command.runEpoch);
|
|
1378
|
+
reject(error);
|
|
1379
|
+
return;
|
|
1380
|
+
}
|
|
1381
|
+
// Accumulated wheel input precedes this command in event order.
|
|
1382
|
+
this.flushPendingWheel();
|
|
1383
|
+
this.statistics.inputCommandsEnqueued++;
|
|
1384
|
+
this.inputCommands.push(command);
|
|
1385
|
+
}
|
|
1386
|
+
this.processInputCommands();
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
/**
|
|
1391
|
+
* Coalescable commands never exhaust capacity: they merge into a same-kind
|
|
1392
|
+
* queue tail or evict the oldest queued coalescable command under pressure.
|
|
1393
|
+
*/
|
|
1394
|
+
private enqueueCoalescableCommand(commandArg: IInputCommand): void {
|
|
1395
|
+
// Wheel submissions are counted per DOM event before accumulation.
|
|
1396
|
+
if (commandArg.kind === 'mouseMove') {
|
|
1397
|
+
this.statistics.inputCommandsEnqueued++;
|
|
1398
|
+
}
|
|
1399
|
+
const lastCommand = this.inputCommands.at(-1);
|
|
1400
|
+
if (lastCommand && lastCommand.kind === commandArg.kind) {
|
|
1401
|
+
this.statistics.inputCommandsCoalesced++;
|
|
1402
|
+
if (commandArg.kind === 'wheel' && lastCommand.wheelInput && commandArg.wheelInput) {
|
|
1403
|
+
mergeWheelInput(lastCommand.wheelInput, commandArg.wheelInput);
|
|
1404
|
+
commandArg.resolve();
|
|
1405
|
+
} else {
|
|
1406
|
+
lastCommand.resolve();
|
|
1407
|
+
this.inputCommands[this.inputCommands.length - 1] = commandArg;
|
|
1408
|
+
}
|
|
1409
|
+
return;
|
|
1410
|
+
}
|
|
1411
|
+
const queuedCoalescableCommands = this.inputCommands.filter((queuedCommand) => (
|
|
1412
|
+
isCoalescableInputCommand(queuedCommand.kind)
|
|
1413
|
+
));
|
|
1414
|
+
if (queuedCoalescableCommands.length >= maxQueuedCoalescableCommands) {
|
|
1415
|
+
this.dropOldestCoalescableCommand(queuedCoalescableCommands[0]);
|
|
1416
|
+
}
|
|
1417
|
+
this.inputCommands.push(commandArg);
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
private dropOldestCoalescableCommand(oldestCommandArg: IInputCommand): void {
|
|
1421
|
+
const index = this.inputCommands.indexOf(oldestCommandArg);
|
|
1422
|
+
if (index < 0) {
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1425
|
+
this.inputCommands.splice(index, 1);
|
|
1426
|
+
this.statistics.inputCommandsCoalesced++;
|
|
1427
|
+
if (oldestCommandArg.kind === 'wheel' && oldestCommandArg.wheelInput) {
|
|
1428
|
+
const nextWheelCommand = this.inputCommands.find((queuedCommand) => (
|
|
1429
|
+
queuedCommand.kind === 'wheel' && queuedCommand.wheelInput
|
|
1430
|
+
));
|
|
1431
|
+
if (nextWheelCommand?.wheelInput) {
|
|
1432
|
+
const nextWheelInput = nextWheelCommand.wheelInput;
|
|
1433
|
+
nextWheelInput.deltaX = clampWheelDelta(
|
|
1434
|
+
nextWheelInput.deltaX + oldestCommandArg.wheelInput.deltaX,
|
|
1435
|
+
);
|
|
1436
|
+
nextWheelInput.deltaY = clampWheelDelta(
|
|
1437
|
+
nextWheelInput.deltaY + oldestCommandArg.wheelInput.deltaY,
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
oldestCommandArg.resolve();
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/**
|
|
1445
|
+
* Dispatches queued commands in order with a bounded number in flight. The
|
|
1446
|
+
* remote session processes operations sequentially, so order is preserved.
|
|
1447
|
+
*/
|
|
1448
|
+
private processInputCommands(): void {
|
|
1449
|
+
while (
|
|
1450
|
+
this.inputCommands.length > 0
|
|
1451
|
+
&& this.inFlightInputCommands < maxInFlightInputCommands
|
|
1452
|
+
) {
|
|
1453
|
+
const command = this.inputCommands.shift()!;
|
|
1454
|
+
if (!this.isRunActive(command.runEpoch)) {
|
|
1455
|
+
command.reject(new LiveBrowserRunAbortedError(
|
|
1456
|
+
`Live browser input belongs to inactive run ${command.runEpoch}`,
|
|
1457
|
+
));
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
this.dispatchInputCommand(command);
|
|
1461
|
+
}
|
|
1462
|
+
if (this.inputCommands.length === 0 && this.inFlightInputCommands === 0) {
|
|
1463
|
+
const resolvers = this.inputIdleResolvers;
|
|
1464
|
+
this.inputIdleResolvers = [];
|
|
1465
|
+
for (const resolver of resolvers) {
|
|
1466
|
+
resolver();
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
private dispatchInputCommand(commandArg: IInputCommand): void {
|
|
1472
|
+
this.inFlightInputCommands++;
|
|
1473
|
+
if (commandArg.kind === 'wheel') {
|
|
1474
|
+
this.inFlightWheelCommands++;
|
|
1475
|
+
}
|
|
1476
|
+
const startedAt = performance.now();
|
|
1477
|
+
let dispatch: Promise<void>;
|
|
1478
|
+
try {
|
|
1479
|
+
commandArg.onAttempt?.();
|
|
1480
|
+
dispatch = this.runClientOperation(commandArg.execute, 'input dispatch', commandArg.runEpoch);
|
|
1481
|
+
} catch (error) {
|
|
1482
|
+
dispatch = Promise.reject(error);
|
|
1483
|
+
}
|
|
1484
|
+
void dispatch.then(() => {
|
|
1485
|
+
if (!this.isRunActive(commandArg.runEpoch)) {
|
|
1486
|
+
throw new LiveBrowserRunAbortedError(
|
|
1487
|
+
`Live browser input belongs to inactive run ${commandArg.runEpoch}`,
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
this.statistics.lastInputRoundTripMs = performance.now() - startedAt;
|
|
1491
|
+
commandArg.onSuccess?.();
|
|
1492
|
+
commandArg.resolve();
|
|
1493
|
+
}).catch((error) => {
|
|
1494
|
+
commandArg.reject(error);
|
|
1495
|
+
this.handleInputDispatchFailure(error, commandArg.runEpoch);
|
|
1496
|
+
}).finally(() => {
|
|
1497
|
+
this.inFlightInputCommands--;
|
|
1498
|
+
if (commandArg.kind === 'wheel') {
|
|
1499
|
+
this.inFlightWheelCommands--;
|
|
1500
|
+
if (this.inFlightWheelCommands === 0) {
|
|
1501
|
+
this.tryFlushPendingWheel();
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
this.processInputCommands();
|
|
1505
|
+
});
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
private handleInputDispatchFailure(errorArg: unknown, runEpochArg: number): void {
|
|
1509
|
+
if (errorArg instanceof LiveBrowserRunAbortedError) {
|
|
1510
|
+
return;
|
|
1511
|
+
}
|
|
1512
|
+
if (errorArg instanceof LiveBrowserOperationTimeoutError) {
|
|
1513
|
+
this.requestSuspend(runEpochArg);
|
|
1514
|
+
}
|
|
1515
|
+
this.reportError({
|
|
1516
|
+
code: 'input_dispatch_failed',
|
|
1517
|
+
message: `Could not dispatch live browser input: ${getErrorMessage(errorArg)}`,
|
|
1518
|
+
cause: errorArg,
|
|
1519
|
+
}, runEpochArg);
|
|
1520
|
+
this.discardPendingWheel();
|
|
1521
|
+
const abandonedCommands = this.inputCommands.splice(0);
|
|
1522
|
+
for (const abandonedCommand of abandonedCommands) {
|
|
1523
|
+
abandonedCommand.reject(
|
|
1524
|
+
new Error('Live browser input was abandoned after a dispatch failure'),
|
|
1525
|
+
);
|
|
1526
|
+
}
|
|
1527
|
+
if (!(errorArg instanceof LiveBrowserOperationTimeoutError) && !this.stopping) {
|
|
1528
|
+
void this.scheduleInputReset(true, runEpochArg);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
private waitForInputIdle(): Promise<void> {
|
|
1533
|
+
if (this.inFlightInputCommands === 0 && this.inputCommands.length === 0) {
|
|
1534
|
+
return Promise.resolve();
|
|
1535
|
+
}
|
|
1536
|
+
return new Promise<void>((resolve) => {
|
|
1537
|
+
this.inputIdleResolvers.push(resolve);
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
private scheduleInputReset(
|
|
1542
|
+
clearDisplayArg: boolean,
|
|
1543
|
+
runEpochArg: number,
|
|
1544
|
+
): Promise<void> {
|
|
1545
|
+
if (runEpochArg !== this.runEpoch || this.stopping) {
|
|
1546
|
+
return Promise.resolve();
|
|
1547
|
+
}
|
|
1548
|
+
this.inputBlocked = true;
|
|
1549
|
+
this.releasePointerCaptures();
|
|
1550
|
+
if (clearDisplayArg) {
|
|
1551
|
+
this.displayedFrame = undefined;
|
|
1552
|
+
this.clearCanvas();
|
|
1553
|
+
}
|
|
1554
|
+
if (this.inputResetPromise) {
|
|
1555
|
+
return this.inputResetPromise;
|
|
1556
|
+
}
|
|
1557
|
+
this.inputResetCount++;
|
|
1558
|
+
let resetOperation: Promise<void>;
|
|
1559
|
+
resetOperation = this.inputResetTail.then(async () => {
|
|
1560
|
+
await this.waitForInputIdle();
|
|
1561
|
+
await this.releasePressedInput(runEpochArg);
|
|
1562
|
+
}).finally(() => {
|
|
1563
|
+
this.inputResetCount--;
|
|
1564
|
+
if (this.inputResetPromise === resetOperation) {
|
|
1565
|
+
this.inputResetPromise = undefined;
|
|
1566
|
+
}
|
|
1567
|
+
this.updateInputAvailability();
|
|
1568
|
+
});
|
|
1569
|
+
this.inputResetPromise = resetOperation;
|
|
1570
|
+
this.inputResetTail = resetOperation.then(() => undefined, () => undefined);
|
|
1571
|
+
return resetOperation;
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
private async releasePressedInput(runEpochArg: number): Promise<void> {
|
|
1575
|
+
for (const [keyIdentity, pressedKey] of [...this.pressedKeys.entries()]) {
|
|
1576
|
+
const released = await this.tryInputRelease(
|
|
1577
|
+
(optionsArg) => this.client.dispatchKey({
|
|
1578
|
+
...pressedKey,
|
|
1579
|
+
type: 'up',
|
|
1580
|
+
text: undefined,
|
|
1581
|
+
autoRepeat: false,
|
|
1582
|
+
modifiers: {},
|
|
1583
|
+
}, optionsArg),
|
|
1584
|
+
runEpochArg,
|
|
1585
|
+
);
|
|
1586
|
+
if (released && this.pressedKeys.get(keyIdentity) === pressedKey) {
|
|
1587
|
+
this.pressedKeys.delete(keyIdentity);
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
for (const [button, pressedButton] of [...this.pressedMouseButtons.entries()]) {
|
|
1591
|
+
const released = await this.tryInputRelease(
|
|
1592
|
+
(optionsArg) => this.client.dispatchMouse({
|
|
1593
|
+
...pressedButton,
|
|
1594
|
+
type: 'up',
|
|
1595
|
+
buttons: 0,
|
|
1596
|
+
clickCount: 1,
|
|
1597
|
+
modifiers: {},
|
|
1598
|
+
}, optionsArg),
|
|
1599
|
+
runEpochArg,
|
|
1600
|
+
);
|
|
1601
|
+
if (released && this.pressedMouseButtons.get(button) === pressedButton) {
|
|
1602
|
+
this.pressedMouseButtons.delete(button);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
private async tryInputRelease(
|
|
1608
|
+
operationArg: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<void>,
|
|
1609
|
+
runEpochArg: number,
|
|
1610
|
+
): Promise<boolean> {
|
|
1611
|
+
let lastError: unknown;
|
|
1612
|
+
for (let attempt = 0; attempt < maxInputReleaseAttempts; attempt++) {
|
|
1613
|
+
try {
|
|
1614
|
+
await this.runClientOperation(
|
|
1615
|
+
operationArg,
|
|
1616
|
+
'input release',
|
|
1617
|
+
runEpochArg,
|
|
1618
|
+
);
|
|
1619
|
+
return true;
|
|
1620
|
+
} catch (error) {
|
|
1621
|
+
if (error instanceof LiveBrowserRunAbortedError) {
|
|
1622
|
+
return false;
|
|
1623
|
+
}
|
|
1624
|
+
lastError = error;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
this.requestSuspend(runEpochArg);
|
|
1628
|
+
this.reportInputReleaseError(lastError, runEpochArg);
|
|
1629
|
+
return false;
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
private reportInputReleaseError(errorArg: unknown, runEpochArg: number): void {
|
|
1633
|
+
this.reportError({
|
|
1634
|
+
code: 'input_dispatch_failed',
|
|
1635
|
+
message: `Could not release live browser input: ${getErrorMessage(errorArg)}`,
|
|
1636
|
+
cause: errorArg,
|
|
1637
|
+
}, runEpochArg);
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
private queueViewport(widthArg: number, heightArg: number, runEpochArg: number): void {
|
|
1641
|
+
if (
|
|
1642
|
+
!this.running
|
|
1643
|
+
|| this.stopping
|
|
1644
|
+
|| this.interruptionRequested
|
|
1645
|
+
|| runEpochArg !== this.runEpoch
|
|
1646
|
+
) {
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
const viewport = this.normalizeGeneratedViewport(widthArg, heightArg, runEpochArg);
|
|
1650
|
+
if (!viewport) return;
|
|
1651
|
+
if (this.pendingViewport && viewportEquals(viewport, this.pendingViewport)) {
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
if (this.inFlightViewport && viewportEquals(viewport, this.inFlightViewport)) {
|
|
1655
|
+
this.pendingViewport = undefined;
|
|
1656
|
+
return;
|
|
1657
|
+
}
|
|
1658
|
+
if (!this.inFlightViewport && this.lastRequestedViewport && viewportEquals(viewport, this.lastRequestedViewport)) {
|
|
1659
|
+
this.pendingViewport = undefined;
|
|
1660
|
+
return;
|
|
1661
|
+
}
|
|
1662
|
+
if (
|
|
1663
|
+
!this.inFlightViewport
|
|
1664
|
+
&& this.resizeFence
|
|
1665
|
+
&& !this.resizeFence.negotiated
|
|
1666
|
+
&& viewportEquals(viewport, this.resizeFence.target)
|
|
1667
|
+
) {
|
|
1668
|
+
this.pendingViewport = undefined;
|
|
1669
|
+
return;
|
|
1670
|
+
}
|
|
1671
|
+
if (
|
|
1672
|
+
!this.inFlightViewport
|
|
1673
|
+
&& !this.resizeFence
|
|
1674
|
+
&& !this.lastRequestedViewport
|
|
1675
|
+
&& this.state
|
|
1676
|
+
&& viewportEquals(viewport, this.state.viewport)
|
|
1677
|
+
) {
|
|
1678
|
+
this.pendingViewport = undefined;
|
|
1679
|
+
this.lastRequestedViewport = { ...viewport };
|
|
1680
|
+
this.updateInputAvailability();
|
|
1681
|
+
return;
|
|
1682
|
+
}
|
|
1683
|
+
this.pendingViewport = viewport;
|
|
1684
|
+
this.inputBlocked = true;
|
|
1685
|
+
this.ensureViewportProcessor(runEpochArg);
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
private normalizeGeneratedViewport(
|
|
1689
|
+
widthArg: number,
|
|
1690
|
+
heightArg: number,
|
|
1691
|
+
runEpochArg: number,
|
|
1692
|
+
): ILiveBrowserViewport | undefined {
|
|
1693
|
+
if (
|
|
1694
|
+
!Number.isFinite(widthArg)
|
|
1695
|
+
|| !Number.isFinite(heightArg)
|
|
1696
|
+
|| widthArg <= 0
|
|
1697
|
+
|| heightArg <= 0
|
|
1698
|
+
) {
|
|
1699
|
+
this.reportError({
|
|
1700
|
+
code: 'viewport_update_failed',
|
|
1701
|
+
message: 'Live browser viewport dimensions must be positive finite numbers',
|
|
1702
|
+
}, runEpochArg);
|
|
1703
|
+
return undefined;
|
|
1704
|
+
}
|
|
1705
|
+
const measuredWidth = Math.max(1, Math.floor(widthArg));
|
|
1706
|
+
const measuredHeight = Math.max(1, Math.floor(heightArg));
|
|
1707
|
+
const reductionFactor = Math.min(
|
|
1708
|
+
1,
|
|
1709
|
+
maxViewportWidth / measuredWidth,
|
|
1710
|
+
maxViewportHeight / measuredHeight,
|
|
1711
|
+
);
|
|
1712
|
+
const width = Math.max(1, Math.floor(measuredWidth * reductionFactor));
|
|
1713
|
+
const height = Math.max(1, Math.floor(measuredHeight * reductionFactor));
|
|
1714
|
+
const providedScale = this.getDeviceScaleFactor();
|
|
1715
|
+
if (!Number.isFinite(providedScale) || providedScale <= 0) {
|
|
1716
|
+
this.reportError({
|
|
1717
|
+
code: 'viewport_update_failed',
|
|
1718
|
+
message: 'Live browser device scale factor must be a positive finite number',
|
|
1719
|
+
}, runEpochArg);
|
|
1720
|
+
return undefined;
|
|
1721
|
+
}
|
|
1722
|
+
const desiredScale = Math.min(
|
|
1723
|
+
maxDeviceScaleFactor,
|
|
1724
|
+
Math.max(minDeviceScaleFactor, providedScale),
|
|
1725
|
+
);
|
|
1726
|
+
const physicalAreaIsValid = (scaleArg: number): boolean => (
|
|
1727
|
+
Math.ceil(width * scaleArg) * Math.ceil(height * scaleArg) <= maxFramePixelArea
|
|
1728
|
+
);
|
|
1729
|
+
let deviceScaleFactor = desiredScale;
|
|
1730
|
+
if (!physicalAreaIsValid(deviceScaleFactor)) {
|
|
1731
|
+
let lower = minDeviceScaleFactor;
|
|
1732
|
+
let upper = desiredScale;
|
|
1733
|
+
for (let iteration = 0; iteration < 40; iteration += 1) {
|
|
1734
|
+
const candidate = (lower + upper) / 2;
|
|
1735
|
+
if (physicalAreaIsValid(candidate)) lower = candidate;
|
|
1736
|
+
else upper = candidate;
|
|
1737
|
+
}
|
|
1738
|
+
deviceScaleFactor = lower;
|
|
1739
|
+
}
|
|
1740
|
+
return { width, height, deviceScaleFactor };
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
private ensureViewportProcessor(runEpochArg: number): void {
|
|
1744
|
+
if (this.viewportProcessingPromise || !this.pendingViewport) {
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
const processingPromise = this.processViewportQueue(runEpochArg).finally(() => {
|
|
1748
|
+
if (this.viewportProcessingPromise === processingPromise) {
|
|
1749
|
+
this.viewportProcessingPromise = undefined;
|
|
1750
|
+
if (
|
|
1751
|
+
this.pendingViewport
|
|
1752
|
+
&& this.running
|
|
1753
|
+
&& !this.stopping
|
|
1754
|
+
&& runEpochArg === this.runEpoch
|
|
1755
|
+
) {
|
|
1756
|
+
this.ensureViewportProcessor(runEpochArg);
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
});
|
|
1760
|
+
this.viewportProcessingPromise = processingPromise;
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
private async processViewportQueue(runEpochArg: number): Promise<void> {
|
|
1764
|
+
this.viewportProcessing = true;
|
|
1765
|
+
try {
|
|
1766
|
+
while (
|
|
1767
|
+
this.pendingViewport
|
|
1768
|
+
&& this.running
|
|
1769
|
+
&& !this.stopping
|
|
1770
|
+
&& runEpochArg === this.runEpoch
|
|
1771
|
+
) {
|
|
1772
|
+
const viewport = this.pendingViewport;
|
|
1773
|
+
this.pendingViewport = undefined;
|
|
1774
|
+
this.inFlightViewport = viewport;
|
|
1775
|
+
if (!this.resizeFence && !this.lastRequestedViewport && this.state && viewportEquals(viewport, this.state.viewport)) {
|
|
1776
|
+
this.inFlightViewport = undefined;
|
|
1777
|
+
this.lastRequestedViewport = { ...viewport };
|
|
1778
|
+
continue;
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
await this.scheduleInputReset(false, runEpochArg);
|
|
1782
|
+
if (!this.running || this.stopping || runEpochArg !== this.runEpoch) {
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
const currentRevision = this.state?.viewportRevision ?? 0;
|
|
1786
|
+
const previousMinimumRevision = this.resizeFence?.minimumViewportRevision ?? currentRevision;
|
|
1787
|
+
try {
|
|
1788
|
+
const result = await this.runClientOperation(
|
|
1789
|
+
(optionsArg) => this.client.setViewport(viewport, optionsArg),
|
|
1790
|
+
'viewport update',
|
|
1791
|
+
runEpochArg,
|
|
1792
|
+
);
|
|
1793
|
+
if (!this.running || this.stopping || runEpochArg !== this.runEpoch) {
|
|
1794
|
+
return;
|
|
1795
|
+
}
|
|
1796
|
+
if (result !== undefined) {
|
|
1797
|
+
this.validateViewport(result.viewport, 'viewport result');
|
|
1798
|
+
this.validateProtocolInteger(result.viewportRevision, 'viewport result revision', Math.max(1, currentRevision));
|
|
1799
|
+
this.resizeFence = { target: { ...result.viewport }, minimumViewportRevision: result.viewportRevision, negotiated: true };
|
|
1800
|
+
this.followNegotiatedViewport();
|
|
1801
|
+
} else {
|
|
1802
|
+
this.resizeFence = { target: viewport, minimumViewportRevision: Math.max(currentRevision, previousMinimumRevision) + 1 };
|
|
1803
|
+
}
|
|
1804
|
+
this.lastRequestedViewport = { ...viewport };
|
|
1805
|
+
} catch (error) {
|
|
1806
|
+
if (runEpochArg === this.runEpoch && !(error instanceof LiveBrowserRunAbortedError)) {
|
|
1807
|
+
this.resizeFence = undefined;
|
|
1808
|
+
if (error instanceof LiveBrowserOperationTimeoutError) {
|
|
1809
|
+
this.requestSuspend(runEpochArg);
|
|
1810
|
+
}
|
|
1811
|
+
this.reportError({
|
|
1812
|
+
code: 'viewport_update_failed',
|
|
1813
|
+
message: `Could not update live browser viewport: ${getErrorMessage(error)}`,
|
|
1814
|
+
cause: error,
|
|
1815
|
+
}, runEpochArg);
|
|
1816
|
+
}
|
|
1817
|
+
} finally {
|
|
1818
|
+
if (this.inFlightViewport && viewportEquals(this.inFlightViewport, viewport)) {
|
|
1819
|
+
this.inFlightViewport = undefined;
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
} finally {
|
|
1824
|
+
this.viewportProcessing = false;
|
|
1825
|
+
this.updateInputAvailability();
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
private updateInputAvailability(): void {
|
|
1830
|
+
if (this.resizeFence && !this.viewportProcessing && this.displayedFrame && this.isDisplayedFrameCurrent()
|
|
1831
|
+
&& this.displayedFrame.viewportRevision >= this.resizeFence.minimumViewportRevision
|
|
1832
|
+
&& viewportEquals(this.displayedFrame.viewport, this.resizeFence.target)) {
|
|
1833
|
+
this.resizeFence = undefined;
|
|
1834
|
+
}
|
|
1835
|
+
this.inputBlocked = !(
|
|
1836
|
+
this.running
|
|
1837
|
+
&& this.acceptingFrames
|
|
1838
|
+
&& !this.viewportProcessing
|
|
1839
|
+
&& !this.resizeFence
|
|
1840
|
+
&& this.inputResetCount === 0
|
|
1841
|
+
&& this.isDisplayedFrameCurrent()
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
private followNegotiatedViewport(): void {
|
|
1846
|
+
if (this.resizeFence?.negotiated && this.state && this.state.viewportRevision > this.resizeFence.minimumViewportRevision) {
|
|
1847
|
+
this.resizeFence = { target: { ...this.state.viewport }, minimumViewportRevision: this.state.viewportRevision, negotiated: true };
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
private makeFocusTargetFocusable(): void {
|
|
1852
|
+
this.focusWasAdjusted = false;
|
|
1853
|
+
if (this.focusTarget.tabIndex >= 0) {
|
|
1854
|
+
return;
|
|
1855
|
+
}
|
|
1856
|
+
this.focusHadTabIndex = this.focusTarget.hasAttribute('tabindex');
|
|
1857
|
+
this.focusTabIndexValue = this.focusTarget.getAttribute('tabindex');
|
|
1858
|
+
this.focusTarget.setAttribute('tabindex', '0');
|
|
1859
|
+
this.focusWasAdjusted = true;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
private restoreFocusTarget(): void {
|
|
1863
|
+
if (!this.focusWasAdjusted) {
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
if (this.focusHadTabIndex && this.focusTabIndexValue !== null) {
|
|
1867
|
+
this.focusTarget.setAttribute('tabindex', this.focusTabIndexValue);
|
|
1868
|
+
} else {
|
|
1869
|
+
this.focusTarget.removeAttribute('tabindex');
|
|
1870
|
+
}
|
|
1871
|
+
this.focusWasAdjusted = false;
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
private clearCanvas(): void {
|
|
1875
|
+
if (this.bitmapContext) {
|
|
1876
|
+
this.bitmapContext.transferFromImageBitmap(null);
|
|
1877
|
+
return;
|
|
1878
|
+
}
|
|
1879
|
+
this.canvasContext?.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
/**
|
|
1883
|
+
* Runs work on the next animation frame, or after one nominal frame period
|
|
1884
|
+
* when the document is not visible and animation frames would not fire.
|
|
1885
|
+
*/
|
|
1886
|
+
private scheduleAnimationFrame(callbackArg: () => void): IScheduledFrame {
|
|
1887
|
+
const schedule: IScheduledFrame = {};
|
|
1888
|
+
if (
|
|
1889
|
+
typeof document !== 'undefined'
|
|
1890
|
+
&& document.visibilityState === 'visible'
|
|
1891
|
+
&& typeof globalThis.requestAnimationFrame === 'function'
|
|
1892
|
+
) {
|
|
1893
|
+
schedule.animationFrameId = globalThis.requestAnimationFrame(() => {
|
|
1894
|
+
schedule.animationFrameId = undefined;
|
|
1895
|
+
callbackArg();
|
|
1896
|
+
});
|
|
1897
|
+
} else {
|
|
1898
|
+
schedule.timeoutId = globalThis.setTimeout(() => {
|
|
1899
|
+
schedule.timeoutId = undefined;
|
|
1900
|
+
callbackArg();
|
|
1901
|
+
}, animationFrameFallbackMs);
|
|
1902
|
+
}
|
|
1903
|
+
return schedule;
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
private cancelScheduledFrame(scheduleArg: IScheduledFrame): void {
|
|
1907
|
+
if (scheduleArg.animationFrameId !== undefined) {
|
|
1908
|
+
globalThis.cancelAnimationFrame(scheduleArg.animationFrameId);
|
|
1909
|
+
scheduleArg.animationFrameId = undefined;
|
|
1910
|
+
}
|
|
1911
|
+
if (scheduleArg.timeoutId !== undefined) {
|
|
1912
|
+
globalThis.clearTimeout(scheduleArg.timeoutId);
|
|
1913
|
+
scheduleArg.timeoutId = undefined;
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
private callFrameRendered(frameArg: ILiveBrowserFrame, runEpochArg: number): void {
|
|
1918
|
+
if (!this.onFrameRendered || runEpochArg !== this.runEpoch || !this.running) {
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
try {
|
|
1922
|
+
this.onFrameRendered(frameArg);
|
|
1923
|
+
} catch (error) {
|
|
1924
|
+
this.reportError({
|
|
1925
|
+
code: 'renderer_callback_failed',
|
|
1926
|
+
message: `Live browser frame callback failed: ${getErrorMessage(error)}`,
|
|
1927
|
+
cause: error,
|
|
1928
|
+
frame: frameArg,
|
|
1929
|
+
}, runEpochArg);
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
private reportError(errorArg: ILiveBrowserCanvasError, runEpochArg: number): void {
|
|
1934
|
+
if (!this.onError || runEpochArg !== this.runEpoch) {
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
try {
|
|
1938
|
+
this.onError(errorArg);
|
|
1939
|
+
} catch {
|
|
1940
|
+
// Consumer error reporting must not break renderer cleanup.
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
private requestSuspend(runEpochArg: number): void {
|
|
1945
|
+
if (!this.isRunActive(runEpochArg) || this.interruptionRequested) {
|
|
1946
|
+
return;
|
|
1947
|
+
}
|
|
1948
|
+
this.interruptRun(runEpochArg);
|
|
1949
|
+
void this.suspend();
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
private validateTimeout(timeoutArg: number, nameArg: string): number {
|
|
1953
|
+
if (!Number.isFinite(timeoutArg) || timeoutArg <= 0) {
|
|
1954
|
+
throw new Error(`${nameArg} must be a positive finite number`);
|
|
1955
|
+
}
|
|
1956
|
+
return timeoutArg;
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
private runClientOperation<T>(
|
|
1960
|
+
operationArg: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<T>,
|
|
1961
|
+
operationNameArg: string,
|
|
1962
|
+
runEpochArg: number,
|
|
1963
|
+
): Promise<T> {
|
|
1964
|
+
return this.runAbortableOperation(
|
|
1965
|
+
(signalArg) => operationArg({ signal: signalArg }),
|
|
1966
|
+
this.operationTimeoutMs,
|
|
1967
|
+
operationNameArg,
|
|
1968
|
+
runEpochArg,
|
|
1969
|
+
);
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
private runAbortableOperation<T>(
|
|
1973
|
+
operationArg: (signalArg: AbortSignal) => Promise<T>,
|
|
1974
|
+
timeoutMsArg: number,
|
|
1975
|
+
operationNameArg: string,
|
|
1976
|
+
runEpochArg: number,
|
|
1977
|
+
): Promise<T> {
|
|
1978
|
+
const runController = this.runController;
|
|
1979
|
+
if (
|
|
1980
|
+
!runController
|
|
1981
|
+
|| !this.isRunActive(runEpochArg)
|
|
1982
|
+
|| runController.signal.aborted
|
|
1983
|
+
) {
|
|
1984
|
+
return Promise.reject(new LiveBrowserRunAbortedError(
|
|
1985
|
+
`Cannot start ${operationNameArg} for inactive run ${runEpochArg}`,
|
|
1986
|
+
));
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
const operationController = new AbortController();
|
|
1990
|
+
this.operationControllers.add(operationController);
|
|
1991
|
+
const abortFromRun = () => {
|
|
1992
|
+
const reason = runController.signal.reason instanceof Error
|
|
1993
|
+
? runController.signal.reason
|
|
1994
|
+
: new LiveBrowserRunAbortedError(
|
|
1995
|
+
`Live browser renderer run ${runEpochArg} was interrupted`,
|
|
1996
|
+
);
|
|
1997
|
+
operationController.abort(reason);
|
|
1998
|
+
};
|
|
1999
|
+
runController.signal.addEventListener('abort', abortFromRun, { once: true });
|
|
2000
|
+
const timeout = globalThis.setTimeout(() => {
|
|
2001
|
+
operationController.abort(new LiveBrowserOperationTimeoutError(
|
|
2002
|
+
`${operationNameArg} timed out after ${timeoutMsArg}ms`,
|
|
2003
|
+
));
|
|
2004
|
+
}, timeoutMsArg);
|
|
2005
|
+
|
|
2006
|
+
let abortOperation = () => undefined;
|
|
2007
|
+
const operation = new Promise<T>((resolve, reject) => {
|
|
2008
|
+
let settled = false;
|
|
2009
|
+
const settle = (callbackArg: () => void) => {
|
|
2010
|
+
if (!settled) {
|
|
2011
|
+
settled = true;
|
|
2012
|
+
callbackArg();
|
|
2013
|
+
}
|
|
2014
|
+
};
|
|
2015
|
+
abortOperation = () => {
|
|
2016
|
+
const reason = operationController.signal.reason instanceof Error
|
|
2017
|
+
? operationController.signal.reason
|
|
2018
|
+
: new LiveBrowserRunAbortedError(`${operationNameArg} was aborted`);
|
|
2019
|
+
settle(() => reject(reason));
|
|
2020
|
+
};
|
|
2021
|
+
operationController.signal.addEventListener('abort', abortOperation, { once: true });
|
|
2022
|
+
|
|
2023
|
+
let operationResult: Promise<T>;
|
|
2024
|
+
try {
|
|
2025
|
+
operationResult = operationArg(operationController.signal);
|
|
2026
|
+
} catch (error) {
|
|
2027
|
+
settle(() => reject(error));
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
operationResult.then(
|
|
2031
|
+
(valueArg) => {
|
|
2032
|
+
settle(() => resolve(valueArg));
|
|
2033
|
+
},
|
|
2034
|
+
(errorArg) => {
|
|
2035
|
+
settle(() => reject(errorArg));
|
|
2036
|
+
},
|
|
2037
|
+
);
|
|
2038
|
+
}).finally(() => {
|
|
2039
|
+
globalThis.clearTimeout(timeout);
|
|
2040
|
+
runController.signal.removeEventListener('abort', abortFromRun);
|
|
2041
|
+
operationController.signal.removeEventListener('abort', abortOperation);
|
|
2042
|
+
this.operationControllers.delete(operationController);
|
|
2043
|
+
});
|
|
2044
|
+
return operation;
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
private async decodeFrame(blobArg: Blob, runEpochArg: number): Promise<ImageBitmap> {
|
|
2048
|
+
let decodeOperation: Promise<ImageBitmap>;
|
|
2049
|
+
try {
|
|
2050
|
+
decodeOperation = createImageBitmap(blobArg);
|
|
2051
|
+
} catch (error) {
|
|
2052
|
+
throw error;
|
|
2053
|
+
}
|
|
2054
|
+
let rawDecodeSettlementPromise: Promise<void>;
|
|
2055
|
+
rawDecodeSettlementPromise = decodeOperation.then(
|
|
2056
|
+
() => undefined,
|
|
2057
|
+
() => undefined,
|
|
2058
|
+
).finally(() => {
|
|
2059
|
+
this.rawDecodeSettlementPromises.delete(rawDecodeSettlementPromise);
|
|
2060
|
+
});
|
|
2061
|
+
this.rawDecodeSettlementPromises.add(rawDecodeSettlementPromise);
|
|
2062
|
+
try {
|
|
2063
|
+
return await this.runAbortableOperation(
|
|
2064
|
+
async () => decodeOperation,
|
|
2065
|
+
this.frameDecodeTimeoutMs,
|
|
2066
|
+
'frame decode',
|
|
2067
|
+
runEpochArg,
|
|
2068
|
+
);
|
|
2069
|
+
} catch (error) {
|
|
2070
|
+
if (
|
|
2071
|
+
error instanceof LiveBrowserOperationTimeoutError
|
|
2072
|
+
|| error instanceof LiveBrowserRunAbortedError
|
|
2073
|
+
) {
|
|
2074
|
+
void decodeOperation.then((lateBitmapArg) => lateBitmapArg.close(), () => undefined);
|
|
2075
|
+
throw error;
|
|
2076
|
+
}
|
|
2077
|
+
throw new LiveBrowserFrameIntegrityError(
|
|
2078
|
+
`frame bytes could not be decoded: ${getErrorMessage(error)}`,
|
|
2079
|
+
{ cause: error },
|
|
2080
|
+
);
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
private isRunActive(runEpochArg: number): boolean {
|
|
2085
|
+
return Boolean(
|
|
2086
|
+
this.running
|
|
2087
|
+
&& !this.stopping
|
|
2088
|
+
&& runEpochArg === this.runEpoch
|
|
2089
|
+
&& this.runController
|
|
2090
|
+
&& !this.runController.signal.aborted,
|
|
2091
|
+
);
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
private validateFrame(frameArg: ILiveBrowserFrame): void {
|
|
2095
|
+
if (!frameArg || typeof frameArg !== 'object') {
|
|
2096
|
+
throw new LiveBrowserFrameProtocolError('frame must be an object');
|
|
2097
|
+
}
|
|
2098
|
+
if (
|
|
2099
|
+
typeof frameArg.tabId !== 'string'
|
|
2100
|
+
|| frameArg.tabId.length === 0
|
|
2101
|
+
|| frameArg.tabId.length > 4096
|
|
2102
|
+
) {
|
|
2103
|
+
throw new LiveBrowserFrameProtocolError('frame.tabId must be a non-empty bounded string');
|
|
2104
|
+
}
|
|
2105
|
+
this.validateProtocolInteger(frameArg.sequence, 'frame.sequence', 1);
|
|
2106
|
+
this.validateProtocolInteger(frameArg.generation, 'frame.generation', 1);
|
|
2107
|
+
this.validateProtocolInteger(frameArg.viewportRevision, 'frame.viewportRevision', 1);
|
|
2108
|
+
this.validateViewport(frameArg.viewport, 'frame.viewport');
|
|
2109
|
+
if (frameArg.format !== 'jpeg' && frameArg.format !== 'png') {
|
|
2110
|
+
throw new LiveBrowserFrameProtocolError('frame.format must be jpeg or png');
|
|
2111
|
+
}
|
|
2112
|
+
const expectedMimeType = frameArg.format === 'jpeg' ? 'image/jpeg' : 'image/png';
|
|
2113
|
+
if (frameArg.mimeType !== expectedMimeType) {
|
|
2114
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2115
|
+
`frame.mimeType must be ${expectedMimeType} for ${frameArg.format}`,
|
|
2116
|
+
);
|
|
2117
|
+
}
|
|
2118
|
+
this.validateProtocolInteger(frameArg.width, 'frame.width', 1, maxFrameDimension);
|
|
2119
|
+
this.validateProtocolInteger(frameArg.height, 'frame.height', 1, maxFrameDimension);
|
|
2120
|
+
if (frameArg.width * frameArg.height > maxFramePixelArea) {
|
|
2121
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2122
|
+
`frame pixel area must not exceed ${maxFramePixelArea}`,
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
2125
|
+
if (!(frameArg.data instanceof Uint8Array)) {
|
|
2126
|
+
throw new LiveBrowserFrameProtocolError('frame.data must be a Uint8Array');
|
|
2127
|
+
}
|
|
2128
|
+
if (frameArg.data.byteLength === 0 || frameArg.data.byteLength > maxFrameByteLength) {
|
|
2129
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2130
|
+
`frame.data byte length must be between 1 and ${maxFrameByteLength}`,
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
const metadata = frameArg.metadata;
|
|
2135
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
2136
|
+
throw new LiveBrowserFrameProtocolError('frame.metadata must be an object');
|
|
2137
|
+
}
|
|
2138
|
+
this.validateProtocolNumber(metadata.offsetTop, 'frame.metadata.offsetTop');
|
|
2139
|
+
this.validateProtocolNumber(
|
|
2140
|
+
metadata.pageScaleFactor,
|
|
2141
|
+
'frame.metadata.pageScaleFactor',
|
|
2142
|
+
true,
|
|
2143
|
+
);
|
|
2144
|
+
this.validateProtocolNumber(metadata.deviceWidth, 'frame.metadata.deviceWidth', true);
|
|
2145
|
+
this.validateProtocolNumber(metadata.deviceHeight, 'frame.metadata.deviceHeight', true);
|
|
2146
|
+
if (
|
|
2147
|
+
metadata.deviceWidth > maxFrameDimension
|
|
2148
|
+
|| metadata.deviceHeight > maxFrameDimension
|
|
2149
|
+
|| metadata.deviceWidth * metadata.deviceHeight > maxFramePixelArea
|
|
2150
|
+
) {
|
|
2151
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2152
|
+
`frame metadata dimensions must fit within ${maxFramePixelArea} pixels`,
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
this.validateProtocolNumber(metadata.scrollOffsetX, 'frame.metadata.scrollOffsetX');
|
|
2156
|
+
this.validateProtocolNumber(metadata.scrollOffsetY, 'frame.metadata.scrollOffsetY');
|
|
2157
|
+
if (metadata.timestamp !== undefined) {
|
|
2158
|
+
this.validateProtocolNumber(metadata.timestamp, 'frame.metadata.timestamp');
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
private validateViewport(viewportArg: ILiveBrowserViewport, nameArg: string): void {
|
|
2163
|
+
if (!viewportArg || typeof viewportArg !== 'object') {
|
|
2164
|
+
throw new LiveBrowserFrameProtocolError(`${nameArg} must be an object`);
|
|
2165
|
+
}
|
|
2166
|
+
this.validateProtocolInteger(viewportArg.width, `${nameArg}.width`, 1, maxViewportWidth);
|
|
2167
|
+
this.validateProtocolInteger(viewportArg.height, `${nameArg}.height`, 1, maxViewportHeight);
|
|
2168
|
+
this.validateProtocolNumber(viewportArg.deviceScaleFactor, `${nameArg}.deviceScaleFactor`, true);
|
|
2169
|
+
if (viewportArg.deviceScaleFactor > maxDeviceScaleFactor) {
|
|
2170
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2171
|
+
`${nameArg}.deviceScaleFactor must not exceed ${maxDeviceScaleFactor}`,
|
|
2172
|
+
);
|
|
2173
|
+
}
|
|
2174
|
+
const physicalArea = viewportArg.width
|
|
2175
|
+
* viewportArg.height
|
|
2176
|
+
* viewportArg.deviceScaleFactor
|
|
2177
|
+
* viewportArg.deviceScaleFactor;
|
|
2178
|
+
if (physicalArea > maxFramePixelArea) {
|
|
2179
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2180
|
+
`${nameArg} physical pixel area must not exceed ${maxFramePixelArea}`,
|
|
2181
|
+
);
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
private validateProtocolInteger(
|
|
2186
|
+
valueArg: number,
|
|
2187
|
+
nameArg: string,
|
|
2188
|
+
minimumArg: number,
|
|
2189
|
+
maximumArg = Number.MAX_SAFE_INTEGER,
|
|
2190
|
+
): void {
|
|
2191
|
+
if (
|
|
2192
|
+
!Number.isSafeInteger(valueArg)
|
|
2193
|
+
|| valueArg < minimumArg
|
|
2194
|
+
|| valueArg > maximumArg
|
|
2195
|
+
) {
|
|
2196
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2197
|
+
`${nameArg} must be an integer between ${minimumArg} and ${maximumArg}`,
|
|
2198
|
+
);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
private validateProtocolNumber(
|
|
2203
|
+
valueArg: number,
|
|
2204
|
+
nameArg: string,
|
|
2205
|
+
positiveArg = false,
|
|
2206
|
+
): void {
|
|
2207
|
+
if (!Number.isFinite(valueArg) || (positiveArg && valueArg <= 0)) {
|
|
2208
|
+
throw new LiveBrowserFrameProtocolError(
|
|
2209
|
+
`${nameArg} must be a ${positiveArg ? 'positive ' : ''}finite number`,
|
|
2210
|
+
);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
private releasePointerCapture(pointerIdArg: number): void {
|
|
2215
|
+
this.capturedPointerIds.delete(pointerIdArg);
|
|
2216
|
+
try {
|
|
2217
|
+
if (this.canvas.hasPointerCapture(pointerIdArg)) {
|
|
2218
|
+
this.canvas.releasePointerCapture(pointerIdArg);
|
|
2219
|
+
}
|
|
2220
|
+
} catch {
|
|
2221
|
+
// Pointer capture may already have been released by the browser.
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
private releasePointerCaptures(): void {
|
|
2226
|
+
const pointerIds = [...this.capturedPointerIds];
|
|
2227
|
+
this.capturedPointerIds.clear();
|
|
2228
|
+
for (const pointerId of pointerIds) {
|
|
2229
|
+
this.releasePointerCapture(pointerId);
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
}
|