@hyperframes/studio 0.8.209 → 0.8.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{hyperframes-player-CMH8OeX2.js → hyperframes-player-CzrqgMUX.js} +1 -1
- package/dist/assets/{index-DjHzuJGR.js → index-B3SPHPi1.js} +1 -1
- package/dist/assets/{index-CYbvOVp-.css → index-RUPGNyct.css} +1 -1
- package/dist/assets/{index-DQam8zzE.js → index-k4nVfS2D.js} +123 -123
- package/dist/{chunk-HPIJDMWG.js → chunk-HD6NXWGI.js} +6 -5
- package/dist/index.d.ts +4 -0
- package/dist/index.html +2 -2
- package/dist/index.js +485 -241
- package/dist/player.js +1 -1
- package/dist/styles.css +1 -1
- package/package.json +8 -8
- package/src/components/DesktopGuideCard.tsx +1 -1
- package/src/components/nle/NLEPreview.test.ts +9 -0
- package/src/components/nle/NLEPreview.tsx +1 -1
- package/src/player/components/Player.test.ts +10 -0
- package/src/player/components/Player.tsx +0 -12
- package/src/player/components/VideoThumbnail.tsx +6 -0
- package/src/player/hooks/previewMessageRouter.test.ts +4 -0
- package/src/player/lib/mediaFetch.test.ts +466 -0
- package/src/player/lib/mediaFetch.ts +133 -26
- package/src/player/lib/thumbnailFrameCache.ts +69 -0
- package/src/player/lib/thumbnailPlaybackHold.test.ts +86 -0
- package/src/player/lib/thumbnailPlaybackHold.ts +15 -0
- package/src/player/lib/thumbnailScheduler.test.ts +49 -0
- package/src/player/lib/thumbnailScheduler.ts +22 -6
- package/src/player/lib/thumbnailSourcePool.ts +163 -0
- package/src/player/lib/thumbnailVideoDecoder.test.ts +492 -8
- package/src/player/lib/thumbnailVideoDecoder.ts +67 -75
- package/src/player/lib/timelineViewportBudgets.test.ts +1 -0
- package/src/player/lib/timelineViewportBudgets.ts +4 -0
- package/src/player/store/previewQualitySlice.ts +11 -0
- package/src/styles/hex-baseline.json +0 -1
|
@@ -4,7 +4,19 @@ import {
|
|
|
4
4
|
type ThumbnailLoadedResult,
|
|
5
5
|
type ThumbnailValue,
|
|
6
6
|
} from "./thumbnailScheduler";
|
|
7
|
-
import {
|
|
7
|
+
import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch";
|
|
8
|
+
import {
|
|
9
|
+
acquireThumbnailSource,
|
|
10
|
+
timedOut,
|
|
11
|
+
type SourceInfo,
|
|
12
|
+
type ThumbnailSource,
|
|
13
|
+
} from "./thumbnailSourcePool";
|
|
14
|
+
import {
|
|
15
|
+
releaseSharedFrames,
|
|
16
|
+
shareFrame,
|
|
17
|
+
takeSharedFrame,
|
|
18
|
+
type HeldFrame,
|
|
19
|
+
} from "./thumbnailFrameCache";
|
|
8
20
|
|
|
9
21
|
export interface VideoThumbnailDecodeRequest {
|
|
10
22
|
source: string;
|
|
@@ -33,75 +45,40 @@ export function videoThumbnailTimestamps(
|
|
|
33
45
|
return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)];
|
|
34
46
|
}
|
|
35
47
|
|
|
36
|
-
interface SharedFrame {
|
|
37
|
-
url: string;
|
|
38
|
-
decodedAt: number;
|
|
39
|
-
users: number;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
interface HeldFrame {
|
|
43
|
-
key: string;
|
|
44
|
-
frame: SharedFrame;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
48
|
interface FrameWindow {
|
|
48
49
|
sourceStart: number;
|
|
49
50
|
lead: number;
|
|
50
51
|
end: number;
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
const sharedFrames = new Map<string, SharedFrame[]>();
|
|
54
54
|
const sourceInfos = new Map<string, { version: string | undefined; info: SourceInfo }>();
|
|
55
55
|
|
|
56
56
|
const accepts = (window: FrameWindow, time: number, decodedAt: number) =>
|
|
57
57
|
decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead);
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt));
|
|
61
|
-
if (frame) frame.users += 1;
|
|
62
|
-
return frame;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function shareFrame(key: string, decodedAt: number, url: string): SharedFrame {
|
|
66
|
-
const frames = sharedFrames.get(key) ?? [];
|
|
67
|
-
const existing = frames.find((shared) => shared.decodedAt === decodedAt);
|
|
68
|
-
if (existing) {
|
|
69
|
-
existing.users += 1;
|
|
70
|
-
URL.revokeObjectURL(url);
|
|
71
|
-
return existing;
|
|
72
|
-
}
|
|
73
|
-
const frame = { url, decodedAt, users: 1 };
|
|
74
|
-
sharedFrames.set(key, [...frames, frame]);
|
|
75
|
-
return frame;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function releaseSharedFrames(held: HeldFrame[]): void {
|
|
79
|
-
for (const { key, frame } of held.splice(0)) {
|
|
80
|
-
if (--frame.users > 0) continue;
|
|
81
|
-
const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? [];
|
|
82
|
-
if (rest.length > 0) sharedFrames.set(key, rest);
|
|
83
|
-
else sharedFrames.delete(key);
|
|
84
|
-
URL.revokeObjectURL(frame.url);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
59
|
+
class ClipResultError extends Error {}
|
|
87
60
|
|
|
88
61
|
async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {
|
|
89
62
|
if (canvas instanceof HTMLCanvasElement) {
|
|
90
63
|
return new Promise((resolve, reject) => {
|
|
91
64
|
canvas.toBlob(
|
|
92
|
-
(blob) =>
|
|
65
|
+
(blob) =>
|
|
66
|
+
blob ? resolve(blob) : reject(new ClipResultError("Video thumbnail encode failed")),
|
|
93
67
|
"image/jpeg",
|
|
94
68
|
0.72,
|
|
95
69
|
);
|
|
96
70
|
});
|
|
97
71
|
}
|
|
98
|
-
return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 })
|
|
72
|
+
return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 }).catch(() => {
|
|
73
|
+
throw new ClipResultError("Video thumbnail encode failed");
|
|
74
|
+
});
|
|
99
75
|
}
|
|
100
76
|
|
|
101
77
|
interface DecodedResources {
|
|
102
78
|
urls: (string | undefined)[];
|
|
103
79
|
held: HeldFrame[];
|
|
104
80
|
canvases: Set<HTMLCanvasElement | OffscreenCanvas>;
|
|
81
|
+
releasedFrameBytes: number;
|
|
105
82
|
}
|
|
106
83
|
|
|
107
84
|
interface ThumbnailCanvasSink {
|
|
@@ -110,18 +87,19 @@ interface ThumbnailCanvasSink {
|
|
|
110
87
|
): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
|
|
111
88
|
}
|
|
112
89
|
|
|
113
|
-
|
|
114
|
-
aspect: number;
|
|
115
|
-
metadataDuration: number | null;
|
|
116
|
-
}
|
|
90
|
+
const timeoutWithoutProgress = () => new MediaFetchError("thumbnail", null, "timeout");
|
|
117
91
|
|
|
92
|
+
const isAbortError = (error: unknown) =>
|
|
93
|
+
(error as { name?: unknown } | null)?.name === "AbortError";
|
|
94
|
+
const isSourceFailure = (error: unknown) =>
|
|
95
|
+
!isAbortError(error) && !(error instanceof ClipResultError);
|
|
118
96
|
function throwIfAborted(signal: AbortSignal): void {
|
|
119
97
|
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
|
120
98
|
}
|
|
121
99
|
|
|
122
100
|
function releaseDecodedResources(resources: DecodedResources): void {
|
|
123
101
|
resources.urls.length = 0;
|
|
124
|
-
releaseSharedFrames(resources.held);
|
|
102
|
+
releaseSharedFrames(resources.held, resources.releasedFrameBytes);
|
|
125
103
|
for (const canvas of resources.canvases) {
|
|
126
104
|
canvas.width = 0;
|
|
127
105
|
canvas.height = 0;
|
|
@@ -182,7 +160,7 @@ function takeDecodedFrames(
|
|
|
182
160
|
const missing: number[] = [];
|
|
183
161
|
timestamps.forEach((time, slot) => {
|
|
184
162
|
const key = keyOf(time);
|
|
185
|
-
const frame = takeSharedFrame(key, time,
|
|
163
|
+
const frame = takeSharedFrame(key, (decodedAt) => accepts(window, time, decodedAt));
|
|
186
164
|
if (!frame) return void missing.push(slot);
|
|
187
165
|
resources.urls[slot] = frame.url;
|
|
188
166
|
resources.held.push({ key, frame });
|
|
@@ -196,6 +174,7 @@ async function decodeFrames(
|
|
|
196
174
|
slots: { slot: number; key: string; decodedAt?: number }[],
|
|
197
175
|
signal: AbortSignal,
|
|
198
176
|
resources: DecodedResources,
|
|
177
|
+
progress: ThumbnailSource["progress"],
|
|
199
178
|
): Promise<void> {
|
|
200
179
|
let next = 0;
|
|
201
180
|
for await (const wrapped of sink.canvasesAtTimestamps(times)) {
|
|
@@ -205,7 +184,8 @@ async function decodeFrames(
|
|
|
205
184
|
resources.canvases.add(wrapped.canvas);
|
|
206
185
|
const blob = await canvasToBlob(wrapped.canvas);
|
|
207
186
|
throwIfAborted(signal);
|
|
208
|
-
const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob));
|
|
187
|
+
const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob), blob.size);
|
|
188
|
+
progress.framesFromAnyJob += 1;
|
|
209
189
|
resources.urls[target.slot] = frame.url;
|
|
210
190
|
resources.held.push({ key: target.key, frame });
|
|
211
191
|
}
|
|
@@ -220,7 +200,7 @@ function loadedResult(
|
|
|
220
200
|
): ThumbnailLoadedResult {
|
|
221
201
|
const decoded = resources.urls.filter((url): url is string => url !== undefined);
|
|
222
202
|
const firstUrl = decoded[0];
|
|
223
|
-
if (!firstUrl) throw new
|
|
203
|
+
if (!firstUrl) throw new ClipResultError("Video source returned no thumbnail frames");
|
|
224
204
|
let nearest = firstUrl;
|
|
225
205
|
const urls = Array.from(
|
|
226
206
|
{ length: frameCount },
|
|
@@ -247,7 +227,12 @@ export async function decodeVideoThumbnail(
|
|
|
247
227
|
const fit = request.fit ?? "cover";
|
|
248
228
|
const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion });
|
|
249
229
|
const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`;
|
|
250
|
-
const resources: DecodedResources = {
|
|
230
|
+
const resources: DecodedResources = {
|
|
231
|
+
urls: [],
|
|
232
|
+
held: [],
|
|
233
|
+
canvases: new Set(),
|
|
234
|
+
releasedFrameBytes: budgets.sharedFrameCacheBytes,
|
|
235
|
+
};
|
|
251
236
|
const entry = sourceInfos.get(request.source);
|
|
252
237
|
const known = entry?.version === request.contentVersion ? entry?.info : undefined;
|
|
253
238
|
try {
|
|
@@ -273,28 +258,29 @@ async function decodeMissingFrames(
|
|
|
273
258
|
keyOf: (time: number) => string,
|
|
274
259
|
resources: DecodedResources,
|
|
275
260
|
): Promise<ThumbnailLoadedResult> {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
261
|
+
let source: ThumbnailSource;
|
|
262
|
+
try {
|
|
263
|
+
source = await acquireThumbnailSource(
|
|
264
|
+
request.source,
|
|
265
|
+
request.contentVersion,
|
|
266
|
+
signal,
|
|
267
|
+
budgets.thumbnailSourceIdleMs,
|
|
268
|
+
);
|
|
269
|
+
} catch (error) {
|
|
270
|
+
if (timedOut(signal)) reportMediaFetchFailure(timeoutWithoutProgress());
|
|
271
|
+
throw error;
|
|
272
|
+
}
|
|
273
|
+
const framesBefore = source.progress.framesFromAnyJob;
|
|
274
|
+
const releaseOnAbort = () => {
|
|
275
|
+
const madeProgress = source.waitedForOpen || source.progress.framesFromAnyJob > framesBefore;
|
|
276
|
+
const failed = timedOut(signal) && !madeProgress;
|
|
277
|
+
if (failed) reportMediaFetchFailure(timeoutWithoutProgress());
|
|
278
|
+
source.release(failed);
|
|
279
|
+
};
|
|
280
|
+
signal.addEventListener("abort", releaseOnAbort, { once: true });
|
|
283
281
|
try {
|
|
284
|
-
const track = await input.getPrimaryVideoTrack();
|
|
285
|
-
throwIfAborted(signal);
|
|
286
|
-
if (!track) throw new Error("Video source has no decodable video track");
|
|
287
|
-
const [displayWidth, displayHeight] = await Promise.all([
|
|
288
|
-
track.getDisplayWidth(),
|
|
289
|
-
track.getDisplayHeight(),
|
|
290
|
-
]);
|
|
291
|
-
throwIfAborted(signal);
|
|
292
|
-
if (!(displayWidth > 0 && displayHeight > 0)) {
|
|
293
|
-
throw new Error("Video source has invalid dimensions");
|
|
294
|
-
}
|
|
295
|
-
const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
|
|
296
282
|
throwIfAborted(signal);
|
|
297
|
-
const
|
|
283
|
+
const { mediabunny, track, info } = source;
|
|
298
284
|
sourceInfos.set(request.source, { version: request.contentVersion, info });
|
|
299
285
|
const strip = stripTimes(request, info, budgets);
|
|
300
286
|
const { timestamps, window } = strip;
|
|
@@ -321,10 +307,16 @@ async function decodeMissingFrames(
|
|
|
321
307
|
fit: request.fit ?? "cover",
|
|
322
308
|
poolSize: 1,
|
|
323
309
|
});
|
|
324
|
-
|
|
310
|
+
const times = decodeTimesAtNearbyKeyframes();
|
|
311
|
+
await decodeFrames(sink, times, slots, signal, resources, source.progress);
|
|
325
312
|
}
|
|
326
|
-
|
|
313
|
+
const result = loadedResult(resources, info.aspect, budgets, timestamps.length);
|
|
314
|
+
source.release(false);
|
|
315
|
+
return result;
|
|
316
|
+
} catch (error) {
|
|
317
|
+
source.release(isSourceFailure(error));
|
|
318
|
+
throw error;
|
|
327
319
|
} finally {
|
|
328
|
-
|
|
320
|
+
signal.removeEventListener("abort", releaseOnAbort);
|
|
329
321
|
}
|
|
330
322
|
}
|
|
@@ -15,6 +15,7 @@ describe("timeline viewport budgets", () => {
|
|
|
15
15
|
maxMountedClipRootsPerRow: 128,
|
|
16
16
|
maxMountedTimelineDescendants: 5_000,
|
|
17
17
|
thumbnailCacheBytes: 64 * 1024 * 1024,
|
|
18
|
+
sharedFrameCacheBytes: 32 * 1024 * 1024,
|
|
18
19
|
waveformCacheBytes: 16 * 1024 * 1024,
|
|
19
20
|
interactionP95Ms: 50,
|
|
20
21
|
constrainedInteractionP95Ms: 75,
|
|
@@ -16,6 +16,8 @@ export interface TimelineViewportBudgets {
|
|
|
16
16
|
concurrentServerPages: number;
|
|
17
17
|
thumbnailLoadTimeoutMs: number;
|
|
18
18
|
thumbnailCacheBytes: number;
|
|
19
|
+
sharedFrameCacheBytes: number;
|
|
20
|
+
thumbnailSourceIdleMs: number;
|
|
19
21
|
thumbnailCacheEntries: number;
|
|
20
22
|
thumbnailCacheEntriesPerProject: number;
|
|
21
23
|
metadataRegistryEntries: number;
|
|
@@ -76,6 +78,8 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
|
|
|
76
78
|
concurrentServerPages: 1,
|
|
77
79
|
thumbnailLoadTimeoutMs: 30_000,
|
|
78
80
|
thumbnailCacheBytes: 64 * MEBIBYTE,
|
|
81
|
+
sharedFrameCacheBytes: 32 * MEBIBYTE,
|
|
82
|
+
thumbnailSourceIdleMs: 10_000,
|
|
79
83
|
thumbnailCacheEntries: 256,
|
|
80
84
|
thumbnailCacheEntriesPerProject: 96,
|
|
81
85
|
metadataRegistryEntries: 512,
|
|
@@ -18,6 +18,10 @@ export interface PlaybackHealth {
|
|
|
18
18
|
choppy: boolean;
|
|
19
19
|
fps: number;
|
|
20
20
|
worstFrameMs: number;
|
|
21
|
+
skippedShare?: number;
|
|
22
|
+
waitingShare?: number;
|
|
23
|
+
gapP50Ms?: number;
|
|
24
|
+
gapP95Ms?: number;
|
|
21
25
|
mode: "play";
|
|
22
26
|
}
|
|
23
27
|
|
|
@@ -104,8 +108,11 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null {
|
|
|
104
108
|
};
|
|
105
109
|
}
|
|
106
110
|
|
|
111
|
+
const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined);
|
|
112
|
+
|
|
107
113
|
export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
|
|
108
114
|
if (typeof value !== "object" || value === null) return null;
|
|
115
|
+
const extra = value as Record<string, unknown>;
|
|
109
116
|
const { droppedShare, seconds, choppy, fps, worstFrameMs } = value as Record<string, unknown>;
|
|
110
117
|
const numbers = [droppedShare, seconds, fps, worstFrameMs].every((n) => typeof n === "number");
|
|
111
118
|
if (!numbers || typeof choppy !== "boolean") return null;
|
|
@@ -115,6 +122,10 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
|
|
|
115
122
|
choppy,
|
|
116
123
|
fps: fps as number,
|
|
117
124
|
worstFrameMs: worstFrameMs as number,
|
|
125
|
+
skippedShare: optionalNumber(extra.skippedShare),
|
|
126
|
+
waitingShare: optionalNumber(extra.waitingShare),
|
|
127
|
+
gapP50Ms: optionalNumber(extra.gapP50Ms),
|
|
128
|
+
gapP95Ms: optionalNumber(extra.gapP95Ms),
|
|
118
129
|
mode: "play",
|
|
119
130
|
};
|
|
120
131
|
}
|
|
@@ -42,7 +42,6 @@
|
|
|
42
42
|
"packages/studio/src/player/components/AudioWaveform.tsx": 2,
|
|
43
43
|
"packages/studio/src/player/components/BeatStrip.tsx": 6,
|
|
44
44
|
"packages/studio/src/player/components/CompositionThumbnail.tsx": 2,
|
|
45
|
-
"packages/studio/src/player/components/Player.tsx": 2,
|
|
46
45
|
"packages/studio/src/player/components/PlayerControls.tsx": 1,
|
|
47
46
|
"packages/studio/src/player/components/SpeedMenu.tsx": 2,
|
|
48
47
|
"packages/studio/src/styles/components.css": 48,
|