@hyperframes/studio 0.8.210 → 0.8.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,19 @@ import {
4
4
  type ThumbnailLoadedResult,
5
5
  type ThumbnailValue,
6
6
  } from "./thumbnailScheduler";
7
- import { mediaUrlSourceOptions, reportMediaFetchFailure } from "./mediaFetch";
7
+ import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch";
8
+ import {
9
+ acquireThumbnailSource,
10
+ timedOut,
11
+ type SourceInfo,
12
+ type ThumbnailSource,
13
+ } from "./thumbnailSourcePool";
14
+ import {
15
+ releaseSharedFrames,
16
+ shareFrame,
17
+ takeSharedFrame,
18
+ type HeldFrame,
19
+ } from "./thumbnailFrameCache";
8
20
 
9
21
  export interface VideoThumbnailDecodeRequest {
10
22
  source: string;
@@ -33,75 +45,40 @@ export function videoThumbnailTimestamps(
33
45
  return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)];
34
46
  }
35
47
 
36
- interface SharedFrame {
37
- url: string;
38
- decodedAt: number;
39
- users: number;
40
- }
41
-
42
- interface HeldFrame {
43
- key: string;
44
- frame: SharedFrame;
45
- }
46
-
47
48
  interface FrameWindow {
48
49
  sourceStart: number;
49
50
  lead: number;
50
51
  end: number;
51
52
  }
52
53
 
53
- const sharedFrames = new Map<string, SharedFrame[]>();
54
54
  const sourceInfos = new Map<string, { version: string | undefined; info: SourceInfo }>();
55
55
 
56
56
  const accepts = (window: FrameWindow, time: number, decodedAt: number) =>
57
57
  decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead);
58
58
 
59
- function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined {
60
- const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt));
61
- if (frame) frame.users += 1;
62
- return frame;
63
- }
64
-
65
- function shareFrame(key: string, decodedAt: number, url: string): SharedFrame {
66
- const frames = sharedFrames.get(key) ?? [];
67
- const existing = frames.find((shared) => shared.decodedAt === decodedAt);
68
- if (existing) {
69
- existing.users += 1;
70
- URL.revokeObjectURL(url);
71
- return existing;
72
- }
73
- const frame = { url, decodedAt, users: 1 };
74
- sharedFrames.set(key, [...frames, frame]);
75
- return frame;
76
- }
77
-
78
- function releaseSharedFrames(held: HeldFrame[]): void {
79
- for (const { key, frame } of held.splice(0)) {
80
- if (--frame.users > 0) continue;
81
- const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? [];
82
- if (rest.length > 0) sharedFrames.set(key, rest);
83
- else sharedFrames.delete(key);
84
- URL.revokeObjectURL(frame.url);
85
- }
86
- }
59
+ class ClipResultError extends Error {}
87
60
 
88
61
  async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {
89
62
  if (canvas instanceof HTMLCanvasElement) {
90
63
  return new Promise((resolve, reject) => {
91
64
  canvas.toBlob(
92
- (blob) => (blob ? resolve(blob) : reject(new Error("Video thumbnail encode failed"))),
65
+ (blob) =>
66
+ blob ? resolve(blob) : reject(new ClipResultError("Video thumbnail encode failed")),
93
67
  "image/jpeg",
94
68
  0.72,
95
69
  );
96
70
  });
97
71
  }
98
- return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 });
72
+ return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 }).catch(() => {
73
+ throw new ClipResultError("Video thumbnail encode failed");
74
+ });
99
75
  }
100
76
 
101
77
  interface DecodedResources {
102
78
  urls: (string | undefined)[];
103
79
  held: HeldFrame[];
104
80
  canvases: Set<HTMLCanvasElement | OffscreenCanvas>;
81
+ releasedFrameBytes: number;
105
82
  }
106
83
 
107
84
  interface ThumbnailCanvasSink {
@@ -110,18 +87,19 @@ interface ThumbnailCanvasSink {
110
87
  ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
111
88
  }
112
89
 
113
- interface SourceInfo {
114
- aspect: number;
115
- metadataDuration: number | null;
116
- }
90
+ const timeoutWithoutProgress = () => new MediaFetchError("thumbnail", null, "timeout");
117
91
 
92
+ const isAbortError = (error: unknown) =>
93
+ (error as { name?: unknown } | null)?.name === "AbortError";
94
+ const isSourceFailure = (error: unknown) =>
95
+ !isAbortError(error) && !(error instanceof ClipResultError);
118
96
  function throwIfAborted(signal: AbortSignal): void {
119
97
  if (signal.aborted) throw new DOMException("Aborted", "AbortError");
120
98
  }
121
99
 
122
100
  function releaseDecodedResources(resources: DecodedResources): void {
123
101
  resources.urls.length = 0;
124
- releaseSharedFrames(resources.held);
102
+ releaseSharedFrames(resources.held, resources.releasedFrameBytes);
125
103
  for (const canvas of resources.canvases) {
126
104
  canvas.width = 0;
127
105
  canvas.height = 0;
@@ -182,7 +160,7 @@ function takeDecodedFrames(
182
160
  const missing: number[] = [];
183
161
  timestamps.forEach((time, slot) => {
184
162
  const key = keyOf(time);
185
- const frame = takeSharedFrame(key, time, window);
163
+ const frame = takeSharedFrame(key, (decodedAt) => accepts(window, time, decodedAt));
186
164
  if (!frame) return void missing.push(slot);
187
165
  resources.urls[slot] = frame.url;
188
166
  resources.held.push({ key, frame });
@@ -196,6 +174,7 @@ async function decodeFrames(
196
174
  slots: { slot: number; key: string; decodedAt?: number }[],
197
175
  signal: AbortSignal,
198
176
  resources: DecodedResources,
177
+ progress: ThumbnailSource["progress"],
199
178
  ): Promise<void> {
200
179
  let next = 0;
201
180
  for await (const wrapped of sink.canvasesAtTimestamps(times)) {
@@ -205,7 +184,8 @@ async function decodeFrames(
205
184
  resources.canvases.add(wrapped.canvas);
206
185
  const blob = await canvasToBlob(wrapped.canvas);
207
186
  throwIfAborted(signal);
208
- const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob));
187
+ const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob), blob.size);
188
+ progress.framesFromAnyJob += 1;
209
189
  resources.urls[target.slot] = frame.url;
210
190
  resources.held.push({ key: target.key, frame });
211
191
  }
@@ -220,7 +200,7 @@ function loadedResult(
220
200
  ): ThumbnailLoadedResult {
221
201
  const decoded = resources.urls.filter((url): url is string => url !== undefined);
222
202
  const firstUrl = decoded[0];
223
- if (!firstUrl) throw new Error("Video source returned no thumbnail frames");
203
+ if (!firstUrl) throw new ClipResultError("Video source returned no thumbnail frames");
224
204
  let nearest = firstUrl;
225
205
  const urls = Array.from(
226
206
  { length: frameCount },
@@ -247,7 +227,12 @@ export async function decodeVideoThumbnail(
247
227
  const fit = request.fit ?? "cover";
248
228
  const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion });
249
229
  const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`;
250
- const resources: DecodedResources = { urls: [], held: [], canvases: new Set() };
230
+ const resources: DecodedResources = {
231
+ urls: [],
232
+ held: [],
233
+ canvases: new Set(),
234
+ releasedFrameBytes: budgets.sharedFrameCacheBytes,
235
+ };
251
236
  const entry = sourceInfos.get(request.source);
252
237
  const known = entry?.version === request.contentVersion ? entry?.info : undefined;
253
238
  try {
@@ -273,28 +258,29 @@ async function decodeMissingFrames(
273
258
  keyOf: (time: number) => string,
274
259
  resources: DecodedResources,
275
260
  ): Promise<ThumbnailLoadedResult> {
276
- const mediabunny = await import("mediabunny");
277
- throwIfAborted(signal);
278
-
279
- const input = new mediabunny.Input({
280
- source: new mediabunny.UrlSource(request.source, mediaUrlSourceOptions("thumbnail")),
281
- formats: mediabunny.ALL_FORMATS,
282
- });
261
+ let source: ThumbnailSource;
262
+ try {
263
+ source = await acquireThumbnailSource(
264
+ request.source,
265
+ request.contentVersion,
266
+ signal,
267
+ budgets.thumbnailSourceIdleMs,
268
+ );
269
+ } catch (error) {
270
+ if (timedOut(signal)) reportMediaFetchFailure(timeoutWithoutProgress());
271
+ throw error;
272
+ }
273
+ const framesBefore = source.progress.framesFromAnyJob;
274
+ const releaseOnAbort = () => {
275
+ const madeProgress = source.waitedForOpen || source.progress.framesFromAnyJob > framesBefore;
276
+ const failed = timedOut(signal) && !madeProgress;
277
+ if (failed) reportMediaFetchFailure(timeoutWithoutProgress());
278
+ source.release(failed);
279
+ };
280
+ signal.addEventListener("abort", releaseOnAbort, { once: true });
283
281
  try {
284
- const track = await input.getPrimaryVideoTrack();
285
- throwIfAborted(signal);
286
- if (!track) throw new Error("Video source has no decodable video track");
287
- const [displayWidth, displayHeight] = await Promise.all([
288
- track.getDisplayWidth(),
289
- track.getDisplayHeight(),
290
- ]);
291
- throwIfAborted(signal);
292
- if (!(displayWidth > 0 && displayHeight > 0)) {
293
- throw new Error("Video source has invalid dimensions");
294
- }
295
- const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
296
282
  throwIfAborted(signal);
297
- const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration };
283
+ const { mediabunny, track, info } = source;
298
284
  sourceInfos.set(request.source, { version: request.contentVersion, info });
299
285
  const strip = stripTimes(request, info, budgets);
300
286
  const { timestamps, window } = strip;
@@ -321,10 +307,16 @@ async function decodeMissingFrames(
321
307
  fit: request.fit ?? "cover",
322
308
  poolSize: 1,
323
309
  });
324
- await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources);
310
+ const times = decodeTimesAtNearbyKeyframes();
311
+ await decodeFrames(sink, times, slots, signal, resources, source.progress);
325
312
  }
326
- return loadedResult(resources, info.aspect, budgets, timestamps.length);
313
+ const result = loadedResult(resources, info.aspect, budgets, timestamps.length);
314
+ source.release(false);
315
+ return result;
316
+ } catch (error) {
317
+ source.release(isSourceFailure(error));
318
+ throw error;
327
319
  } finally {
328
- input.dispose();
320
+ signal.removeEventListener("abort", releaseOnAbort);
329
321
  }
330
322
  }
@@ -15,6 +15,7 @@ describe("timeline viewport budgets", () => {
15
15
  maxMountedClipRootsPerRow: 128,
16
16
  maxMountedTimelineDescendants: 5_000,
17
17
  thumbnailCacheBytes: 64 * 1024 * 1024,
18
+ sharedFrameCacheBytes: 32 * 1024 * 1024,
18
19
  waveformCacheBytes: 16 * 1024 * 1024,
19
20
  interactionP95Ms: 50,
20
21
  constrainedInteractionP95Ms: 75,
@@ -16,6 +16,8 @@ export interface TimelineViewportBudgets {
16
16
  concurrentServerPages: number;
17
17
  thumbnailLoadTimeoutMs: number;
18
18
  thumbnailCacheBytes: number;
19
+ sharedFrameCacheBytes: number;
20
+ thumbnailSourceIdleMs: number;
19
21
  thumbnailCacheEntries: number;
20
22
  thumbnailCacheEntriesPerProject: number;
21
23
  metadataRegistryEntries: number;
@@ -76,6 +78,8 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
76
78
  concurrentServerPages: 1,
77
79
  thumbnailLoadTimeoutMs: 30_000,
78
80
  thumbnailCacheBytes: 64 * MEBIBYTE,
81
+ sharedFrameCacheBytes: 32 * MEBIBYTE,
82
+ thumbnailSourceIdleMs: 10_000,
79
83
  thumbnailCacheEntries: 256,
80
84
  thumbnailCacheEntriesPerProject: 96,
81
85
  metadataRegistryEntries: 512,
@@ -18,6 +18,10 @@ export interface PlaybackHealth {
18
18
  choppy: boolean;
19
19
  fps: number;
20
20
  worstFrameMs: number;
21
+ skippedShare?: number;
22
+ waitingShare?: number;
23
+ gapP50Ms?: number;
24
+ gapP95Ms?: number;
21
25
  mode: "play";
22
26
  }
23
27
 
@@ -104,8 +108,11 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null {
104
108
  };
105
109
  }
106
110
 
111
+ const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined);
112
+
107
113
  export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
108
114
  if (typeof value !== "object" || value === null) return null;
115
+ const extra = value as Record<string, unknown>;
109
116
  const { droppedShare, seconds, choppy, fps, worstFrameMs } = value as Record<string, unknown>;
110
117
  const numbers = [droppedShare, seconds, fps, worstFrameMs].every((n) => typeof n === "number");
111
118
  if (!numbers || typeof choppy !== "boolean") return null;
@@ -115,6 +122,10 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
115
122
  choppy,
116
123
  fps: fps as number,
117
124
  worstFrameMs: worstFrameMs as number,
125
+ skippedShare: optionalNumber(extra.skippedShare),
126
+ waitingShare: optionalNumber(extra.waitingShare),
127
+ gapP50Ms: optionalNumber(extra.gapP50Ms),
128
+ gapP95Ms: optionalNumber(extra.gapP95Ms),
118
129
  mode: "play",
119
130
  };
120
131
  }