@banou/media-player 0.8.21 → 0.10.0

package/src/lib/engine/thumbnails.ts

@@ -29,6 +29,17 @@ const KEYFRAME_TIMEOUT = 10_000
 export type ThumbnailGenerator = {
   /** Report which byte ranges are readable. Called with no argument when the whole file is. */
   update: (ranges?: [number, number][]) => void
+  /**
+   * Where the viewer is pointing, so that preview is decoded next. `undefined` when they stop.
+   *
+   * Only the slot covering `time` jumps the queue, and only for as long as it is still waiting, so
+   * this moves one preview forward rather than re-ordering the run. Everything behind it keeps the
+   * order it was claimed in and carries on the moment the jumped slot is done.
+   *
+   * It cannot interrupt a decode that has already started, so the wait is the tail of the one in
+   * flight and not the whole backlog.
+   */
+  prioritize: (time: number | undefined) => void
   destroy: () => void
 }
 
@@ -68,7 +79,35 @@ export const createThumbnailGenerator = async (options: ThumbnailGeneratorOption
 
     let thumbnails: ThumbnailImage[] = []
     let destroyed = false
-    let queue = Promise.resolve()
+
+    /*
+     * Slots claimed for decoding but not yet started, in the order they were claimed.
+     *
+     * A promise chain used to be this queue, which fixed the running order at the moment each slot
+     * was chained on and left nothing a hover could reach: the preview under the pointer waited
+     * behind every slot already queued, which on a long file is the rest of the run. The order is
+     * the same, it is just held somewhere a pick can look into.
+     */
+    const pending: Slot[] = []
+    let running = false
+    /** Where the pointer is on the seekbar, or undefined when it is off it. */
+    let priorityTime: number | undefined
+
+    /*
+     * The slot to decode next: the one under the pointer when it is still waiting, else the oldest claim.
+     *
+     * Requiring the slot to COVER the time is what keeps this a single jump rather than a re-sort
+     * around the cursor. Once that slot is decoded nothing covers the pointer any more, so the very
+     * next pick is the oldest claim again and the sequential walk carries on where it left off.
+     */
+    const nextIndex = () => {
+      const at = priorityTime
+      if (at !== undefined) {
+        const hit = pending.findIndex(({ timestamp, endTime }) => timestamp <= at && at < endTime)
+        if (hit >= 0) return hit
+      }
+      return 0
+    }
 
     // the slider assumes a gapless storyboard, so gaps get sentinels the UI hides
     const emit = () => {
@@ -88,29 +127,44 @@ export const createThumbnailGenerator = async (options: ThumbnailGeneratorOption
       onThumbnails(display)
     }
 
-    const generate = (slot: Slot) => {
-      slot.done = true
-      queue = queue
-        .then(async () => {
-          if (destroyed) return
-          const png = await Promise.race([
-            remuxer.readKeyframe(slot.timestamp),
-            new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), KEYFRAME_TIMEOUT)),
-          ])
-          const bitmap = await createImageBitmap(new Blob([png], { type: 'image/png' }))
-          const canvas = new OffscreenCanvas(width, Math.max(1, Math.round(bitmap.height * (width / bitmap.width))))
-          canvas.getContext('2d')!.drawImage(bitmap, 0, 0, canvas.width, canvas.height)
-          bitmap.close()
-          const blob = await canvas.convertToBlob({ type: 'image/webp', quality: 0.7 })
-          if (destroyed) return
-          thumbnails = [...thumbnails, { url: URL.createObjectURL(blob), startTime: slot.timestamp, endTime: slot.endTime }]
-            .sort((a, b) => a.startTime - b.startTime)
-          emit()
-        })
+    const decode = async (slot: Slot) => {
+      if (destroyed) return
+      const png = await Promise.race([
+        remuxer.readKeyframe(slot.timestamp),
+        new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), KEYFRAME_TIMEOUT)),
+      ])
+      const bitmap = await createImageBitmap(new Blob([png], { type: 'image/png' }))
+      const canvas = new OffscreenCanvas(width, Math.max(1, Math.round(bitmap.height * (width / bitmap.width))))
+      canvas.getContext('2d')!.drawImage(bitmap, 0, 0, canvas.width, canvas.height)
+      bitmap.close()
+      const blob = await canvas.convertToBlob({ type: 'image/webp', quality: 0.7 })
+      if (destroyed) return
+      thumbnails = [...thumbnails, { url: URL.createObjectURL(blob), startTime: slot.timestamp, endTime: slot.endTime }]
+        .sort((a, b) => a.startTime - b.startTime)
+      emit()
+    }
+
+    // one decode at a time, because there is one wasm worker behind them all
+    const pump = () => {
+      if (running || destroyed || !pending.length) return
+      const slot = pending.splice(nextIndex(), 1)[0]!
+      running = true
+      void decode(slot)
         .catch(() => {
           slot.attempts += 1
+          // left claimable again, so a later `update` retries it
           slot.done = slot.attempts >= MAX_ATTEMPTS
         })
+        .finally(() => {
+          running = false
+          pump()
+        })
+    }
+
+    const claim = (slot: Slot) => {
+      slot.done = true
+      pending.push(slot)
+      pump()
     }
 
     emit()
@@ -120,11 +174,16 @@ export const createThumbnailGenerator = async (options: ThumbnailGeneratorOption
         if (destroyed) return
         for (const slot of slots) {
           if (slot.done) continue
-          if (!ranges || ranges.some(([from, to]) => from <= slot.startByte && slot.endByte <= to)) generate(slot)
+          if (!ranges || ranges.some(([from, to]) => from <= slot.startByte && slot.endByte <= to)) claim(slot)
         }
       },
+      prioritize: (time) => {
+        if (destroyed) return
+        priorityTime = time
+      },
       destroy: () => {
         destroyed = true
+        pending.length = 0
         for (const t of thumbnails) URL.revokeObjectURL(t.url)
         thumbnails = []
         terminateRemuxer(remuxer)

package/src/lib/react/components/chrome.tsx

@@ -49,6 +49,19 @@ const style = css`
     z-index: 2;
   }
 
+  /* The subtitle layer sits UNDER the title and the spinner, where the canvas itself used to sit.
+     As a plain child div it is matched by the rule above and raised to the overlay items' level,
+     which paints subtitles over the title's gradient, over the spinner and over the error text.
+     Naming the type as well is what takes it back: :not() carries the specificity of its argument,
+     so the rule above is (0,2,1) and a bare .subtitles at (0,2,0) loses to it whatever the source
+     order, while div.subtitles ties at (0,2,1) and second place in the same block then wins.
+     subtitle-layering.browser.test.tsx measures the computed value rather than trusting this. */
+  & > div.subtitles {
+    inset: 0;
+    z-index: 1;
+    pointer-events: none;
+  }
+
   canvas {
     height: 100%;
     width: 100%;
@@ -93,7 +106,7 @@ export type ChromeProps = {
   ref?: Ref<HTMLDivElement> | ((element: HTMLDivElement | null) => void)
   /** Absent means render no video element: the media belongs to someone else and arrives as children. */
   onVideoRef?: (element: HTMLVideoElement | null) => void
-  onCanvasRef: (element: HTMLCanvasElement | null) => void
+  onSubtitleRef: (element: HTMLDivElement | null) => void
   /** The app's own content, over the video and outside the click-to-pause region, unlike `children`. */
   overlay?: ReactNode
   /** False draws no control bar at all, leaving the picture, the title and the overlay. */
@@ -101,7 +114,7 @@ export type ChromeProps = {
   children?: ReactNode
 }
 
-export const Chrome = ({ ref, onVideoRef, onCanvasRef, overlay, controls, children }: ChromeProps) => {
+export const Chrome = ({ ref, onVideoRef, onSubtitleRef, overlay, controls, children }: ChromeProps) => {
   const player = usePlayer()
   const hideUI = usePlayer((state) => state.hideUI)
   const setHideUI = usePlayer((state) => state.setHideUI)
@@ -207,7 +220,7 @@ export const Chrome = ({ ref, onVideoRef, onCanvasRef, overlay, controls, childr
       onMouseOut={onMouseOut}
       className={hideUI ? 'hide' : ''}
     >
-      <Overlay onCanvasRef={onCanvasRef} />
+      <Overlay onSubtitleRef={onSubtitleRef} />
       {overlayItems(overlay).map(({ key, item }) => (
         <div
           key={key}

package/src/lib/react/components/overlay.tsx

@@ -6,13 +6,32 @@ import { css, keyframes } from '@emotion/react'
 import { usePlayer } from '../player'
 import { fonts } from '../../utils/fonts'
 
+/**
+ * The subtitle layer.
+ *
+ * A container rather than the `<canvas>` this used to render. From jassub 2 the canvas belongs to
+ * the renderer: the constructor transfers it to a worker, which an element accepts exactly once for
+ * its whole life, and `destroy()` removes it from the document. React can own neither, and this
+ * pipeline is rebuilt in place on an audio track change and on an element recovery, so the canvas is
+ * created per jassub instance inside this box instead.
+ *
+ * The geometry is unchanged. The layer covers the picture and centres its child the way the chrome
+ * root centred the canvas directly, so jassub's inline pixel size still wins over the percentages
+ * and its inline `top` and `left` are still neutralised.
+ */
 const style = css`
-  top: unset !important;
-  left: unset !important;
-  width: 100%;
-  height: 100%;
-  margin: auto;
-  pointer-events: none;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+
+  canvas {
+    top: unset !important;
+    left: unset !important;
+    width: 100%;
+    height: 100%;
+    margin: auto;
+    pointer-events: none;
+  }
 `
 
 const titleStyle = css`
@@ -88,7 +107,7 @@ const errorMessage = (error: unknown) =>
     ? error.message
     : typeof error === 'string' ? error : 'Playback failed'
 
-export const Overlay = ({ onCanvasRef }: { onCanvasRef: (element: HTMLCanvasElement | null) => void }) => {
+export const Overlay = ({ onSubtitleRef }: { onSubtitleRef: (element: HTMLDivElement | null) => void }) => {
   const title = usePlayer((state) => state.title)
   const hideUI = usePlayer((state) => state.hideUI)
   const playbackError = usePlayer((state) => state.playbackError)
@@ -120,7 +139,7 @@ export const Overlay = ({ onCanvasRef }: { onCanvasRef: (element: HTMLCanvasElem
       {playbackError
         ? <div css={errorStyle}>{errorMessage(playbackError)}</div>
         : undefined}
-      <canvas ref={onCanvasRef} css={style} />
+      <div className="subtitles" ref={onSubtitleRef} css={style} />
     </>
   )
 }

package/src/lib/react/components/progress-bar.tsx

@@ -194,12 +194,25 @@ export const ProgressBar = () => {
   const indexes = usePlayer((state) => state.indexes)
   const thumbnails = usePlayer((state) => state.thumbnails)
   const thumbnailAt = usePlayer((state) => state.thumbnailAt)
+  const requestThumbnail = usePlayer((state) => state.requestThumbnail)
 
   const progressBarRef = useRef<HTMLDivElement>(null)
 
   const [seekFraction, setSeekFraction] = useState<number | undefined>(undefined)
   const [progressBarHoverTime, setProgressBarOverTime] = useState<number | undefined>(undefined)
 
+  /*
+   * Move the preview and the generator's next pick together.
+   *
+   * The frame under the pointer is both the one drawn and the one worth decoding first, and every
+   * place that opens or closes the preview goes through here so the two can never disagree. The
+   * request is a bare assignment inside the generator, so a pointermove costs nothing extra.
+   */
+  const showPreviewAt = (time: number | undefined) => {
+    setProgressBarOverTime(time)
+    requestThumbnail(time)
+  }
+
   // onChange reports a bare fraction, so the device that opened the gesture is recorded on press
   const dragPointerType = useRef<string | undefined>(undefined)
 
@@ -235,7 +248,7 @@ export const ProgressBar = () => {
     }
     setSeekFraction(fraction)
     if (dragPointerType.current === 'mouse') return
-    setProgressBarOverTime(fraction * duration)
+    showPreviewAt(fraction * duration)
   }
 
   const { dragging, handlers } = useDragValue({ ref: progressBarRef, onChange: onSeekDrag })
@@ -269,7 +282,7 @@ export const ProgressBar = () => {
     pressFraction.current = undefined
     // a lifted finger leaves nothing over the bar, so the preview it opened closes with it
     if (ev.pointerType === 'mouse') return
-    setProgressBarOverTime(undefined)
+    showPreviewAt(undefined)
   }
 
   // offsetX is relative to whichever child is under the pointer, and a captured pointer has none
@@ -281,12 +294,12 @@ export const ProgressBar = () => {
   }
 
   const onProgressBarOver: DOMAttributes<HTMLDivElement>['onMouseMove'] = (ev) => {
-    setProgressBarOverTime(timeAtClientX(ev.clientX))
+    showPreviewAt(timeAtClientX(ev.clientX))
   }
 
   const hideProgressBarTime = () => {
     if (!progressBarRef.current) return
-    setProgressBarOverTime(undefined)
+    showPreviewAt(undefined)
   }
 
   // duration is 0 until metadata lands, which is never a divisor

package/src/lib/react/hooks/use-picture-in-picture.ts

@@ -19,10 +19,14 @@ export type PictureInPicture = {
  * hidden mirror. Where it does not, and the engine is Gecko, the same composite becomes the picture
  * in the page so that the BROWSER'S own picture in picture control carries the subtitles with it,
  * which it otherwise cannot: it takes a video element, and the subtitles live on a canvas above one.
+ *
+ * The second argument is the subtitle LAYER, not that canvas. From jassub 2 the canvas belongs to the
+ * renderer and is replaced on every pipeline rebuild, so only the layer lives long enough to hold.
  */
 export const usePictureInPicture = (
   video: HTMLVideoElement | null,
-  canvas: HTMLCanvasElement | null,
+  /** The subtitle layer, not the canvas: see `PictureInPictureOptions.subtitles`. */
+  subtitles: HTMLElement | null,
 ): PictureInPicture => {
   const controller = useRef<PictureInPictureController | null>(null)
   const [burnedIn, setBurnedIn] = useState(false)
@@ -31,10 +35,10 @@ export const usePictureInPicture = (
   const [mode] = useState<PictureInPictureMode | null>(() => pictureInPictureMode())
 
   useEffect(() => {
-    if (!video || !canvas || !mode) return
+    if (!video || !subtitles || !mode) return
     const instance = createPictureInPicture({
       video,
-      canvas,
+      subtitles,
       mode,
       onBurnedInChange: setBurnedIn,
     })
@@ -44,7 +48,7 @@ export const usePictureInPicture = (
       controller.current = null
       setBurnedIn(false)
     }
-  }, [video, canvas, mode])
+  }, [video, subtitles, mode])
 
   const toggle = useCallback(() => {
     void controller.current?.toggle().catch((error) => {
@@ -53,6 +57,6 @@ export const usePictureInPicture = (
   }, [])
 
   // null rather than a dead callback: the chrome hides the control instead of offering one that
-  // cannot work, and there is nothing to composite without both an element and a canvas.
-  return { toggle: video && canvas && mode ? toggle : null, mode, burnedIn }
+  // cannot work, and there is nothing to composite without both a video and a subtitle layer.
+  return { toggle: video && subtitles && mode ? toggle : null, mode, burnedIn }
 }

package/src/lib/react/hooks/use-playback.ts

@@ -64,7 +64,7 @@ const causeChain = (error: unknown) => {
  */
 export const usePlayback = (
   video: HTMLVideoElement | null,
-  canvas: HTMLCanvasElement | null,
+  subtitles: HTMLElement | null,
   /** null when the media is remote: there are no bytes, so there is no pipeline to run. */
   options: MediaPlayerLocalOptions | null,
 ) => {
@@ -227,7 +227,7 @@ export const usePlayback = (
   }, [setSourceState, selectSubtitleTrack, selectAudioTrack, requestSeek])
 
   useEffect(() => {
-    if (!video || !canvas || !size || !read) return
+    if (!video || !subtitles || !size || !read) return
     let cancelled = false
     player.setSourceState({ playbackError: null, ready: false })
 
@@ -293,7 +293,7 @@ export const usePlayback = (
       try {
         const controller = await startPlayback({
           videoElement: video,
-          canvasElement: canvas,
+          subtitleContainer: subtitles,
           read: (offset, length) => readRef.current!(offset, length),
           length: size,
           publicPath,
@@ -355,7 +355,7 @@ export const usePlayback = (
     // identity. A streaming consumer passes a fresh closure on every state update, which is several
     // times a second, and the restart loop reads as "Loading metadata" forever at a flat 0 B/s.
   }, [
-    player, video, canvas, size, publicPath, libavWorkerUrl, jassubWorkerUrl, jassubWasmUrl,
+    player, video, subtitles, size, publicPath, libavWorkerUrl, jassubWorkerUrl, jassubWasmUrl,
     jassubLegacyWasmUrl, defaultFontUrl, bufferSize, audioStreamIndex, autoplay, restartToken,
   ])
 }

package/src/lib/react/hooks/use-thumbnails.ts

@@ -1,7 +1,7 @@
 import type { ThumbnailGenerator, ThumbnailImage } from '../../engine'
 import type { DownloadedRange } from '../source-feature'
 
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 
 import { createThumbnailGenerator } from '../../engine'
 
@@ -19,9 +19,21 @@ export type UseSeekThumbnailsOptions = {
   downloadedRanges?: DownloadedRange[]
 }
 
+export type SeekThumbnails = {
+  thumbnails: ThumbnailImage[]
+  /**
+   * Where the viewer is pointing on the seekbar, so that preview is decoded next.
+   *
+   * Stable for the life of the hook, so it can be published to the store once and called from a
+   * pointermove without re-rendering anything. A no-op until the generator boots, and on a source
+   * that brings its own storyboard.
+   */
+  requestThumbnail: (time: number | undefined) => void
+}
+
 export const useSeekThumbnails = ({
   publicPath, workerUrl, length, read, downloadedRanges,
-}: UseSeekThumbnailsOptions): ThumbnailImage[] => {
+}: UseSeekThumbnailsOptions): SeekThumbnails => {
   const [thumbnails, setThumbnails] = useState<ThumbnailImage[]>([])
   const generatorRef = useRef<ThumbnailGenerator | null>(null)
   const readRef = useRef(read)
@@ -74,5 +86,9 @@ export const useSeekThumbnails = ({
 
   useEffect(() => { generatorRef.current?.update(ranges) }, [ranges])
 
-  return thumbnails
+  const requestThumbnail = useCallback((time: number | undefined) => {
+    generatorRef.current?.prioritize(time)
+  }, [])
+
+  return { thumbnails, requestThumbnail }
 }

package/src/lib/react/source-feature.ts

@@ -83,6 +83,17 @@ export type SourceState = {
    * Falls back to scanning `thumbnails` when absent, which is what the engine's generator fills.
    */
   thumbnailAt?: (time: number) => ThumbnailImage | undefined
+  /**
+   * Where the pointer is on the seekbar, so the preview under it is generated before the rest.
+   *
+   * Generation otherwise walks the file start to end, so pointing at the last third of a long video
+   * means waiting out everything before it. Called on every pointermove, which is why it is a
+   * callback on the store rather than a piece of state: a hover time held in the store would
+   * re-render every subscriber for each move.
+   *
+   * A no-op on a source that brings its own storyboard, and until the generator has booted.
+   */
+  requestThumbnail: (time: number | undefined) => void
 
   /**
    * Both selectors may answer with a promise, and the menu waits on it.
@@ -168,6 +179,7 @@ export type SourceState = {
 const initialState: SourceState = {
   indexes: [],
   thumbnails: [],
+  requestThumbnail: () => {},
   subtitleTracks: [],
   selectedSubtitleTrack: undefined,
   selectSubtitleTrack: () => {},

package/src/lib/react/video-player.tsx

@@ -194,7 +194,7 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
   const setContainer = useContainerAttach()
 
   const [video, setVideo] = useState<HTMLVideoElement | null>(null)
-  const [canvas, setCanvas] = useState<HTMLCanvasElement | null>(null)
+  const [subtitleLayer, setSubtitleLayer] = useState<HTMLDivElement | null>(null)
 
   // Attaching is not optional in either arm: the store installs `setSourceState` in `attach`, and
   // video.js only runs attach once media is non-null, so skipping it would leave every write below a
@@ -235,9 +235,9 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
 
   // Each of these no-ops on null inputs, which is what a remote arm supplies: it renders no <video>,
   // so there is nothing for them to attach to and nothing to guard at the call site.
-  usePlayback(video, canvas, local)
+  usePlayback(video, subtitleLayer, local)
 
-  const generatedThumbnails = useSeekThumbnails({
+  const { thumbnails: generatedThumbnails, requestThumbnail } = useSeekThumbnails({
     publicPath,
     workerUrl: libavWorkerUrl,
     length: thumbnailsEnabled === false ? undefined : size,
@@ -249,7 +249,7 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
     toggle: togglePictureInPicture,
     mode: pictureInPictureMode,
     burnedIn: burnedInSubtitles,
-  } = usePictureInPicture(video, canvas)
+  } = usePictureInPicture(video, subtitleLayer)
 
   // Subscribed rather than read off the store, because it is a no-op until the media element
   // attaches: when attach swaps in the real setter the identity changes and these publish again.
@@ -264,12 +264,14 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
     setSourceState({
       thumbnails,
       thumbnailAt,
+      requestThumbnail,
       togglePictureInPicture,
       pictureInPictureMode,
       burnedInSubtitles,
     })
   }, [
-    setSourceState, thumbnails, thumbnailAt, togglePictureInPicture, pictureInPictureMode, burnedInSubtitles,
+    setSourceState, thumbnails, thumbnailAt, requestThumbnail, togglePictureInPicture, pictureInPictureMode,
+    burnedInSubtitles,
   ])
 
   // A delegated track list writes the same store fields the engine writes, so the menus never learn
@@ -305,7 +307,7 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
       // No element in the remote arm: the media is somebody else's, and whatever renders it is
       // passed in as children. Rendering an idle <video> here would sit over it.
       onVideoRef={remote ? undefined : setVideo}
-      onCanvasRef={setCanvas}
+      onSubtitleRef={setSubtitleLayer}
       overlay={options.overlay}
       controls={options.controls}
     >