@banou/media-player 0.6.2 → 0.7.0

package/src/lib/engine/subtitles.ts

@@ -0,0 +1,210 @@
+import type { ParsedASS, ParsedASSStyles } from 'ass-compiler'
+import type { ASS_Event } from 'jassub'
+import type { Attachment, SubtitleFragment } from 'libav-wasm/build/worker'
+
+import JASSUB from 'jassub'
+import { parse, stringify } from 'ass-compiler'
+
+export type SubtitleStream = { streamIndex: number, title: string, language: string }
+
+/** -1 turns subtitles off. It frees the track and matches no header, so nothing is set. */
+export const SUBTITLES_OFF = -1
+
+type SubtitleHeaderPart = { type: 'header', streamIndex: number, content: string, eventsContent: string, parsed: ParsedASS }
+type SubtitleDialoguePart = { type: 'dialogue', streamIndex: number, index: number, assEvent: ASS_Event }
+
+export type SubtitleRendererOptions = {
+  video: HTMLVideoElement
+  canvas: HTMLCanvasElement
+  workerUrl: string
+  /** The SIMD build, `jassub-worker-modern.wasm`. Used wherever WebAssembly SIMD is available. */
+  wasmUrl: string
+  /**
+   * The non-SIMD build, `jassub-worker.wasm`, for Safari before 16.4 and anything else without SIMD.
+   *
+   * Not optional in practice, only in the type. jassub picks `wasmUrl ?? 'jassub-worker.wasm'` when SIMD
+   * is missing, and that bare relative name resolves against the blob: url the worker is built from,
+   * which throws. So leaving this unset does not degrade to the slower build, it fails outright.
+   */
+  legacyWasmUrl?: string
+  /** Fallback face for `liberation sans`. Without it jassub falls back to whatever the wasm build embeds. */
+  defaultFontUrl?: string
+  onStreams?: (streams: SubtitleStream[]) => void
+}
+
+const convertTimestamp = (ms: number) => new Date(ms).toISOString().slice(11, 22)
+
+const appendParsedStyle = (jassub: JASSUB, style: ParsedASSStyles['style'][number]) =>
+  jassub.createStyle({
+    ...style,
+    treat_fontname_as_pattern: 0,
+    Blur: 0,
+    Justify: 0,
+    FontName: style.Fontname,
+    FontSize: Number(style.Fontsize),
+    PrimaryColour: Number(style.PrimaryColour),
+    SecondaryColour: Number(style.SecondaryColour),
+    OutlineColour: Number(style.OutlineColour),
+    BackColour: Number(style.BackColour),
+    Bold: Number(style.Bold),
+    Italic: Number(style.Italic),
+    Underline: Number(style.Underline),
+    StrikeOut: Number(style.StrikeOut),
+    ScaleX: Number(style.ScaleX),
+    ScaleY: Number(style.ScaleY),
+    Spacing: Number(style.Spacing),
+    Angle: Number(style.Angle),
+    BorderStyle: Number(style.BorderStyle),
+    Outline: Number(style.Outline),
+    Shadow: Number(style.Shadow),
+    Alignment: Number(style.Alignment),
+    MarginL: Number(style.MarginL),
+    MarginR: Number(style.MarginR),
+    MarginV: Number(style.MarginV),
+    Encoding: Number(style.Encoding),
+  } as Parameters<JASSUB['createStyle']>[0])
+
+// cleared so jassub scales the script to the canvas, not to the authored resolution
+const renderable = (content: string) => {
+  const parsed = parse(content)
+  return stringify({ ...parsed, info: { ...parsed.info, ScaledBorderAndShadow: 'no', LayoutResX: '', LayoutResY: '' } })
+}
+
+/**
+ * `\r?\n`, not `\r\n`: an ASS header muxed straight out of a matroska file uses CRLF, but one libav
+ * CONVERTED from another format (an srt track, most commonly) is LF only, and requiring CRLF rejected it.
+ *
+ * Returns null rather than throwing. A header this cannot read is a reason to have no subtitles, and it
+ * used to be a reason to have no VIDEO: the throw crossed pushFragments into startPlayback's try, so one
+ * unreadable track failed the whole file with "playback failed".
+ */
+const toHeaderPart = (fragment: SubtitleFragment & { type: 'header' }): SubtitleHeaderPart | null => {
+  const eventsContent = fragment.content.match(/\r?\n\[Events\]\r?\nFormat: (.*)/)?.[0]
+  if (!eventsContent) {
+    console.warn(`subtitle stream ${fragment.streamIndex} has no Events format, ignoring the track`)
+    return null
+  }
+  return { type: 'header', streamIndex: fragment.streamIndex, content: fragment.content, eventsContent, parsed: parse(fragment.content) }
+}
+
+const toDialoguePart = (header: SubtitleHeaderPart, fragment: SubtitleFragment & { type: 'dialogue' }): SubtitleDialoguePart => {
+  const [dialogueIndexString, layer] = fragment.content.split(',')
+  const dialogueIndex = Number(dialogueIndexString)
+  const start = convertTimestamp(fragment.start)
+  const end = convertTimestamp(fragment.end)
+  const rest = fragment.content.replace(`${dialogueIndex},${layer},`, '')
+  const dialogueContent = `Dialogue: ${layer},${start},${end},${rest}`
+  const event = parse(`${header.eventsContent}\r\n${dialogueContent}`).events.dialogue[0]
+  if (!event) throw new Error('dialogue event is undefined')
+  return {
+    type: 'dialogue',
+    streamIndex: fragment.streamIndex,
+    index: dialogueIndex,
+    assEvent: {
+      ...event,
+      Effect: event.Effect ?? '',
+      Text: event.Text.raw,
+      Duration: (event.End - event.Start) * 1000,
+      Start: event.Start * 1000,
+      End: event.End * 1000,
+      ReadOrder: dialogueIndex,
+      _index: dialogueIndex,
+    } as ASS_Event,
+  }
+}
+
+export type SubtitleRenderer = ReturnType<typeof createSubtitleRenderer>
+
+export const createSubtitleRenderer = (options: SubtitleRendererOptions) => {
+  const { video, canvas, workerUrl, wasmUrl, legacyWasmUrl, defaultFontUrl } = options
+  let jassub: JASSUB | undefined
+  let attachments: [string, Uint8Array][] = []
+  const headers = new Map<number, SubtitleHeaderPart>()
+  const streams: SubtitleStream[] = []
+  const dialogues = new Map<number, Map<number, SubtitleDialoguePart>>()
+  let selected: number | undefined
+  let onStreams = options.onStreams
+
+  const tick = setInterval(() => {
+    jassub?.setCurrentTime(video.paused, video.currentTime, video.playbackRate)
+  }, 100)
+
+  const onRateChange = () => jassub?.setRate(video.playbackRate)
+
+  const bootJassub = (header: SubtitleHeaderPart) => {
+    jassub = new JASSUB({
+      onDemandRender: false,
+      video,
+      canvas,
+      subContent: renderable(header.content),
+      workerUrl,
+      modernWasmUrl: wasmUrl,
+      ...legacyWasmUrl ? { wasmUrl: legacyWasmUrl } : {},
+      fonts: attachments.map(([, data]) => data),
+      availableFonts: {
+        ...Object.fromEntries(attachments),
+        ...(defaultFontUrl ? { 'liberation sans': defaultFontUrl } : {}),
+      },
+    })
+    // jassub 1.8.x binds setRate as the ratechange listener, so the Event becomes the rate
+    video.removeEventListener('ratechange', (jassub as unknown as { _boundSetRate: EventListener })._boundSetRate)
+    video.addEventListener('ratechange', onRateChange)
+    for (const style of header.parsed.styles.style) appendParsedStyle(jassub, style)
+  }
+
+  const pushAttachments = (incoming: Attachment[]) => {
+    attachments = [...attachments, ...incoming.map((a) => [a.filename, new Uint8Array(a.data)] as [string, Uint8Array])]
+  }
+
+  const pushFragments = (fragments: SubtitleFragment[]) => {
+    for (const fragment of fragments) {
+      if (fragment.type === 'header') {
+        if (headers.has(fragment.streamIndex)) continue
+        const header = toHeaderPart(fragment)
+        if (!header) continue
+        headers.set(fragment.streamIndex, header)
+        streams.push({ streamIndex: fragment.streamIndex, title: fragment.title, language: fragment.language })
+        onStreams?.([...streams])
+        if (selected === undefined) selected = fragment.streamIndex
+        if (!jassub) bootJassub(header)
+      } else {
+        const header = headers.get(fragment.streamIndex)
+        if (!header) continue
+        let byIndex = dialogues.get(fragment.streamIndex)
+        if (!byIndex) { byIndex = new Map(); dialogues.set(fragment.streamIndex, byIndex) }
+        const part = toDialoguePart(header, fragment)
+        if (byIndex.has(part.index)) continue
+        byIndex.set(part.index, part)
+        if (selected === fragment.streamIndex) jassub?.createEvent(part.assEvent)
+      }
+    }
+  }
+
+  const selectStream = (streamIndex: number | undefined) => {
+    const next = streamIndex ?? SUBTITLES_OFF
+    if (next === selected || !jassub) return
+    selected = next
+    jassub.freeTrack()
+    const header = headers.get(next)
+    if (!header) return
+    jassub.setTrack(renderable(header.content))
+    for (const style of header.parsed.styles.style) appendParsedStyle(jassub, style)
+    for (const part of dialogues.get(next)?.values() ?? []) jassub.createEvent(part.assEvent)
+    jassub.setCurrentTime(video.paused, video.currentTime, video.playbackRate)
+  }
+
+  return {
+    pushAttachments,
+    pushFragments,
+    selectStream,
+    getStreams: () => [...streams],
+    getSelectedStream: () => selected,
+    setOnStreams: (cb: (streams: SubtitleStream[]) => void) => { onStreams = cb },
+    destroy: () => {
+      clearInterval(tick)
+      video.removeEventListener('ratechange', onRateChange)
+      jassub?.destroy()
+      jassub = undefined
+    },
+  }
+}

package/src/lib/engine/thumbnails.ts

@@ -0,0 +1,137 @@
+import { makeThumbnailer } from 'libav-wasm'
+
+import { terminateRemuxer } from './playback'
+
+export type ThumbnailImage = { url: string, startTime: number, endTime: number }
+
+export type ThumbnailGeneratorOptions = {
+  publicPath: string
+  workerUrl: string
+  length: number
+  /** The same reader playback uses, so generation shares its fetch order rather than competing. */
+  read: (offset: number, size: number) => Promise<ArrayBuffer>
+  onThumbnails: (thumbnails: ThumbnailImage[]) => void
+  /** Seconds between thumbnails. Widened automatically on long content to stay under MAX_THUMBNAILS. */
+  interval?: number
+  width?: number
+}
+
+const INTERVAL = 5
+// the interval widens on long content rather than paying thousands of decodes and blobs
+const MAX_THUMBNAILS = 500
+const WIDTH = 320
+// avio reads up to bufferSize past the slot span, so require that margin
+const READAHEAD = 1_000_000
+const MAX_ATTEMPTS = 3
+// a keyframe decode can hang without ever settling
+const KEYFRAME_TIMEOUT = 10_000
+
+export type ThumbnailGenerator = {
+  /** Report which byte ranges are readable. Called with no argument when the whole file is. */
+  update: (ranges?: [number, number][]) => void
+  destroy: () => void
+}
+
+export const createThumbnailGenerator = async (options: ThumbnailGeneratorOptions): Promise<ThumbnailGenerator> => {
+  const { publicPath, workerUrl, length, read, onThumbnails, width = WIDTH } = options
+  // a thumbnailer, not a remuxer: readKeyframe seeks backward, which an output muxer cannot follow
+  const remuxer = await makeThumbnailer({
+    publicPath,
+    workerUrl,
+    workerOptions: { type: 'module' },
+    length,
+    read,
+  })
+  // init and the index walk both throw on a file that is not readable yet, so the worker must go too
+  try {
+    const metadata = await remuxer.init()
+    const duration = metadata.duration
+    const interval = Math.max(options.interval ?? INTERVAL, duration / MAX_THUMBNAILS)
+
+    type Slot = { timestamp: number, endTime: number, startByte: number, endByte: number, done: boolean, attempts: number }
+    const slots: Slot[] = []
+    for (const [i, index] of metadata.indexes.entries()) {
+      const last = slots.at(-1)
+      if (last && index.timestamp - last.timestamp < interval) continue
+      slots.push({
+        timestamp: index.timestamp,
+        endTime: duration,
+        startByte: index.pos,
+        endByte: Math.min((metadata.indexes[i + 1]?.pos ?? length) + READAHEAD, length),
+        done: false,
+        attempts: 0,
+      })
+    }
+    for (const [i, slot] of slots.entries()) slot.endTime = slots[i + 1]?.timestamp ?? duration
+    // reading the last keyframe runs the demuxer into EOF, which crashes the libav build
+    if (slots.length > 1 && (slots.at(-1)!.timestamp > duration - interval * 2)) slots.pop()
+
+    let thumbnails: ThumbnailImage[] = []
+    let destroyed = false
+    let queue = Promise.resolve()
+
+    // the slider assumes a gapless storyboard, so gaps get sentinels the UI hides
+    const emit = () => {
+      const display: ThumbnailImage[] = []
+      for (const [i, t] of thumbnails.entries()) {
+        if (t.startTime - (display.at(-1)?.endTime ?? 0) > 0.01) {
+          display.push({ url: '', startTime: display.at(-1)?.endTime ?? 0, endTime: t.startTime })
+        }
+        display.push(t)
+        const next = thumbnails[i + 1]
+        if (next && next.startTime - t.endTime > 0.01) {
+          display.push({ url: '', startTime: t.endTime, endTime: next.startTime })
+        }
+      }
+      const tailEnd = display.at(-1)?.endTime ?? 0
+      if (duration - tailEnd > 0.01) display.push({ url: '', startTime: tailEnd, endTime: duration })
+      onThumbnails(display)
+    }
+
+    const generate = (slot: Slot) => {
+      slot.done = true
+      queue = queue
+        .then(async () => {
+          if (destroyed) return
+          const png = await Promise.race([
+            remuxer.readKeyframe(slot.timestamp),
+            new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), KEYFRAME_TIMEOUT)),
+          ])
+          const bitmap = await createImageBitmap(new Blob([png], { type: 'image/png' }))
+          const canvas = new OffscreenCanvas(width, Math.max(1, Math.round(bitmap.height * (width / bitmap.width))))
+          canvas.getContext('2d')!.drawImage(bitmap, 0, 0, canvas.width, canvas.height)
+          bitmap.close()
+          const blob = await canvas.convertToBlob({ type: 'image/webp', quality: 0.7 })
+          if (destroyed) return
+          thumbnails = [...thumbnails, { url: URL.createObjectURL(blob), startTime: slot.timestamp, endTime: slot.endTime }]
+            .sort((a, b) => a.startTime - b.startTime)
+          emit()
+        })
+        .catch(() => {
+          slot.attempts += 1
+          slot.done = slot.attempts >= MAX_ATTEMPTS
+        })
+    }
+
+    emit()
+
+    return {
+      update: (ranges) => {
+        if (destroyed) return
+        for (const slot of slots) {
+          if (slot.done) continue
+          if (!ranges || ranges.some(([from, to]) => from <= slot.startByte && slot.endByte <= to)) generate(slot)
+        }
+      },
+      destroy: () => {
+        destroyed = true
+        for (const t of thumbnails) URL.revokeObjectURL(t.url)
+        thumbnails = []
+        terminateRemuxer(remuxer)
+      },
+    }
+  } catch (error) {
+    terminateRemuxer(remuxer)
+    throw error
+  }
+}

package/src/lib/globals.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The library ships an inline SVG asset, so it declares the module shape itself rather than relying
+ * on `vite/client`, which belongs to the app and is not in the library's type surface.
+ */
+declare module '*.svg' {
+  const src: string
+  export default src
+}

package/src/lib/index.tsx

@@ -0,0 +1,36 @@
+/// <reference types="@emotion/react/types/css-prop" />
+
+export { MediaPlayer, default } from './react/video-player'
+export type { MediaPlayerOptions, MediaPlayerSource } from './react/video-player'
+
+export { Player, usePlayer } from './react/player'
+export type { PlayerStore } from './react/player'
+
+// Source state lives on the player store next to the built-in playback state, so `usePlayer` is the
+// only hook the chrome needs.
+export { sourceFeature } from './react/source-feature'
+export type { DownloadedRange, SourceState } from './react/source-feature'
+
+export { useSeekThumbnails } from './react/hooks/use-thumbnails'
+export { usePictureInPicture } from './react/hooks/use-picture-in-picture'
+
+export { inputToRemuxerInput } from './utils/source'
+export type { RemuxerInput } from './utils/source'
+
+// The engine is also published on its own subpath for consumers that want the pipeline with no React.
+export type {
+  AudioStream,
+  MediaIndex,
+  PictureInPictureController,
+  PlaybackController,
+  PlaybackOptions,
+  SubtitleStream,
+  ThumbnailImage,
+} from './engine'
+export {
+  startPlayback,
+  createThumbnailGenerator,
+  createSubtitleRenderer,
+  createPictureInPicture,
+  SUBTITLES_OFF,
+} from './engine'

package/src/lib/react/components/chrome.tsx

@@ -0,0 +1,126 @@
+/// <reference types="@emotion/react/types/css-prop" />
+import type { ReactNode, Ref } from 'react'
+
+import { useEffect, useRef } from 'react'
+import { css } from '@emotion/react'
+
+import { usePlayer } from '../player'
+import { Overlay } from './overlay'
+import ControlBar from './control-bar'
+
+const AUTO_HIDE_DELAY = 3_000
+
+const style = css`
+  position: relative;
+  background-color: black;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100%;
+  width: 100%;
+
+  & > div:not(:last-of-type) {
+    position: absolute;
+    z-index: 2;
+  }
+
+  canvas {
+    height: 100%;
+    width: 100%;
+    pointer-events: none;
+    position: absolute;
+    z-index: 1;
+  }
+
+  .video, video {
+    height: 100%;
+    width: 100%;
+    background-color: black;
+    object-fit: contain;
+  }
+
+  &.hide {
+    cursor: none;
+  }
+`
+
+export type ChromeProps = {
+  ref?: Ref<HTMLDivElement> | ((element: HTMLDivElement | null) => void)
+  onVideoRef: (element: HTMLVideoElement | null) => void
+  onCanvasRef: (element: HTMLCanvasElement | null) => void
+  children?: ReactNode
+}
+
+export const Chrome = ({ ref, onVideoRef, onCanvasRef, children }: ChromeProps) => {
+  const player = usePlayer()
+  const hideUI = usePlayer((state) => state.hideUI)
+  const setHideUI = usePlayer((state) => state.setHideUI)
+  const autoHide = useRef<ReturnType<typeof setTimeout>>(undefined)
+  // a tap and a click mean different things, so the last pointer kind is remembered
+  const lastPointerType = useRef<string>('mouse')
+
+  useEffect(() => () => clearTimeout(autoHide.current), [])
+
+  const reveal = () => {
+    setHideUI(false)
+    clearTimeout(autoHide.current)
+    autoHide.current = setTimeout(() => setHideUI(true), AUTO_HIDE_DELAY)
+  }
+
+  // Hides after the delay whether or not playback is running. A finger produces no move, so this is
+  // the mouse path only and touch reveals on tap instead.
+  const onPointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
+    lastPointerType.current = event.pointerType
+    if (event.pointerType !== 'mouse') return
+    reveal()
+  }
+
+  const onPointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
+    lastPointerType.current = event.pointerType
+  }
+
+  // play/pause with a mouse, show or hide the controls with a finger, which has no other way back
+  const onVideoClick = () => {
+    if (lastPointerType.current === 'mouse') {
+      player.togglePaused()
+      return
+    }
+    if (hideUI) reveal()
+    else {
+      clearTimeout(autoHide.current)
+      setHideUI(true)
+    }
+  }
+
+  // relatedTarget alone is not enough: the browser fires mouseout with a null one while the pointer
+  // still sits over the video, so a null is confirmed against the pointer's own coordinates.
+  const onMouseOut: React.DOMAttributes<HTMLDivElement>['onMouseOut'] = (ev) => {
+    const related = ev.relatedTarget as Node | null
+    if (related && ev.currentTarget.contains(related)) return
+    const { left, right, top, bottom } = ev.currentTarget.getBoundingClientRect()
+    const inside = ev.clientX >= left && ev.clientX < right && ev.clientY >= top && ev.clientY < bottom
+    if (inside) return
+    clearTimeout(autoHide.current)
+    setHideUI(true)
+  }
+
+  return (
+    <div
+      css={style}
+      ref={ref}
+      onPointerMove={onPointerMove}
+      onPointerDown={onPointerDown}
+      onMouseOut={onMouseOut}
+      className={hideUI ? 'hide' : ''}
+    >
+      <Overlay onCanvasRef={onCanvasRef} />
+      <ControlBar />
+      <div className="video" onClick={onVideoClick}>
+        <video ref={onVideoRef} playsInline />
+        {children}
+      </div>
+    </div>
+  )
+}
+
+export default Chrome