@banou/media-player 0.10.20 → 0.10.22

package/src/lib/audio/volume-normalizer.ts

@@ -1,8 +1,10 @@
 import { createMediaElementSource } from '@banou/ponyfill/web-audio'
 
-/** The worklet's processorOptions, read once when the node is made; its header has the defaults. */
+/**
+ * The worklet's processorOptions, read once when the node is made: `enabled` false starts it bypassed, and
+ * `inputGainDb` is the gain already applied upstream (the element's volume), which loudness is judged without.
+ */
 export type VolumeNormalizerOptions = {
-  targetLufs?: number
   enabled?: boolean
   inputGainDb?: number
 }
@@ -106,7 +108,7 @@ export const attachVolumeNormalizer = (
     created.port.postMessage({ enabled, inputGainDb: inputGainDb() })
     source.connect(created).connect(context.destination)
     element.addEventListener('volumechange', () => created.port.postMessage({ inputGainDb: inputGainDb() }))
-    // the 3 s window describes where the playhead was, and a new file owes nothing to the last one's cut
+    // the live detector describes where the playhead was, and a new file owes nothing to the last one's level
     element.addEventListener('seeking', () => created.port.postMessage({ reset: 'seek' }))
     element.addEventListener('emptied', () => created.port.postMessage({ reset: 'source' }))
     node = created

package/src/lib/engine/index.ts

@@ -1,5 +1,5 @@
 export { startPlayback, terminateRemuxer, MediaElementError, isMediaElementError, DEFAULT_BUFFER_SIZE } from './playback'
-export type { PlaybackOptions, PlaybackController, MediaIndex, MediaChapter, AudioStream } from './playback'
+export type { PlaybackOptions, PlaybackController, PlaybackOutput, MediaIndex, MediaChapter, AudioStream } from './playback'
 
 export { createSubtitleRenderer, SUBTITLES_OFF } from './subtitles'
 export type { SubtitleRenderer, SubtitleRendererOptions, SubtitleStream } from './subtitles'

package/src/lib/engine/playback.ts

@@ -23,6 +23,18 @@ export type MediaIndex = { pos: number, timestamp: number }
  */
 export type MediaChapter = { start: number, end: number, title: string }
 
+/**
+ * What the remuxer produced, in the order it was appended: the init segment and the input audio stream it carries,
+ * a run at load and at every remuxer seek (`pts` is the timestampOffset its fragments are appended at), each
+ * fragment, and the end of the file once read. The buffers are playback's own and may be appended again, so a
+ * consumer copies what it keeps.
+ */
+export type PlaybackOutput =
+  | { type: 'init', data: ArrayBuffer, audioMimeType: string, audioStreamIndex: number }
+  | { type: 'run', pts: number }
+  | { type: 'fragment', data: ArrayBuffer }
+  | { type: 'end' }
+
 export type PlaybackOptions = {
   videoElement: HTMLVideoElement
   /**
@@ -51,6 +63,7 @@ export type PlaybackOptions = {
   onSeek?: (fraction: number) => void
   onSubtitleStreams?: (streams: SubtitleStream[]) => void
   onAudioStreams?: (streams: AudioStream[], selected: number) => void
+  onOutput?: (output: PlaybackOutput) => void
 }
 
 export type PlaybackController = {
@@ -146,7 +159,7 @@ export const startPlayback = async (options: PlaybackOptions): Promise<PlaybackC
     videoElement, subtitleContainer, read, length, publicPath, libavWorkerUrl,
     jassubWorkerUrl, jassubWasmUrl, jassubLegacyWasmUrl, defaultFontUrl, bufferSize = DEFAULT_BUFFER_SIZE,
     audioStreamIndex, onReady, onError, onRecovered, onSeek, onSubtitleStreams,
-    onAudioStreams,
+    onAudioStreams, onOutput,
   } = options
 
   const remuxer = await makeRemuxer({
@@ -253,6 +266,8 @@ export const startPlayback = async (options: PlaybackOptions): Promise<PlaybackC
 
     const { appendBuffer, unbufferRange, updateTimestampOffset, endOfStream } = updateSourceBuffer(sourceBuffer, mediaSource)
     await appendBuffer(metadata.data)
+    onOutput?.({ type: 'init', data: metadata.data, audioMimeType: metadata.info.output.audioMimeType, audioStreamIndex: selectedAudio })
+    onOutput?.({ type: 'run', pts: 0 })
     if (metadata.subtitles?.length) subtitles.pushFragments(metadata.subtitles)
     onReady?.()
 
@@ -400,6 +415,9 @@ export const startPlayback = async (options: PlaybackOptions): Promise<PlaybackC
         if (fragments?.length) subtitles.pushFragments(fragments)
         await updateTimestampOffset(pts)
         if (destroyed || generation !== seekGeneration) return
+        onOutput?.({ type: 'run', pts })
+        if (data.byteLength) onOutput?.({ type: 'fragment', data })
+        if (done) onOutput?.({ type: 'end' })
         if (data.byteLength) {
           await appendBuffer(data)
           trackAppendedEnd()
@@ -458,6 +476,8 @@ export const startPlayback = async (options: PlaybackOptions): Promise<PlaybackC
           if (stale()) return
           if (done) finished = true
           if (fragments?.length) subtitles.pushFragments(fragments)
+          if (data.byteLength) onOutput?.({ type: 'fragment', data })
+          if (done) onOutput?.({ type: 'end' })
           if (!data.byteLength) return
           pending = data
           pendingAttempts = 0

package/src/lib/index.tsx

@@ -43,6 +43,7 @@ export type {
   PictureInPictureController,
   PlaybackController,
   PlaybackOptions,
+  PlaybackOutput,
   SubtitleStream,
   ThumbnailImage,
 } from './engine'

package/src/lib/react/hooks/use-playback.ts

@@ -1,4 +1,4 @@
-import type { PlaybackController } from '../../engine'
+import type { PlaybackController, PlaybackOutput } from '../../engine'
 import type { PlaybackErrorEntry } from '../source-feature'
 import type { MediaPlayerLocalOptions } from '../video-player'
 
@@ -74,6 +74,7 @@ export const usePlayback = (
   subtitles: HTMLElement | null,
   /** null when the media is remote: there are no bytes, so there is no pipeline to run. */
   options: MediaPlayerLocalOptions | null,
+  onOutput?: (output: PlaybackOutput) => void,
 ) => {
   const player = usePlayer()
   // The four asset urls are required on the local arm, so a default only ever applies when options is
@@ -147,6 +148,8 @@ export const usePlayback = (
   onSeekRef.current = options?.onSeek
   const onPlaybackErrorRef = useRef(options?.onPlaybackError)
   onPlaybackErrorRef.current = options?.onPlaybackError
+  const onOutputRef = useRef(onOutput)
+  onOutputRef.current = onOutput
 
   // The store's ids are opaque, but everything this hook writes into it came from libav, so every id
   // that comes back is one of its own stream indices.
@@ -355,6 +358,7 @@ export const usePlayback = (
           onError: fail,
           onRecovered: () => { if (!cancelled) player.setSourceState({ playbackError: null }) },
           onSeek: (fraction) => onSeekRef.current?.(fraction),
+          onOutput: (output) => onOutputRef.current?.(output),
           onSubtitleStreams: (streams) => {
             if (cancelled) return
             player.setSourceState({

package/src/lib/react/hooks/use-volume-normalizer.ts

@@ -1,24 +1,39 @@
 import type { VolumeNormalizer } from '../../audio/volume-normalizer'
+import type { LoudnessSource } from '../../audio/loudness-analysis'
+import type { PlaybackTap } from '../../audio/playback-tap'
 
 import { useEffect, useRef, useState } from 'react'
 
 import { attachVolumeNormalizer } from '../../audio/volume-normalizer'
+import { startLoudnessAnalysis } from '../../audio/loudness-analysis'
 
 /**
  * Normalizes the element's volume while `enabled`. Nothing is built until it first is, and turning it off
  * afterwards bypasses rather than unroutes, since an element cannot be handed back. `refused` is true once the
  * engine turned the element down, which leaves it playing exactly as before.
+ *
+ * With a `source`, the file is also analysed ahead of the playhead while it is on, reading only the byte `ranges`
+ * given (all of them when undefined), or with a `tap`, from the audio playback remuxed.
  */
-export const useVolumeNormalizer = (video: HTMLVideoElement | null, workletUrl: string | undefined, enabled: boolean) => {
+export const useVolumeNormalizer = (
+  video: HTMLVideoElement | null,
+  workletUrl: string | undefined,
+  enabled: boolean,
+  source?: LoudnessSource | { tap: PlaybackTap, audioStreamIndex: number },
+  ranges?: [number, number][],
+) => {
   const handle = useRef<VolumeNormalizer | null>(null)
   const [refused, setRefused] = useState(false)
+  const [node, setNode] = useState<AudioWorkletNode | null>(null)
 
   useEffect(() => {
     if (!video || !workletUrl) return
     if (enabled && !handle.current) {
       const attached = attachVolumeNormalizer(video, workletUrl, { enabled })
       handle.current = attached
-      attached.routed.catch((error) => {
+      attached.routed.then((routed) => {
+        if (handle.current === attached) setNode(routed)
+      }, (error) => {
         // released first, which closes a context that had not run yet: nothing was refused
         if (handle.current !== attached) return
         console.warn('[media-player] the volume normalizer is unavailable:', error)
@@ -31,7 +46,30 @@ export const useVolumeNormalizer = (video: HTMLVideoElement | null, workletUrl:
   useEffect(() => () => {
     handle.current?.release()
     handle.current = null
+    setNode(null)
   }, [video])
 
+  const rangesRef = useRef(ranges)
+  rangesRef.current = ranges
+  const analysis = useRef<ReturnType<typeof startLoudnessAnalysis> | null>(null)
+  const sourceRef = useRef(source)
+  sourceRef.current = source
+  const { length, publicPath, libavWorkerUrl } = source && 'read' in source ? source : {}
+  const tap = source && 'tap' in source ? source.tap : undefined
+  const audioStreamIndex = source?.audioStreamIndex
+  useEffect(() => {
+    const current = sourceRef.current
+    if (!node || !video || !workletUrl || !enabled || !current) return
+    const started = startLoudnessAnalysis({ node, element: video, scriptUrl: workletUrl, source: current })
+    started.update(rangesRef.current)
+    analysis.current = started
+    return () => {
+      started.stop()
+      analysis.current = null
+    }
+  }, [node, video, workletUrl, enabled, length, publicPath, libavWorkerUrl, audioStreamIndex, tap])
+
+  useEffect(() => { analysis.current?.update(ranges) }, [ranges])
+
   return { refused }
 }

package/src/lib/react/video-player.tsx

@@ -15,15 +15,20 @@ import { useSeekThumbnails } from './hooks/use-thumbnails'
 import { usePictureInPicture } from './hooks/use-picture-in-picture'
 import { usePassThroughPictureInPicture } from './hooks/use-pass-through-picture-in-picture'
 import { useVolumeNormalizer } from './hooks/use-volume-normalizer'
+import { createPlaybackTap } from '../audio/playback-tap'
 import Chrome from './components/chrome'
 
 /**
  * The source, read a range at a time: the player never downloads the whole file. Both fields travel
  * together, so passing one without the other is a type error.
+ *
+ * `readsDownload` marks a source each read of which is a download, a file over HTTP, so the volume normalizer
+ * analyses the audio playback fetched instead of reading the file again. Without it, and with no
+ * `downloadedRanges`, every byte is taken to be readable for free, as a local file's are.
  */
 export type MediaPlayerSource =
-  | { read: (offset: number, size: number) => Promise<ArrayBuffer>, size: number }
-  | { read?: undefined, size?: undefined }
+  | { read: (offset: number, size: number) => Promise<ArrayBuffer>, size: number, readsDownload?: boolean }
+  | { read?: undefined, size?: undefined, readsDownload?: undefined }
 
 /** Shared by both arms: nothing here depends on who owns the media. */
 type CommonOptions = {
@@ -156,11 +161,13 @@ export type MediaPlayerLocalOptions =
     downloadedRanges?: DownloadedRange[]
 
     /**
-     * Reader for the thumbnail engine, when it should differ from playback's.
+     * Reader for the thumbnail engine and the volume normalizer's analysis, when it should differ from playback's.
      *
      * They are the same by default so generation shares playback's fetch order. A consumer whose
      * reads are not free wants them apart: a torrent hands this a non-prioritising, fail-fast reader
-     * so generating previews cannot steal download order from the bytes playback is blocked on.
+     * so generating previews cannot steal download order from the bytes playback is blocked on. The
+     * analysis asks it only for bytes inside `downloadedRanges`, with none given for the whole file, and for
+     * nothing when the source `readsDownload`.
      */
     thumbnailRead?: (offset: number, size: number) => Promise<ArrayBuffer>
     /** Off entirely. A second wasm worker during pipeline boot is worth avoiding on a slow source. */
@@ -176,10 +183,14 @@ export type MediaPlayerLocalOptions =
      */
     normalizerWorkletUrl?: string
     /**
-     * The settings menu's "Normalize volume" switch: holds the loudness steady, so a music cue is turned down to
-     * the dialogue around it. Off by default. The player keeps the switch's state from here and from the viewer's
-     * clicks; nothing is saved, so an app that wants it to stick saves what `onNormalizeVolumeChange` reports and
-     * passes it back.
+     * The settings menu's "Normalize volume" switch: keeps the file's normal level and turns down what goes well
+     * over it, a loud burst before it starts. On by default; pass false for off. The player keeps the switch's
+     * state from here and from the viewer's clicks; nothing is saved, so an app that wants it to stick saves what
+     * `onNormalizeVolumeChange` reports and passes it back.
+     *
+     * While it is on the file is read a second time, in a worker, as far as `downloadedRanges` allows, to find the
+     * loud parts ahead of the playhead; a source that `readsDownload` is not read again, and its loud parts are found
+     * in what playback fetched. The sound plays 10 ms behind its input, on or off, once routed.
      *
      * The first time it is on, the element is routed through Web Audio for the rest of its life (off then
      * bypasses). If autoplay rules hold the audio back, that waits for the page's next click, tap or key press, and
@@ -188,7 +199,7 @@ export type MediaPlayerLocalOptions =
      * the element is never routed.
      */
     normalizeVolume?: boolean
-    /** The viewer flipped the switch. */
+    /** The viewer flipped the switch, to `normalizeVolume`: what an app saves to pass back as `normalizeVolume`. */
     onNormalizeVolumeChange?: (normalizeVolume: boolean) => void
   }
 
@@ -246,7 +257,7 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
   // local arm and is absent when the media is remote.
   const { title, chapters } = options
   const {
-    size, downloadedRanges, publicPath, libavWorkerUrl, read, thumbnailRead, thumbnailsEnabled,
+    size, downloadedRanges, publicPath, libavWorkerUrl, read, thumbnailRead, thumbnailsEnabled, readsDownload,
   } = local ?? ({} as Partial<MediaPlayerLocalOptions>)
 
   const setMedia = useMediaAttach()
@@ -294,17 +305,46 @@ const PlayerRoot = ({ options, children }: { options: MediaPlayerOptions, childr
     return () => { dropped = true; stop?.() }
   }, [media, !!expose, exposeId, exposeOrigin, exposeChannel])
 
+  // one per file, as `size` is the source's identity everywhere here
+  const tap = useMemo(() => createPlaybackTap(), [size])
+
   // Each of these no-ops on null inputs, which is what a remote arm supplies: it renders no <video>,
   // so there is nothing for them to attach to and nothing to guard at the call site.
-  usePlayback(video, subtitleLayer, local)
+  usePlayback(video, subtitleLayer, local, readsDownload ? tap.push : undefined)
 
-  const normalizeVolumeProp = local?.normalizeVolume
-  const [normalizeVolume, setNormalizeVolume] = useState(!!normalizeVolumeProp)
-  useEffect(() => { setNormalizeVolume(!!normalizeVolumeProp) }, [normalizeVolumeProp])
+  const normalizeVolumeProp = local?.normalizeVolume ?? true
+  const [normalizeVolume, setNormalizeVolume] = useState(normalizeVolumeProp)
+  useEffect(() => { setNormalizeVolume(normalizeVolumeProp) }, [normalizeVolumeProp])
   const onNormalizeVolumeChange = useRef(local?.onNormalizeVolumeChange)
   onNormalizeVolumeChange.current = local?.onNormalizeVolumeChange
   const normalizerWorkletUrl = local?.normalizerWorkletUrl
-  const { refused: normalizerRefused } = useVolumeNormalizer(video, normalizerWorkletUrl, normalizeVolume)
+  // the background reader, through a ref: a streaming consumer passes a fresh closure several times a second
+  const analysisRead = useRef(thumbnailRead ?? read)
+  analysisRead.current = thumbnailRead ?? read
+  const playingAudio = usePlayer((state) => state.selectedAudioTrack)
+  const hasRead = !!read
+  const normalizerSource = useMemo(
+    () => hasRead && size && publicPath && libavWorkerUrl && typeof playingAudio === 'number' && playingAudio >= 0
+      ? readsDownload
+        ? { tap, audioStreamIndex: playingAudio }
+        : {
+          read: (offset: number, length: number) => analysisRead.current!(offset, length),
+          length: size,
+          publicPath,
+          libavWorkerUrl,
+          audioStreamIndex: playingAudio,
+        }
+      : undefined,
+    [hasRead, size, publicPath, libavWorkerUrl, playingAudio, readsDownload, tap],
+  )
+  const normalizerRanges = useMemo(
+    () => downloadedRanges?.map(({ startByteOffset, endByteOffset }) => [startByteOffset, endByteOffset] as [number, number]),
+    // a streaming consumer changes the array identity every tick, so compare contents
+    [downloadedRanges?.map(({ startByteOffset, endByteOffset }) => `${startByteOffset}-${endByteOffset}`).join(',')],
+  )
+  const { refused: normalizerRefused } = useVolumeNormalizer(
+    video, normalizerWorkletUrl, normalizeVolume, normalizerSource, normalizerRanges,
+  )
   const volumeNormalizer = useMemo<VolumeNormalizerControl | null>(
     () => normalizerWorkletUrl && typeof AudioWorkletNode === 'function'
       ? {

package/src/lib/utils/source.ts

@@ -42,14 +42,18 @@ const fromUrl = async (
     if (!response.ok) throw new Error(`The source could not be read: HTTP ${response.status}`)
     return response.arrayBuffer()
   }
-  return { length, name, read }
+  return { length, name, read, readsDownload: true }
 }
 
-/** A source resolved down to what the remuxer needs: a total length and a byte-range reader. */
+/**
+ * A source resolved down to what the remuxer needs: a total length and a byte-range reader, and whether each read
+ * is a download, which the player's `readsDownload` takes.
+ */
 export type RemuxerInput = {
   length: number
   name?: string
   read: (offset: number, size: number) => Promise<ArrayBuffer>
+  readsDownload?: boolean
 }
 
 export const inputToRemuxerInput = async (