@greendrake/mse-video 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Eugene Greendrake
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,58 @@
-# Temporary Holding Version
+# @greendrake/mse-video
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Live camera video in the browser over Media Source Extensions: H.265 in fragmented MP4
+and the camera's G.711 audio, every stream of a page over one WebSocket, recovering on
+its own from stalls and dropped sockets. A framework-free core and a thin Vue component.
+The server end is `greendrake.nz/go/webcast`, whose README has the socket's protocol.
+
+```sh
+bun add @greendrake/mse-video
+```
+
+## Component
+
+`Video`, from `@greendrake/mse-video/vue` (`vue` is an optional peer, needed for this
+entry only), takes props `hubUrl` (the stream hub's WebSocket URL), `streamKey` (which
+stream on it, a key the server knows) and `audio` (whether the camera's sound plays). It
+fills its box, stretched; a browser that cannot play the stream gets a line saying so
+instead of a player. Its root carries the player's state as `data-state`, or
+`unsupported`. A changed `streamKey` switches the stream in place.
+
+The video element stays muted: the sound plays through the page's audio context, which
+browsers keep silent until a user gesture. Call `unlockAudio()` from the click that turns
+sound on.
+
+## Core
+
+- `streamCodec` is the content type of every stream a webcast server casts, H.265 in
+  fMP4. `canPlay()` says whether the browser can play it, through Apple's
+  `ManagedMediaSource` where it exists (Safari 17+, iPhones) and `MediaSource` otherwise.
+- `MSEVideo(videoEl, hubURL, key, options?)` plays one stream into a video element;
+  construct one only where `canPlay()` holds. `setKey` switches the stream, `setAudio`
+  turns the sound on or off, `close` lets go of the stream and the element. Its `state`
+  (`closed`, `opening`, `listening`, `playing`, `resetting`, `closing`) is emitted as
+  `state` on each change; `error` reports a failure it is recovering from.
+- Every player pointed at one hub URL shares one socket, which goes with the last of
+  them. A dropped socket reconnects with exponential backoff, from `reopenDelayMs` (1 s)
+  to `maxReopenDelayMs` (30 s) — `MSEVideoOptions.hub`, a `StreamHubOptions`, taken from
+  whichever player opens the socket — and subscribes every stream again.
+
+## Recovery
+
+`MSEVideoOptions`:
+
+- Playing with no packet for `noDataTimeoutMs` (3 s), the player rebuilds its
+  MediaSource in place and asks the server to reset the stream, which restarts it from a
+  fresh initialisation segment.
+- Nothing within `startTimeoutMs` (9 s) of subscribing or resetting, a subscription not
+  confirmed within `connectTimeoutMs` (10 s), a stream the server refuses or stops, or a
+  MediaSource that cannot be built: the player drops the subscription and takes out a
+  new one, backing off from `reopenDelayMs` (3 s) to `maxReopenDelayMs` (30 s) until a
+  packet arrives.
+- Playback is held near the newest frame.
+
+## Audio
+
+Audio arrives as frames of its own, G.711 A-law or µ-law, decoded here (no browser media
+pipeline plays G.711) and scheduled against the video element's clock: each frame plays
+when the video reaches it, and nothing plays while the video stands still.

package/package.json

@@ -1,6 +1,56 @@
 {
-  "name": "@greendrake/mse-video",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+    "name": "@greendrake/mse-video",
+    "version": "0.1.0",
+    "description": "MSE live video over WebSocket: framework-free playback core with stall watchdogs and a server reset protocol, plus a thin Vue wrapper",
+    "license": "MIT",
+    "author": "Eugene Greendrake <eugene@greendrake.nz>",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/greendrake/lib.git",
+        "directory": "mse-video"
+    },
+    "homepage": "https://github.com/greendrake/lib/tree/main/mse-video#readme",
+    "bugs": {
+        "url": "https://github.com/greendrake/lib/issues"
+    },
+    "type": "module",
+    "main": "src/main.ts",
+    "types": "src/main.ts",
+    "exports": {
+        ".": {
+            "types": "./src/main.ts",
+            "default": "./src/main.ts"
+        },
+        "./vue": {
+            "types": "./src/vue.ts",
+            "default": "./src/vue.ts"
+        }
+    },
+    "files": [
+        "src"
+    ],
+    "publishConfig": {
+        "access": "public"
+    },
+    "scripts": {
+        "check": "vue-tsc --noEmit"
+    },
+    "prettier": "@greendrake/dev-config/prettier",
+    "dependencies": {
+        "@greendrake/util": "^0.9.0"
+    },
+    "peerDependencies": {
+        "vue": "^3.5.34"
+    },
+    "peerDependenciesMeta": {
+        "vue": {
+            "optional": true
+        }
+    },
+    "devDependencies": {
+        "@greendrake/dev-config": "0.7.0",
+        "typescript": "6.0.3",
+        "vue": "^3.5.34",
+        "vue-tsc": "^2.2.0"
+    }
+}

package/src/AudioPlayer.ts

@@ -0,0 +1,131 @@
+import { decodeG711, type G711Codec } from './g711'
+
+// One frame of a camera's audio, as the stream hub delivers it.
+export interface AudioFrame {
+    // When the frame is due on the timeline of the video it belongs to.
+    ptsMs: number
+    sampleRate: number
+    codec: G711Codec
+    samples: Uint8Array
+}
+
+// What the player needs of the video element showing the stream: its clock, and the
+// events that say whether that clock is running.
+export type AudioClock = EventTarget & Pick<HTMLVideoElement, 'currentTime' | 'paused' | 'seeking' | 'readyState'>
+
+// Where the player's samples go: a clock, and the scheduling of samples on it.
+export interface AudioOutput {
+    readonly currentTime: number
+    // Plays the samples at the given time on the output's clock, calling ended once
+    // they have played or been stopped. Returns what stops them.
+    schedule(samples: Float32Array<ArrayBuffer>, sampleRate: number, at: number, ended: () => void): () => void
+}
+
+// How far the next frame may be due from where the last one ends and still follow it
+// without a gap. Beyond it the frame is put where the video says, at the cost of a gap
+// or of what was queued.
+const TOLERANCE_S = 0.06
+
+// The video stops showing what is due on its clock on these.
+const HALTS = ['waiting', 'pause', 'seeking', 'emptied'] as const
+
+// HTMLMediaElement.HAVE_FUTURE_DATA: the video has what it needs to keep playing.
+const HAVE_FUTURE_DATA = 3
+
+let context: AudioContext | undefined
+
+// The page's one audio context.
+const audioContext = (): AudioContext => {
+    context ??= new AudioContext()
+    return context
+}
+
+// Browsers keep an audio context silent until a user gesture lets it play, and some
+// only during the gesture itself: call this from the handler of the click that turns
+// sound on.
+export const unlockAudio = (): void => {
+    void audioContext().resume()
+}
+
+const contextOutput = (context: AudioContext): AudioOutput => ({
+    get currentTime() {
+        return context.currentTime
+    },
+    schedule(samples, sampleRate, at, ended) {
+        const buffer = context.createBuffer(1, samples.length, sampleRate)
+        buffer.copyToChannel(samples, 0)
+        const source = context.createBufferSource()
+        source.buffer = buffer
+        source.connect(context.destination)
+        source.addEventListener('ended', ended)
+        source.start(at)
+        return () => source.stop()
+    }
+})
+
+// Plays a stream's audio in step with the video element showing it. The video's clock
+// is the master: each frame is scheduled for when the video will reach it, frames
+// follow each other back to back while they stay within reach of that, and nothing
+// plays while the video stands still.
+export class AudioPlayer {
+    readonly #video: AudioClock
+    readonly #output: AudioOutput
+    readonly #abort = new AbortController()
+    #running: boolean
+    // Output time at which the last scheduled frame ends.
+    #queuedUntil = 0
+    // What stops each scheduled frame that has yet to end.
+    readonly #queued = new Set<() => void>()
+
+    constructor(video: AudioClock, output: AudioOutput = contextOutput(audioContext())) {
+        this.#video = video
+        this.#output = output
+        this.#running = !video.paused && !video.seeking && video.readyState >= HAVE_FUTURE_DATA
+        const signal = this.#abort.signal
+        video.addEventListener('playing', () => (this.#running = true), { signal })
+        for (const event of HALTS) {
+            video.addEventListener(
+                event,
+                () => {
+                    this.#running = false
+                    this.flush()
+                },
+                { signal }
+            )
+        }
+    }
+
+    push(frame: AudioFrame): void {
+        if (!this.#running) {
+            return
+        }
+        const now = this.#output.currentTime
+        const due = now + frame.ptsMs / 1000 - this.#video.currentTime
+        if (due < now) {
+            return
+        }
+        let at = this.#queuedUntil
+        if (Math.abs(due - at) > TOLERANCE_S) {
+            if (due < at) {
+                // The video has moved on from what is queued.
+                this.flush()
+            }
+            at = due
+        }
+        const stop = this.#output.schedule(decodeG711(frame.codec, frame.samples), frame.sampleRate, at, () => this.#queued.delete(stop))
+        this.#queued.add(stop)
+        this.#queuedUntil = at + frame.samples.length / frame.sampleRate
+    }
+
+    // Silences whatever is queued.
+    flush(): void {
+        this.#queued.forEach(stop => stop())
+        this.#queued.clear()
+        this.#queuedUntil = 0
+    }
+
+    close(): void {
+        this.#abort.abort()
+        this.flush()
+    }
+}

package/src/Backoff.ts

@@ -0,0 +1,24 @@
+// Exponential backoff for retry loops. The first delay after a reset is zero, so a
+// one-off failure recovers immediately; each further failure without a reset doubles
+// the wait up to a cap.
+export class Backoff {
+    readonly #baseMs: number
+    readonly #maxMs: number
+    #delayMs = 0
+
+    constructor(baseMs: number, maxMs: number) {
+        this.#baseMs = baseMs
+        this.#maxMs = maxMs
+    }
+
+    // How long to wait before the attempt being made now, advancing the sequence.
+    next(): number {
+        const delayMs = this.#delayMs
+        this.#delayMs = delayMs === 0 ? this.#baseMs : Math.min(delayMs * 2, this.#maxMs)
+        return delayMs
+    }
+
+    reset(): void {
+        this.#delayMs = 0
+    }
+}

package/src/MSEVideo.ts

@@ -0,0 +1,365 @@
+import { Emitter } from '@greendrake/util'
+import { AudioPlayer } from './AudioPlayer'
+import { acquireStreamHub, type StreamHub, type StreamHubOptions, type StreamSubscription } from './StreamHub'
+import { Backoff } from './Backoff'
+import { canPlay, mediaSourceOf, streamCodec } from './codec'
+import { followLiveEdge } from './liveEdge'
+
+export type MSEVideoState = 'closed' | 'opening' | 'listening' | 'playing' | 'resetting' | 'closing'
+
+export interface MSEVideoEvents {
+    // Fired on every state transition.
+    state: [state: MSEVideoState]
+    // Fired when a recoverable failure triggers a reset or reopen; the player
+    // recovers by itself, so this is diagnostic, not fatal.
+    error: [reason: string]
+    [key: string]: unknown[]
+}
+
+export interface MSEVideoOptions {
+    // The shared socket's, taken from whichever player opens it: every player on
+    // the page pointed at one hub URL shares that one socket.
+    hub?: StreamHubOptions
+    // Base delay before rebuilding the pipeline, doubling on each reopen that brings
+    // no data and capping at maxReopenDelayMs. The first reopen after a packet has
+    // arrived is immediate.
+    reopenDelayMs?: number
+    maxReopenDelayMs?: number
+    // While playing: max silence between appended packets before an in-place reset.
+    noDataTimeoutMs?: number
+    // After subscribing or resetting: max wait for the first packet before a full reopen.
+    startTimeoutMs?: number
+    // Max wait for the hub to confirm the subscription, which covers the shared socket
+    // connecting or reconnecting underneath it.
+    connectTimeoutMs?: number
+}
+
+const DEFAULT_REOPEN_DELAY_MS = 3000
+const DEFAULT_MAX_REOPEN_DELAY_MS = 30000
+const DEFAULT_NO_DATA_TIMEOUT_MS = 3000
+const DEFAULT_START_TIMEOUT_MS = 9000
+const DEFAULT_CONNECT_TIMEOUT_MS = 10000
+
+const concatArrayBuffers = (...bufs: ArrayBuffer[]): ArrayBuffer => {
+    const result = new Uint8Array(bufs.reduce((totalSize, buf) => totalSize + buf.byteLength, 0))
+    bufs.reduce((offset, buf) => {
+        result.set(new Uint8Array(buf), offset)
+        return offset + buf.byteLength
+    }, 0)
+    return result.buffer
+}
+
+// MSE live playback engine, fed by one stream of a StreamHub's shared socket.
+//
+// The server streams an fMP4 initialization segment followed by media segments;
+// each payload the hub routes here is appended to a single SourceBuffer.
+//
+// On Safari the source is a ManagedMediaSource, which leaves buffer eviction to
+// the user agent. Its startstreaming/endstreaming throttling hints are not
+// honoured: this is a live feed with no seekable past, so there is nothing to
+// prefetch and nothing to defer.
+//
+// Reset protocol (application-level): when playback stalls (no packet appended
+// within noDataTimeoutMs while playing), the player keeps the subscription but
+// tears down and rebuilds the MediaSource + SourceBuffer, then asks the server to
+// reset. The server acknowledges with a zero-length payload and restarts the
+// stream from a fresh initialization segment; on receiving the ack while
+// 'resetting' the player returns to 'listening' and the next payload resumes
+// playback. If nothing arrives within startTimeoutMs, the player falls back to a
+// full reopen: drop the subscription and take out a new one.
+//
+// Every other failure is a full reopen too: a subscription the hub does not
+// confirm within connectTimeoutMs, a stream the server refuses or stops serving,
+// a MediaSource or SourceBuffer that cannot be built. Consecutive reopens back off
+// exponentially, and the first packet to arrive clears the backoff.
+//
+// A socket that drops is the hub's business, not this class's: it reconnects and
+// re-subscribes, and the 'subscribed' event that follows rebuilds the pipeline here
+// around the fresh initialization segment the server starts with.
+//
+// Playback is kept close to the newest frame by followLiveEdge.
+//
+// Audio, when on, arrives as frames of its own and plays through an AudioPlayer slaved
+// to the video element's clock.
+export class MSEVideo extends Emitter<MSEVideoEvents> {
+    readonly #videoEl: HTMLVideoElement
+    // Safari is the only engine that attaches a media source object directly,
+    // and the only one with ManagedMediaSource, so one flag selects both. Read
+    // as a player is made rather than as the module loads, which may be where
+    // there is no window.
+    readonly #managed = Boolean(window.ManagedMediaSource)
+    readonly #backoff: Backoff
+    readonly #noDataTimeoutMs: number
+    readonly #startTimeoutMs: number
+    readonly #connectTimeoutMs: number
+    readonly #hub: StreamHub
+    // Removes the listeners this player puts on the video element.
+    readonly #elementAbort = new AbortController()
+    #key: string
+    #subscription?: StreamSubscription
+    #state: MSEVideoState = 'closed'
+    #mse?: MediaSource
+    #sourceBuffer?: SourceBuffer
+    #bufferAbort?: AbortController
+    #pendingBuffer?: ArrayBuffer
+    #watchdog?: ReturnType<typeof setTimeout>
+    #reopenTimer?: ReturnType<typeof setTimeout>
+    // Exists while the audio is on.
+    #audio?: AudioPlayer
+
+    // Construct one only in a browser canPlay says can play the stream.
+    constructor(videoEl: HTMLVideoElement, hubURL: string, key: string, options: MSEVideoOptions = {}) {
+        super()
+        // Before anything that has to be handed back, the hub reference in particular.
+        if (!canPlay()) {
+            throw new Error(`Unsupported codec: ${streamCodec}`)
+        }
+        this.#videoEl = videoEl
+        this.#hub = acquireStreamHub(hubURL, options.hub)
+        this.#key = key
+        this.#backoff = new Backoff(
+            options.reopenDelayMs ?? DEFAULT_REOPEN_DELAY_MS,
+            options.maxReopenDelayMs ?? DEFAULT_MAX_REOPEN_DELAY_MS
+        )
+        this.#noDataTimeoutMs = options.noDataTimeoutMs ?? DEFAULT_NO_DATA_TIMEOUT_MS
+        this.#startTimeoutMs = options.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS
+        this.#connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS
+        // A MediaSource cannot be handed to an AirPlay receiver, so WebKit keeps
+        // a ManagedMediaSource inert — sourceopen never fires — unless the element
+        // either opts out of remote playback or offers an HLS <source> alternative.
+        // Must be set before any src assignment.
+        videoEl.disableRemotePlayback = true
+        const signal = this.#elementAbort.signal
+        // play() is interrupted by the load() a reset/close performs; that
+        // rejection is part of normal teardown, not a failure to surface.
+        videoEl.addEventListener('loadeddata', () => void videoEl.play().catch(() => undefined), { signal })
+        videoEl.addEventListener('timeupdate', () => followLiveEdge(videoEl), { signal })
+        videoEl.addEventListener('progress', () => followLiveEdge(videoEl), { signal })
+        this.#open()
+    }
+
+    get state(): MSEVideoState {
+        return this.#state
+    }
+
+    setKey(key: string): void {
+        this.#key = key
+        // A different stream is a fresh start, not a retry of the failing one.
+        this.#backoff.reset()
+        this.#restart(0)
+    }
+
+    // Turns the stream's audio on or off. Call unlockAudio from the user gesture that
+    // turns it on, or the browser may keep it silent.
+    setAudio(on: boolean): void {
+        if (on === Boolean(this.#audio)) {
+            return
+        }
+        if (on) {
+            this.#audio = new AudioPlayer(this.#videoEl)
+        } else {
+            this.#audio!.close()
+            this.#audio = undefined
+        }
+        this.#subscription?.setAudio(on)
+    }
+
+    close(): void {
+        this.#elementAbort.abort()
+        this.#audio?.close()
+        this.#audio = undefined
+        this.#teardown()
+        this.#hub.release()
+    }
+
+    #teardown(): void {
+        if (this.#reopenTimer) {
+            clearTimeout(this.#reopenTimer)
+            this.#reopenTimer = undefined
+        }
+        this.#clearWatchdog()
+        this.#setState('closing')
+        this.#closeVideo()
+        this.#subscription?.close()
+        this.#subscription = undefined
+        this.#setState('closed')
+    }
+
+    #setState(state: MSEVideoState): void {
+        if (this.#state === state) {
+            return
+        }
+        this.#state = state
+        if (state === 'opening') {
+            this.#armWatchdog(this.#connectTimeoutMs)
+        } else if (state === 'listening' || state === 'resetting') {
+            this.#armWatchdog(this.#startTimeoutMs)
+        }
+        this.emit('state', state)
+    }
+
+    // The MediaSource is built only once the subscription is live, because that is when
+    // the server starts a fresh initialization segment — and it does so again on every
+    // re-subscribe the hub performs after a socket reconnect.
+    #open(): void {
+        this.#setState('opening')
+        const subscription = this.#hub.subscribe(this.#key)
+        this.#subscription = subscription
+        subscription.setAudio(Boolean(this.#audio))
+        subscription.on('subscribed', () => this.#onSubscribed())
+        subscription.on('packet', payload => this.#readPacket(payload))
+        subscription.on('audio', frame => {
+            // While resetting, what arrives belongs to the timeline being discarded.
+            if (this.#state !== 'resetting') {
+                this.#audio?.push(frame)
+            }
+        })
+        subscription.on('failed', reason => this.#reopen(reason))
+    }
+
+    #onSubscribed(): void {
+        this.#closeVideo()
+        this.#setState('opening')
+        this.#armWatchdog(this.#startTimeoutMs)
+        this.#setUp()
+    }
+
+    #setUp(): void {
+        this.#mse = new (mediaSourceOf(window)!)()
+        if (this.#managed) {
+            this.#videoEl.srcObject = this.#mse
+        } else {
+            this.#videoEl.src = URL.createObjectURL(this.#mse)
+            this.#videoEl.srcObject = null
+        }
+        this.#mse.addEventListener('sourceopen', () => this.#onSourceOpen(), { once: true })
+    }
+
+    #onSourceOpen(): void {
+        if (!this.#managed) {
+            URL.revokeObjectURL(this.#videoEl.src)
+        }
+        try {
+            this.#sourceBuffer = this.#mse!.addSourceBuffer(streamCodec)
+        } catch {
+            this.#reopen('Could not create SourceBuffer')
+            return
+        }
+        this.#sourceBuffer.mode = 'segments'
+        this.#bufferAbort = new AbortController()
+        this.#sourceBuffer.addEventListener('updateend', () => this.#eatPacket(), { signal: this.#bufferAbort.signal })
+        if (this.#state === 'opening') {
+            this.#setState('listening')
+            // Whatever arrived while the pipeline was being built, initialization
+            // segment included.
+            this.#eatPacket()
+        } else if (this.#state === 'resetting') {
+            this.#subscription!.reset()
+        } else {
+            throw new Error(`Invalid state to set up: ${this.#state}`)
+        }
+    }
+
+    #readPacket(packet: ArrayBuffer): void {
+        // The server is alive and talking, so the next failure starts a fresh backoff.
+        this.#backoff.reset()
+        if (this.#state === 'resetting') {
+            if (packet.byteLength === 0) {
+                // The server's reset acknowledgement.
+                this.#setState('listening')
+            }
+            // Anything ahead of it belongs to the stream being discarded.
+            return
+        }
+        // The subscription goes live a turn of the event loop before the SourceBuffer
+        // it feeds exists, so packets accumulate whether or not there is one yet.
+        this.#pendingBuffer = this.#pendingBuffer ? concatArrayBuffers(this.#pendingBuffer, packet) : packet
+        this.#eatPacket()
+    }
+
+    // Append the accumulated bytes to the SourceBuffer once there is one and it is not
+    // busy; until then packets pile up in #pendingBuffer, and the 'updateend' listener
+    // re-enters here to flush them.
+    #eatPacket(): void {
+        if (!this.#sourceBuffer || !this.#pendingBuffer) {
+            return
+        }
+        // Data with a pipeline to take it: playing, whether it has just arrived or has
+        // been waiting for the SourceBuffer to be built or to finish updating.
+        this.#setState('playing')
+        if (this.#sourceBuffer.updating) {
+            return
+        }
+        this.#armWatchdog(this.#noDataTimeoutMs)
+        const packet = this.#pendingBuffer
+        this.#pendingBuffer = undefined
+        try {
+            this.#sourceBuffer.appendBuffer(packet)
+        } catch {
+            this.#reset('Append buffer error')
+        }
+    }
+
+    // In-place reset: rebuild the MediaSource pipeline but keep the WebSocket,
+    // asking the server (via #onSourceOpen) to restart the stream.
+    #reset(reason: string): void {
+        if (this.#state !== 'playing') {
+            return
+        }
+        this.#clearWatchdog()
+        this.emit('error', reason)
+        this.#setState('resetting')
+        this.#closeVideo()
+        this.#setUp()
+    }
+
+    // Full recovery: drop the subscription and take out a new one. Repeated reopens back
+    // off, so a camera that is down does not have every player watching it churning a
+    // subscription — and a Stream node, and a camera dial — on every timeout.
+    #reopen(reason: string): void {
+        this.emit('error', reason)
+        this.#restart(this.#backoff.next())
+    }
+
+    #restart(delayMs: number): void {
+        this.#teardown()
+        if (delayMs > 0) {
+            this.#reopenTimer = setTimeout(() => this.#open(), delayMs)
+        } else {
+            this.#open()
+        }
+    }
+
+    #closeVideo(): void {
+        this.#bufferAbort?.abort()
+        this.#bufferAbort = undefined
+        this.#pendingBuffer = undefined
+        this.#sourceBuffer = undefined
+        this.#videoEl.pause()
+        if (this.#managed) {
+            this.#videoEl.srcObject = null
+        } else {
+            this.#videoEl.removeAttribute('src')
+        }
+        this.#videoEl.load()
+        this.#mse = undefined
+    }
+
+    #armWatchdog(ms: number): void {
+        this.#clearWatchdog()
+        this.#watchdog = setTimeout(() => {
+            if (this.#state === 'playing') {
+                this.#reset('No video timeout')
+            } else {
+                this.#reopen(`No video timeout on ${this.#state}`)
+            }
+        }, ms)
+    }
+
+    #clearWatchdog(): void {
+        if (this.#watchdog) {
+            clearTimeout(this.#watchdog)
+            this.#watchdog = undefined
+        }
+    }
+}

package/src/StreamHub.ts

@@ -0,0 +1,316 @@
+import { Emitter } from '@greendrake/util'
+import type { AudioFrame } from './AudioPlayer'
+import { Backoff } from './Backoff'
+import type { G711Codec } from './g711'
+
+export interface StreamSubscriptionEvents {
+    // The subscription is live on the server. Fired on the first subscribe and again
+    // every time the hub re-establishes it on a new socket, where the server restarts
+    // the stream from a fresh initialization segment: the consumer must rebuild its
+    // decoding pipeline each time.
+    subscribed: []
+    // One fMP4 payload. A zero-length payload is the server's reset acknowledgement.
+    packet: [payload: ArrayBuffer]
+    // One frame of the camera's audio, while the subscription has it on.
+    audio: [frame: AudioFrame]
+    // The server will not serve this stream, or has stopped serving it. The consumer
+    // decides whether and when to subscribe again.
+    failed: [reason: string]
+    [key: string]: unknown[]
+}
+
+export interface StreamHubOptions {
+    // Base delay before reconnecting a dropped socket, doubling up to maxReopenDelayMs.
+    reopenDelayMs?: number
+    maxReopenDelayMs?: number
+}
+
+const DEFAULT_REOPEN_DELAY_MS = 1000
+const DEFAULT_MAX_REOPEN_DELAY_MS = 30000
+
+interface ServerMessage {
+    op: 'subbed' | 'error' | 'gone'
+    id?: number
+    ch?: number
+    key?: string
+}
+
+// What a binary frame's payload is: the byte after the channel byte.
+const KIND_VIDEO = 0
+const KIND_AUDIO = 1
+
+// An audio payload's codec byte.
+const AUDIO_CODECS: G711Codec[] = ['alaw', 'ulaw']
+
+// An audio payload is [u32 ms][u16 sample rate][u8 codec][samples], big-endian.
+const AUDIO_HEADER = 7
+
+const audioFrame = (payload: DataView): AudioFrame => ({
+    ptsMs: payload.getUint32(0),
+    sampleRate: payload.getUint16(4),
+    codec: AUDIO_CODECS[payload.getUint8(6)],
+    samples: new Uint8Array(payload.buffer, payload.byteOffset + AUDIO_HEADER, payload.byteLength - AUDIO_HEADER)
+})
+
+export class StreamSubscription extends Emitter<StreamSubscriptionEvents> {
+    readonly key: string
+    readonly id: number
+    readonly #hub: StreamHub
+    // Assigned by the server, and cleared whenever the socket carrying it goes away.
+    channel?: number
+    // Whether the server is to send the audio: off until asked for, on every socket.
+    #audio = false
+
+    constructor(hub: StreamHub, key: string, id: number) {
+        super()
+        this.#hub = hub
+        this.key = key
+        this.id = id
+    }
+
+    get audio(): boolean {
+        return this.#audio
+    }
+
+    setAudio(on: boolean): void {
+        this.#audio = on
+        this.#hub.sendAudio(this)
+    }
+
+    // Ask the server to restart this stream from a fresh initialization segment. It
+    // answers with a zero-length packet.
+    reset(): void {
+        this.#hub.resetSubscription(this)
+    }
+
+    close(): void {
+        this.#hub.unsubscribe(this)
+    }
+}
+
+// One WebSocket per host, shared by every player on the page and demultiplexed by the
+// channel byte the server puts in front of each payload.
+//
+// Browsers hold at most one connection per host in CONNECTING (RFC 6455 4.1), so a grid
+// of N tiles on N sockets pays N serialized handshakes plus the browser's own growing
+// inter-handshake delay. One socket removes that entirely.
+export class StreamHub {
+    readonly #url: string
+    readonly #onIdle: () => void
+    readonly #backoff: Backoff
+    #ws?: WebSocket
+    #abort?: AbortController
+    #reopenTimer?: ReturnType<typeof setTimeout>
+    #refs = 0
+    #nextId = 1
+    #subscriptions = new Set<StreamSubscription>()
+    #byChannel = new Map<number, StreamSubscription>()
+    // Subscribe requests awaiting an answer, by the id they were sent with.
+    #awaiting = new Map<number, StreamSubscription>()
+
+    constructor(url: string, onIdle: () => void, options: StreamHubOptions = {}) {
+        this.#url = url
+        this.#onIdle = onIdle
+        this.#backoff = new Backoff(options.reopenDelayMs ?? DEFAULT_REOPEN_DELAY_MS, options.maxReopenDelayMs ?? DEFAULT_MAX_REOPEN_DELAY_MS)
+    }
+
+    retain(): void {
+        this.#refs++
+    }
+
+    release(): void {
+        this.#refs--
+        if (this.#refs <= 0) {
+            this.#close()
+            this.#onIdle()
+        }
+    }
+
+    subscribe(key: string): StreamSubscription {
+        const subscription = new StreamSubscription(this, key, this.#nextId++)
+        this.#subscriptions.add(subscription)
+        if (this.#ws?.readyState === WebSocket.OPEN) {
+            this.#request(subscription)
+        } else {
+            this.#connect()
+        }
+        return subscription
+    }
+
+    unsubscribe(subscription: StreamSubscription): void {
+        this.#subscriptions.delete(subscription)
+        this.#awaiting.delete(subscription.id)
+        if (subscription.channel !== undefined) {
+            this.#byChannel.delete(subscription.channel)
+            this.#send({ op: 'unsub', ch: subscription.channel })
+            subscription.channel = undefined
+        }
+    }
+
+    resetSubscription(subscription: StreamSubscription): void {
+        this.#sendOn(subscription, { op: 'reset' })
+    }
+
+    // Tells the server whether to send the subscription's audio. One without a channel
+    // yet gets it with its next one.
+    sendAudio(subscription: StreamSubscription): void {
+        this.#sendOn(subscription, { op: 'audio', on: subscription.audio })
+    }
+
+    // Sends an op about a subscription's channel, if it has one.
+    #sendOn(subscription: StreamSubscription, op: object): void {
+        if (subscription.channel !== undefined) {
+            this.#send({ ...op, ch: subscription.channel })
+        }
+    }
+
+    #request(subscription: StreamSubscription): void {
+        this.#awaiting.set(subscription.id, subscription)
+        this.#send({
+            op: 'sub',
+            id: subscription.id,
+            key: subscription.key
+        })
+    }
+
+    #send(message: object): void {
+        if (this.#ws?.readyState === WebSocket.OPEN) {
+            this.#ws.send(JSON.stringify(message))
+        }
+    }
+
+    #connect(): void {
+        if (this.#ws || this.#reopenTimer) {
+            return
+        }
+        try {
+            this.#ws = new WebSocket(this.#url)
+        } catch {
+            this.#onDown()
+            return
+        }
+        this.#ws.binaryType = 'arraybuffer'
+        this.#abort = new AbortController()
+        const signal = this.#abort.signal
+        this.#ws.addEventListener('open', () => this.#onOpen(), { signal })
+        this.#ws.addEventListener('message', event => this.#onMessage(event.data), { signal })
+        this.#ws.addEventListener('error', () => this.#onDown(), { signal })
+        this.#ws.addEventListener('close', () => this.#onDown(), { signal })
+    }
+
+    #onOpen(): void {
+        this.#subscriptions.forEach(subscription => this.#request(subscription))
+    }
+
+    #onMessage(data: ArrayBuffer | string): void {
+        // The socket is not merely open but carrying traffic, so the next drop starts a
+        // fresh backoff. Clearing it on the handshake alone would leave no backoff at all
+        // against a server that accepts connections and then drops them.
+        this.#backoff.reset()
+        if (typeof data === 'string') {
+            this.#onServerMessage(JSON.parse(data) as ServerMessage)
+            return
+        }
+        const frame = new DataView(data)
+        const subscription = this.#byChannel.get(frame.getUint8(0))
+        if (!subscription) {
+            return
+        }
+        if (data.byteLength === 1) {
+            // A bare channel byte acknowledges a reset.
+            subscription.emit('packet', new ArrayBuffer(0))
+            return
+        }
+        const kind = frame.getUint8(1)
+        if (kind === KIND_VIDEO) {
+            subscription.emit('packet', data.slice(2))
+        } else if (kind === KIND_AUDIO) {
+            subscription.emit('audio', audioFrame(new DataView(data, 2)))
+        }
+    }
+
+    #onServerMessage(message: ServerMessage): void {
+        if (message.op === 'gone') {
+            const subscription = this.#byChannel.get(message.ch!)
+            if (subscription) {
+                this.#byChannel.delete(message.ch!)
+                this.#subscriptions.delete(subscription)
+                subscription.channel = undefined
+                subscription.emit('failed', 'Stream gone')
+            }
+            return
+        }
+        const subscription = this.#awaiting.get(message.id!)
+        this.#awaiting.delete(message.id!)
+        if (!subscription) {
+            // A subscription cancelled while its request was in flight. The channel the
+            // server just allocated for it would otherwise stay open forever.
+            if (message.op === 'subbed') {
+                this.#send({ op: 'unsub', ch: message.ch })
+            }
+            return
+        }
+        if (message.op === 'subbed') {
+            subscription.channel = message.ch
+            this.#byChannel.set(message.ch!, subscription)
+            // A new channel starts with its audio off.
+            if (subscription.audio) {
+                this.sendAudio(subscription)
+            }
+            subscription.emit('subscribed')
+        } else {
+            this.#subscriptions.delete(subscription)
+            subscription.emit('failed', `Server refused ${subscription.key}`)
+        }
+    }
+
+    #onDown(): void {
+        this.#dropSocket()
+        this.#byChannel.clear()
+        this.#awaiting.clear()
+        this.#subscriptions.forEach(subscription => {
+            subscription.channel = undefined
+        })
+        if (this.#refs <= 0 || this.#subscriptions.size === 0) {
+            return
+        }
+        // Every live subscription is re-sent once the socket is back, and each one is
+        // told it has restarted so it can rebuild around the fresh init segment.
+        this.#reopenTimer = setTimeout(() => {
+            this.#reopenTimer = undefined
+            this.#connect()
+        }, this.#backoff.next())
+    }
+
+    #dropSocket(): void {
+        this.#abort?.abort()
+        this.#abort = undefined
+        this.#ws?.close()
+        this.#ws = undefined
+    }
+
+    #close(): void {
+        if (this.#reopenTimer) {
+            clearTimeout(this.#reopenTimer)
+            this.#reopenTimer = undefined
+        }
+        this.#dropSocket()
+        this.#subscriptions.clear()
+        this.#byChannel.clear()
+        this.#awaiting.clear()
+    }
+}
+
+const hubs = new Map<string, StreamHub>()
+
+// Hubs are shared by URL and reference counted: every player on a page talks to the same
+// host over the same socket, and the socket goes away with the last of them.
+export const acquireStreamHub = (url: string, options?: StreamHubOptions): StreamHub => {
+    let hub = hubs.get(url)
+    if (!hub) {
+        hub = new StreamHub(url, () => hubs.delete(url), options)
+        hubs.set(url, hub)
+    }
+    hub.retain()
+    return hub
+}

package/src/Video.vue

@@ -0,0 +1,63 @@
+<template>
+    <div class="Video" :data-state="state">
+        <video v-if="supported" ref="videoEl" muted playsinline></video>
+        <p v-else class="unsupported">This browser cannot play the camera's H.265 video.</p>
+    </div>
+</template>
+<script setup lang="ts">
+import { onMounted, onUnmounted, ref, watch } from 'vue'
+import { canPlay } from './codec'
+import { MSEVideo, type MSEVideoState } from './MSEVideo'
+
+const props = defineProps<{
+    // WebSocket URL of the stream hub. Every player pointed at the same one shares a
+    // single socket.
+    hubUrl: string
+    // Which stream on that hub: a key its server knows.
+    streamKey: string
+    // Whether to play the camera's audio. The element itself stays muted: the audio
+    // plays through the page's audio context, which unlockAudio has to be called for,
+    // from the click that turns this on.
+    audio: boolean
+}>()
+
+// A browser that cannot play the stream gets no player at all, and says so.
+const supported = canPlay()
+const state = ref<MSEVideoState | 'unsupported'>(supported ? 'closed' : 'unsupported')
+const videoEl = ref<HTMLVideoElement | null>(null)
+let player: MSEVideo | undefined
+
+onMounted(() => {
+    if (!supported) {
+        return
+    }
+    player = new MSEVideo(videoEl.value!, props.hubUrl, props.streamKey)
+    player.on('state', s => (state.value = s))
+    state.value = player.state
+    player.setAudio(props.audio)
+})
+onUnmounted(() => player?.close())
+watch(
+    () => props.streamKey,
+    key => player?.setKey(key)
+)
+watch(
+    () => props.audio,
+    on => player?.setAudio(on)
+)
+</script>
+<style scoped>
+.Video,
+video {
+    width: 100%;
+    height: 100%;
+}
+video {
+    object-fit: fill;
+}
+.unsupported {
+    margin: 0;
+    padding: 1em;
+    text-align: center;
+}
+</style>

package/src/codec.ts

@@ -0,0 +1,28 @@
+declare global {
+    interface Window {
+        // Apple's Managed Media Source (Safari 17+), the only Media Source flavour
+        // iPhones expose: plain MediaSource is undefined there. TypeScript's DOM
+        // library has no definition yet; it is a MediaSource subclass, and the
+        // constructor type covers all of it used here — its extra streaming API is
+        // not. https://w3c.github.io/media-source/#managedmediasource
+        ManagedMediaSource?: typeof MediaSource
+    }
+}
+
+// The content type of every stream a webcast server casts: H.265 in fMP4.
+export const streamCodec = 'video/mp4; codecs="hvc1.1.6.L153.B0"'
+
+// A browser's Media Source constructors, as its globals have them.
+export interface MediaSources {
+    ManagedMediaSource?: typeof MediaSource
+    MediaSource?: typeof MediaSource
+}
+
+// The Media Source a browser has: Apple's Managed one where it exists, the standard one
+// otherwise, and none at all on an iPhone before Safari 17.
+export const mediaSourceOf = (browser: MediaSources): typeof MediaSource | undefined =>
+    browser.ManagedMediaSource ?? browser.MediaSource
+
+// Whether a browser can play the stream at all.
+export const canPlay = (browser: MediaSources = window): boolean =>
+    mediaSourceOf(browser)?.isTypeSupported(streamCodec) ?? false

package/src/g711.ts

@@ -0,0 +1,38 @@
+// G.711 decoding (ITU-T G.711), which no browser media pipeline offers: a table from
+// each 8-bit code to its sample.
+
+export type G711Codec = 'alaw' | 'ulaw'
+
+const alawSample = (code: number): number => {
+    const a = code ^ 0x55
+    const segment = (a & 0x70) >> 4
+    let magnitude = (a & 0x0f) << 4
+    magnitude += segment === 0 ? 8 : 0x108
+    if (segment > 1) {
+        magnitude <<= segment - 1
+    }
+    return a & 0x80 ? magnitude : -magnitude
+}
+
+const ulawSample = (code: number): number => {
+    const u = ~code & 0xff
+    const magnitude = (((u & 0x0f) << 3) + 0x84) << ((u & 0x70) >> 4)
+    return u & 0x80 ? 0x84 - magnitude : magnitude - 0x84
+}
+
+const table = (sample: (code: number) => number): Float32Array => Float32Array.from({ length: 256 }, (_, code) => sample(code) / 32768)
+
+const tables: Record<G711Codec, Float32Array> = {
+    alaw: table(alawSample),
+    ulaw: table(ulawSample)
+}
+
+// One sample per byte, scaled to [-1, 1).
+export const decodeG711 = (codec: G711Codec, codes: Uint8Array): Float32Array<ArrayBuffer> => {
+    const decode = tables[codec]
+    const samples = new Float32Array(codes.length)
+    for (let i = 0; i < codes.length; i++) {
+        samples[i] = decode[codes[i]]
+    }
+    return samples
+}

package/src/liveEdge.ts

@@ -0,0 +1,32 @@
+// Keeping a live stream's playback close to the newest frame it has. The server times
+// frames on the camera's own clock, so playback keeps pace with the camera, and the lag
+// a viewer starts with or picks up from a stall would otherwise stay.
+
+// In seconds of video buffered ahead of what is on screen: past CATCH_UP_LEAD playback
+// speeds up to CATCH_UP_RATE until the lead is back under CAUGHT_UP_LEAD; past
+// JUMP_LEAD it jumps to JUMP_TO_LEAD behind the newest frame.
+const CATCH_UP_LEAD = 0.5
+const CAUGHT_UP_LEAD = 0.2
+const CATCH_UP_RATE = 1.1
+const JUMP_LEAD = 2.5
+const JUMP_TO_LEAD = 0.3
+
+export type LiveVideo = Pick<HTMLVideoElement, 'buffered' | 'currentTime' | 'playbackRate'>
+
+// Steers playback towards the live edge; meant to run on every timeupdate and
+// progress event.
+export const followLiveEdge = (video: LiveVideo): void => {
+    const buffered = video.buffered
+    if (buffered.length === 0) {
+        return
+    }
+    const end = buffered.end(buffered.length - 1)
+    const lead = end - video.currentTime
+    if (lead > JUMP_LEAD) {
+        video.currentTime = end - JUMP_TO_LEAD
+    } else if (lead > CATCH_UP_LEAD) {
+        video.playbackRate = CATCH_UP_RATE
+    } else if (lead < CAUGHT_UP_LEAD) {
+        video.playbackRate = 1
+    }
+}

package/src/main.ts

@@ -0,0 +1,4 @@
+export { unlockAudio } from './AudioPlayer'
+export { canPlay, streamCodec } from './codec'
+export { MSEVideo, type MSEVideoEvents, type MSEVideoOptions, type MSEVideoState } from './MSEVideo'
+export type { StreamHubOptions } from './StreamHub'

package/src/vue.ts

@@ -0,0 +1,2 @@
+// Its own entry, so a page without Vue takes the core alone.
+export { default as Video } from './Video.vue'