@synapse-sdk/media 0.0.0-stage → 0.3.0

package/README.md

@@ -1,3 +1,3 @@
-# Temporary Holding Version
+# @synapse-sdk/media
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Part of the Synapse SDK for JavaScript. Documentation: the repository.

package/dist/annexb.d.ts

@@ -0,0 +1,93 @@
+import type { MediaFrame } from "@synapse-sdk/core";
+/** NAL unit types this package cares about (H.264 Table 7-1). */
+export declare const NAL_TYPE: {
+    readonly SLICE: 1;
+    readonly SLICE_DPA: 2;
+    readonly SLICE_DPB: 3;
+    readonly SLICE_DPC: 4;
+    readonly IDR: 5;
+    readonly SEI: 6;
+    readonly SPS: 7;
+    readonly PPS: 8;
+    readonly AUD: 9;
+};
+/** SEI payload type of a recovery point (H.264 D.1.8). */
+export declare const SEI_RECOVERY_POINT = 6;
+/** One NAL unit inside an Annex-B buffer. */
+export interface Nal {
+    /** Offset of the start code in the buffer. */
+    offset: number;
+    /** 3 or 4 (bytes of `00 00 01` / `00 00 00 01`). */
+    startCodeLen: number;
+    /** The NAL unit bytes after the start code: header byte, then the payload (still escaped). A view, not a copy. */
+    bytes: Uint8Array;
+}
+/** `nal_unit_type` (low 5 bits of the header byte); 0 for an empty NAL. */
+export declare function nalType(n: Nal): number;
+/** Payload after the 1-byte NAL header (escaped RBSP). */
+export declare function nalPayload(n: Nal): Uint8Array;
+/** A VCL NAL type: a coded slice (types 1-5). */
+export declare function isVclType(t: number): boolean;
+export declare function nalIsVcl(n: Nal): boolean;
+/** Start code plus NAL unit, as it sits in the stream (a view). */
+export declare function nalWithStartCode(buf: Uint8Array, n: Nal): Uint8Array;
+/** Next Annex-B start code (`00 00 01` or `00 00 00 01`) at or after `from`: `[offset, len]`. */
+export declare function findStartCode(buf: Uint8Array, from: number): [offset: number, len: number] | undefined;
+/**
+ * The NAL units of an Annex-B buffer, in order. Bytes before the first start code are skipped;
+ * trailing zero bytes (`trailing_zero_8bits`) stay attached to the NAL before them, which is
+ * harmless for type/flag decisions and keeps the bytes of a walked AU identical to the input.
+ */
+export declare function nalUnits(buf: Uint8Array): Nal[];
+/** The SPS + PPS NAL units of an access unit (with their start codes, in stream order), if it carries any. Defined ⇒ the AU deserves the `CONFIG` flag. */
+export declare function auParamSets(au: Uint8Array): Uint8Array | undefined;
+/**
+ * The sync points an Annex-B access unit offers.
+ * - `idr`: an IDR slice (NAL type 5) is present → instantaneous start point (`KEY`).
+ * - `recovery`: a recovery point SEI (NAL 6, payload type 6) is present → gradual start point,
+ *   clean one refresh period later (`RECOVERY`; x264 and nvenc both emit it at each intra-refresh sweep).
+ * Decided from the bitstream rather than the encoder's keyframe flag, which libx264 raises for
+ * both kinds and nvenc only for IDRs.
+ */
+export declare function auSyncPoints(au: Uint8Array): {
+    idr: boolean;
+    recovery: boolean;
+};
+/** An IDR is present: the AU is a keyframe in the `KEY` sense. */
+export declare function isKeyframe(au: Uint8Array): boolean;
+/** The AU carries at least one coded slice (otherwise it is parameter sets / SEI only). */
+export declare function hasVcl(au: Uint8Array): boolean;
+/** Whether an SEI NAL payload (after the NAL header byte) contains a recovery point message (payload type 6). */
+export declare function seiHasRecoveryPoint(payload: Uint8Array): boolean;
+/** The payload types of the SEI messages in an SEI NAL payload (D.1.1), in order. Emulation-prevention bytes are removed first. */
+export declare function seiPayloadTypes(payload: Uint8Array): number[];
+/** Strip H.264 emulation-prevention bytes (the `03` in `00 00 03`), giving the RBSP. */
+export declare function rbspUnescape(nal: Uint8Array): Uint8Array;
+/** The SPS of an AU (NAL bytes without start code), if present. */
+export declare function findSps(au: Uint8Array): Uint8Array | undefined;
+/** The WebCodecs / RFC 6381 codec string for an AU that carries an SPS: `avc1.` + profile_idc + constraint flags + level_idc as hex (e.g. `avc1.42c01e`). */
+export declare function codecStringFromSps(au: Uint8Array): string | undefined;
+/** `parts` joined into one new buffer (a single part is copied too, so the result never aliases the input). */
+export declare function concatBytes(parts: readonly Uint8Array[]): Uint8Array;
+/**
+ * Splits a continuous Annex-B byte stream (a file, an encoder pipe) into access units. Rule
+ * (dylithium's reader loop): an AU ends with its coded slice; the next NAL of any kind starts a
+ * new AU, so non-VCL NALs (SPS, PPS, SEI, AUD) are attached to the picture that follows them.
+ * Sufficient for one-slice-per-picture streams (x264 `slices=1`, nvenc, WebCodecs), which is what
+ * the encoder options enforce.
+ */
+export declare class AuSplitter {
+    #private;
+    /** Feeds bytes; returns every access unit completed by them (in order). */
+    push(bytes: Uint8Array): Uint8Array[];
+    /** End of stream: the last NAL (whose end no start code marks) closes the last AU, or starts one more if the current AU already has its slice. */
+    finish(): Uint8Array[];
+}
+/** Splits a whole Annex-B buffer into access units (see {@link AuSplitter}). */
+export declare function splitAccessUnits(stream: Uint8Array): Uint8Array[];
+/**
+ * A `MediaFrame` for an H.264 Annex-B access unit with `key`/`config`/`recovery` derived from the
+ * NAL units (IDR present / SPS+PPS present / recovery-point SEI present), so the header flags
+ * always agree with the bytes (R27 §2 publisher rule). `tUs` is the capture time (0 = stamp at publish).
+ */
+export declare function mediaFrameFromAnnexb(au: Uint8Array, tUs?: number): MediaFrame;

package/dist/annexb.js

@@ -0,0 +1,227 @@
+/** NAL unit types this package cares about (H.264 Table 7-1). */
+export const NAL_TYPE = { SLICE: 1, SLICE_DPA: 2, SLICE_DPB: 3, SLICE_DPC: 4, IDR: 5, SEI: 6, SPS: 7, PPS: 8, AUD: 9 };
+/** SEI payload type of a recovery point (H.264 D.1.8). */
+export const SEI_RECOVERY_POINT = 6;
+/** `nal_unit_type` (low 5 bits of the header byte); 0 for an empty NAL. */
+export function nalType(n) { return n.bytes.length ? n.bytes[0] & 0x1f : 0; }
+/** Payload after the 1-byte NAL header (escaped RBSP). */
+export function nalPayload(n) { return n.bytes.subarray(1); }
+/** A VCL NAL type: a coded slice (types 1-5). */
+export function isVclType(t) { return t >= NAL_TYPE.SLICE && t <= NAL_TYPE.IDR; }
+export function nalIsVcl(n) { return isVclType(nalType(n)); }
+/** Start code plus NAL unit, as it sits in the stream (a view). */
+export function nalWithStartCode(buf, n) { return buf.subarray(n.offset, n.offset + n.startCodeLen + n.bytes.length); }
+/** Next Annex-B start code (`00 00 01` or `00 00 00 01`) at or after `from`: `[offset, len]`. */
+export function findStartCode(buf, from) {
+    const n = buf.length;
+    for (let i = from; i + 3 <= n; i++) {
+        if (buf[i] === 0 && buf[i + 1] === 0) {
+            if (buf[i + 2] === 1)
+                return [i, 3];
+            if (i + 4 <= n && buf[i + 2] === 0 && buf[i + 3] === 1)
+                return [i, 4];
+        }
+    }
+    return undefined;
+}
+/**
+ * The NAL units of an Annex-B buffer, in order. Bytes before the first start code are skipped;
+ * trailing zero bytes (`trailing_zero_8bits`) stay attached to the NAL before them, which is
+ * harmless for type/flag decisions and keeps the bytes of a walked AU identical to the input.
+ */
+export function nalUnits(buf) {
+    const out = [];
+    let sc = findStartCode(buf, 0);
+    while (sc) {
+        const [s, l] = sc;
+        const start = s + l;
+        const next = findStartCode(buf, start);
+        out.push({ offset: s, startCodeLen: l, bytes: buf.subarray(start, next ? next[0] : buf.length) });
+        sc = next;
+    }
+    return out;
+}
+/** The SPS + PPS NAL units of an access unit (with their start codes, in stream order), if it carries any. Defined ⇒ the AU deserves the `CONFIG` flag. */
+export function auParamSets(au) {
+    const parts = [];
+    for (const n of nalUnits(au)) {
+        const t = nalType(n);
+        if (t === NAL_TYPE.SPS || t === NAL_TYPE.PPS)
+            parts.push(nalWithStartCode(au, n));
+    }
+    return parts.length ? concatBytes(parts) : undefined;
+}
+/**
+ * The sync points an Annex-B access unit offers.
+ * - `idr`: an IDR slice (NAL type 5) is present → instantaneous start point (`KEY`).
+ * - `recovery`: a recovery point SEI (NAL 6, payload type 6) is present → gradual start point,
+ *   clean one refresh period later (`RECOVERY`; x264 and nvenc both emit it at each intra-refresh sweep).
+ * Decided from the bitstream rather than the encoder's keyframe flag, which libx264 raises for
+ * both kinds and nvenc only for IDRs.
+ */
+export function auSyncPoints(au) {
+    let idr = false, recovery = false;
+    for (const n of nalUnits(au)) {
+        const t = nalType(n);
+        if (t === NAL_TYPE.IDR)
+            idr = true;
+        else if (t === NAL_TYPE.SEI)
+            recovery ||= seiHasRecoveryPoint(nalPayload(n));
+    }
+    return { idr, recovery };
+}
+/** An IDR is present: the AU is a keyframe in the `KEY` sense. */
+export function isKeyframe(au) { return auSyncPoints(au).idr; }
+/** The AU carries at least one coded slice (otherwise it is parameter sets / SEI only). */
+export function hasVcl(au) { return nalUnits(au).some(nalIsVcl); }
+/** Whether an SEI NAL payload (after the NAL header byte) contains a recovery point message (payload type 6). */
+export function seiHasRecoveryPoint(payload) { return seiPayloadTypes(payload).includes(SEI_RECOVERY_POINT); }
+/** The payload types of the SEI messages in an SEI NAL payload (D.1.1), in order. Emulation-prevention bytes are removed first. */
+export function seiPayloadTypes(payload) {
+    const rbsp = rbspUnescape(payload);
+    const out = [];
+    let i = 0;
+    // 0x80 is the rbsp_trailing_bits byte that ends the SEI NAL.
+    while (i < rbsp.length && rbsp[i] !== 0x80) {
+        // payloadType and payloadSize: runs of 0xFF each adding 255, then a final byte.
+        let ty = 0;
+        while (i < rbsp.length && rbsp[i] === 0xff) {
+            ty += 255;
+            i++;
+        }
+        if (i >= rbsp.length)
+            break;
+        ty += rbsp[i];
+        i++;
+        let sz = 0;
+        while (i < rbsp.length && rbsp[i] === 0xff) {
+            sz += 255;
+            i++;
+        }
+        if (i >= rbsp.length)
+            break;
+        sz += rbsp[i];
+        i++;
+        out.push(ty);
+        i += sz;
+    }
+    return out;
+}
+/** Strip H.264 emulation-prevention bytes (the `03` in `00 00 03`), giving the RBSP. */
+export function rbspUnescape(nal) {
+    const out = new Uint8Array(nal.length);
+    let o = 0, zeros = 0;
+    for (const b of nal) {
+        if (zeros >= 2 && b === 3) {
+            zeros = 0;
+            continue;
+        }
+        out[o++] = b;
+        zeros = b === 0 ? zeros + 1 : 0;
+    }
+    return out.subarray(0, o);
+}
+/** The SPS of an AU (NAL bytes without start code), if present. */
+export function findSps(au) {
+    for (const n of nalUnits(au))
+        if (nalType(n) === NAL_TYPE.SPS)
+            return n.bytes;
+    return undefined;
+}
+/** The WebCodecs / RFC 6381 codec string for an AU that carries an SPS: `avc1.` + profile_idc + constraint flags + level_idc as hex (e.g. `avc1.42c01e`). */
+export function codecStringFromSps(au) {
+    const sps = findSps(au);
+    if (!sps || sps.length < 4)
+        return undefined;
+    const hex = (b) => b.toString(16).padStart(2, "0");
+    return `avc1.${hex(sps[1])}${hex(sps[2])}${hex(sps[3])}`;
+}
+/** `parts` joined into one new buffer (a single part is copied too, so the result never aliases the input). */
+export function concatBytes(parts) {
+    let n = 0;
+    for (const p of parts)
+        n += p.length;
+    const out = new Uint8Array(n);
+    let o = 0;
+    for (const p of parts) {
+        out.set(p, o);
+        o += p.length;
+    }
+    return out;
+}
+/**
+ * Splits a continuous Annex-B byte stream (a file, an encoder pipe) into access units. Rule
+ * (dylithium's reader loop): an AU ends with its coded slice; the next NAL of any kind starts a
+ * new AU, so non-VCL NALs (SPS, PPS, SEI, AUD) are attached to the picture that follows them.
+ * Sufficient for one-slice-per-picture streams (x264 `slices=1`, nvenc, WebCodecs), which is what
+ * the encoder options enforce.
+ */
+export class AuSplitter {
+    #pending = new Uint8Array(0);
+    #au = [];
+    #auHasVcl = false;
+    /** Feeds bytes; returns every access unit completed by them (in order). */
+    push(bytes) {
+        this.#pending = this.#pending.length ? concatBytes([this.#pending, bytes]) : bytes.slice();
+        const out = [];
+        const p = this.#pending;
+        let pos = 0;
+        for (;;) {
+            const sc0 = findStartCode(p, pos);
+            if (!sc0)
+                break; // no start code yet (maybe a partial one): keep the bytes
+            const [s0, l0] = sc0;
+            // The NAL is complete only once the next start code is visible.
+            const sc1 = findStartCode(p, s0 + l0);
+            if (!sc1) {
+                pos = s0;
+                break;
+            }
+            const isVcl = s0 + l0 < p.length && isVclType(p[s0 + l0] & 0x1f);
+            if (this.#auHasVcl)
+                out.push(this.#take()); // the previous AU already has its slice: this NAL belongs to the next one
+            this.#au.push(p.slice(s0, sc1[0]));
+            if (isVcl)
+                this.#auHasVcl = true;
+            pos = sc1[0];
+        }
+        this.#pending = pos > 0 ? p.slice(pos) : p;
+        return out;
+    }
+    /** End of stream: the last NAL (whose end no start code marks) closes the last AU, or starts one more if the current AU already has its slice. */
+    finish() {
+        const out = [];
+        const sc0 = findStartCode(this.#pending, 0);
+        if (sc0) {
+            if (this.#auHasVcl)
+                out.push(this.#take());
+            this.#au.push(this.#pending.slice(sc0[0]));
+        }
+        if (this.#au.length)
+            out.push(this.#take());
+        this.#pending = new Uint8Array(0);
+        return out;
+    }
+    #take() {
+        const au = this.#au.length === 1 ? this.#au[0] : concatBytes(this.#au);
+        this.#au = [];
+        this.#auHasVcl = false;
+        return au;
+    }
+}
+/** Splits a whole Annex-B buffer into access units (see {@link AuSplitter}). */
+export function splitAccessUnits(stream) {
+    const sp = new AuSplitter();
+    const out = sp.push(stream);
+    out.push(...sp.finish());
+    return out;
+}
+/**
+ * A `MediaFrame` for an H.264 Annex-B access unit with `key`/`config`/`recovery` derived from the
+ * NAL units (IDR present / SPS+PPS present / recovery-point SEI present), so the header flags
+ * always agree with the bytes (R27 §2 publisher rule). `tUs` is the capture time (0 = stamp at publish).
+ */
+export function mediaFrameFromAnnexb(au, tUs = 0) {
+    const { idr, recovery } = auSyncPoints(au);
+    return { data: au, tUs, key: idr, config: auParamSets(au) !== undefined, recovery };
+}

package/dist/camera.d.ts

@@ -0,0 +1,12 @@
+import type { H264Encoder } from "./encoder.ts";
+/** `MediaStreamTrackProcessor` is available (Chrome 94+). */
+export declare function trackProcessorSupported(): boolean;
+/** The `VideoFrame`s of a video track as a `ReadableStream` (`maxBufferSize` 1: the newest frame wins, older ones are dropped by the browser). */
+export declare function trackFrames(track: MediaStreamTrack): ReadableStream<VideoFrame>;
+/**
+ * Pumps a camera track into the encoder until the track ends or `signal` aborts. Resolves with
+ * the number of frames offered to the encoder (dropped ones included; see `encoder.stats()`).
+ */
+export declare function pumpTrack(track: MediaStreamTrack, encoder: H264Encoder, opts?: {
+    signal?: AbortSignal;
+}): Promise<number>;

package/dist/camera.js

@@ -0,0 +1,43 @@
+// Camera frames into the encoder: `MediaStreamTrackProcessor` (Chrome-only, not in TypeScript's
+// DOM lib, so it is reached through `globalThis`) turns a `MediaStreamTrack` into a stream of
+// `VideoFrame`s. Each frame is encoded with `tUs = nowUs()` at dequeue (stamp at capture, as
+// dylithium did) and closed right after.
+import { SynapseError, nowUs } from "@synapse-sdk/core";
+/** `MediaStreamTrackProcessor` is available (Chrome 94+). */
+export function trackProcessorSupported() { return typeof globalThis.MediaStreamTrackProcessor === "function"; }
+/** The `VideoFrame`s of a video track as a `ReadableStream` (`maxBufferSize` 1: the newest frame wins, older ones are dropped by the browser). */
+export function trackFrames(track) {
+    const Ctor = globalThis.MediaStreamTrackProcessor;
+    if (!Ctor)
+        throw new SynapseError("not_supported", "trackFrames: MediaStreamTrackProcessor is not available (Chrome 94+ only)");
+    return new Ctor({ track, maxBufferSize: 1 }).readable;
+}
+/**
+ * Pumps a camera track into the encoder until the track ends or `signal` aborts. Resolves with
+ * the number of frames offered to the encoder (dropped ones included; see `encoder.stats()`).
+ */
+export async function pumpTrack(track, encoder, opts = {}) {
+    const reader = trackFrames(track).getReader();
+    const onAbort = () => { void reader.cancel(); };
+    opts.signal?.addEventListener("abort", onAbort, { once: true });
+    let n = 0;
+    try {
+        for (;;) {
+            const { value, done } = await reader.read();
+            if (done || !value)
+                break;
+            try {
+                encoder.encode(value, { tUs: nowUs() });
+                n++;
+            }
+            finally {
+                value.close();
+            }
+        }
+    }
+    finally {
+        opts.signal?.removeEventListener("abort", onAbort);
+        reader.releaseLock();
+    }
+    return n;
+}

package/dist/canvas.d.ts

@@ -0,0 +1,19 @@
+import type { VideoSink } from "./decoder.ts";
+export interface CanvasSinkOptions {
+    /** Coalesce to the display refresh (drops superseded pictures). Default false: draw immediately. */
+    vsync?: boolean;
+    /** Resize the canvas bitmap to the picture (default true). `false` scales the picture into the canvas as it is. */
+    resize?: boolean;
+}
+export interface CanvasSink {
+    /** Pictures drawn. */
+    readonly drawn: number;
+    /** Pictures closed undrawn because a newer one arrived first (`vsync` only). */
+    readonly dropped: number;
+    detach(): void;
+}
+/** Anything with a decoder-style `attachSink` (an `H264Decoder`, or a `VideoSubscriber` through its decoder). */
+export interface VideoSource {
+    attachSink(sink: VideoSink): () => void;
+}
+export declare function attachCanvas(source: VideoSource, canvas: HTMLCanvasElement, opts?: CanvasSinkOptions): CanvasSink;

package/dist/canvas.js

@@ -0,0 +1,63 @@
+// Canvas sink — dylithium-web-2 `react.tsx:233-252` as a framework-free helper. Per-frame
+// presentation bypasses React: decoder output → `drawImage(vf)` → `vf.close()`.
+//
+// Default (`vsync: false`): draw the moment the decoder outputs, the lowest-latency path
+// (`drawImage` of a `VideoFrame` is synchronous). `vsync: true`: hold the newest picture and draw
+// it on the next animation frame; a picture superseded before it was drawn is closed unseen and
+// counted `dropped`, so a decoder that runs ahead of the display never queues pictures.
+import { SynapseError } from "@synapse-sdk/core";
+export function attachCanvas(source, canvas, opts = {}) {
+    const ctx = canvas.getContext("2d");
+    if (!ctx)
+        throw new SynapseError("not_supported", "attachCanvas: the canvas has no 2d context (already used with another context type?)");
+    const resize = opts.resize ?? true;
+    let drawn = 0, dropped = 0;
+    let latest;
+    let raf = 0;
+    const draw = (vf) => {
+        try {
+            if (resize) {
+                if (canvas.width !== vf.displayWidth || canvas.height !== vf.displayHeight) {
+                    canvas.width = vf.displayWidth;
+                    canvas.height = vf.displayHeight;
+                }
+                ctx.drawImage(vf, 0, 0);
+            }
+            else
+                ctx.drawImage(vf, 0, 0, canvas.width, canvas.height);
+            drawn++;
+        }
+        finally {
+            vf.close();
+        }
+    };
+    const detachSink = source.attachSink(vf => {
+        if (!opts.vsync) {
+            draw(vf);
+            return;
+        }
+        if (latest) {
+            latest.close();
+            dropped++;
+        }
+        latest = vf;
+        if (!raf)
+            raf = requestAnimationFrame(() => { raf = 0; const f = latest; latest = undefined; if (f)
+                draw(f); });
+    });
+    return {
+        get drawn() { return drawn; },
+        get dropped() { return dropped; },
+        detach() {
+            detachSink();
+            if (raf) {
+                cancelAnimationFrame(raf);
+                raf = 0;
+            }
+            if (latest) {
+                latest.close();
+                latest = undefined;
+            }
+        },
+    };
+}

package/dist/decoder.d.ts

@@ -0,0 +1,57 @@
+import { type MediaFrame } from "@synapse-sdk/core";
+import { DecoderSync, type SyncConfig, type SyncState, type SyncStats } from "./sync.ts";
+/** What travelled with a decoded frame. */
+export interface DecodedFrameMeta {
+    /** The access unit this picture came from (header facts included). */
+    frame: MediaFrame;
+    /** `nowUs()` when the AU was pushed into the decoder. */
+    recvUs: number;
+    /** `decode()` call to output, ms. */
+    decodeMs: number;
+}
+/** Receives decoded pictures; owns `vf` and must `vf.close()` it. */
+export type VideoSink = (vf: VideoFrame, meta: DecodedFrameMeta) => void;
+export interface H264DecoderOptions {
+    /** `"no-preference"` (default) lets Chrome pick the hardware decoder; `"prefer-software"` is about 1 ms at low resolutions. */
+    hardwareAcceleration?: HardwareAcceleration;
+    /** Sync policy overrides; the defaults are the shared 10/150/60 (`SYNC_DEFAULT`) with `startAtRecovery: false` (see the constructor). */
+    sync?: Partial<SyncConfig>;
+    /** The decoder has produced nothing for `stuckFrames` chunks: ask the publisher for an IDR (`VideoSubscriber` wires this to the `<topic>/idr` service). */
+    onRequestIdr?: () => void;
+    /** Diagnostics (configure, sync, reset, error lines). */
+    onLog?: (msg: string) => void;
+}
+export interface H264DecoderStats extends SyncStats {
+    state: SyncState;
+    /** Decoder queue depth right now. */
+    queue: number;
+    /** Smoothed `decode()`-to-output time, ms. */
+    decodeMs: number;
+    width: number;
+    height: number;
+    /** The codec string the decoder is configured with. */
+    codec: string | undefined;
+    /** Decoded pictures closed unseen because no sink was attached. */
+    sinkless: number;
+    /** WebCodecs `error` callbacks and `decode()` throws (each one resets). */
+    errors: number;
+}
+export declare class H264Decoder {
+    #private;
+    private readonly opts;
+    /** The shared sync policy (read `sync.state`, `sync.stats`). */
+    readonly sync: DecoderSync;
+    /** Stuck signal; set by the options or later by the owner (`VideoSubscriber` does). */
+    onRequestIdr: (() => void) | undefined;
+    /** Observer of every decoded picture (stats, latency), called before the sink. */
+    onOutput: ((meta: DecodedFrameMeta) => void) | undefined;
+    constructor(opts?: H264DecoderOptions);
+    /** Sets the sink that owns decoded pictures (one at a time; replacing detaches the previous). Returns a detach function. */
+    attachSink(sink: VideoSink): () => void;
+    /** One received access unit. Never throws: decoder failures reset the decoder and count. */
+    push(frame: MediaFrame): void;
+    /** Drop the decoder now and wait for the next start point (counts as a reset). */
+    reset(reason?: string): void;
+    stats(): H264DecoderStats;
+    close(): void;
+}

package/dist/decoder.js

@@ -0,0 +1,131 @@
+// WebCodecs H.264 playback tuned for a live, low-latency stream — dylithium-web-2 `player.ts`
+// (`H264Player`) rebuilt on the Synapse `MediaFrame` and the shared `DecoderSync` state machine.
+//
+// What stays from dylithium (lessons register): `optimizeForLatency: true`; Annex-B chunks with
+// SPS/PPS in-band and **no** `description` (avcC), so the decoder takes the bytes as they travel;
+// the codec string read from the SPS (`avc1.` + profile/constraints/level) so any encoder profile
+// decodes; the first chunk after `configure()` labelled `key` even when it is a recovery point;
+// prime a recovery point with the cached SPS/PPS; stuck → ask for an IDR; wedged or decoder error
+// → drop the decoder and wait for the next start point. Frames are handed to one sink as they
+// decode; the sink owns them and must `close()` them (`attachCanvas` does).
+import { nowUs } from "@synapse-sdk/core";
+import { codecStringFromSps } from "./annexb.js";
+import { DecoderSync } from "./sync.js";
+import { DEFAULT_H264_CODEC, requireWebCodecs } from "./webcodecs.js";
+export class H264Decoder {
+    opts;
+    /** The shared sync policy (read `sync.state`, `sync.stats`). */
+    sync;
+    /** Stuck signal; set by the options or later by the owner (`VideoSubscriber` does). */
+    onRequestIdr;
+    /** Observer of every decoded picture (stats, latency), called before the sink. */
+    onOutput;
+    #dec;
+    #codec;
+    #sink;
+    #ts = 0;
+    /** Pending chunks by synthetic timestamp (the key WebCodecs hands back on output). */
+    #meta = new Map();
+    #dims = { width: 0, height: 0 };
+    #decodeMs = 0;
+    #sinkless = 0;
+    #errors = 0;
+    #closed = false;
+    constructor(opts = {}) {
+        this.opts = opts;
+        requireWebCodecs("H264Decoder");
+        // Chromium's VideoDecoder checks that a `key` chunk carries an IDR (Annex-B without `description`):
+        // a recovery-point start throws "A key frame is required after configure()", so WebCodecs waits for
+        // an IDR (the owner asks for one through `onRequestIdr`; `VideoSubscriber` does so while waiting).
+        this.sync = new DecoderSync({ startAtRecovery: false, ...opts.sync });
+        this.onRequestIdr = opts.onRequestIdr;
+    }
+    /** Sets the sink that owns decoded pictures (one at a time; replacing detaches the previous). Returns a detach function. */
+    attachSink(sink) {
+        this.#sink = sink;
+        return () => { if (this.#sink === sink)
+            this.#sink = undefined; };
+    }
+    /** One received access unit. Never throws: decoder failures reset the decoder and count. */
+    push(frame) {
+        if (this.#closed)
+            return;
+        const recvUs = nowUs();
+        const d = this.sync.push(frame);
+        if (d.feed) {
+            if (d.started || !this.#dec)
+                this.#configure(d.feed, d.started);
+            const ts = this.#ts++ * 33_333; // timestamps are only keys to pair outputs with inputs (any spacing works)
+            this.#meta.set(ts, { frame, recvUs, pushedMs: performance.now() });
+            try {
+                this.#dec.decode(new EncodedVideoChunk({ type: d.keyChunk ? "key" : "delta", timestamp: ts, data: d.feed }));
+            }
+            catch (e) {
+                this.#meta.delete(ts);
+                this.#fail(`decode() threw: ${e instanceof Error ? e.message : String(e)}`);
+                return;
+            }
+        }
+        if (d.signal === "reset") {
+            this.#log(`decoder produced nothing for ${this.sync.cfg.wedgedFrames} chunks: dropping it`);
+            this.#drop();
+        }
+        else if (d.signal === "request_idr") {
+            this.#log(`no output for ${this.sync.cfg.stuckFrames} chunks: requesting an IDR`);
+            this.onRequestIdr?.();
+        }
+    }
+    #configure(feed, restart) {
+        if (this.#dec)
+            this.#drop();
+        const codec = codecStringFromSps(feed) ?? this.#codec ?? DEFAULT_H264_CODEC;
+        const dec = new VideoDecoder({ output: vf => this.#onOutput(vf), error: e => this.#fail(`decoder error: ${e.message}`) });
+        dec.configure({ codec, optimizeForLatency: true, hardwareAcceleration: this.opts.hardwareAcceleration ?? "no-preference" });
+        this.#dec = dec;
+        this.#codec = codec;
+        this.#log(`decoder configured: ${codec}${restart ? " (start point)" : ""}`);
+    }
+    #onOutput(vf) {
+        this.sync.onOutput();
+        const m = this.#meta.get(vf.timestamp);
+        this.#meta.delete(vf.timestamp);
+        this.#dims = { width: vf.displayWidth, height: vf.displayHeight };
+        const decodeMs = m ? performance.now() - m.pushedMs : 0;
+        if (m)
+            this.#decodeMs = this.#decodeMs === 0 ? decodeMs : this.#decodeMs * 0.9 + decodeMs * 0.1;
+        const meta = { frame: m?.frame ?? { data: new Uint8Array(0), tUs: 0, key: false, config: false, recovery: false }, recvUs: m?.recvUs ?? 0, decodeMs };
+        this.onOutput?.(meta);
+        if (this.#sink)
+            this.#sink(vf, meta);
+        else {
+            vf.close();
+            this.#sinkless++;
+        }
+    }
+    /** A decoder error or throw: count it, drop the decoder, wait for the next start point. */
+    #fail(reason) {
+        this.#errors++;
+        this.#log(`reset: ${reason}`);
+        this.sync.onError();
+        this.#drop();
+    }
+    #drop() {
+        try {
+            this.#dec?.close();
+        }
+        catch { /* already closed */ }
+        this.#dec = undefined;
+        this.#meta.clear();
+    }
+    /** Drop the decoder now and wait for the next start point (counts as a reset). */
+    reset(reason = "requested") { this.#log(`reset: ${reason}`); this.sync.onError(); this.#drop(); }
+    stats() {
+        return { ...this.sync.stats, state: this.sync.state, queue: this.#dec?.decodeQueueSize ?? 0, decodeMs: this.#decodeMs, ...this.#dims, codec: this.#codec, sinkless: this.#sinkless, errors: this.#errors };
+    }
+    close() {
+        this.#closed = true;
+        this.#drop();
+        this.#sink = undefined;
+    }
+    #log(msg) { this.opts.onLog?.(msg); }
+}