@synapse-sdk/media 0.0.0-stage → 0.3.3

package/README.md

@@ -1,3 +1,3 @@
-# Temporary Holding Version
+# @synapse-sdk/media
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Part of the Synapse SDK for JavaScript. Documentation: the repository.

package/dist/camera.d.ts

@@ -0,0 +1,11 @@
+/** `MediaStreamTrackProcessor` is available (Chrome 94+). */
+export declare function trackProcessorSupported(): boolean;
+/** The `VideoFrame`s of a video track as a `ReadableStream` (`maxBufferSize` 1: the newest frame wins, older ones are dropped by the browser). */
+export declare function trackFrames(track: MediaStreamTrack): ReadableStream<VideoFrame>;
+/**
+ * Pumps a camera track into `publish` (a video publisher's) until the track ends or `signal`
+ * aborts; each frame is closed after the call. Resolves with the number of frames handed over.
+ */
+export declare function pumpTrack(track: MediaStreamTrack, publish: (frame: VideoFrame, tUs: number) => void, opts?: {
+    signal?: AbortSignal;
+}): Promise<number>;

package/dist/camera.js

@@ -0,0 +1,45 @@
+// Camera frames into a video publisher: `MediaStreamTrackProcessor` (Chrome-only, not in
+// TypeScript's DOM lib, so it is reached through `globalThis`) turns a `MediaStreamTrack` into a
+// stream of `VideoFrame`s. Each frame is published with `tUs = nowUs()` at dequeue (stamp at
+// capture, as dylithium did) and closed right after.
+import { SynapseError, nowUs } from "@synapse-sdk/core";
+/** `MediaStreamTrackProcessor` is available (Chrome 94+). */
+export function trackProcessorSupported() { return typeof globalThis.MediaStreamTrackProcessor === "function"; }
+/** The `VideoFrame`s of a video track as a `ReadableStream` (`maxBufferSize` 1: the newest frame wins, older ones are dropped by the browser). */
+export function trackFrames(track) {
+    const Ctor = globalThis.MediaStreamTrackProcessor;
+    if (!Ctor)
+        throw new SynapseError("not_supported", "trackFrames: MediaStreamTrackProcessor is not available (Chrome 94+ only)");
+    return new Ctor({ track, maxBufferSize: 1 }).readable;
+}
+/**
+ * Pumps a camera track into `publish` (a video publisher's) until the track ends or `signal`
+ * aborts; each frame is closed after the call. Resolves with the number of frames handed over.
+ */
+export async function pumpTrack(track, publish, opts = {}) {
+    const reader = trackFrames(track).getReader();
+    const onAbort = () => { void reader.cancel(); };
+    if (opts.signal?.aborted)
+        onAbort();
+    opts.signal?.addEventListener("abort", onAbort, { once: true });
+    let n = 0;
+    try {
+        for (;;) {
+            const { value, done } = await reader.read();
+            if (done || !value)
+                break;
+            try {
+                publish(value, nowUs());
+                n++;
+            }
+            finally {
+                value.close();
+            }
+        }
+    }
+    finally {
+        opts.signal?.removeEventListener("abort", onAbort);
+        reader.releaseLock();
+    }
+    return n;
+}

package/dist/canvas.d.ts

@@ -0,0 +1,18 @@
+export interface CanvasSinkOptions {
+    /** Coalesce to the display refresh (drops superseded pictures). Default false: draw immediately. */
+    vsync?: boolean;
+    /** Resize the canvas bitmap to the picture (default true). `false` scales the picture into the canvas as it is. */
+    resize?: boolean;
+}
+export interface CanvasSink {
+    /** Pictures drawn. */
+    readonly drawn: number;
+    /** Pictures closed undrawn because a newer one arrived first (`vsync` only). */
+    readonly dropped: number;
+    detach(): void;
+}
+/** Anything with a decoder-style `attachSink` that hands over pictures the sink then owns (an `H264Decoder`). */
+export interface VideoSource {
+    attachSink(sink: (vf: VideoFrame) => void): () => void;
+}
+export declare function attachCanvas(source: VideoSource, canvas: HTMLCanvasElement, opts?: CanvasSinkOptions): CanvasSink;

package/dist/canvas.js

@@ -0,0 +1,63 @@
+// Canvas sink — dylithium-web-2 `react.tsx:233-252` as a framework-free helper. Per-frame
+// presentation bypasses React: decoder output → `drawImage(vf)` → `vf.close()`.
+//
+// Default (`vsync: false`): draw the moment the decoder outputs, the lowest-latency path
+// (`drawImage` of a `VideoFrame` is synchronous). `vsync: true`: hold the newest picture and draw
+// it on the next animation frame; a picture superseded before it was drawn is closed unseen and
+// counted `dropped`, so a decoder that runs ahead of the display never queues pictures.
+import { SynapseError } from "@synapse-sdk/core";
+export function attachCanvas(source, canvas, opts = {}) {
+    const ctx = canvas.getContext("2d");
+    if (!ctx)
+        throw new SynapseError("not_supported", "attachCanvas: the canvas has no 2d context (already used with another context type?)");
+    const resize = opts.resize ?? true;
+    let drawn = 0, dropped = 0;
+    let latest;
+    let raf = 0;
+    const draw = (vf) => {
+        try {
+            if (resize) {
+                if (canvas.width !== vf.displayWidth || canvas.height !== vf.displayHeight) {
+                    canvas.width = vf.displayWidth;
+                    canvas.height = vf.displayHeight;
+                }
+                ctx.drawImage(vf, 0, 0);
+            }
+            else
+                ctx.drawImage(vf, 0, 0, canvas.width, canvas.height);
+            drawn++;
+        }
+        finally {
+            vf.close();
+        }
+    };
+    const detachSink = source.attachSink(vf => {
+        if (!opts.vsync) {
+            draw(vf);
+            return;
+        }
+        if (latest) {
+            latest.close();
+            dropped++;
+        }
+        latest = vf;
+        if (!raf)
+            raf = requestAnimationFrame(() => { raf = 0; const f = latest; latest = undefined; if (f)
+                draw(f); });
+    });
+    return {
+        get drawn() { return drawn; },
+        get dropped() { return dropped; },
+        detach() {
+            detachSink();
+            if (raf) {
+                cancelAnimationFrame(raf);
+                raf = 0;
+            }
+            if (latest) {
+                latest.close();
+                latest = undefined;
+            }
+        },
+    };
+}

package/dist/decoder.d.ts

@@ -0,0 +1,57 @@
+import { type MediaFrame } from "@synapse-sdk/core";
+import { DecoderSync, type SyncConfig, type SyncState, type SyncStats } from "./sync.ts";
+/** What travelled with a decoded frame. */
+export interface DecodedFrameMeta {
+    /** The access unit this picture came from (header facts included). */
+    frame: MediaFrame;
+    /** `nowUs()` when the AU was pushed into the decoder. */
+    recvUs: number;
+    /** `decode()` call to output, ms. */
+    decodeMs: number;
+}
+/** Receives decoded pictures; owns `vf` and must `vf.close()` it. */
+export type VideoSink = (vf: VideoFrame, meta: DecodedFrameMeta) => void;
+export interface H264DecoderOptions {
+    /** `"no-preference"` (default) lets Chrome pick the hardware decoder; `"prefer-software"` is about 1 ms at low resolutions. */
+    hardwareAcceleration?: HardwareAcceleration;
+    /** Sync policy overrides; the defaults are the shared 10/150/60 (`SYNC_DEFAULT`) with `startAtRecovery: false` (see the constructor). */
+    sync?: Partial<SyncConfig>;
+    /** The decoder has produced nothing for `stuckFrames` chunks: ask the publisher for an IDR (`videoSubscriber` wires this to the `<topic>/idr` service). */
+    onRequestIdr?: () => void;
+    /** Diagnostics (configure, sync, reset, error lines). */
+    onLog?: (msg: string) => void;
+}
+export interface H264DecoderStats extends SyncStats {
+    state: SyncState;
+    /** Decoder queue depth right now. */
+    queue: number;
+    /** Smoothed `decode()`-to-output time, ms. */
+    decodeMs: number;
+    width: number;
+    height: number;
+    /** The codec string the decoder is configured with. */
+    codec: string | undefined;
+    /** Decoded pictures closed unseen because no sink was attached. */
+    sinkless: number;
+    /** WebCodecs `error` callbacks and `decode()` throws (each one resets). */
+    errors: number;
+}
+export declare class H264Decoder {
+    #private;
+    private readonly opts;
+    /** The shared sync policy (read `sync.state`, `sync.stats`). */
+    readonly sync: DecoderSync;
+    /** Stuck signal; set by the options or later by the owner (`videoSubscriber` does). */
+    onRequestIdr: (() => void) | undefined;
+    /** Observer of every decoded picture (stats, latency), called before the sink. */
+    onOutput: ((meta: DecodedFrameMeta) => void) | undefined;
+    constructor(opts?: H264DecoderOptions);
+    /** Sets the sink that owns decoded pictures (one at a time; replacing detaches the previous). Returns a detach function. */
+    attachSink(sink: VideoSink): () => void;
+    /** One received access unit. Never throws: decoder failures reset the decoder and count. */
+    push(frame: MediaFrame): void;
+    /** Drop the decoder now and wait for the next start point (counts as a reset). */
+    reset(reason?: string): void;
+    stats(): H264DecoderStats;
+    close(): void;
+}

package/dist/decoder.js

@@ -0,0 +1,131 @@
+// WebCodecs H.264 playback tuned for a live, low-latency stream — dylithium-web-2 `player.ts`
+// (`H264Player`) rebuilt on the Synapse `MediaFrame` and the shared `DecoderSync` state machine.
+//
+// What stays from dylithium (lessons register): `optimizeForLatency: true`; Annex-B chunks with
+// SPS/PPS in-band and **no** `description` (avcC), so the decoder takes the bytes as they travel;
+// the codec string read from the SPS (`avc1.` + profile/constraints/level) so any encoder profile
+// decodes; the first chunk after `configure()` labelled `key` even when it is a recovery point;
+// prime a recovery point with the cached SPS/PPS; stuck → ask for an IDR; wedged or decoder error
+// → drop the decoder and wait for the next start point. Frames are handed to one sink as they
+// decode; the sink owns them and must `close()` them (`attachCanvas` does).
+import { nowUs } from "@synapse-sdk/core";
+import { codecStringFromSps } from "@synapse-sdk/core";
+import { DecoderSync } from "./sync.js";
+import { DEFAULT_H264_CODEC, requireWebCodecs } from "./webcodecs.js";
+export class H264Decoder {
+    opts;
+    /** The shared sync policy (read `sync.state`, `sync.stats`). */
+    sync;
+    /** Stuck signal; set by the options or later by the owner (`videoSubscriber` does). */
+    onRequestIdr;
+    /** Observer of every decoded picture (stats, latency), called before the sink. */
+    onOutput;
+    #dec;
+    #codec;
+    #sink;
+    #ts = 0;
+    /** Pending chunks by synthetic timestamp (the key WebCodecs hands back on output). */
+    #meta = new Map();
+    #dims = { width: 0, height: 0 };
+    #decodeMs = 0;
+    #sinkless = 0;
+    #errors = 0;
+    #closed = false;
+    constructor(opts = {}) {
+        this.opts = opts;
+        requireWebCodecs("H264Decoder");
+        // Chromium's VideoDecoder checks that a `key` chunk carries an IDR (Annex-B without `description`):
+        // a recovery-point start throws "A key frame is required after configure()", so WebCodecs waits for
+        // an IDR (the owner asks for one through `onRequestIdr`; the sync policy raises it while waiting too).
+        this.sync = new DecoderSync({ startAtRecovery: false, ...opts.sync });
+        this.onRequestIdr = opts.onRequestIdr;
+    }
+    /** Sets the sink that owns decoded pictures (one at a time; replacing detaches the previous). Returns a detach function. */
+    attachSink(sink) {
+        this.#sink = sink;
+        return () => { if (this.#sink === sink)
+            this.#sink = undefined; };
+    }
+    /** One received access unit. Never throws: decoder failures reset the decoder and count. */
+    push(frame) {
+        if (this.#closed)
+            return;
+        const recvUs = nowUs();
+        const d = this.sync.push(frame);
+        if (d.feed) {
+            if (d.started || !this.#dec)
+                this.#configure(d.feed, d.started);
+            const ts = this.#ts++ * 33_333; // timestamps are only keys to pair outputs with inputs (any spacing works)
+            this.#meta.set(ts, { frame, recvUs, pushedMs: performance.now() });
+            try {
+                this.#dec.decode(new EncodedVideoChunk({ type: d.keyChunk ? "key" : "delta", timestamp: ts, data: d.feed }));
+            }
+            catch (e) {
+                this.#meta.delete(ts);
+                this.#fail(`decode() threw: ${e instanceof Error ? e.message : String(e)}`);
+                return;
+            }
+        }
+        if (d.signal === "reset") {
+            this.#log(`decoder produced nothing for ${this.sync.cfg.wedgedFrames} chunks: dropping it`);
+            this.#drop();
+        }
+        else if (d.signal === "request_idr") {
+            this.#log(`no output for ${this.sync.cfg.stuckFrames} chunks: requesting an IDR`);
+            this.onRequestIdr?.();
+        }
+    }
+    #configure(feed, restart) {
+        if (this.#dec)
+            this.#drop();
+        const codec = codecStringFromSps(feed) ?? this.#codec ?? DEFAULT_H264_CODEC;
+        const dec = new VideoDecoder({ output: vf => this.#onOutput(vf), error: e => this.#fail(`decoder error: ${e.message}`) });
+        dec.configure({ codec, optimizeForLatency: true, hardwareAcceleration: this.opts.hardwareAcceleration ?? "no-preference" });
+        this.#dec = dec;
+        this.#codec = codec;
+        this.#log(`decoder configured: ${codec}${restart ? " (start point)" : ""}`);
+    }
+    #onOutput(vf) {
+        this.sync.onOutput();
+        const m = this.#meta.get(vf.timestamp);
+        this.#meta.delete(vf.timestamp);
+        this.#dims = { width: vf.displayWidth, height: vf.displayHeight };
+        const decodeMs = m ? performance.now() - m.pushedMs : 0;
+        if (m)
+            this.#decodeMs = this.#decodeMs === 0 ? decodeMs : this.#decodeMs * 0.9 + decodeMs * 0.1;
+        const meta = { frame: m?.frame ?? { data: new Uint8Array(0), tUs: 0, key: false, config: false, recovery: false }, recvUs: m?.recvUs ?? 0, decodeMs };
+        this.onOutput?.(meta);
+        if (this.#sink)
+            this.#sink(vf, meta);
+        else {
+            vf.close();
+            this.#sinkless++;
+        }
+    }
+    /** A decoder error or throw: count it, drop the decoder, wait for the next start point. */
+    #fail(reason) {
+        this.#errors++;
+        this.#log(`reset: ${reason}`);
+        this.sync.onError();
+        this.#drop();
+    }
+    #drop() {
+        try {
+            this.#dec?.close();
+        }
+        catch { /* already closed */ }
+        this.#dec = undefined;
+        this.#meta.clear();
+    }
+    /** Drop the decoder now and wait for the next start point (counts as a reset). */
+    reset(reason = "requested") { this.#log(`reset: ${reason}`); this.sync.onError(); this.#drop(); }
+    stats() {
+        return { ...this.sync.stats, state: this.sync.state, queue: this.#dec?.decodeQueueSize ?? 0, decodeMs: this.#decodeMs, ...this.#dims, codec: this.#codec, sinkless: this.#sinkless, errors: this.#errors };
+    }
+    close() {
+        this.#closed = true;
+        this.#drop();
+        this.#sink = undefined;
+    }
+    #log(msg) { this.opts.onLog?.(msg); }
+}

package/dist/encoder.d.ts

@@ -0,0 +1,74 @@
+import { type MediaFrame } from "@synapse-sdk/core";
+export interface H264EncoderOptions {
+    width: number;
+    height: number;
+    /** Frames per second the source will deliver (rate control and level selection); default 30. */
+    fps?: number;
+    /** Target bits per second; default `width × height × fps × 0.15` (about 260 kbps at 320×180@30). */
+    bitrate?: number;
+    /** Codec string; default Baseline (`avc1.4200XX`) with the level chosen from width × height × fps × bitrate (H.264 Table A-1). */
+    codec?: string;
+    /** Default `"no-preference"`. */
+    hardwareAcceleration?: HardwareAcceleration;
+    /** Default `"realtime"`. */
+    latencyMode?: LatencyMode;
+    /** Default `"constant"` (CBR, the dylithium encoder option block). */
+    bitrateMode?: VideoEncoderBitrateMode;
+    /** Force a key frame every N encoded frames; 0 (default) = only the first frame, `forceIdr()` and the IDR service. */
+    keyframeInterval?: number;
+    /** Captured frames are dropped, not queued, while the encoder holds this many (default 2): drop captured, never encoded. */
+    maxQueue?: number;
+    onError?: (e: unknown) => void;
+    onLog?: (msg: string) => void;
+}
+export interface H264EncoderStats {
+    /** Access units produced. */
+    encoded: number;
+    keyframes: number;
+    /** Captured frames dropped because the encoder was behind (`maxQueue`). */
+    dropped: number;
+    /** `encode()` throws and encoder `error` callbacks. */
+    errors: number;
+    /** Encoder queue depth right now. */
+    queue: number;
+    /** Bits per second in force. */
+    bitrate: number;
+    /** Smoothed `encode()`-to-chunk time, ms. */
+    encodeMs: number;
+    /** Encoded bytes produced. */
+    bytes: number;
+    codec: string;
+}
+/** Receives encoded access units (`stages.encodedUs` stamped, `stages.sentUs` 0 until the publisher writes). */
+export type MediaSink = (frame: MediaFrame) => void;
+/** Lowest H.264 level (level_idc) whose Table A-1 limits cover the picture rate, frame size and Baseline bitrate. */
+export declare function h264LevelIdc(width: number, height: number, fps: number, bitrateBps: number): number;
+export declare class H264Encoder {
+    #private;
+    private readonly opts;
+    readonly cfg: VideoEncoderConfig;
+    constructor(opts: H264EncoderOptions);
+    /** Whether WebCodecs can encode this configuration here (Annex-B, realtime). */
+    static isSupported(opts: Pick<H264EncoderOptions, "width" | "height" | "fps" | "bitrate" | "codec" | "hardwareAcceleration">): Promise<boolean>;
+    /** Adds a consumer of encoded access units; returns a detach function. */
+    attachSink(sink: MediaSink): () => void;
+    /**
+     * Encodes one picture: a `VideoFrame` (camera, `MediaStreamTrackProcessor`) or any
+     * `CanvasImageSource` (a canvas, video element, image bitmap). `tUs` is the capture time in the
+     * node clock (default `nowUs()` now) and rides through `VideoFrame.timestamp` to the chunk, so
+     * the published `t_us` is the capture stamp. The caller keeps ownership of a `VideoFrame` source
+     * (close it afterwards). Returns `false` when the frame was dropped (encoder behind, or closed).
+     */
+    encode(source: VideoFrame | CanvasImageSource, opts?: {
+        tUs?: number;
+        keyFrame?: boolean;
+    }): boolean;
+    /** The next encoded frame is an IDR (the `<topic>/idr` service calls this). */
+    forceIdr(): void;
+    /** Changes the target bitrate (reconfigures; the next frame is an IDR so the new rate starts clean). */
+    setBitrate(bps: number): void;
+    /** Waits for every queued frame to come out. */
+    flush(): Promise<void>;
+    stats(): H264EncoderStats;
+    close(): void;
+}

package/dist/encoder.js

@@ -0,0 +1,171 @@
+// WebCodecs H.264 encoder for a live stream (R27 §4; dylithium had no browser encoder).
+//
+// `avc: { format: "annexb" }` puts SPS/PPS in-band on every key chunk, so the bytes are exactly
+// what a `video/h264` topic carries (no avcC, no repackaging) and the flags are read off the
+// bitstream like every other Synapse publisher. `latencyMode: "realtime"`, `bitrateMode:
+// "constant"`, `hardwareAcceleration: "no-preference"` (Chrome picks; software OpenH264 in
+// headless Chromium). WebCodecs has no GOP and no intra-refresh: keyframes happen on the first
+// frame, on `forceIdr()` (the `<topic>/idr` service `videoPublisher` serves) and optionally every `keyframeInterval`
+// frames; a receiver's policy for a browser publisher is therefore IDR-only, which is fine.
+// Captured frames are dropped when the encoder queue is behind, encoded ones never (lessons register).
+import { auParamSets, auSyncPoints, nowUs } from "@synapse-sdk/core";
+import { requireWebCodecs } from "./webcodecs.js";
+/** Lowest H.264 level (level_idc) whose Table A-1 limits cover the picture rate, frame size and Baseline bitrate. */
+export function h264LevelIdc(width, height, fps, bitrateBps) {
+    const mbs = Math.ceil(width / 16) * Math.ceil(height / 16);
+    const mbps = mbs * fps;
+    const kbps = bitrateBps / 1000;
+    // [level_idc, MaxMBPS, MaxFS, MaxBR (kbit/s, Baseline/Main)]
+    const table = [
+        [0x0a, 1485, 99, 64], [0x0b, 3000, 396, 192], [0x0c, 6000, 396, 384], [0x0d, 11880, 396, 768], [0x14, 11880, 396, 2000],
+        [0x15, 19800, 792, 4000], [0x16, 20250, 1620, 4000], [0x1e, 40500, 1620, 10000], [0x1f, 108000, 3600, 14000], [0x20, 216000, 5120, 20000],
+        [0x28, 245760, 8192, 20000], [0x29, 245760, 8192, 50000], [0x2a, 522240, 8704, 50000], [0x32, 589824, 22080, 135000], [0x33, 983040, 36864, 240000], [0x34, 2073600, 36864, 240000],
+    ];
+    for (const [idc, maxMbps, maxFs, maxBr] of table)
+        if (mbps <= maxMbps && mbs <= maxFs && kbps <= maxBr)
+            return idc;
+    return 0x34;
+}
+export class H264Encoder {
+    opts;
+    cfg;
+    #enc;
+    #sinks = new Set();
+    #forceKey = true;
+    #sinceKey = 0;
+    #inflight = new Map();
+    #encodeMs = 0;
+    #stats = { encoded: 0, keyframes: 0, dropped: 0, errors: 0, bytes: 0 };
+    #closed = false;
+    constructor(opts) {
+        this.opts = opts;
+        requireWebCodecs("H264Encoder");
+        const fps = opts.fps ?? 30;
+        const bitrate = opts.bitrate ?? Math.round(opts.width * opts.height * fps * 0.15);
+        const codec = opts.codec ?? `avc1.4200${h264LevelIdc(opts.width, opts.height, fps, bitrate).toString(16).padStart(2, "0")}`;
+        this.cfg = {
+            codec, width: opts.width, height: opts.height, framerate: fps, bitrate,
+            bitrateMode: opts.bitrateMode ?? "constant", latencyMode: opts.latencyMode ?? "realtime", hardwareAcceleration: opts.hardwareAcceleration ?? "no-preference",
+            avc: { format: "annexb" },
+        };
+        this.#enc = new VideoEncoder({ output: (chunk, meta) => this.#onChunk(chunk, meta), error: e => { this.#stats.errors++; this.#log(`encoder error: ${e.message}`); this.opts.onError?.(e); } });
+        this.#enc.configure(this.cfg);
+        this.#log(`encoder configured: ${codec} ${opts.width}x${opts.height}@${fps} ${bitrate} bps`);
+    }
+    /** Whether WebCodecs can encode this configuration here (Annex-B, realtime). */
+    static async isSupported(opts) {
+        if (typeof VideoEncoder !== "function")
+            return false;
+        const fps = opts.fps ?? 30;
+        const bitrate = opts.bitrate ?? Math.round(opts.width * opts.height * fps * 0.15);
+        const codec = opts.codec ?? `avc1.4200${h264LevelIdc(opts.width, opts.height, fps, bitrate).toString(16).padStart(2, "0")}`;
+        try {
+            const r = await VideoEncoder.isConfigSupported({ codec, width: opts.width, height: opts.height, framerate: fps, bitrate, latencyMode: "realtime", hardwareAcceleration: opts.hardwareAcceleration ?? "no-preference", avc: { format: "annexb" } });
+            return r.supported === true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Adds a consumer of encoded access units; returns a detach function. */
+    attachSink(sink) { this.#sinks.add(sink); return () => { this.#sinks.delete(sink); }; }
+    /**
+     * Encodes one picture: a `VideoFrame` (camera, `MediaStreamTrackProcessor`) or any
+     * `CanvasImageSource` (a canvas, video element, image bitmap). `tUs` is the capture time in the
+     * node clock (default `nowUs()` now) and rides through `VideoFrame.timestamp` to the chunk, so
+     * the published `t_us` is the capture stamp. The caller keeps ownership of a `VideoFrame` source
+     * (close it afterwards). Returns `false` when the frame was dropped (encoder behind, or closed).
+     */
+    encode(source, opts = {}) {
+        if (this.#closed)
+            return false;
+        if (this.#enc.encodeQueueSize >= (this.opts.maxQueue ?? 2)) {
+            this.#stats.dropped++;
+            return false;
+        }
+        const tUs = Math.round(opts.tUs ?? nowUs());
+        const interval = this.opts.keyframeInterval ?? 0;
+        const key = this.#forceKey || opts.keyFrame === true || (interval > 0 && this.#sinceKey >= interval);
+        let vf;
+        try {
+            vf = new VideoFrame(source, { timestamp: tUs });
+        }
+        catch (e) {
+            this.#stats.errors++;
+            this.opts.onError?.(e);
+            return false;
+        }
+        try {
+            this.#enc.encode(vf, { keyFrame: key });
+        }
+        catch (e) {
+            this.#stats.errors++;
+            this.#log(`encode() threw: ${e instanceof Error ? e.message : String(e)}`);
+            this.opts.onError?.(e);
+            return false;
+        }
+        finally {
+            vf.close();
+        }
+        this.#inflight.set(tUs, performance.now());
+        this.#forceKey = false;
+        this.#sinceKey = key ? 1 : this.#sinceKey + 1;
+        return true;
+    }
+    /** The next encoded frame is an IDR (the `<topic>/idr` service calls this). */
+    forceIdr() { this.#forceKey = true; }
+    /** Changes the target bitrate (reconfigures; the next frame is an IDR so the new rate starts clean). */
+    setBitrate(bps) {
+        if (this.#closed || bps <= 0)
+            return;
+        this.cfg.bitrate = Math.round(bps);
+        try {
+            this.#enc.configure(this.cfg);
+            this.#forceKey = true;
+            this.#log(`bitrate → ${Math.round(bps)} bps`);
+        }
+        catch (e) {
+            this.#stats.errors++;
+            this.opts.onError?.(e);
+        }
+    }
+    /** Waits for every queued frame to come out. */
+    async flush() { if (!this.#closed && this.#enc.state === "configured")
+        await this.#enc.flush(); }
+    #onChunk(chunk, _meta) {
+        const data = new Uint8Array(chunk.byteLength);
+        chunk.copyTo(data);
+        // Flags from the bitstream like every Synapse publisher; R27 §4: a key chunk is an IDR (the bitstream says so too).
+        const { idr, recovery } = auSyncPoints(data);
+        const f = { data, tUs: chunk.timestamp, key: idr || chunk.type === "key", config: auParamSets(data) !== undefined, recovery, stages: { encodedUs: nowUs(), sentUs: 0 } };
+        const started = this.#inflight.get(chunk.timestamp);
+        if (started !== undefined) {
+            this.#inflight.delete(chunk.timestamp);
+            const ms = performance.now() - started;
+            this.#encodeMs = this.#encodeMs === 0 ? ms : this.#encodeMs * 0.9 + ms * 0.1;
+        }
+        this.#stats.encoded++;
+        if (f.key) {
+            this.#stats.keyframes++;
+            this.#sinceKey = Math.min(this.#sinceKey, 1);
+        }
+        this.#stats.bytes += data.length;
+        for (const s of this.#sinks)
+            s(f);
+    }
+    stats() {
+        return { ...this.#stats, queue: this.#closed ? 0 : this.#enc.encodeQueueSize, bitrate: this.cfg.bitrate ?? 0, encodeMs: this.#encodeMs, codec: this.cfg.codec };
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        try {
+            this.#enc.close();
+        }
+        catch { /* already closed */ }
+        this.#sinks.clear();
+        this.#inflight.clear();
+    }
+    #log(msg) { this.opts.onLog?.(msg); }
+}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export { AuSplitter, NAL_TYPE, SEI_RECOVERY_POINT, auParamSets, auSyncPoints, codecStringFromSps, concatBytes, findSps, findStartCode, hasVcl, isKeyframe, isVclType, nalIsVcl, nalPayload, nalType, nalUnits, nalWithStartCode, rbspUnescape, seiHasRecoveryPoint, seiPayloadTypes, splitAccessUnits, type Nal, } from "@synapse-sdk/core";
+export * from "./reinsert.ts";
+export * from "./sync.ts";
+export * from "./webcodecs.ts";
+export * from "./decoder.ts";
+export * from "./canvas.ts";
+export * from "./encoder.ts";
+export * from "./video.ts";
+export { trackFrames, trackProcessorSupported } from "./camera.ts";
+export type { MediaFrame, Stages } from "@synapse-sdk/core";
+export { isStartPoint, mediaFlags, mediaFrameFromHeader, mediaHeader, stagesOf } from "@synapse-sdk/core";

package/dist/index.js

@@ -0,0 +1,16 @@
+// @synapse-sdk/media: H.264 video over Synapse in the browser.
+// - `videoPublisher(node, topic)` / `videoSubscriber(node, topic)`: pictures in, pictures out.
+// - `H264Encoder` / `H264Decoder`: WebCodecs, for a two-step path of your own (constructors throw `not_supported` without WebCodecs).
+// - `ParamSetCache` / `DecoderSync`: the publisher re-insertion and decoder sync policies, identical in every SDK.
+// - The Annex-B walkers (flags from the bitstream), `attachCanvas`, camera helpers.
+// The React hook lives in `@synapse-sdk/media/react`.
+export { AuSplitter, NAL_TYPE, SEI_RECOVERY_POINT, auParamSets, auSyncPoints, codecStringFromSps, concatBytes, findSps, findStartCode, hasVcl, isKeyframe, isVclType, nalIsVcl, nalPayload, nalType, nalUnits, nalWithStartCode, rbspUnescape, seiHasRecoveryPoint, seiPayloadTypes, splitAccessUnits, } from "@synapse-sdk/core";
+export * from "./reinsert.js";
+export * from "./sync.js";
+export * from "./webcodecs.js";
+export * from "./decoder.js";
+export * from "./canvas.js";
+export * from "./encoder.js";
+export * from "./video.js";
+export { trackFrames, trackProcessorSupported } from "./camera.js";
+export { isStartPoint, mediaFlags, mediaFrameFromHeader, mediaHeader, stagesOf } from "@synapse-sdk/core";

package/dist/react.d.ts

@@ -0,0 +1,15 @@
+import type { Synapse, TopicRef } from "@synapse-sdk/client";
+import { type SynapseError } from "@synapse-sdk/core";
+import { type CanvasSinkOptions } from "./canvas.ts";
+export interface UseVideoOptions extends CanvasSinkOptions {
+    /** The node to use instead of the one from `<SynapseProvider>`. */
+    node?: Synapse;
+}
+export interface UseVideoResult {
+    /** Put this on the `<canvas>`: `<canvas ref={ref} />`. */
+    ref: (el: HTMLCanvasElement | null) => void;
+    /** `not_supported` when WebCodecs is missing, or whatever failed while setting up. */
+    error: SynapseError | undefined;
+}
+/** Decodes a video topic into a canvas. `topic` may be `"skip"` to render nothing without subscribing. */
+export declare function useVideo(topic: TopicRef<Uint8Array, Uint8Array> | "skip", opts?: UseVideoOptions): UseVideoResult;

package/dist/react.js

@@ -0,0 +1,52 @@
+// `@synapse-sdk/media/react`: `useVideo(topic)` — a canvas ref callback for a video topic, on
+// `videoSubscriber`. The per-frame path (decoder → canvas) never touches React. Needs a
+// `<SynapseProvider>` (from `@synapse-sdk/react`) or `opts.node`.
+import { useCallback, useEffect, useRef, useState } from "react";
+import { toSynapseError } from "@synapse-sdk/core";
+import { useSynapseOptional } from "@synapse-sdk/react";
+import { attachCanvas } from "./canvas.js";
+import { videoSubscriber } from "./video.js";
+/** Decodes a video topic into a canvas. `topic` may be `"skip"` to render nothing without subscribing. */
+export function useVideo(topic, opts = {}) {
+    const ctxNode = useSynapseOptional();
+    const node = opts.node ?? ctxNode;
+    const [canvas, setCanvas] = useState(null);
+    const [error, setError] = useState(undefined);
+    const optsRef = useRef(opts);
+    optsRef.current = opts;
+    const ref = useCallback((el) => setCanvas(el), []);
+    const topicName = topic === "skip" ? undefined : typeof topic === "string" ? topic : topic.name;
+    useEffect(() => {
+        if (!canvas || !node || topic === "skip")
+            return;
+        const o = optsRef.current;
+        try {
+            // The canvas sink owns each picture handed to it (draws, then closes it).
+            let sink;
+            const drawn = attachCanvas({ attachSink: s => { sink = s; return () => { if (sink === s)
+                    sink = undefined; }; } }, canvas, { vsync: o.vsync, resize: o.resize });
+            let video;
+            try {
+                video = videoSubscriber(node, topic);
+            }
+            catch (e) {
+                drawn.detach();
+                throw e;
+            }
+            void (async () => { for await (const vf of video) {
+                if (sink)
+                    sink(vf);
+                else
+                    vf.close();
+            } })();
+            setError(undefined);
+            return () => { video.close(); drawn.detach(); };
+        }
+        catch (e) {
+            setError(toSynapseError(e, "not_supported", "sdk"));
+            return;
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [node, canvas, topicName]);
+    return { ref, error };
+}

package/dist/reinsert.d.ts

@@ -0,0 +1,24 @@
+import type { MediaFrame } from "@synapse-sdk/core";
+/** Remembers the most recent SPS+PPS (with start codes) and re-inserts them where a decoder needs them. */
+export declare class ParamSetCache {
+    #private;
+    /** Frames the publisher policy re-inserted into (diagnostics). */
+    reinserted: number;
+    /** The cached SPS+PPS bytes, if any were seen. */
+    get paramSets(): Uint8Array | undefined;
+    get isPrimed(): boolean;
+    /** Looks at an AU: caches its parameter sets when it carries some. Returns `true` if it did. */
+    observe(au: Uint8Array): boolean;
+    /**
+     * Publisher policy on an outgoing frame: cache parameter sets when present; when absent on a
+     * recovery point (or when `force`), prepend the cached ones and set `config`. Frames that need
+     * nothing pass through unchanged (no copy).
+     */
+    prepare(frame: MediaFrame, force?: boolean): MediaFrame;
+    /**
+     * Receiver side: the AU with parameter sets in front when it lacks them and the cache has some
+     * (`undefined` if neither is available, i.e. the AU cannot start a decoder on its own). Caches
+     * the AU's own parameter sets first, so the newest ones always win.
+     */
+    prime(au: Uint8Array): Uint8Array | undefined;
+}

package/dist/reinsert.js

@@ -0,0 +1,41 @@
+import { auParamSets, concatBytes } from "@synapse-sdk/core";
+/** Remembers the most recent SPS+PPS (with start codes) and re-inserts them where a decoder needs them. */
+export class ParamSetCache {
+    #paramSets;
+    /** Frames the publisher policy re-inserted into (diagnostics). */
+    reinserted = 0;
+    /** The cached SPS+PPS bytes, if any were seen. */
+    get paramSets() { return this.#paramSets; }
+    get isPrimed() { return this.#paramSets !== undefined; }
+    /** Looks at an AU: caches its parameter sets when it carries some. Returns `true` if it did. */
+    observe(au) {
+        const ps = auParamSets(au);
+        if (!ps)
+            return false;
+        this.#paramSets = ps;
+        return true;
+    }
+    /**
+     * Publisher policy on an outgoing frame: cache parameter sets when present; when absent on a
+     * recovery point (or when `force`), prepend the cached ones and set `config`. Frames that need
+     * nothing pass through unchanged (no copy).
+     */
+    prepare(frame, force = false) {
+        if (this.observe(frame.data))
+            return frame.config ? frame : { ...frame, config: true };
+        if (!(frame.recovery || force) || !this.#paramSets)
+            return frame;
+        this.reinserted++;
+        return { ...frame, data: concatBytes([this.#paramSets, frame.data]), config: true };
+    }
+    /**
+     * Receiver side: the AU with parameter sets in front when it lacks them and the cache has some
+     * (`undefined` if neither is available, i.e. the AU cannot start a decoder on its own). Caches
+     * the AU's own parameter sets first, so the newest ones always win.
+     */
+    prime(au) {
+        if (this.observe(au))
+            return au;
+        return this.#paramSets ? concatBytes([this.#paramSets, au]) : undefined;
+    }
+}

package/dist/sync.d.ts

@@ -0,0 +1,60 @@
+import type { MediaFrame } from "@synapse-sdk/core";
+import { ParamSetCache } from "./reinsert.ts";
+export interface SyncConfig {
+    /** Frames fed without output before an IDR is requested (about a third of a second at 30 fps). */
+    stuckFrames: number;
+    /** Frames fed without output before the decoder is dropped outright. */
+    wedgedFrames: number;
+    /** Silence tolerated after a start at a recovery point (a refresh period plus slack). */
+    recoverySilenceFrames: number;
+    /**
+     * A recovery point with parameter sets is a start point (default true, the Rust/libav rule).
+     * `false` for decoders that can only start on an IDR: WebCodecs in Chromium verifies that a
+     * chunk labelled `key` carries an IDR and rejects the recovery-point start (`H264Decoder` sets this).
+     */
+    startAtRecovery: boolean;
+}
+/** `synapse_media::SyncConfig::default()`. */
+export declare const SYNC_DEFAULT: Readonly<SyncConfig>;
+/** `waiting`: no start point yet, frames are dropped. `primed`: started, nothing decoded yet. `streaming`: output seen since the start. */
+export type SyncState = "waiting" | "primed" | "streaming";
+/** What the owner should do about the decoder after a frame: ask the publisher for an IDR, or drop the decoder (the machine is back in `waiting`). */
+export type SyncSignal = "request_idr" | "reset";
+/** The verdict on one pushed frame. */
+export interface SyncDecision {
+    /** Bytes to hand to the decoder (parameter sets prepended when needed), or `undefined` = dropped. */
+    feed: Uint8Array | undefined;
+    /** Label the chunk `key` for the decoder (an IDR, or the first chunk after a start). */
+    keyChunk: boolean;
+    /** The decoder was just (re)started with this frame: (re)configure it first. */
+    started: boolean;
+    signal: SyncSignal | undefined;
+}
+export interface SyncStats {
+    frames: number;
+    dropped: number;
+    fed: number;
+    output: number;
+    starts: number;
+    startsAtRecovery: number;
+    resets: number;
+    idrRequests: number;
+}
+export declare class DecoderSync {
+    #private;
+    readonly cfg: SyncConfig;
+    readonly stats: SyncStats;
+    /** The SPS/PPS cache (primed from every AU that carries them, even while waiting). */
+    readonly cache: ParamSetCache;
+    constructor(cfg?: Partial<SyncConfig>);
+    get state(): SyncState;
+    get paramSets(): Uint8Array | undefined;
+    /** Frames fed since the decoder last produced output. */
+    get fedSinceOutput(): number;
+    /** A received media frame. Decides whether to feed it and how to label it, and raises the stuck/wedged signals. */
+    push(frame: MediaFrame): SyncDecision;
+    /** The decoder produced a frame. */
+    onOutput(): void;
+    /** The decoder reported an error: drop it and wait for the next start point. */
+    onError(): void;
+}

package/dist/sync.js

@@ -0,0 +1,101 @@
+import { ParamSetCache } from "./reinsert.js";
+/** `synapse_media::SyncConfig::default()`. */
+export const SYNC_DEFAULT = { stuckFrames: 10, wedgedFrames: 150, recoverySilenceFrames: 60, startAtRecovery: true };
+export class DecoderSync {
+    cfg;
+    stats = { frames: 0, dropped: 0, fed: 0, output: 0, starts: 0, startsAtRecovery: 0, resets: 0, idrRequests: 0 };
+    /** The SPS/PPS cache (primed from every AU that carries them, even while waiting). */
+    cache = new ParamSetCache();
+    #state = "waiting";
+    #startedAtRecovery = false;
+    #syncChunk = false;
+    #fedSinceOutput = 0;
+    /** Frames since the last IDR request (or since the output or start): the stuck rule. */
+    #fedSinceRequest = 0;
+    constructor(cfg = {}) { this.cfg = { ...SYNC_DEFAULT, ...cfg }; }
+    get state() { return this.#state; }
+    get paramSets() { return this.cache.paramSets; }
+    /** Frames fed since the decoder last produced output. */
+    get fedSinceOutput() { return this.#fedSinceOutput; }
+    /** A received media frame. Decides whether to feed it and how to label it, and raises the stuck/wedged signals. */
+    push(frame) {
+        this.stats.frames++;
+        let started = false;
+        // Parameter sets are cached from every frame, synced or not (the cache primes later starts).
+        const inBand = this.cache.observe(frame.data);
+        let data = frame.data;
+        if (this.#state === "waiting") {
+            if (frame.key) {
+                this.#startedAtRecovery = false;
+            }
+            else if (this.cfg.startAtRecovery && frame.recovery && (inBand || this.cache.isPrimed)) {
+                if (!inBand) {
+                    const primed = this.cache.prime(frame.data);
+                    if (primed)
+                        data = primed;
+                }
+                this.#startedAtRecovery = true;
+                this.stats.startsAtRecovery++;
+            }
+            else {
+                // still waiting: after a stuck window of dropped frames, ask for an IDR
+                this.stats.dropped++;
+                this.#fedSinceOutput++;
+                this.#fedSinceRequest++;
+                let signal;
+                if (this.#fedSinceRequest >= this.cfg.stuckFrames) {
+                    this.#fedSinceRequest = 0;
+                    this.stats.idrRequests++;
+                    signal = "request_idr";
+                }
+                return { feed: undefined, keyChunk: false, started: false, signal };
+            }
+            this.#state = "primed";
+            this.#syncChunk = true;
+            this.#fedSinceOutput = 0;
+            this.#fedSinceRequest = 0;
+            this.stats.starts++;
+            started = true;
+        }
+        const keyChunk = frame.key || this.#syncChunk;
+        this.#syncChunk = false;
+        this.#fedSinceOutput++;
+        this.#fedSinceRequest++;
+        this.stats.fed++;
+        return { feed: data, keyChunk, started, signal: this.#checkStuck() };
+    }
+    #checkStuck() {
+        const tolerated = this.#startedAtRecovery ? this.cfg.recoverySilenceFrames : this.cfg.stuckFrames;
+        if (this.#fedSinceOutput >= this.cfg.wedgedFrames) {
+            this.#reset();
+            return "reset";
+        }
+        if (this.#fedSinceRequest >= tolerated) {
+            // Give the IDR a full window before asking again (the wedged count runs on).
+            this.#fedSinceRequest = 0;
+            this.#startedAtRecovery = false;
+            this.stats.idrRequests++;
+            return "request_idr";
+        }
+        return undefined;
+    }
+    /** The decoder produced a frame. */
+    onOutput() {
+        this.#fedSinceOutput = 0;
+        this.#fedSinceRequest = 0;
+        this.#startedAtRecovery = false;
+        this.stats.output++;
+        if (this.#state === "primed")
+            this.#state = "streaming";
+    }
+    /** The decoder reported an error: drop it and wait for the next start point. */
+    onError() { this.#reset(); }
+    #reset() {
+        this.#state = "waiting";
+        this.#syncChunk = false;
+        this.#startedAtRecovery = false;
+        this.#fedSinceOutput = 0;
+        this.#fedSinceRequest = 0;
+        this.stats.resets++;
+    }
+}

package/dist/video.d.ts

@@ -0,0 +1,47 @@
+import { type TopicRef } from "@synapse-sdk/core";
+import type { Envelope, Synapse } from "@synapse-sdk/client";
+import { H264Encoder, type H264EncoderOptions } from "./encoder.ts";
+/** `videoPublisher` settings: the `H264Encoder` knobs a caller may want, all with defaults. */
+export type VideoPublisherOptions = Pick<H264EncoderOptions, "fps" | "bitrate" | "hardwareAcceleration">;
+/** Pictures in, `video/h264` out (`videoPublisher`). */
+export interface VideoPublisher {
+    /**
+     * Encodes and publishes `frame`, stamped `tUs` (the capture time, if you have one; default now).
+     * Synchronous; the encoder drops a picture rather than queue behind (see `encoder.stats()`). The
+     * first picture opens the encoder at its size. The frame stays yours to close.
+     */
+    publish(frame: VideoFrame, tUs?: number): void;
+    /** Publishes a camera track's frames until it ends or `signal` aborts (Chrome's `MediaStreamTrackProcessor`); resolves with the frames handed over. */
+    pumpTrack(track: MediaStreamTrack, opts?: {
+        signal?: AbortSignal;
+    }): Promise<number>;
+    /** Ends the topic, the `<topic>/idr` service and the encoder. */
+    close(): void;
+    /** The encoder, once the first picture made it. */
+    readonly encoder: H264Encoder | undefined;
+}
+/**
+ * A video publisher on `topic` (a name, or a `video/h264` topic: datagrams with FEC unless the
+ * contract says otherwise). Resolves once the topic and its `<topic>/idr` service are admitted.
+ */
+export declare function videoPublisher(node: Synapse, topic: TopicRef<Uint8Array, Uint8Array>, opts?: VideoPublisherOptions): Promise<VideoPublisher>;
+/** Decoded pictures of a `video/h264` topic (`videoSubscriber`): `for await` them (you close each), or get them through `onFrame`. */
+export interface VideoSubscriber extends AsyncIterable<VideoFrame> {
+    /** Stops the subscription and the decoder; pictures not yet read are closed. */
+    close(): void;
+}
+export interface VideoSubscriberOptions {
+    /**
+     * Each decoded picture with the envelope of the access unit it came from (`tUs` = capture time,
+     * `key`, `dom`, publisher). The picture is closed when the callback returns; `clone()` it to keep
+     * it. With `onFrame` the subscriber yields nothing.
+     */
+    onFrame?: (frame: VideoFrame, meta: Envelope<Uint8Array>) => void;
+}
+/**
+ * Decoded pictures of the video topic `topic` (by name; the established descriptor says what it
+ * carries). Every frame is fed to the decoder in order and every picture it gives is delivered.
+ * Frames before a start point are dropped by the decoder's policy, which asks the publisher for an
+ * IDR when stuck or waiting. Throws `not_supported` where WebCodecs is missing.
+ */
+export declare function videoSubscriber(node: Synapse, topic: TopicRef<Uint8Array, Uint8Array>, opts?: VideoSubscriberOptions): VideoSubscriber;

package/dist/video.js

@@ -0,0 +1,172 @@
+// Video in one call each way (PLAN v0.25 §2), the browser spelling of the Rust client's `video.rs`:
+// `videoPublisher(node, topic)` takes `VideoFrame`s (WebCodecs H.264 encode + publish in the
+// caller's `publish`), `videoSubscriber(node, topic)` gives decoded `VideoFrame`s. Underneath they
+// are the two-step path anyone may still take: an `H264Encoder` feeding a publisher on a
+// `video/h264` topic, and a subscription feeding an `H264Decoder` (its `DecoderSync` policy decides
+// where decoding starts and when to ask for an IDR).
+//
+// The publisher serves `<topic>/idr`: the next picture is an IDR with SPS/PPS. The subscriber calls
+// it when its decoder's policy says `request_idr` (one call in flight at a time).
+import { SynapseError, defineTopic } from "@synapse-sdk/core";
+import { pumpTrack } from "./camera.js";
+import { H264Decoder } from "./decoder.js";
+import { H264Encoder } from "./encoder.js";
+const H264 = "video/h264";
+/** The key of a client publisher handle's encoder-frame path (`@synapse-sdk/client` `PUBLISH_MEDIA_FRAME`): flags and `STAGES` as the encoder made them. */
+const PUBLISH_MEDIA_FRAME = Symbol.for("@synapse-sdk/client:publishMediaFrame");
+/** The `video/h264` contract of `topic`: a name gets one; a contract must already be `video/h264`. */
+function videoContract(topic) {
+    if (typeof topic === "string")
+        return defineTopic({ name: topic, contentType: H264 });
+    if (topic.contentType !== H264)
+        throw new SynapseError("invalid_input", `topic ${topic.name}: a video publisher needs contentType ${H264} (it has ${topic.contentType})`);
+    return topic;
+}
+/**
+ * A video publisher on `topic` (a name, or a `video/h264` topic: datagrams with FEC unless the
+ * contract says otherwise). Resolves once the topic and its `<topic>/idr` service are admitted.
+ */
+export async function videoPublisher(node, topic, opts = {}) {
+    const contract = videoContract(topic);
+    const publisher = await node.publisher(contract);
+    let encoder;
+    let requests = 0;
+    let idr;
+    try {
+        idr = await node.serve(`${contract.name}/idr`, () => { requests++; encoder?.forceIdr(); return { ok: true, requests }; }, { minRole: "observer", description: "force the next picture to be an IDR with SPS/PPS" });
+    }
+    catch (e) {
+        publisher.close();
+        throw e;
+    }
+    let closed = false;
+    const send = publisher;
+    const publish = (frame, tUs) => {
+        if (closed)
+            throw new SynapseError("not_connected", "video publisher closed");
+        if (encoder && (encoder.cfg.width !== frame.displayWidth || encoder.cfg.height !== frame.displayHeight)) {
+            // TODO(video, D 2026-10-08): one publisher per video topic is assumed, not enforced;
+            // resolution and settings do not change while a publisher lives. Codify later.
+            throw new SynapseError("invalid_input", "frame size changed; one publisher, one size");
+        }
+        if (!encoder) {
+            encoder = new H264Encoder({ width: frame.displayWidth, height: frame.displayHeight, ...opts, onError: e => node.warn(`video ${contract.name}: encoder: ${e instanceof Error ? e.message : String(e)}`) });
+            // flags and `STAGES` as the encoder made them; `sentUs` 0 is stamped at the write
+            encoder.attachSink(f => {
+                try {
+                    send[PUBLISH_MEDIA_FRAME](f);
+                }
+                catch (e) {
+                    node.warn(`video ${contract.name}: publish failed: ${e instanceof Error ? e.message : String(e)}`);
+                }
+            });
+        }
+        encoder.encode(frame, tUs === undefined ? {} : { tUs });
+    };
+    return {
+        publish,
+        pumpTrack: (track, o = {}) => pumpTrack(track, publish, o),
+        close() {
+            if (closed)
+                return;
+            closed = true;
+            encoder?.close();
+            idr.unregister();
+            publisher.close();
+        },
+        get encoder() { return encoder; },
+    };
+}
+/**
+ * Decoded pictures of the video topic `topic` (by name; the established descriptor says what it
+ * carries). Every frame is fed to the decoder in order and every picture it gives is delivered.
+ * Frames before a start point are dropped by the decoder's policy, which asks the publisher for an
+ * IDR when stuck or waiting. Throws `not_supported` where WebCodecs is missing.
+ */
+export function videoSubscriber(node, topic, opts = {}) {
+    const name = typeof topic === "string" ? topic : topic.name;
+    const idrService = `${name}/idr`;
+    const decoder = new H264Decoder();
+    // every frame reaches the decoder (a P picture references the one before it)
+    const sub = node.subscribe(name, { queueDepth: 0 });
+    const pending = [];
+    let wake;
+    let ended = false;
+    const signal = () => { const w = wake; wake = undefined; w?.(); };
+    let idrInFlight = false;
+    // `<topic>/idr` unless a call is still in flight; a failure is the policy's business (it asks again after another stuck window).
+    decoder.onRequestIdr = () => {
+        if (idrInFlight || ended)
+            return;
+        let run;
+        try {
+            run = node.call(idrService, {}, { deadlineMs: 1000 });
+        }
+        catch {
+            return;
+        } // no session: the next window asks again
+        idrInFlight = true;
+        run.result.then(() => { }, () => { }).finally(() => { idrInFlight = false; });
+    };
+    decoder.attachSink((vf, meta) => {
+        if (ended) {
+            vf.close();
+            return;
+        }
+        if (!opts.onFrame) {
+            pending.push(vf);
+            signal();
+            return;
+        }
+        const env = meta.frame;
+        const m = "seq" in env ? env : { data: env.data, tUs: env.tUs, seq: 0, publisher: "", dom: "", flags: 0, type: 0, tlvs: [], key: env.key, config: env.config, recovery: env.recovery };
+        try {
+            opts.onFrame(vf, m);
+        }
+        finally {
+            vf.close();
+        }
+    });
+    const close = () => {
+        if (ended)
+            return;
+        ended = true;
+        sub.unsubscribe();
+        decoder.close();
+        for (const vf of pending.splice(0))
+            vf.close();
+        signal();
+    };
+    void (async () => {
+        for (;;) {
+            const batch = await sub.next();
+            if (!batch || ended)
+                break;
+            // an envelope is the decoder's `MediaFrame` (data, tUs, key, config, recovery): the picture's meta is the envelope itself
+            for (const env of batch)
+                if (env.data instanceof Uint8Array)
+                    decoder.push(env);
+        }
+        close();
+    })();
+    return {
+        close,
+        async *[Symbol.asyncIterator]() {
+            try {
+                for (;;) {
+                    const vf = pending.shift();
+                    if (vf) {
+                        yield vf;
+                        continue;
+                    }
+                    if (ended)
+                        return;
+                    await new Promise(r => { wake = r; });
+                }
+            }
+            finally {
+                close();
+            }
+        },
+    };
+}

package/dist/webcodecs.d.ts

@@ -0,0 +1,10 @@
+/** `VideoDecoder`, `VideoEncoder`, `EncodedVideoChunk` and `VideoFrame` all exist in this runtime. */
+export declare function webCodecsSupported(): boolean;
+/** Throws `SynapseError("not_supported")` naming `what` when WebCodecs is missing. */
+export declare function requireWebCodecs(what: string): void;
+/** WebCodecs H.264 decode support for a codec string (default Constrained Baseline L3.1); `false` where WebCodecs is missing. */
+export declare function h264DecodeSupported(codec?: string): Promise<boolean>;
+/** WebCodecs H.264 *Annex-B* encode support for a frame size; `false` where WebCodecs is missing. */
+export declare function h264EncodeSupported(width: number, height: number, codec?: string): Promise<boolean>;
+/** Constrained Baseline, level 3.1: what a decoder is configured with when the stream has not shown its SPS yet (dylithium's default). */
+export declare const DEFAULT_H264_CODEC = "avc1.42e01f";

package/dist/webcodecs.js

@@ -0,0 +1,39 @@
+// WebCodecs availability. The package must import cleanly where WebCodecs does not exist (Bun,
+// Node, Firefox before 130, Safari before 16.4): nothing touches `VideoDecoder`/`VideoEncoder` at
+// module load; the classes that need them throw `not_supported` from their constructors.
+import { SynapseError } from "@synapse-sdk/core";
+/** `VideoDecoder`, `VideoEncoder`, `EncodedVideoChunk` and `VideoFrame` all exist in this runtime. */
+export function webCodecsSupported() {
+    const g = globalThis;
+    return typeof g.VideoDecoder === "function" && typeof g.VideoEncoder === "function" && typeof g.EncodedVideoChunk === "function" && typeof g.VideoFrame === "function";
+}
+/** Throws `SynapseError("not_supported")` naming `what` when WebCodecs is missing. */
+export function requireWebCodecs(what) {
+    if (webCodecsSupported())
+        return;
+    throw new SynapseError("not_supported", `${what}: WebCodecs (VideoDecoder/VideoEncoder/VideoFrame) is not available in this runtime; H.264 in the browser needs Chrome/Chromium 94+ (the project decision is Chrome-only)`);
+}
+/** WebCodecs H.264 decode support for a codec string (default Constrained Baseline L3.1); `false` where WebCodecs is missing. */
+export async function h264DecodeSupported(codec = DEFAULT_H264_CODEC) {
+    if (!webCodecsSupported())
+        return false;
+    try {
+        return (await VideoDecoder.isConfigSupported({ codec, optimizeForLatency: true })).supported === true;
+    }
+    catch {
+        return false;
+    }
+}
+/** WebCodecs H.264 *Annex-B* encode support for a frame size; `false` where WebCodecs is missing. */
+export async function h264EncodeSupported(width, height, codec = DEFAULT_H264_CODEC) {
+    if (!webCodecsSupported())
+        return false;
+    try {
+        return (await VideoEncoder.isConfigSupported({ codec, width, height, avc: { format: "annexb" }, latencyMode: "realtime" })).supported === true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Constrained Baseline, level 3.1: what a decoder is configured with when the stream has not shown its SPS yet (dylithium's default). */
+export const DEFAULT_H264_CODEC = "avc1.42e01f";

package/package.json

@@ -1,6 +1,44 @@
 {
   "name": "@synapse-sdk/media",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.3.3",
+  "description": "Synapse SDK media: H.264 video in the browser — encoder, decoder, canvas sink, video publisher/subscriber and the React useVideo hook.",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./react": {
+      "types": "./dist/react.d.ts",
+      "default": "./dist/react.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@synapse-sdk/core": "0.3.3",
+    "@synapse-sdk/client": "0.3.3"
+  },
+  "peerDependencies": {
+    "@synapse-sdk/react": "0.3.3",
+    "react": ">=18"
+  },
+  "peerDependenciesMeta": {
+    "@synapse-sdk/react": {
+      "optional": true
+    },
+    "react": {
+      "optional": true
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/taiga-robotics/synapse"
+  }
+}