@synapse-sdk/media 0.0.0-stage → 0.3.0

package/dist/video.d.ts

@@ -0,0 +1,124 @@
+import { type MediaFrame, type MediaLayout, type TopicContract, type TopicRef } from "@synapse-sdk/core";
+import type { Envelope, Publisher, Registration, Subscription, Synapse } from "@synapse-sdk/client";
+import type { H264Decoder } from "./decoder.ts";
+import type { H264Encoder } from "./encoder.ts";
+import type { SyncState } from "./sync.ts";
+export declare const H264 = "video/h264";
+export interface H264TopicOptions {
+    /** RaptorQ repair percentage; default 30 (`repairPct: 0` = source symbols only). */
+    repairPct?: number;
+    /** FEC give-up budget, ms; default 50 (video). */
+    deadlineMs?: number;
+    /** One MoQ group per frame with the relay caching the latest, instead of datagrams (a poster-frame topic, not live video). */
+    retain?: boolean;
+}
+/** A raw topic for H.264 access units: `contentType: "video/h264"`, `layout: {codec: "h264", stream: "annexb", ...}`, datagrams with FEC. */
+export declare function h264Topic(name: string, layout?: Omit<MediaLayout, "codec" | "stream">, opts?: H264TopicOptions): TopicContract<Uint8Array, Uint8Array>;
+/** The IDR-on-demand service a video publisher serves (decision 4): `<topic>/idr`, no arguments, `{ ok: true }`. */
+export declare function idrServiceName(topic: string): string;
+export interface VideoPublisherOptions {
+    /** Serve `<topic>/idr` → `encoder.forceIdr()`; default true. */
+    idrService?: boolean;
+}
+export interface VideoPublisherStats {
+    /** Access units handed to `publishMedia`. */
+    sent: number;
+    /** `<topic>/idr` calls served. */
+    idrCalls: number;
+    /** Frames the SPS/PPS re-insertion policy touched (WebCodecs streams: none, key chunks carry them). */
+    reinserted: number;
+}
+/**
+ * Encoder output → `publisher.publishMedia()`: `t_us` = capture (the `VideoFrame.timestamp`),
+ * `STAGES` = encoder stamp + `sentUs` 0 (the publisher stamps it at the write, WP §4.1), SPS/PPS
+ * re-insertion at recovery points, plus the `<topic>/idr` service that forces the next frame to be
+ * an IDR. `VideoPublisher.open()` resolves once the runtime admitted the topic (and the service).
+ */
+export declare class VideoPublisher {
+    #private;
+    readonly node: Synapse;
+    readonly topic: TopicContract<Uint8Array, Uint8Array>;
+    readonly publisher: Publisher<Uint8Array>;
+    readonly registration: Registration | undefined;
+    readonly encoder: H264Encoder;
+    private constructor();
+    static open(node: Synapse, topic: TopicRef<Uint8Array, Uint8Array>, encoder: H264Encoder, opts?: VideoPublisherOptions): Promise<VideoPublisher>;
+    stats(): VideoPublisherStats;
+    /** Stops publishing (the encoder stays yours to close). */
+    close(): void;
+}
+export interface VideoSubscriberOptions {
+    /** Call `<topic>/idr` when the decoder is stuck and while it waits for a start point; default true. */
+    requestIdr?: boolean;
+    /** Minimum spacing between IDR requests, ms; default 1000. */
+    idrMinIntervalMs?: number;
+    /** Subscription priority 0..7; default 2. */
+    priority?: number;
+}
+/**
+ * Latency split over the last stats window (averages, ms). Undefined when the stamps needed are
+ * not there. `e2eMs` and `netMs` cross the publisher's clock and ours: exact when it shares our time
+ * domain, otherwise through the broker's clock (`node.toLocalUs`, SDK §4.4) and reported with
+ * `epsMs`, the error bound; undefined while the publisher's domain has no offset yet (no judgment).
+ */
+export interface VideoLatency {
+    /** Capture → decoded picture, `± epsMs`. */
+    e2eMs: number | undefined;
+    /** Capture → encoder output (`STAGES.encoded_us`; one clock). */
+    encMs: number | undefined;
+    /** Encoder output → transport write (`STAGES.sent_us`; one clock). */
+    sendMs: number | undefined;
+    /** Transport write → our receive, `± epsMs`. */
+    netMs: number | undefined;
+    /** `decode()` → output. */
+    decodeMs: number | undefined;
+    /** Error bound on `e2eMs`/`netMs` from the clock conversion: 0 in our own domain, the two estimates' bounds added across domains, undefined when the clocks are not comparable (then both are undefined too). */
+    epsMs: number | undefined;
+}
+export interface VideoStats {
+    /** Decoded pictures per second over the window. */
+    fps: number;
+    /** Received bit rate over the window. */
+    kbps: number;
+    /** Totals since the subscriber was created. */
+    received: number;
+    decoded: number;
+    /** Frames not decoded: dropped while waiting for a start point, plus frames superseded before they were read. */
+    dropped: number;
+    superseded: number;
+    resets: number;
+    idrRequests: number;
+    /** Decoder queue depth right now. */
+    queue: number;
+    width: number;
+    height: number;
+    state: SyncState;
+    latency: VideoLatency;
+}
+/** A received envelope on a raw media topic as a `MediaFrame` (the header's flags and `STAGES`). */
+export declare function mediaFrameOf(env: Envelope<Uint8Array>): MediaFrame;
+/**
+ * A raw subscription feeding an `H264Decoder`. Every envelope becomes a `MediaFrame` and goes to
+ * `decoder.push()`; the decoder's stuck signal and the waiting state (a late joiner on an IDR-only
+ * stream sees nothing until an IDR comes) both call the publisher's `<topic>/idr` service, at most
+ * once per `idrMinIntervalMs`. Stats: fps, totals, and the latency split from `t_us` and `STAGES`.
+ */
+export declare class VideoSubscriber {
+    #private;
+    readonly node: Synapse;
+    readonly decoder: H264Decoder;
+    private readonly opts;
+    readonly topic: TopicContract<Uint8Array, Uint8Array>;
+    readonly sub: Subscription<Uint8Array>;
+    constructor(node: Synapse, topic: TopicRef<Uint8Array, Uint8Array>, decoder: H264Decoder, opts?: VideoSubscriberOptions);
+    /** Window averages since the previous call plus totals; resets the window. */
+    stats(): VideoStats;
+    /** IDR calls that failed (no publisher serving `<topic>/idr`, or timed out). */
+    get idrFailed(): number;
+    /**
+     * Stops the subscription and gives the decoder back its previous callbacks (the decoder stays
+     * yours to close). Close subscribers sharing one decoder newest first: a subscriber whose
+     * callbacks another one wrapped since cannot take them back, so the newer one still calls them.
+     */
+    close(): void;
+}

package/dist/video.js

@@ -0,0 +1,250 @@
+// Glue between the codecs and a Synapse node: `VideoPublisher` (encoder → `publishMedia`, serves
+// `<topic>/idr`), `VideoSubscriber` (raw subscription → `H264Decoder`, calls `<topic>/idr` when
+// stuck or waiting, keeps stats), `h264Topic` (the raw `fec` datagram topic with an H.264 layout).
+import { SynapseError, defineTopic, nowUs, resolveTopic } from "@synapse-sdk/core";
+import { ParamSetCache } from "./reinsert.js";
+export const H264 = "video/h264";
+/** A raw topic for H.264 access units: `contentType: "video/h264"`, `layout: {codec: "h264", stream: "annexb", ...}`, datagrams with FEC. */
+export function h264Topic(name, layout = {}, opts = {}) {
+    const l = { codec: "h264", stream: "annexb", ...layout };
+    if (opts.retain)
+        return defineTopic({ name, contentType: H264, layout: l, delivery: "stream" });
+    return defineTopic({ name, contentType: H264, layout: l, delivery: "datagram", fec: { repairPct: opts.repairPct ?? 30, deadlineMs: opts.deadlineMs ?? 50 } });
+}
+/** The IDR-on-demand service a video publisher serves (decision 4): `<topic>/idr`, no arguments, `{ ok: true }`. */
+export function idrServiceName(topic) { return `${topic}/idr`; }
+function rawContract(topic, who) {
+    const c = resolveTopic(topic);
+    if (!c.raw)
+        throw new SynapseError("invalid_input", `${who}: topic ${c.name} must be a raw topic (use h264Topic())`);
+    return c;
+}
+/**
+ * Encoder output → `publisher.publishMedia()`: `t_us` = capture (the `VideoFrame.timestamp`),
+ * `STAGES` = encoder stamp + `sentUs` 0 (the publisher stamps it at the write, WP §4.1), SPS/PPS
+ * re-insertion at recovery points, plus the `<topic>/idr` service that forces the next frame to be
+ * an IDR. `VideoPublisher.open()` resolves once the runtime admitted the topic (and the service).
+ */
+export class VideoPublisher {
+    node;
+    topic;
+    publisher;
+    registration;
+    encoder;
+    #cache = new ParamSetCache();
+    #detach;
+    #sent = 0;
+    #idrCalls = 0;
+    constructor(node, topic, publisher, registration, encoder) {
+        this.node = node;
+        this.topic = topic;
+        this.publisher = publisher;
+        this.registration = registration;
+        this.encoder = encoder;
+        this.#detach = encoder.attachSink(f => this.#onFrame(f));
+    }
+    static async open(node, topic, encoder, opts = {}) {
+        const contract = rawContract(topic, "VideoPublisher");
+        const publisher = await node.publisher(contract);
+        let reg;
+        if (opts.idrService !== false) {
+            let self;
+            try {
+                reg = await node.serve(idrServiceName(contract.name), () => { if (self)
+                    self.#idrCalls++; encoder.forceIdr(); return { ok: true }; }, { minRole: "observer", description: `force the next ${contract.name} frame to be an IDR` });
+            }
+            catch (e) {
+                publisher.close();
+                throw e;
+            }
+            self = new VideoPublisher(node, contract, publisher, reg, encoder);
+            return self;
+        }
+        return new VideoPublisher(node, contract, publisher, undefined, encoder);
+    }
+    #onFrame(f) {
+        const p = this.#cache.prepare(f, false);
+        p.stages = { encodedUs: f.stages?.encodedUs ?? nowUs(), sentUs: 0 };
+        this.#sent++;
+        try {
+            this.publisher.publishMedia(p);
+        }
+        catch (e) {
+            this.node.warn(`VideoPublisher ${this.topic.name}: publish failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    stats() { return { sent: this.#sent, idrCalls: this.#idrCalls, reinserted: this.#cache.reinserted }; }
+    /** Stops publishing (the encoder stays yours to close). */
+    close() {
+        this.#detach();
+        this.registration?.unregister();
+        this.publisher.close();
+    }
+}
+const freshWindow = () => ({ t0: performance.now(), frames: 0, bytes: 0, e2e: 0, e2eN: 0, enc: 0, encN: 0, send: 0, sendN: 0, net: 0, netN: 0, dec: 0, decN: 0, eps: 0, epsN: 0 });
+/** A received envelope on a raw media topic as a `MediaFrame` (the header's flags and `STAGES`). */
+export function mediaFrameOf(env) {
+    const f = { data: env.data, tUs: env.tUs, key: (env.flags & 1) !== 0, config: (env.flags & 32) !== 0, recovery: (env.flags & 64) !== 0 };
+    for (const t of env.tlvs)
+        if (t.kind === "stages") {
+            f.stages = { encodedUs: Number(t.encodedUs), sentUs: Number(t.sentUs) };
+            break;
+        }
+    return f;
+}
+/**
+ * A raw subscription feeding an `H264Decoder`. Every envelope becomes a `MediaFrame` and goes to
+ * `decoder.push()`; the decoder's stuck signal and the waiting state (a late joiner on an IDR-only
+ * stream sees nothing until an IDR comes) both call the publisher's `<topic>/idr` service, at most
+ * once per `idrMinIntervalMs`. Stats: fps, totals, and the latency split from `t_us` and `STAGES`.
+ */
+export class VideoSubscriber {
+    node;
+    decoder;
+    opts;
+    topic;
+    sub;
+    #win = freshWindow();
+    #received = 0;
+    #idrRequests = 0;
+    #idrFailed = 0;
+    #lastIdrMs = -Infinity;
+    #waitingDropped = 0;
+    /** Capture stamp of each frame in flight, converted to our clock (`toLocalUs`); absent when the clocks are not comparable. */
+    #captureLocal = new WeakMap();
+    #closed = false;
+    constructor(node, topic, decoder, opts = {}) {
+        this.node = node;
+        this.decoder = decoder;
+        this.opts = opts;
+        this.topic = rawContract(topic, "VideoSubscriber");
+        const prevIdr = decoder.onRequestIdr;
+        const onIdr = () => { prevIdr?.(); this.#requestIdr("stuck"); };
+        decoder.onRequestIdr = onIdr;
+        const prevOut = decoder.onOutput;
+        const onOut = (meta) => { prevOut?.(meta); this.#onOutput(meta); };
+        decoder.onOutput = onOut;
+        // `close()` puts the decoder's previous callbacks back when ours are still the ones installed.
+        this.#restore = () => {
+            if (decoder.onRequestIdr === onIdr)
+                decoder.onRequestIdr = prevIdr;
+            if (decoder.onOutput === onOut)
+                decoder.onOutput = prevOut;
+        };
+        // Every frame reaches the decoder (queueDepth 0): the decoder's own queue sheds, in picture order.
+        this.sub = node.subscribe(this.topic, { priority: opts.priority ?? 2, queueDepth: 0 });
+        void this.#read();
+    }
+    async #read() {
+        for (;;) {
+            const batch = await this.sub.next();
+            if (!batch || this.#closed)
+                return;
+            for (const env of batch)
+                this.#onEnvelope(env);
+        }
+    }
+    #onEnvelope(env) {
+        if (this.#closed)
+            return;
+        const f = mediaFrameOf(env);
+        this.#received++;
+        this.#win.frames++;
+        this.#win.bytes += f.data.length;
+        const recvUs = nowUs();
+        // Stamps within the publisher's clock need no conversion; those compared with ours go through `toLocalUs` (SDK §4.4).
+        if (f.stages) {
+            if (f.stages.encodedUs > 0 && f.tUs > 0) {
+                this.#win.enc += (f.stages.encodedUs - f.tUs) / 1000;
+                this.#win.encN++;
+            }
+            if (f.stages.sentUs > 0 && f.stages.encodedUs > 0) {
+                this.#win.send += (f.stages.sentUs - f.stages.encodedUs) / 1000;
+                this.#win.sendN++;
+            }
+            if (f.stages.sentUs > 0) {
+                const sent = this.node.toLocalUs(f.stages.sentUs, env.dom);
+                if (sent) {
+                    this.#win.net += (recvUs - sent.localUs) / 1000;
+                    this.#win.netN++;
+                }
+            }
+        }
+        if (f.tUs > 0) {
+            const capture = this.node.toLocalUs(f.tUs, env.dom);
+            if (capture) {
+                this.#captureLocal.set(f, capture);
+                this.#win.eps += capture.epsUs / 1000;
+                this.#win.epsN++;
+            }
+        }
+        const droppedBefore = this.decoder.sync.stats.dropped;
+        this.decoder.push(f);
+        if (this.decoder.sync.stats.dropped > droppedBefore) {
+            // Waiting for a start point: a browser publisher emits IDRs only on demand, so ask (first frame, then every stuck window).
+            this.#waitingDropped++;
+            if (this.#waitingDropped === 1 || this.#waitingDropped % this.decoder.sync.cfg.stuckFrames === 0)
+                this.#requestIdr("waiting");
+        }
+        else
+            this.#waitingDropped = 0;
+    }
+    #onOutput(meta) {
+        this.#win.dec += meta.decodeMs;
+        this.#win.decN++;
+        const capture = this.#captureLocal.get(meta.frame);
+        if (capture) {
+            this.#win.e2e += (nowUs() - capture.localUs) / 1000;
+            this.#win.e2eN++;
+        }
+    }
+    #requestIdr(why) {
+        if (this.opts.requestIdr === false || this.#closed)
+            return;
+        const now = performance.now();
+        if (now - this.#lastIdrMs < (this.opts.idrMinIntervalMs ?? 1000))
+            return;
+        this.#lastIdrMs = now;
+        this.#idrRequests++;
+        try {
+            this.node.call(idrServiceName(this.topic.name), {}, { deadlineMs: 1000 }).result.catch(() => { this.#idrFailed++; });
+        }
+        catch {
+            this.#idrFailed++;
+        }
+        void why;
+    }
+    /** Window averages since the previous call plus totals; resets the window. */
+    stats() {
+        const w = this.#win;
+        const dt = (performance.now() - w.t0) / 1000;
+        const avg = (sum, n) => (n ? sum / n : undefined);
+        const d = this.decoder.stats();
+        const superseded = this.sub.stats.superseded;
+        const s = {
+            fps: dt > 0 ? (d.output - this.#lastOutput) / dt : 0, kbps: dt > 0 ? (w.bytes * 8) / dt / 1000 : 0,
+            received: this.#received, decoded: d.output, dropped: d.dropped + superseded, superseded, resets: d.resets, idrRequests: this.#idrRequests,
+            queue: d.queue, width: d.width, height: d.height, state: d.state,
+            latency: { e2eMs: avg(w.e2e, w.e2eN), encMs: avg(w.enc, w.encN), sendMs: avg(w.send, w.sendN), netMs: avg(w.net, w.netN), decodeMs: avg(w.dec, w.decN), epsMs: avg(w.eps, w.epsN) },
+        };
+        this.#lastOutput = d.output;
+        this.#win = freshWindow();
+        return s;
+    }
+    #lastOutput = 0;
+    /** IDR calls that failed (no publisher serving `<topic>/idr`, or timed out). */
+    get idrFailed() { return this.#idrFailed; }
+    /**
+     * Stops the subscription and gives the decoder back its previous callbacks (the decoder stays
+     * yours to close). Close subscribers sharing one decoder newest first: a subscriber whose
+     * callbacks another one wrapped since cannot take them back, so the newer one still calls them.
+     */
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        this.sub.unsubscribe();
+        this.#restore();
+    }
+    #restore;
+}

package/dist/webcodecs.d.ts

@@ -0,0 +1,10 @@
+/** `VideoDecoder`, `VideoEncoder`, `EncodedVideoChunk` and `VideoFrame` all exist in this runtime. */
+export declare function webCodecsSupported(): boolean;
+/** Throws `SynapseError("not_supported")` naming `what` when WebCodecs is missing. */
+export declare function requireWebCodecs(what: string): void;
+/** WebCodecs H.264 decode support for a codec string (default Constrained Baseline L3.1); `false` where WebCodecs is missing. */
+export declare function h264DecodeSupported(codec?: string): Promise<boolean>;
+/** WebCodecs H.264 *Annex-B* encode support for a frame size; `false` where WebCodecs is missing. */
+export declare function h264EncodeSupported(width: number, height: number, codec?: string): Promise<boolean>;
+/** Constrained Baseline, level 3.1: what a decoder is configured with when the stream has not shown its SPS yet (dylithium's default). */
+export declare const DEFAULT_H264_CODEC = "avc1.42e01f";

package/dist/webcodecs.js

@@ -0,0 +1,39 @@
+// WebCodecs availability. The package must import cleanly where WebCodecs does not exist (Bun,
+// Node, Firefox before 130, Safari before 16.4): nothing touches `VideoDecoder`/`VideoEncoder` at
+// module load; the classes that need them throw `not_supported` from their constructors.
+import { SynapseError } from "@synapse-sdk/core";
+/** `VideoDecoder`, `VideoEncoder`, `EncodedVideoChunk` and `VideoFrame` all exist in this runtime. */
+export function webCodecsSupported() {
+    const g = globalThis;
+    return typeof g.VideoDecoder === "function" && typeof g.VideoEncoder === "function" && typeof g.EncodedVideoChunk === "function" && typeof g.VideoFrame === "function";
+}
+/** Throws `SynapseError("not_supported")` naming `what` when WebCodecs is missing. */
+export function requireWebCodecs(what) {
+    if (webCodecsSupported())
+        return;
+    throw new SynapseError("not_supported", `${what}: WebCodecs (VideoDecoder/VideoEncoder/VideoFrame) is not available in this runtime; H.264 in the browser needs Chrome/Chromium 94+ (the project decision is Chrome-only)`);
+}
+/** WebCodecs H.264 decode support for a codec string (default Constrained Baseline L3.1); `false` where WebCodecs is missing. */
+export async function h264DecodeSupported(codec = DEFAULT_H264_CODEC) {
+    if (!webCodecsSupported())
+        return false;
+    try {
+        return (await VideoDecoder.isConfigSupported({ codec, optimizeForLatency: true })).supported === true;
+    }
+    catch {
+        return false;
+    }
+}
+/** WebCodecs H.264 *Annex-B* encode support for a frame size; `false` where WebCodecs is missing. */
+export async function h264EncodeSupported(width, height, codec = DEFAULT_H264_CODEC) {
+    if (!webCodecsSupported())
+        return false;
+    try {
+        return (await VideoEncoder.isConfigSupported({ codec, width, height, avc: { format: "annexb" }, latencyMode: "realtime" })).supported === true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Constrained Baseline, level 3.1: what a decoder is configured with when the stream has not shown its SPS yet (dylithium's default). */
+export const DEFAULT_H264_CODEC = "avc1.42e01f";

package/package.json

@@ -1,6 +1,44 @@
 {
   "name": "@synapse-sdk/media",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.3.0",
+  "description": "Synapse SDK media: H.264 video in the browser — encoder, decoder, canvas sink, video publisher/subscriber and the React useVideo hook.",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./react": {
+      "types": "./dist/react.d.ts",
+      "default": "./dist/react.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@synapse-sdk/core": "0.3.0",
+    "@synapse-sdk/client": "0.3.0"
+  },
+  "peerDependencies": {
+    "@synapse-sdk/react": "0.3.0",
+    "react": ">=18"
+  },
+  "peerDependenciesMeta": {
+    "@synapse-sdk/react": {
+      "optional": true
+    },
+    "react": {
+      "optional": true
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/taiga-robotics/synapse"
+  }
+}