@eventual-ai/mcap-viewer 0.1.0-alpha.0 → 0.1.0-alpha.2

package/dist/core/playback/player.js

@@ -0,0 +1,344 @@
+import { snapshot } from "./snapshot.js";
+import { VideoSequence, videoSequence, decodeOrder, isVideo, } from "./video-sequence.js";
+const defaultScheduler = {
+    now: () => performance.now(),
+    request: (callback) => requestAnimationFrame(callback),
+    cancel: (id) => cancelAnimationFrame(id),
+};
+/** Local playback clock for a recording's native-timestamped video topics. */
+export class Player {
+    recording;
+    scheduler;
+    time;
+    speed = 1;
+    playing = false;
+    status = "paused";
+    closed = false;
+    generation = 0;
+    frame;
+    anchorTime = 0n;
+    anchorWall = 0;
+    waitingForData = false;
+    started = false;
+    seekController;
+    video = new Map();
+    submitted = new Map();
+    unsubscribeRecording;
+    listeners = new Set();
+    constructor(recording, callbacks = {}, scheduler = defaultScheduler) {
+        this.recording = recording;
+        this.scheduler = scheduler;
+        this.time = recording.start;
+        this.listeners.add({ listener: callbacks });
+        this.unsubscribeRecording = recording.subscribe?.(() => this.onLoad());
+    }
+    /**
+     * Subscribe a view or renderer to the shared playback clock and selected topics.
+     * It receives the current state, and once playback has started, a snapshot of
+     * the current frame. Call the returned function to unsubscribe.
+     */
+    subscribe(listener, { topics } = {}) {
+        if (this.closed)
+            throw new Error("Player is closed");
+        const subscription = {
+            listener,
+            topics: topics ? new Set(topics) : undefined,
+        };
+        this.listeners.add(subscription);
+        const unsubscribe = () => {
+            this.listeners.delete(subscription);
+        };
+        listener.onState?.(this.state());
+        if (this.started &&
+            this.status !== "buffering" &&
+            this.status !== "error") {
+            listener.onSnapshotStart?.();
+            try {
+                for (const record of snapshot(this.recording, this.time))
+                    if (!subscription.topics ||
+                        subscription.topics.has(record.channel.topic))
+                        listener.onMessage?.(record.channel, record.time, record.value);
+            }
+            catch (error) {
+                this.fail(error);
+                return unsubscribe;
+            }
+            void listener.onSnapshotEnd?.(!this.playing);
+        }
+        return unsubscribe;
+    }
+    state() {
+        return {
+            time: this.time,
+            status: this.status,
+            playing: this.playing,
+            speed: this.speed,
+        };
+    }
+    emit() {
+        const state = this.state();
+        for (const { listener } of this.listeners)
+            listener.onState?.(state);
+    }
+    publish(record) {
+        this.submitted.set(record.channel.topic, decodeOrder(record));
+        for (const { listener, topics } of this.listeners) {
+            if (this.closed)
+                return;
+            if (!topics || topics.has(record.channel.topic))
+                listener.onMessage?.(record.channel, record.time, record.value);
+        }
+    }
+    cancelFrame() {
+        if (this.frame !== undefined)
+            this.scheduler.cancel(this.frame);
+        this.frame = undefined;
+    }
+    fail(error) {
+        // A recording closed or aborted by its owner ends playback, not as an error.
+        if (error === this.recording.error &&
+            error instanceof Error &&
+            error.name === "AbortError") {
+            this.playing = false;
+            this.status = "paused";
+            this.emit();
+            this.close();
+            return;
+        }
+        this.generation++;
+        this.seekController?.abort();
+        this.waitingForData = false;
+        this.playing = false;
+        this.status = "error";
+        this.cancelFrame();
+        for (const { listener } of this.listeners)
+            listener.onError?.(error instanceof Error ? error : new Error(String(error)));
+        this.emit();
+    }
+    availableEnd() {
+        for (const records of this.recording.byTopic.values()) {
+            const channel = records[0]?.channel;
+            if (channel && isVideo(channel) && !this.video.has(channel.topic))
+                this.video.set(channel.topic, videoSequence(channel, records));
+        }
+        let end = this.recording.complete === false
+            ? (this.recording.bufferedEnd ?? this.recording.start - 1n)
+            : this.recording.end;
+        for (const sequence of this.video.values())
+            end = sequence.bufferedEnd(end, this.recording.complete !== false);
+        return end;
+    }
+    endBuffered() {
+        return (this.recording.complete !== false ||
+            (this.recording.bounds !== undefined &&
+                this.availableEnd() >= this.recording.end));
+    }
+    onLoad() {
+        // After a failure, only a new seek resumes reporting.
+        if (this.closed || this.status === "error")
+            return;
+        if (this.recording.error) {
+            this.fail(this.recording.error);
+            return;
+        }
+        let bufferedEnd;
+        try {
+            bufferedEnd = this.availableEnd();
+        }
+        catch (error) {
+            this.fail(error);
+            return;
+        }
+        if (this.waitingForData &&
+            this.playing &&
+            (this.recording.complete !== false || bufferedEnd > this.time)) {
+            this.waitingForData = false;
+            this.anchorTime = this.time;
+            this.anchorWall = this.scheduler.now();
+            this.status = "playing";
+            this.frame = this.scheduler.request(() => void this.tick());
+        }
+        this.emit();
+    }
+    waitForTime(time, signal) {
+        if (this.recording.error)
+            return Promise.reject(this.recording.error);
+        if (this.availableEnd() >= time || this.recording.complete !== false)
+            return Promise.resolve();
+        return new Promise((resolve, reject) => {
+            const cleanup = () => {
+                unsubscribe?.();
+                signal.removeEventListener("abort", aborted);
+            };
+            const check = () => {
+                if (this.recording.error) {
+                    cleanup();
+                    reject(this.recording.error);
+                }
+                else {
+                    try {
+                        if (this.availableEnd() >= time ||
+                            this.recording.complete !== false) {
+                            cleanup();
+                            resolve();
+                        }
+                    }
+                    catch (error) {
+                        cleanup();
+                        reject(error);
+                    }
+                }
+            };
+            const aborted = () => {
+                cleanup();
+                reject(signal.reason);
+            };
+            const unsubscribe = this.recording.subscribe?.(check);
+            signal.addEventListener("abort", aborted, { once: true });
+            if (signal.aborted)
+                aborted();
+            else
+                check();
+        });
+    }
+    async endSnapshot(paused) {
+        await Promise.all([...this.listeners].map(({ listener }) => listener.onSnapshotEnd?.(paused)));
+    }
+    async seek(target) {
+        if (this.closed)
+            return;
+        const generation = ++this.generation;
+        this.seekController?.abort();
+        const controller = new AbortController();
+        this.seekController = controller;
+        this.waitingForData = false;
+        this.cancelFrame();
+        this.time =
+            target < this.recording.start
+                ? this.recording.start
+                : target > this.recording.end &&
+                    (this.recording.complete !== false ||
+                        this.recording.bounds !== undefined)
+                    ? this.recording.end
+                    : target;
+        this.status = "buffering";
+        this.emit();
+        try {
+            if (this.recording.error)
+                throw this.recording.error;
+            if (this.recording.complete === false && this.availableEnd() < this.time)
+                await this.waitForTime(this.time, controller.signal);
+            if (generation !== this.generation || this.closed)
+                return;
+            if (this.time > this.recording.end)
+                this.time = this.recording.end;
+            this.submitted.clear();
+            for (const { listener } of this.listeners)
+                listener.onSnapshotStart?.();
+            for (const record of snapshot(this.recording, this.time)) {
+                if (generation !== this.generation || this.closed)
+                    return;
+                this.publish(record);
+            }
+            await this.endSnapshot(!this.playing);
+            if (generation !== this.generation || this.closed)
+                return;
+            this.started = true;
+            this.anchorTime = this.time;
+            this.anchorWall = this.scheduler.now();
+            this.status = this.playing ? "playing" : "paused";
+            this.emit();
+            if (this.playing)
+                this.frame = this.scheduler.request(() => void this.tick());
+        }
+        catch (error) {
+            if (generation === this.generation && !this.closed)
+                this.fail(error);
+        }
+    }
+    play() {
+        if (this.closed)
+            return;
+        this.playing = true;
+        let restart;
+        try {
+            restart = this.endBuffered() && this.time >= this.recording.end;
+        }
+        catch (error) {
+            this.fail(error);
+            return Promise.resolve();
+        }
+        return this.seek(restart ? this.recording.start : this.time);
+    }
+    pause() {
+        if (this.closed)
+            return;
+        this.playing = false;
+        return this.seek(this.time);
+    }
+    setSpeed(speed) {
+        if (this.closed)
+            return;
+        if (!Number.isFinite(speed) || speed <= 0)
+            throw new Error("Playback speed must be positive");
+        this.speed = speed;
+        this.anchorTime = this.time;
+        this.anchorWall = this.scheduler.now();
+        this.emit();
+    }
+    async tick() {
+        this.frame = undefined;
+        if (this.closed || !this.playing)
+            return;
+        const generation = this.generation;
+        const target = this.anchorTime +
+            BigInt(Math.round((this.scheduler.now() - this.anchorWall) * 1e6 * this.speed));
+        try {
+            const bufferedEnd = this.availableEnd();
+            const availableEnd = bufferedEnd < this.recording.end ? bufferedEnd : this.recording.end;
+            this.time = target > availableEnd ? availableEnd : target;
+            // State first, so listeners see this tick's time with its packets.
+            this.emit();
+            for (const [topic, sequence] of this.video) {
+                const submitted = this.submitted.get(topic);
+                const records = submitted === undefined
+                    ? sequence.snapshot(this.time)
+                    : sequence.after(this.time, submitted);
+                for (const record of records) {
+                    this.publish(record);
+                    if (generation !== this.generation || this.closed)
+                        return;
+                }
+            }
+            if (this.time === this.recording.end && this.endBuffered()) {
+                this.playing = false;
+                await this.endSnapshot(true);
+                if (generation !== this.generation || this.closed)
+                    return;
+                this.status = "ended";
+                this.emit();
+            }
+            else if (this.recording.complete === false && target >= availableEnd) {
+                this.status = "buffering";
+                this.waitingForData = true;
+                this.emit();
+            }
+            if (this.playing && !this.waitingForData)
+                this.frame = this.scheduler.request(() => void this.tick());
+        }
+        catch (error) {
+            if (generation === this.generation && !this.closed)
+                this.fail(error);
+        }
+    }
+    close() {
+        this.closed = true;
+        this.playing = false;
+        this.generation++;
+        this.seekController?.abort();
+        this.unsubscribeRecording?.();
+        this.waitingForData = false;
+        this.cancelFrame();
+        this.listeners.clear();
+    }
+}

package/dist/{types/core → core}/playback/snapshot.d.ts

@@ -1,3 +1,3 @@
-import type { Recording, ReplayMessage } from "../mcap/types.js";
+import type { Recording, Message } from "../mcap/types.js";
 /** The packets that rebuild each video topic's frame at `time`, in decode order. */
-export declare function snapshot(recording: Recording, time: bigint): readonly ReplayMessage[];
+export declare function snapshot(recording: Recording, time: bigint): readonly Message[];

package/dist/core/playback/snapshot.js

@@ -0,0 +1,11 @@
+import { isVideo, videoSequence } from "./video-sequence.js";
+/** The packets that rebuild each video topic's frame at `time`, in decode order. */
+export function snapshot(recording, time) {
+    const result = [];
+    for (const records of recording.byTopic.values()) {
+        const channel = records[0]?.channel;
+        if (channel && isVideo(channel))
+            result.push(...videoSequence(channel, records).snapshot(time));
+    }
+    return result;
+}

package/dist/{types/core → core}/playback/video-sequence.d.ts

@@ -1,10 +1,10 @@
-import type { ReplayMessage, Topic } from "../mcap/types.js";
+import type { Message, Topic } from "../mcap/types.js";
 export declare function isVideo(channel: Topic): boolean;
-export declare function decodeOrder(record: ReplayMessage): bigint;
-export declare function videoSequence(channel: Topic, records: readonly ReplayMessage[]): VideoSequence;
+export declare function decodeOrder(record: Message): bigint;
+export declare function videoSequence(channel: Topic, records: readonly Message[]): VideoSequence;
 export declare class VideoSequence {
     readonly channel: Topic;
-    readonly records: readonly ReplayMessage[];
+    readonly records: readonly Message[];
     readonly start: bigint;
     readonly end: bigint;
     private readonly lookahead;
@@ -13,12 +13,12 @@ export declare class VideoSequence {
     private count;
     private safeCount;
     private contiguous;
-    constructor(channel: Topic, records: readonly ReplayMessage[]);
+    constructor(channel: Topic, records: readonly Message[]);
     private update;
     private required;
     private target;
     bufferedEnd(time: bigint, complete: boolean): bigint;
-    snapshot(time: bigint): readonly ReplayMessage[];
-    after(time: bigint, submitted: bigint): readonly ReplayMessage[];
+    snapshot(time: bigint): readonly Message[];
+    after(time: bigint, submitted: bigint): readonly Message[];
     private range;
 }

package/dist/core/playback/video-sequence.js

@@ -0,0 +1,116 @@
+import { compressedVideo, videoSchema } from "../schemas.js";
+export function isVideo(channel) {
+    return channel.schemaName === videoSchema;
+}
+export function decodeOrder(record) {
+    const value = compressedVideo(record.value).decode_order;
+    if (typeof value !== "string" || !/^\d+$/.test(value))
+        throw new Error("Video packet is missing a valid decode_order");
+    return BigInt(value);
+}
+const sequences = new WeakMap();
+export function videoSequence(channel, records) {
+    let sequence = sequences.get(records);
+    if (!sequence) {
+        sequence = new VideoSequence(channel, records);
+        sequences.set(records, sequence);
+    }
+    return sequence;
+}
+export class VideoSequence {
+    channel;
+    records;
+    start;
+    end;
+    lookahead;
+    packets = new Map();
+    keyframes = new Map();
+    count = 0;
+    safeCount = 0;
+    contiguous;
+    constructor(channel, records) {
+        this.channel = channel;
+        this.records = records;
+        const integer = (name) => {
+            const value = channel.metadata.get(name);
+            if (value === undefined || !/^\d+$/.test(value))
+                throw new Error(`Missing or invalid ${name} video metadata`);
+            return BigInt(value);
+        };
+        this.start = integer("video.decode_start");
+        this.end = integer("video.decode_end");
+        this.lookahead = integer("video.max_reorder_frames");
+        if (this.start > this.end)
+            throw new Error("Invalid video decode interval");
+        this.contiguous = this.start - 1n;
+    }
+    update() {
+        for (; this.count < this.records.length; this.count++) {
+            const record = this.records[this.count];
+            const order = decodeOrder(record);
+            if (order < this.start || order > this.end || this.packets.has(order))
+                throw new Error("Video decode_order is duplicated or outside the declared interval");
+            this.packets.set(order, record);
+            if (compressedVideo(record.value).keyframe)
+                this.keyframes.set(order, record);
+        }
+        while (this.packets.has(this.contiguous + 1n))
+            this.contiguous++;
+    }
+    required(record) {
+        const order = decodeOrder(record) + this.lookahead;
+        return order > this.end ? this.end : order;
+    }
+    target(time) {
+        let low = 0, high = this.records.length;
+        while (low < high) {
+            const middle = (low + high) >>> 1;
+            if (this.records[middle].time <= time)
+                low = middle + 1;
+            else
+                high = middle;
+        }
+        return this.records[low - 1];
+    }
+    bufferedEnd(time, complete) {
+        this.update();
+        if (complete && this.contiguous !== this.end)
+            throw new Error("Incomplete video decode interval");
+        if (complete)
+            return time;
+        while (this.safeCount < this.records.length &&
+            this.required(this.records[this.safeCount]) <= this.contiguous)
+            this.safeCount++;
+        const blocked = this.records[this.safeCount];
+        return blocked && blocked.time <= time ? blocked.time - 1n : time;
+    }
+    snapshot(time) {
+        this.update();
+        const target = this.target(time);
+        if (!target)
+            return [];
+        const order = decodeOrder(target);
+        let begin;
+        for (const [index, keyframe] of this.keyframes)
+            if (index <= order &&
+                keyframe.time <= time &&
+                (begin === undefined || index > begin))
+                begin = index;
+        if (begin === undefined)
+            throw new Error("MCAP is missing the preceding video keyframe");
+        return this.range(begin, this.required(target));
+    }
+    after(time, submitted) {
+        this.update();
+        const target = this.target(time);
+        return target ? this.range(submitted + 1n, this.required(target)) : [];
+    }
+    range(begin, end) {
+        if (end > this.contiguous)
+            throw new Error("Video decode dependencies are not buffered");
+        const result = [];
+        for (let order = begin; order <= end; order++)
+            result.push(this.packets.get(order));
+        return result;
+    }
+}

package/dist/{types/core → core}/renderers/image.d.ts

@@ -19,6 +19,7 @@ export declare class ImageRenderer implements PlaybackListener {
     private generation;
     private snapshot;
     private imageTopic?;
+    private destroyed;
     private readonly video;
     constructor(canvas: HTMLCanvasElement, options?: ImageRendererOptions);
     onMessage(channel: Topic, time: bigint, value: unknown): void;
@@ -31,5 +32,6 @@ export declare class ImageRenderer implements PlaybackListener {
     private resetSnapshot;
     private clearCanvas;
     reset(): void;
+    /** Release decoder and frame resources; later playback events are ignored. */
     destroy(): void;
 }

package/dist/core/renderers/image.js

@@ -0,0 +1,165 @@
+import { compressedVideo, videoSchema } from "../schemas.js";
+import { CompressedVideoDecoder } from "./video-decoder.js";
+/** Canvas view of one video topic, showing the decoded frame at the playback time. */
+export class ImageRenderer {
+    canvas;
+    options;
+    onFrame;
+    onError;
+    image;
+    frameTime = 0n;
+    presentationTime;
+    frames = new Map();
+    latestInputTime;
+    presentationFloor = 0n;
+    requested;
+    generation = 0;
+    snapshot = false;
+    imageTopic;
+    destroyed = false;
+    video;
+    constructor(canvas, options = {}) {
+        this.canvas = canvas;
+        this.options = options;
+        this.imageTopic = options.imageTopic;
+        this.video = new CompressedVideoDecoder((frame, time) => this.present(frame, time), (error) => this.onError?.(error));
+    }
+    onMessage(channel, time, value) {
+        try {
+            if (this.destroyed || channel.schemaName !== videoSchema)
+                return;
+            this.imageTopic ??= channel.topic;
+            if (this.imageTopic !== channel.topic)
+                return;
+            const video = compressedVideo(value);
+            if ((this.presentationTime === undefined ||
+                time <= this.presentationTime) &&
+                (this.latestInputTime === undefined || time > this.latestInputTime))
+                this.latestInputTime = time;
+            this.video.decode(time, video.data, video.keyframe, video.format);
+            if (video.decode_order === channel.metadata.get("video.decode_end"))
+                void this.video.flush();
+        }
+        catch (error) {
+            this.onError?.(error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    onState(state) {
+        if (this.destroyed)
+            return;
+        this.presentationTime = state.time;
+        if (state.status !== "buffering")
+            this.selectFrame();
+    }
+    onSnapshotStart() {
+        if (this.destroyed)
+            return;
+        this.resetSnapshot();
+        this.snapshot = true;
+    }
+    async onSnapshotEnd(paused) {
+        if (this.destroyed)
+            return;
+        const generation = this.generation;
+        this.presentationFloor = this.latestInputTime ?? 0n;
+        if (paused)
+            await this.video.flush();
+        if (generation !== this.generation)
+            return;
+        this.snapshot = false;
+        if (this.latestInputTime === undefined ||
+            (paused && (!this.image || this.frameTime < this.presentationFloor)))
+            this.clearCanvas();
+        if (this.image && this.frameTime >= this.presentationFloor) {
+            this.render();
+            this.onFrame?.(this.frameTime);
+        }
+    }
+    present(frame, time) {
+        if (this.image && time < this.frameTime) {
+            frame.close();
+            return;
+        }
+        this.frames.get(time)?.close();
+        this.frames.set(time, frame);
+        this.selectFrame();
+    }
+    selectFrame() {
+        let selected;
+        for (const time of this.frames.keys())
+            if ((this.presentationTime === undefined ||
+                time <= this.presentationTime) &&
+                (selected === undefined || time > selected))
+                selected = time;
+        if (selected === undefined)
+            return;
+        const frame = this.frames.get(selected);
+        this.frames.delete(selected);
+        for (const [time, earlier] of this.frames)
+            if (time < selected) {
+                earlier.close();
+                this.frames.delete(time);
+            }
+        this.image?.close();
+        this.image = frame;
+        this.frameTime = selected;
+        if (!this.snapshot && selected >= this.presentationFloor) {
+            this.render();
+            this.onFrame?.(selected);
+        }
+    }
+    render() {
+        if (this.requested !== undefined)
+            return;
+        this.requested = requestAnimationFrame(() => {
+            this.requested = undefined;
+            const context = this.canvas.getContext("2d");
+            if (!this.image ||
+                !context ||
+                this.snapshot ||
+                this.frameTime < this.presentationFloor)
+                return;
+            const width = this.image.displayWidth;
+            const height = this.image.displayHeight;
+            if (this.canvas.width !== width || this.canvas.height !== height) {
+                this.canvas.width = width;
+                this.canvas.height = height;
+                this.canvas.style.aspectRatio = `${width} / ${height}`;
+            }
+            context.drawImage(this.image, 0, 0);
+        });
+    }
+    resetSnapshot() {
+        this.generation++;
+        this.video.reset();
+        for (const frame of this.frames.values())
+            frame.close();
+        this.frames.clear();
+        this.snapshot = false;
+        this.frameTime = 0n;
+        this.latestInputTime = undefined;
+        this.presentationFloor = 0n;
+        this.image?.close();
+        this.image = undefined;
+    }
+    clearCanvas() {
+        this.canvas
+            .getContext("2d")
+            ?.clearRect(0, 0, this.canvas.width, this.canvas.height);
+    }
+    reset() {
+        this.resetSnapshot();
+        // The first video topic stays selected across seeks until a full reset.
+        this.imageTopic = this.options.imageTopic;
+        this.presentationTime = undefined;
+        this.clearCanvas();
+    }
+    /** Release decoder and frame resources; later playback events are ignored. */
+    destroy() {
+        this.destroyed = true;
+        this.reset();
+        if (this.requested !== undefined)
+            cancelAnimationFrame(this.requested);
+        this.requested = undefined;
+    }
+}

package/dist/core/renderers/index.js

@@ -0,0 +1 @@
+export { ImageRenderer } from "./image.js";