jupyterlab_advanced_image_viewer_extension 1.1.8 → 1.2.2

package/README.md

@@ -8,9 +8,9 @@
 [![Brought To You By KOLOMOLO](https://img.shields.io/badge/Brought%20To%20You%20By-KOLOMOLO-00ffff?style=flat)](https://kolomolo.com)
 [![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-blue?style=flat)](https://www.paypal.com/donate/?hosted_button_id=B4KPBJDLLXTSA)
 
-A JupyterLab 4 extension that makes the built-in image viewer interactive. Images open fitted to the panel just like the stock viewer, then you can zoom with the wheel, pan by dragging, and step through a folder of images with the arrow keys - all in a single viewer tab.
+A JupyterLab 4 extension that makes the built-in image viewer interactive and adds audio and video viewers. Images open fitted to the panel just like the stock viewer, then you can zoom with the wheel, pan by dragging, and step through a folder of images with the arrow keys - all in a single viewer tab.
 
-![Advanced image viewer](https://raw.githubusercontent.com/stellarshenson/jupyterlab_advanced_image_viewer_extension/main/.resources/screenshot.png)
+![Advanced image viewer](.resources/screenshot.png)
 
 The viewer toolbar adds zoom in / out / fit controls and a `help` link that lists every keybinding.
 
@@ -25,6 +25,20 @@ The viewer toolbar adds zoom in / out / fit controls and a `help` link that list
 - **Refresh** - a toolbar refresh button reloads the image from disk, the same control the HTML viewer provides
 - **WebP and AVIF support** - `.webp` and `.avif` files open in the image viewer with every feature above; stock JupyterLab does not open them and shows a File Load Error
 
+## Audio and video viewers
+
+Audio files open in a waveform player built on wavesurfer.js and video files in a Plyr player. The extension disables the stock JupyterLab audio and video viewers, so a double click on a media file opens these instead.
+
+![Audio viewer](.resources/audio-viewer.png)
+![Video viewer](.resources/video-viewer.png)
+
+- **Audio controls** - play/pause, back and forward 5 s, Loop, a speed choice of 0.5x to 2x, waveform zoom with - / + / Fit, an overview strip that seeks anywhere in the file, the time under the mouse, and a drag-marked loop region of at least 1 s
+- **Video controls** - play/pause, a seek bar, the current time, mute and volume, a speed menu, picture in picture and full screen, in a player fitted to the panel with the aspect ratio kept
+- **Keys in a focused media tab** - Space plays and pauses, Left and Right seek 5 s, and Page Up / Page Down open the previous and next file of the same kind in the folder, in the order the file browser lists them (the folder navigation setting turns this off)
+- **Help dialog** - a `help` link in each media toolbar lists the keys and the waveform gestures
+- **File details** - the toolbar shows the format, the size and the duration, plus the channel count for audio and the frame size for video
+- **100 MB limit** - a file up to 100 MB is downloaded once from its file URL and played from that copy, so seeking works on a server without byte ranges; above 100 MB the audio waveform is skipped and either file plays from its URL, so seeking there needs a server that serves byte ranges
+
 ## Requirements
 
 - JupyterLab >= 4.6

package/lib/audio.d.ts

@@ -0,0 +1,32 @@
+import { Toolbar } from '@jupyterlab/apputils';
+import type WaveSurfer from 'wavesurfer.js';
+import { MediaViewer } from './media';
+export declare class AudioViewer extends MediaViewer {
+    constructor(options: Omit<MediaViewer.IOptions, 'kind'>);
+    wavesurfer: WaveSurfer | null;
+    load(): Promise<void>;
+    addControls(toolbar: Toolbar): void;
+    zoomBy(factor: number): void;
+    setLoop(on: boolean): void;
+    dispose(): void;
+    private setZoomEnabled;
+    private createPlayer;
+    private colours;
+    private paint;
+    private labelRegion;
+    private showTime;
+    private showView;
+    private showDetails;
+    private wave;
+    private time;
+    private view;
+    private themeObserver;
+    private minimap;
+    private createMinimap;
+    private region;
+    private looping;
+    private loopButton;
+    private zoomButtons;
+    private zoomEnabled;
+    private last;
+}

package/lib/audio.js

@@ -0,0 +1,341 @@
+// Audio viewer: the shared media widget with a wavesurfer.js waveform, an
+// overview strip, a time ruler, the time under the mouse, a loop region and
+// zoom. wavesurfer.js loads when the first audio file opens, not at start-up.
+import { ToolbarButton } from '@jupyterlab/apputils';
+import { jumpBackIcon, jumpForwardIcon, pauseIcon, runIcon } from '@jupyterlab/ui-components';
+import { Widget } from '@lumino/widgets';
+import { MediaViewer } from './media';
+import { drawsWaveform, formatTime, SEEK_STEP, WAVEFORM_LIMIT } from './mediautil';
+import { addZoomButtons } from './toolbar';
+const SPEEDS = [0.5, 0.75, 1, 1.25, 1.5, 2];
+// The shortest drag, in seconds, that makes a loop region: a click that
+// moves a few pixels must not arm Loop on a sub-second region.
+const MIN_LOOP = 1;
+// wavesurfer.js and the plugins the viewer uses.
+const loadPlayer = () => Promise.all([
+    import('wavesurfer.js'),
+    import('wavesurfer.js/dist/plugins/minimap.js'),
+    import('wavesurfer.js/dist/plugins/timeline.js'),
+    import('wavesurfer.js/dist/plugins/hover.js'),
+    import('wavesurfer.js/dist/plugins/regions.js'),
+    import('wavesurfer.js/dist/plugins/zoom.js')
+]);
+export class AudioViewer extends MediaViewer {
+    constructor(options) {
+        super({ ...options, kind: 'audio' });
+        // The player of the waveform; null before the first one and above the
+        // limit.
+        this.wavesurfer = null;
+        this.minimap = null;
+        this.createMinimap = null;
+        this.region = null;
+        this.looping = false;
+        this.loopButton = null;
+        this.zoomButtons = null;
+        // Kept here too, so a load that finishes before addControls() disables the
+        // buttons when they are made.
+        this.zoomEnabled = true;
+        // Playback time at the previous update, to tell a run over the region end
+        // from a seek.
+        this.last = 0;
+        this.addClass('jp-AdvancedMediaViewer-audio');
+        this.wave = document.createElement('div');
+        this.wave.className = 'jp-AdvancedMediaViewer-wave';
+        const status = document.createElement('div');
+        status.className = 'jp-AdvancedMediaViewer-status';
+        this.time = document.createElement('span');
+        this.view = document.createElement('span');
+        status.append(this.time, this.view);
+        this.node.prepend(this.wave, status);
+        for (const type of ['timeupdate', 'durationchange', 'seeking', 'emptied']) {
+            this.media.addEventListener(type, () => this.showTime());
+        }
+        this.media.addEventListener('durationchange', () => this.showDetails());
+        // The waveform colours are painted on a canvas, which cannot read CSS
+        // variables, so they are resolved here and again on every theme change.
+        this.themeObserver = new MutationObserver(() => this.paint());
+        this.themeObserver.observe(document.body, {
+            attributes: true,
+            attributeFilter: ['data-jp-theme-name']
+        });
+    }
+    async load() {
+        var _a, _b;
+        const token = ++this.loads;
+        this.setLoop(false);
+        this.showMessage(this.trans.__('Loading...'));
+        try {
+            const size = await this.readSize();
+            const url = await this.contents.getDownloadUrl(this.context.path);
+            if (this.isDisposed || token !== this.loads) {
+                return;
+            }
+            this.showDetails();
+            if (!drawsWaveform(size)) {
+                // Played from the file URL: seeking needs a server that answers byte
+                // ranges, as the stock file handler does.
+                (_a = this.wavesurfer) === null || _a === void 0 ? void 0 : _a.destroy();
+                this.wavesurfer = null;
+                // There is no waveform to zoom, so the buttons would do nothing.
+                this.setZoomEnabled(false);
+                this.media.controls = true;
+                this.media.src = url;
+                this.showMessage(this.trans.__('The waveform is skipped for files over %1 MB.', WAVEFORM_LIMIT / 1024 / 1024));
+                return;
+            }
+            this.media.controls = false;
+            this.setZoomEnabled(true);
+            // Fetched once and handed over as a blob: the player then seeks in
+            // memory, also on a server that ignores byte ranges.
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`${response.status} ${response.statusText}`);
+            }
+            const blob = await response.blob();
+            const modules = await loadPlayer();
+            // A reload or a close while this one was fetching supersedes it.
+            if (this.isDisposed || token !== this.loads) {
+                return;
+            }
+            // One player per tab: a new player would load the element's previous
+            // source on its own, so a reload hands the existing one the new blob.
+            (_b = this.wavesurfer) !== null && _b !== void 0 ? _b : (this.wavesurfer = this.createPlayer(modules));
+            await this.wavesurfer.loadBlob(blob);
+            this.showMessage('');
+        }
+        catch (reason) {
+            // AbortError: a newer load of the same player took over.
+            if (!this.isDisposed &&
+                token === this.loads &&
+                reason.name !== 'AbortError') {
+                this.showMessage(this.trans.__('This file cannot be played: %1', String(reason)));
+            }
+        }
+    }
+    // The playback controls, zoom, loop and speed, between Reload and the
+    // details on the toolbar.
+    addControls(toolbar) {
+        const trans = this.trans;
+        const play = new ToolbarButton({
+            icon: runIcon,
+            pressedIcon: pauseIcon,
+            tooltip: trans.__('Play'),
+            pressedTooltip: trans.__('Pause'),
+            noFocusOnClick: true,
+            onClick: () => this.togglePlay()
+        });
+        for (const type of ['play', 'pause', 'emptied']) {
+            this.media.addEventListener(type, () => (play.pressed = !this.media.paused));
+        }
+        toolbar.addItem('advanced-play', play);
+        toolbar.addItem('advanced-back', new ToolbarButton({
+            icon: jumpBackIcon,
+            tooltip: trans.__('Back %1 seconds', SEEK_STEP),
+            noFocusOnClick: true,
+            onClick: () => this.seekBy(-SEEK_STEP)
+        }));
+        toolbar.addItem('advanced-forward', new ToolbarButton({
+            icon: jumpForwardIcon,
+            tooltip: trans.__('Forward %1 seconds', SEEK_STEP),
+            noFocusOnClick: true,
+            onClick: () => this.seekBy(SEEK_STEP)
+        }));
+        this.zoomButtons = addZoomButtons(toolbar, trans, () => this.zoomBy(0.5), () => this.zoomBy(2), () => this.zoomBy(0));
+        this.setZoomEnabled(this.zoomEnabled);
+        this.loopButton = new ToolbarButton({
+            label: trans.__('Loop'),
+            tooltip: trans.__('Loop the selected region, or the whole file'),
+            noFocusOnClick: true,
+            onClick: () => this.setLoop(!this.looping)
+        });
+        toolbar.addItem('advanced-loop', this.loopButton);
+        const speed = document.createElement('select');
+        speed.className = 'jp-AdvancedMediaViewer-speed';
+        speed.title = trans.__('Playback speed');
+        for (const rate of SPEEDS) {
+            speed.add(new Option(`${rate}×`, String(rate), rate === 1, rate === 1));
+        }
+        speed.addEventListener('change', () => {
+            this.media.playbackRate = Number(speed.value);
+        });
+        toolbar.addItem('advanced-speed', new Widget({ node: speed }));
+    }
+    // Multiply the shown time scale by `factor`; 0 fits the whole file. The
+    // waveform never zooms out past the fit.
+    zoomBy(factor) {
+        const ws = this.wavesurfer;
+        const duration = ws === null || ws === void 0 ? void 0 : ws.getDuration();
+        if (!(ws === null || ws === void 0 ? void 0 : ws.getDecodedData()) || !duration) {
+            return;
+        }
+        // The level comes from wavesurfer's own setting, not from the wrapper
+        // width: the wrapper keeps the previous zoom's width until its next
+        // render, so a click right after a fit or a wheel zoom would start from
+        // the old level. 0 fits the file.
+        ws.zoom(factor ? (ws.options.minPxPerSec || ws.getWidth() / duration) * factor : 0);
+    }
+    // Loop on repeats the region, or the whole file without one; loop off
+    // removes the region.
+    setLoop(on) {
+        this.looping = on;
+        if (!on && this.region) {
+            this.region.remove();
+            this.region = null;
+        }
+        this.media.loop = on && !this.region;
+        if (this.loopButton) {
+            this.loopButton.pressed = on;
+        }
+    }
+    dispose() {
+        var _a;
+        if (this.isDisposed) {
+            return;
+        }
+        this.themeObserver.disconnect();
+        // Revokes the blob URL it made for the media element.
+        (_a = this.wavesurfer) === null || _a === void 0 ? void 0 : _a.destroy();
+        this.wavesurfer = null;
+        super.dispose();
+    }
+    setZoomEnabled(on) {
+        this.zoomEnabled = on;
+        if (this.zoomButtons) {
+            this.zoomButtons.zoomOut.enabled = on;
+            this.zoomButtons.zoomIn.enabled = on;
+            this.zoomButtons.fit.enabled = on;
+        }
+    }
+    createPlayer([{ default: Player }, { default: Minimap }, { default: Timeline }, { default: Hover }, { default: Regions }, { default: Zoom }]) {
+        this.createMinimap = () => Minimap.create({
+            ...this.colours(),
+            cursorColor: 'var(--jp-ui-font-color0)',
+            height: 44,
+            dragToSeek: true,
+            insertPosition: 'beforebegin',
+            overlayColor: 'color-mix(in srgb, var(--jp-brand-color1) 15%, transparent)'
+        });
+        this.minimap = this.createMinimap();
+        const regions = Regions.create();
+        const ws = Player.create({
+            container: this.wave,
+            media: this.media,
+            height: 200,
+            ...this.colours(),
+            cursorColor: 'var(--jp-ui-font-color0)',
+            cursorWidth: 2,
+            plugins: [
+                this.minimap,
+                Timeline.create({
+                    height: 18,
+                    insertPosition: 'beforebegin',
+                    formatTimeCallback: (seconds) => formatTime(seconds, 0),
+                    style: { color: 'var(--jp-ui-font-color2)' }
+                }),
+                Hover.create({
+                    lineColor: 'var(--jp-ui-font-color2)',
+                    labelColor: 'var(--jp-ui-font-color0)',
+                    labelBackground: 'var(--jp-layout-color3)',
+                    formatTimeCallback: (seconds) => formatTime(seconds)
+                }),
+                regions,
+                Zoom.create()
+            ]
+        });
+        regions.enableDragSelection({
+            color: 'color-mix(in srgb, var(--jp-brand-color1) 20%, transparent)',
+            minLength: MIN_LOOP
+        });
+        // One loop region at a time: a new drag replaces the old one.
+        regions.on('region-created', region => {
+            // wavesurfer stretches a shorter drag to exactly minLength instead of
+            // dropping it; the small margin covers the float error of that sum.
+            if (region.end - region.start < MIN_LOOP + 0.01) {
+                region.remove();
+                return;
+            }
+            regions
+                .getRegions()
+                .filter(other => other !== region)
+                .forEach(other => other.remove());
+            this.region = region;
+            this.labelRegion();
+            this.setLoop(true);
+        });
+        regions.on('region-updated', () => this.labelRegion());
+        // Return to the region start when playback runs over its end, not when
+        // the user seeks past it.
+        ws.on('timeupdate', time => {
+            const r = this.region;
+            if (this.looping &&
+                r &&
+                ws.isPlaying() &&
+                this.last >= r.start &&
+                this.last < r.end &&
+                time >= r.end) {
+                ws.setTime(r.start);
+                time = r.start;
+            }
+            this.last = time;
+        });
+        ws.on('seeking', time => (this.last = time));
+        for (const event of ['zoom', 'scroll', 'redrawcomplete']) {
+            ws.on(event, () => this.showView());
+        }
+        ws.on('decode', () => this.showDetails());
+        return ws;
+    }
+    colours() {
+        const style = getComputedStyle(document.body);
+        return {
+            // color3 is 2.68:1 on white; the waveform is graphical content and
+            // WCAG 1.4.11 asks 3:1, which color2 clears.
+            waveColor: style.getPropertyValue('--jp-ui-font-color2').trim(),
+            progressColor: style.getPropertyValue('--jp-brand-color1').trim()
+        };
+    }
+    paint() {
+        const ws = this.wavesurfer;
+        if (!ws || !this.minimap || !this.createMinimap) {
+            return;
+        }
+        ws.setOptions(this.colours());
+        // The overview copies its colours when it is made, so it is made anew.
+        ws.unregisterPlugin(this.minimap);
+        this.minimap = ws.registerPlugin(this.createMinimap());
+    }
+    labelRegion() {
+        const r = this.region;
+        r === null || r === void 0 ? void 0 : r.setContent(this.trans.__('Loop %1 - %2', formatTime(r.start), formatTime(r.end)));
+    }
+    showTime() {
+        this.time.textContent = `${formatTime(this.media.currentTime)} / ${formatTime(this.media.duration)}`;
+    }
+    showView() {
+        const ws = this.wavesurfer;
+        const duration = ws === null || ws === void 0 ? void 0 : ws.getDuration();
+        if (!ws || !duration) {
+            return;
+        }
+        // From the zoom level, not the rendered width: on the zoom event the
+        // wrapper still has its old width.
+        const width = ws.getWidth();
+        const pxPerSec = Math.max(ws.options.minPxPerSec, width / duration);
+        const shown = Math.min(width / pxPerSec, duration);
+        const start = Math.min(ws.getScroll() / pxPerSec, duration - shown);
+        this.view.textContent = this.trans.__('showing %1 - %2', formatTime(start), formatTime(start + shown));
+    }
+    showDetails() {
+        var _a, _b;
+        const channels = (_b = (_a = this.wavesurfer) === null || _a === void 0 ? void 0 : _a.getDecodedData()) === null || _b === void 0 ? void 0 : _b.numberOfChannels;
+        this.setDetails(channels === undefined
+            ? []
+            : [
+                channels === 1
+                    ? this.trans.__('mono')
+                    : channels === 2
+                        ? this.trans.__('stereo')
+                        : this.trans.__('%1 channels', channels)
+            ]);
+    }
+}