@helios-project/player 0.48.3 → 0.76.5

package/README.md

@@ -60,13 +60,56 @@ The player will automatically attempt to access `window.helios` on the iframe's
 | `controls` | Show the UI controls overlay. | `false` |
 | `export-mode` | Strategy for client-side export: `auto`, `canvas`, or `dom`. | `auto` |
 | `canvas-selector` | CSS selector for the canvas element (used in `canvas` export mode). | `canvas` |
-| `export-format` | Output video format: `mp4` (H.264/AAC) or `webm` (VP9/Opus). | `mp4` |
+| `export-format` | Output format: `mp4`, `webm`, `png`, or `jpeg`. | `mp4` |
 | `poster` | URL of an image to display before playback starts. | - |
 | `preload` | `auto` or `none`. If `none`, defers loading the iframe until interaction. | `auto` |
 | `input-props` | JSON string of properties to pass to the composition. | - |
 | `interactive` | Enable direct interaction with the composition (disables click-to-pause). | `false` |
 | `controlslist` | Space-separated list of features to disable: `nodownload`, `nofullscreen`. | - |
 | `sandbox` | Security flags for the iframe. | `allow-scripts allow-same-origin` |
+| `export-width` | Target width for client-side export. | - |
+| `export-height` | Target height for client-side export. | - |
+| `export-bitrate` | Target bitrate for client-side export (bps). | - |
+| `export-filename` | Filename for client-side export (without extension). | `video` |
+| `export-caption-mode` | Strategy for caption export: `burn-in` or `file`. | `burn-in` |
+| `disablepictureinpicture` | Hides the Picture-in-Picture button. | `false` |
+| `media-title` | Title of the composition for OS Media Session. | - |
+| `media-artist` | Artist name for OS Media Session. | - |
+| `media-album` | Album name for OS Media Session. | - |
+| `media-artwork` | URL of artwork for OS Media Session (defaults to poster). | - |
+
+## User Interface
+
+The player includes a comprehensive set of controls:
+
+- **Playback**: Play/Pause, Scrubber, Time Display.
+- **Audio**: Volume, Mute, and a Track Menu for individual track control.
+- **Settings Menu** (Gear Icon):
+  - **Speed**: Adjust playback rate (0.25x - 2x).
+  - **Loop**: Toggle playback looping.
+  - **Playback Range**: Set In/Out points to loop a specific section.
+  - **Diagnostics**: View environment capabilities (WebCodecs support).
+  - **Shortcuts**: View keyboard shortcuts.
+- **Tools**: Fullscreen, Picture-in-Picture, Captions (CC), Export.
+
+### Keyboard Shortcuts
+
+| Key | Action |
+|-----|--------|
+| `Space` / `K` | Play / Pause |
+| `F` | Toggle Fullscreen |
+| `M` | Mute / Unmute |
+| `C` | Toggle Captions |
+| `?` | Show Shortcuts Help |
+| `←` / `→` | Seek 1 frame |
+| `Shift` + `←` / `→` | Seek 10 frames |
+| `Home` | Go to Start |
+| `End` | Go to End |
+| `I` | Set In Point |
+| `O` | Set Out Point |
+| `X` | Clear Playback Range |
+| `Shift` + `D` | Toggle Diagnostics |
+| `0-9` | Seek to 0-90% |
 
 ## CSS Variables
 
@@ -90,6 +133,10 @@ The `<helios-player>` element implements a subset of the HTMLMediaElement interf
 - `pause(): void` - Pauses playback.
 - `load(): void` - Reloads the iframe (useful if `src` changed or to retry connection).
 - `addTextTrack(kind: string, label?: string, language?: string): TextTrack` - Adds a new text track to the media element.
+- `diagnose(): Promise<DiagnosticReport>` - Runs environment diagnostics (WebCodecs, WebGL) and returns a report.
+- `requestPictureInPicture(): Promise<PictureInPictureWindow>` - Requests Picture-in-Picture mode for the player.
+- `export(options?: HeliosExportOptions): Promise<void>` - Programmatically trigger client-side export.
+- `fastSeek(time: number): void` - Seeks to the specified time as fast as possible (currently equivalent to setting `currentTime`).
 
 ### Properties
 
@@ -101,6 +148,8 @@ The `<helios-player>` element implements a subset of the HTMLMediaElement interf
 - `volume` (number): Audio volume (0.0 to 1.0).
 - `muted` (boolean): Audio mute state.
 - `playbackRate` (number): Playback speed (default 1.0).
+- `width` (number): Reflected width attribute.
+- `height` (number): Reflected height attribute.
 - `videoWidth` (number, read-only): The intrinsic width of the video (from controller state or attributes).
 - `videoHeight` (number, read-only): The intrinsic height of the video (from controller state or attributes).
 - `buffered` (TimeRanges, read-only): Returns a TimeRanges object representing buffered content (always 0-duration).
@@ -111,6 +160,8 @@ The `<helios-player>` element implements a subset of the HTMLMediaElement interf
 - `fps` (number, read-only): Frames per second of the composition.
 - `currentFrame` (number): Current frame index.
 - `inputProps` (object): Get or set the input properties passed to the composition.
+- `playsInline` (boolean): Reflected playsinline attribute.
+- `disablePictureInPicture` (boolean): Hides the Picture-in-Picture button.
 
 ## Events
 
@@ -131,7 +182,7 @@ The element dispatches the following custom events:
 
 ## Client-Side Export
 
-The player supports exporting the composition to video (MP4/WebM) directly in the browser using WebCodecs.
+The player supports exporting the composition to video (MP4/WebM) or image snapshots (PNG/JPEG) directly in the browser using WebCodecs.
 
 - **`export-mode="canvas"`**: captures frames from a `<canvas>` element. Fast and efficient.
 - **`export-mode="dom"`**: captures the entire DOM using `foreignObject` SVG serialization. Useful for compositions using DOM elements (divs, text, css).
@@ -146,3 +197,23 @@ Configuration:
   export-format="webm"
 ></helios-player>
 ```
+
+To take a snapshot (PNG) instead of a video, set `export-format="png"`.
+
+### Audio Fades
+
+To apply audio fades during client-side export, add `data-helios-fade-in` and/or `data-helios-fade-out` attributes to your audio elements within the composition. The value should be the duration in seconds.
+
+```html
+<audio src="music.mp3" data-helios-fade-in="2" data-helios-fade-out="3"></audio>
+```
+
+## Verification
+
+To run the End-to-End (E2E) verification suite:
+
+```bash
+npx tsx tests/e2e/verify-player.ts
+```
+
+This script starts a local server and uses Playwright to verify the player's core functionality (playback, scrubber, menus, volume) using a dependency-free mock composition.

package/dist/bridge.js

@@ -1,6 +1,14 @@
+import { Helios } from "@helios-project/core";
 import { captureDomToBitmap } from "./features/dom-capture";
 import { getAudioAssets } from "./features/audio-utils";
+import { AudioMeter } from "./features/audio-metering";
+import { AudioFader } from "./features/audio-fader";
 export function connectToParent(helios) {
+    let audioMeter = null;
+    let audioMeterRaf = null;
+    const audioFader = new AudioFader();
+    // Connect fader immediately (scans document for audio elements)
+    audioFader.connect(document);
     // 1. Listen for messages from parent
     window.addEventListener('message', async (event) => {
         if (event.source !== window.parent)
@@ -8,7 +16,8 @@ export function connectToParent(helios) {
         const { type, frame } = event.data;
         switch (type) {
             case 'HELIOS_GET_AUDIO_TRACKS':
-                const assets = await getAudioAssets(document);
+                const state = helios.getState();
+                const assets = await getAudioAssets(document, state.availableAudioTracks, state.audioTracks);
                 const buffers = assets.map(a => a.buffer);
                 window.parent.postMessage({ type: 'HELIOS_AUDIO_DATA', assets }, '*', buffers);
                 break;
@@ -26,6 +35,11 @@ export function connectToParent(helios) {
             case 'HELIOS_SEEK':
                 if (typeof frame === 'number') {
                     helios.seek(frame);
+                    requestAnimationFrame(() => {
+                        requestAnimationFrame(() => {
+                            window.parent.postMessage({ type: 'HELIOS_SEEK_DONE', frame }, '*');
+                        });
+                    });
                 }
                 break;
             case 'HELIOS_SET_PLAYBACK_RATE':
@@ -52,6 +66,16 @@ export function connectToParent(helios) {
                     helios.setAudioMuted(event.data.muted);
                 }
                 break;
+            case 'HELIOS_SET_AUDIO_TRACK_VOLUME':
+                if (typeof event.data.trackId === 'string' && typeof event.data.volume === 'number') {
+                    helios.setAudioTrackVolume(event.data.trackId, event.data.volume);
+                }
+                break;
+            case 'HELIOS_SET_AUDIO_TRACK_MUTED':
+                if (typeof event.data.trackId === 'string' && typeof event.data.muted === 'boolean') {
+                    helios.setAudioTrackMuted(event.data.trackId, event.data.muted);
+                }
+                break;
             case 'HELIOS_SET_LOOP':
                 if (typeof event.data.loop === 'boolean') {
                     helios.setLoop(event.data.loop);
@@ -67,9 +91,59 @@ export function connectToParent(helios) {
                     helios.setCaptions(event.data.captions);
                 }
                 break;
+            case 'HELIOS_SET_DURATION':
+                if (typeof event.data.duration === 'number') {
+                    helios.setDuration(event.data.duration);
+                }
+                break;
+            case 'HELIOS_SET_FPS':
+                if (typeof event.data.fps === 'number') {
+                    helios.setFps(event.data.fps);
+                }
+                break;
+            case 'HELIOS_SET_SIZE':
+                if (typeof event.data.width === 'number' && typeof event.data.height === 'number') {
+                    helios.setSize(event.data.width, event.data.height);
+                }
+                break;
+            case 'HELIOS_SET_MARKERS':
+                if (Array.isArray(event.data.markers)) {
+                    helios.setMarkers(event.data.markers);
+                }
+                break;
             case 'HELIOS_GET_SCHEMA':
                 window.parent.postMessage({ type: 'HELIOS_SCHEMA', schema: helios.schema }, '*');
                 break;
+            case 'HELIOS_START_METERING':
+                if (!audioMeter) {
+                    audioMeter = new AudioMeter();
+                }
+                audioMeter.connect(document);
+                audioMeter.enable();
+                if (!audioMeterRaf) {
+                    const loop = () => {
+                        if (audioMeter) {
+                            const levels = audioMeter.getLevels();
+                            window.parent.postMessage({ type: 'HELIOS_AUDIO_LEVELS', levels }, '*');
+                        }
+                        audioMeterRaf = requestAnimationFrame(loop);
+                    };
+                    audioMeterRaf = requestAnimationFrame(loop);
+                }
+                break;
+            case 'HELIOS_STOP_METERING':
+                if (audioMeterRaf) {
+                    cancelAnimationFrame(audioMeterRaf);
+                    audioMeterRaf = null;
+                }
+                if (audioMeter) {
+                    audioMeter.disable();
+                }
+                break;
+            case 'HELIOS_DIAGNOSE':
+                const report = await Helios.diagnose();
+                window.parent.postMessage({ type: 'HELIOS_DIAGNOSE_RESULT', report }, '*');
+                break;
             case 'HELIOS_CAPTURE_FRAME':
                 handleCaptureFrame(helios, event.data);
                 break;
@@ -80,6 +154,12 @@ export function connectToParent(helios) {
     window.parent.postMessage({ type: 'HELIOS_READY', state: helios.getState() }, '*');
     // 3. Subscribe to Helios state changes and broadcast
     helios.subscribe((state) => {
+        if (state.isPlaying) {
+            audioFader.enable();
+        }
+        else {
+            audioFader.disable();
+        }
         window.parent.postMessage({ type: 'HELIOS_STATE', state }, '*');
     });
     // 4. Capture Global Errors
@@ -106,7 +186,7 @@ export function connectToParent(helios) {
     });
 }
 async function handleCaptureFrame(helios, data) {
-    const { frame, selector, mode } = data;
+    const { frame, selector, mode, width, height } = data;
     // 1. Seek
     helios.seek(frame);
     // 2. Wait for render (double RAF to be safe and ensure paint)
@@ -116,7 +196,7 @@ async function handleCaptureFrame(helios, data) {
     // 3. DOM Mode
     if (mode === 'dom') {
         try {
-            const bitmap = await captureDomToBitmap(document.body);
+            const bitmap = await captureDomToBitmap(document.body, { targetWidth: width, targetHeight: height });
             window.parent.postMessage({
                 type: 'HELIOS_FRAME_DATA',
                 frame,
@@ -148,7 +228,28 @@ async function handleCaptureFrame(helios, data) {
     }
     // 4. Create Bitmap
     try {
-        const bitmap = await createImageBitmap(canvas);
+        let source = canvas;
+        if (width && height && (canvas.width !== width || canvas.height !== height)) {
+            if (typeof OffscreenCanvas !== 'undefined') {
+                const offscreen = new OffscreenCanvas(width, height);
+                const ctx = offscreen.getContext('2d');
+                if (ctx) {
+                    ctx.drawImage(canvas, 0, 0, width, height);
+                    source = offscreen;
+                }
+            }
+            else {
+                const tempCanvas = document.createElement('canvas');
+                tempCanvas.width = width;
+                tempCanvas.height = height;
+                const ctx = tempCanvas.getContext('2d');
+                if (ctx) {
+                    ctx.drawImage(canvas, 0, 0, width, height);
+                    source = tempCanvas;
+                }
+            }
+        }
+        const bitmap = await createImageBitmap(source);
         // 5. Send back
         window.parent.postMessage({
             type: 'HELIOS_FRAME_DATA',

package/dist/controllers.d.ts

@@ -1,17 +1,24 @@
-import { Helios, CaptionCue, HeliosSchema } from "@helios-project/core";
+import type { Helios, CaptionCue, HeliosSchema, DiagnosticReport, Marker } from "@helios-project/core";
 import { AudioAsset } from "./features/audio-utils";
+import { AudioLevels } from "./features/audio-metering";
 export interface HeliosController {
     play(): void;
     pause(): void;
-    seek(frame: number): void;
+    seek(frame: number): Promise<void>;
     setAudioVolume(volume: number): void;
     setAudioMuted(muted: boolean): void;
+    setAudioTrackVolume(trackId: string, volume: number): void;
+    setAudioTrackMuted(trackId: string, muted: boolean): void;
     setLoop(loop: boolean): void;
     setPlaybackRate(rate: number): void;
     setPlaybackRange(startFrame: number, endFrame: number): void;
     clearPlaybackRange(): void;
     setCaptions(captions: string | CaptionCue[]): void;
     setInputProps(props: Record<string, any>): void;
+    setDuration(seconds: number): void;
+    setFps(fps: number): void;
+    setSize(width: number, height: number): void;
+    setMarkers(markers: Marker[]): void;
     subscribe(callback: (state: any) => void): () => void;
     onError(callback: (err: any) => void): () => void;
     getState(): any;
@@ -19,73 +26,109 @@ export interface HeliosController {
     captureFrame(frame: number, options?: {
         selector?: string;
         mode?: 'canvas' | 'dom';
+        width?: number;
+        height?: number;
     }): Promise<{
         frame: VideoFrame;
         captions: CaptionCue[];
     } | null>;
     getAudioTracks(): Promise<AudioAsset[]>;
     getSchema(): Promise<HeliosSchema | undefined>;
+    diagnose(): Promise<DiagnosticReport>;
+    startAudioMetering(): void;
+    stopAudioMetering(): void;
+    onAudioMetering(callback: (levels: AudioLevels) => void): () => void;
 }
 export declare class DirectController implements HeliosController {
     instance: Helios;
     private iframe?;
+    private audioMeter;
+    private audioFader;
+    private audioMeteringCallback;
+    private audioMeteringRaf;
     constructor(instance: Helios, iframe?: HTMLIFrameElement | undefined);
     play(): void;
     pause(): void;
-    seek(frame: number): void;
+    seek(frame: number): Promise<void>;
     setAudioVolume(volume: number): void;
     setAudioMuted(muted: boolean): void;
+    setAudioTrackVolume(trackId: string, volume: number): void;
+    setAudioTrackMuted(trackId: string, muted: boolean): void;
     setLoop(loop: boolean): void;
     setPlaybackRate(rate: number): void;
     setPlaybackRange(start: number, end: number): void;
     clearPlaybackRange(): void;
     setCaptions(captions: string | CaptionCue[]): void;
     setInputProps(props: Record<string, any>): void;
+    setDuration(seconds: number): void;
+    setFps(fps: number): void;
+    setSize(width: number, height: number): void;
+    setMarkers(markers: Marker[]): void;
     subscribe(callback: (state: any) => void): () => void;
-    getState(): Readonly<import("@helios-project/core").HeliosState>;
+    getState(): Readonly<import("@helios-project/core").HeliosState<Record<string, any>>>;
     dispose(): void;
+    startAudioMetering(): void;
+    stopAudioMetering(): void;
+    onAudioMetering(callback: (levels: AudioLevels) => void): () => void;
     onError(callback: (err: any) => void): () => void;
     getAudioTracks(): Promise<AudioAsset[]>;
     getSchema(): Promise<HeliosSchema | undefined>;
     captureFrame(frame: number, options?: {
         selector?: string;
         mode?: 'canvas' | 'dom';
+        width?: number;
+        height?: number;
     }): Promise<{
         frame: VideoFrame;
         captions: CaptionCue[];
     } | null>;
+    diagnose(): Promise<DiagnosticReport>;
 }
 export declare class BridgeController implements HeliosController {
     private iframeWindow;
     private listeners;
     private errorListeners;
+    private audioMeteringListeners;
     private lastState;
     constructor(iframeWindow: Window, initialState?: any);
     private handleMessage;
     private notifyListeners;
     private notifyErrorListeners;
+    private notifyAudioMeteringListeners;
     play(): void;
     pause(): void;
-    seek(frame: number): void;
+    seek(frame: number): Promise<void>;
     setAudioVolume(volume: number): void;
     setAudioMuted(muted: boolean): void;
+    setAudioTrackVolume(trackId: string, volume: number): void;
+    setAudioTrackMuted(trackId: string, muted: boolean): void;
     setLoop(loop: boolean): void;
     setPlaybackRate(rate: number): void;
     setPlaybackRange(start: number, end: number): void;
     clearPlaybackRange(): void;
     setCaptions(captions: string | CaptionCue[]): void;
     setInputProps(props: Record<string, any>): void;
+    setDuration(duration: number): void;
+    setFps(fps: number): void;
+    setSize(width: number, height: number): void;
+    setMarkers(markers: Marker[]): void;
+    startAudioMetering(): void;
+    stopAudioMetering(): void;
     subscribe(callback: (state: any) => void): () => void;
     onError(callback: (err: any) => void): () => void;
+    onAudioMetering(callback: (levels: AudioLevels) => void): () => void;
     getState(): any;
     dispose(): void;
     captureFrame(frame: number, options?: {
         selector?: string;
         mode?: 'canvas' | 'dom';
+        width?: number;
+        height?: number;
     }): Promise<{
         frame: VideoFrame;
         captions: CaptionCue[];
     } | null>;
     getAudioTracks(): Promise<AudioAsset[]>;
     getSchema(): Promise<HeliosSchema | undefined>;
+    diagnose(): Promise<DiagnosticReport>;
 }