@needle-tools/engine 4.7.0-alpha → 4.7.0-next.22483b4

package/src/engine-components/VideoPlayer.ts

@@ -1,1025 +1,1025 @@
-import { Material, Mesh, Object3D, ShaderMaterial, SRGBColorSpace, Texture, Vector2, Vector4, VideoTexture } from "three";
-
-import { isDevEnvironment } from "../engine/debug/index.js";
-import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects.js";
-import { awaitInput } from "../engine/engine_input_utils.js";
-import { serializable } from "../engine/engine_serialization_decorator.js";
-import { Context } from "../engine/engine_setup.js";
-import { getWorldScale } from "../engine/engine_three_utils.js";
-import { getParam } from "../engine/engine_utils.js";
-import { Behaviour, GameObject } from "./Component.js";
-import { Renderer } from "./Renderer.js";
-
-const debug = getParam("debugvideo");
-
-
-export enum AspectMode {
-    None = 0,
-    AdjustHeight = 1,
-    AdjustWidth = 2,
-}
-
-export enum VideoSource {
-    /// <summary>
-    ///   <para>Use the current clip as the video content source.</para>
-    /// </summary>
-    VideoClip = 0,
-    /// <summary>
-    ///   <para>Use the current URL as the video content source.</para>
-    /// </summary>
-    Url = 1,
-}
-
-export enum VideoAudioOutputMode {
-    None = 0,
-    AudioSource = 1,
-    Direct = 2,
-    APIOnly = 3,
-}
-
-export enum VideoRenderMode {
-    CameraFarPlane = 0,
-    CameraNearPlane = 1,
-    RenderTexture = 2,
-    MaterialOverride = 3,
-}
-
-/**
- * The VideoPlayer component can be used to playback video clips from urls, streams or m3u8 playlists (livestreams)
- * @example Add a video player component to a game object and set the url to a video file. The video will start playing once the object becomes active in your scene
- * ```typescript
- * // Add a video player component to a game object and set the url to a video file. The video will start playing once the object becomes active in your scene
- * const videoPlayer = addComponent(obj, VideoPlayer, {
- *    url: "https://www.w3schools.com/html/mov_bbb.mp4",
- *    playOnAwake: true,
- * });
- * ```
- * @category Multimedia
- * @group Components
- */
-export class VideoPlayer extends Behaviour {
-
-    /**
-     * When true the video will start playing as soon as the component is enabled
-     */
-    @serializable()
-    playOnAwake: boolean = true;
-
-    /**
-     * The aspect mode to use for the video. If
-     */
-    @serializable()
-    aspectMode: AspectMode = AspectMode.None;
-
-    @serializable(URL)
-    private clip?: string | MediaStream | null = null;
-
-    // set a default src, this should not be undefined
-    @serializable()
-    private source: VideoSource = VideoSource.Url;
-
-    /**
-     * The video clip url to play.
-     */
-    @serializable(URL)
-    get url() { return this._url }
-    /**
-     * The video clip to play. 
-     */
-    set url(val: string | null) {
-        const prev = this._url;
-        const changed = prev !== val;
-        if (this.__didAwake) {
-            if (changed) {
-                this.setClipURL(val ?? "");
-            }
-        }
-        else this._url = val;
-    }
-
-    private _url: string | null = null;
-
-    @serializable()
-    private renderMode?: VideoRenderMode;
-
-    @serializable()
-    private targetMaterialProperty?: string;
-
-    @serializable(Renderer)
-    private targetMaterialRenderer?: Renderer;
-
-    @serializable(Texture)
-    private targetTexture?: Texture;
-
-    @serializable()
-    private time: number = 0;
-
-    private _playbackSpeed: number = 1;
-    /**
-     * Get the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
-     * @default 1
-     */
-    @serializable()
-    get playbackSpeed(): number {
-        return this._videoElement?.playbackRate ?? this._playbackSpeed;
-    }
-    /**
-     * Set the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
-     */
-    set playbackSpeed(val: number) {
-        this._playbackSpeed = val;
-        if (this._videoElement)
-            this._videoElement.playbackRate = val;
-    }
-
-    private _isLooping: boolean = false;
-    get isLooping(): boolean {
-        return this._videoElement?.loop ?? this._isLooping;
-    }
-    @serializable()
-    set isLooping(val: boolean) {
-        this._isLooping = val;
-        if (this._videoElement)
-            this._videoElement.loop = val;
-    }
-
-    /**
-     * @returns the current time of the video in seconds
-     */
-    get currentTime(): number {
-        return this._videoElement?.currentTime ?? this.time;
-    }
-    /**
-     * set the current time of the video in seconds
-     */
-    set currentTime(val: number) {
-        if (this._videoElement) {
-            this._videoElement.currentTime = val;
-        }
-        else this.time = val;
-    }
-
-    /**
-     * @returns true if the video is currently playing
-     */
-    get isPlaying(): boolean {
-        const video = this._videoElement;
-        if (video) {
-            if (video.currentTime > 0 && !video.paused && !video.ended
-                && video.readyState > video.HAVE_CURRENT_DATA)
-                return true;
-            else if (video.srcObject) {
-                const stream = video.srcObject as MediaStream;
-                if (stream.active) return true;
-            }
-        }
-        return false;
-    }
-
-    get crossOrigin(): string | null {
-        return this._videoElement?.crossOrigin ?? this._crossOrigin;
-    }
-    set crossOrigin(val: string | null) {
-        this._crossOrigin = val;
-        if (this._videoElement) {
-            if (val !== null) this._videoElement.setAttribute("crossorigin", val);
-            else this._videoElement.removeAttribute("crossorigin");
-        }
-    }
-
-    /**
-     * the material that is used to render the video
-     */
-    get videoMaterial() {
-        if (!this._videoMaterial) if (!this.create(false)) return null;
-        return this._videoMaterial;
-    }
-
-    /**
-     * the video texture that is used to render the video
-     */
-    get videoTexture() {
-        if (!this._videoTexture) if (!this.create(false)) return null;
-        return this._videoTexture;
-    }
-
-    /**
-     * the HTMLVideoElement that is used to play the video
-     */
-    get videoElement() {
-        if (!this._videoElement) if (!this.create(false)) return null;
-        return this._videoElement!;
-    }
-
-    /**
-     * Request the browser to enter picture in picture mode
-     * @link https://developer.mozilla.org/en-US/docs/Web/API/Picture-in-Picture_API
-     * @returns the promise returned by the browser
-     */
-    requestPictureInPicture() {
-        if (this._videoElement) return this._videoElement.requestPictureInPicture();
-        return null;
-    }
-
-    /**
-     * @returns true if the video is muted
-     */
-    get muted() {
-        return this._videoElement?.muted ?? this._muted;
-    }
-    /**
-     * set the video to be muted
-     */
-    set muted(val: boolean) {
-        this._muted = val;
-        if (this._videoElement) this._videoElement.muted = val;
-    }
-    private _muted: boolean = false;
-
-    /**
-     * The current video clip that is being played
-     */
-    get currentVideo() {
-        return this.clip;
-    }
-
-    @serializable()
-    private set audioOutputMode(mode: VideoAudioOutputMode) {
-        if (mode !== this._audioOutputMode) {
-            if (mode === VideoAudioOutputMode.AudioSource && isDevEnvironment()) console.warn("VideoAudioOutputMode.AudioSource is not yet implemented");
-            this._audioOutputMode = mode;
-            this.updateVideoElementSettings();
-        }
-    }
-    private get audioOutputMode() { return this._audioOutputMode; }
-    private _audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.Direct;
-
-    /** Set this to false to pause video playback while the tab is not active 
-     * @default true
-    */
-    playInBackground: boolean = true;
-
-    private _crossOrigin: string | null = "anonymous";
-
-    private _videoElement: HTMLVideoElement | null = null;
-    private _videoTexture: VideoTexture | null = null;
-    private _videoMaterial: Material | null = null;
-
-    private _isPlaying: boolean = false;
-    private wasPlaying: boolean = false;
-
-    /** ensure's the video element has been created and will start loading the clip */
-    preloadVideo() {
-        if (debug) console.log("Video Preload: " + this.name, this.clip);
-        this.create(false);
-    }
-    /** @deprecated use `preloadVideo()` */
-    preload() { this.preloadVideo(); }
-
-    /** Set a new video stream  
-     * starts to play automatically if the videoplayer hasnt been active before and playOnAwake is true */
-    setVideo(video: MediaStream) {
-        this.clip = video;
-        this.source = VideoSource.VideoClip;
-        if (!this._videoElement) this.create(this.playOnAwake);
-        else {
-            // TODO: how to prevent interruption error when another video is already playing
-            this._videoElement.srcObject = video;
-            if (this._isPlaying)
-                this.play();
-            this.updateAspect();
-        }
-    }
-
-    setClipURL(url: string) {
-        if (this._url === url) return;
-        this._url = url;
-        this.source = VideoSource.Url;
-
-        if (debug) console.log("set url", url);
-        if (!this._videoElement) this.create(this.playOnAwake);
-        else {
-            if (url.endsWith(".m3u8") || url.includes(".m3u")) {
-                this.ensureM3UCanBePlayed();
-            }
-            else {
-                this._videoElement.src = url;
-                if (this._isPlaying) {
-                    this.stop();
-                    this.play();
-                }
-            }
-        }
-    }
-
-    /** @internal */
-    onEnable(): void {
-        if (debug) console.log("VideoPlayer.onEnable", VideoSource[this.source], this.clip, this.url, this)
-        window.addEventListener('visibilitychange', this.visibilityChanged);
-
-        if (this.playOnAwake === true) {
-            this.create(true);
-        }
-        else {
-            this.preloadVideo();
-        }
-
-        if (this.screenspace) {
-            this._overlay?.start();
-        }
-        else this._overlay?.stop();
-    }
-
-    /** @internal */
-    onDisable(): void {
-        window.removeEventListener('visibilitychange', this.visibilityChanged);
-        this._overlay?.stop();
-        this.pause();
-    }
-
-    private visibilityChanged = (_: Event) => {
-        switch (document.visibilityState) {
-            case "hidden":
-                if (!this.playInBackground) {
-                    this.wasPlaying = this._isPlaying;
-                    this.pause();
-                }
-                break;
-            case "visible":
-                if (this.wasPlaying && !this._isPlaying) this.play();
-                break;
-        }
-    }
-
-    /** @internal */
-    onDestroy(): void {
-        if (this._videoElement) {
-            this.videoElement?.remove();
-            this._videoElement = null;
-        }
-        if (this._videoTexture) {
-            this._videoTexture.dispose();
-            this._videoTexture = null;
-        }
-    }
-
-    private _receivedInput: boolean = false;
-
-    /**
-     * @internal
-     */
-    constructor() {
-        super();
-        awaitInput(() => {
-            this._receivedInput = true;
-            this.updateVideoElementSettings();
-        });
-        this._targetObjects = [];
-
-        if (getParam("videoscreenspace")) {
-            window.addEventListener("keydown", evt => {
-                if (evt.key === "f") {
-                    this.screenspace = !this.screenspace;
-                }
-            });
-        }
-    }
-
-    /** start playing the video source */
-    play() {
-        if (!this._videoElement) this.create(false);
-        if (!this._videoElement) {
-            if (debug) console.warn("Can not play: no video element found", this);
-            return
-        }
-        if (this._isPlaying && !this._videoElement?.ended && !this._videoElement?.paused) return;
-        this._isPlaying = true;
-        if (!this._receivedInput) this._videoElement.muted = true;
-        this.handleBeginPlaying(false);
-
-        if (this.shouldUseM3U) {
-            this.ensureM3UCanBePlayed();
-            return;
-        }
-
-        if (debug) console.log("Video Play()", this.clip, this._videoElement, this.time);
-        this._videoElement.currentTime = this.time;
-        this._videoElement.play().catch(err => {
-            console.log(err);
-            // https://developer.chrome.com/blog/play-request-was-interrupted/
-            if (debug)
-                console.error("Error playing video", err, "CODE=" + err.code, this.videoElement?.src, this);
-            setTimeout(() => {
-                if (this._isPlaying && !this.destroyed && this.activeAndEnabled)
-                    this.play();
-            }, 1000);
-        });
-        if (debug) console.log("play", this._videoElement, this.time);
-    }
-
-    /**
-     * Stop the video playback. This will reset the video to the beginning
-     */
-    stop() {
-        this._isPlaying = false;
-        this.time = 0;
-        if (!this._videoElement) return;
-        this._videoElement.currentTime = 0;
-        this._videoElement.pause();
-        if (debug) console.log("STOP", this);
-    }
-
-    /**
-     * Pause the video playback
-     */
-    pause(): void {
-        this.time = this._videoElement?.currentTime ?? 0;
-        this._isPlaying = false;
-        this._videoElement?.pause();
-        if (debug) console.log("PAUSE", this, this.currentTime);
-    }
-
-
-    /** create the video element and assign the video source url or stream */
-    create(playAutomatically: boolean): boolean {
-        let src;
-        switch (this.source) {
-            case VideoSource.VideoClip:
-                src = this.clip;
-                break;
-            case VideoSource.Url:
-                src = this.url;
-                if (!src?.length && typeof this.clip === "string")
-                    src = this.clip;
-                break;
-        }
-
-        if (!src) {
-            if (debug) console.warn("No video source set", this);
-            return false;
-        }
-
-        if (!this._videoElement) {
-            if (debug)
-                console.warn("Create VideoElement", this);
-            this._videoElement = this.createVideoElement();
-            this.context.domElement!.shadowRoot!.prepend(this._videoElement);
-            // hide it because otherwise it would overlay the website with default css
-            this.updateVideoElementStyles();
-        }
-
-        if (typeof src === "string") {
-            if (debug) console.log("Set Video src", src);
-            this._videoElement.src = src;
-            // Nor sure why we did this here, but with this code the video does not restart when being paused / enable toggled
-            // const str = this._videoElement["captureStream"]?.call(this._videoElement);
-            // this.clip = str;
-        }
-        else {
-            if (debug) console.log("Set Video srcObject", src);
-            this._videoElement.srcObject = src;
-        }
-
-
-        if (!this._videoTexture)
-            this._videoTexture = new VideoTexture(this._videoElement);
-        this._videoTexture.flipY = false;
-        this._videoTexture.colorSpace = SRGBColorSpace;
-        if (playAutomatically)
-            this.handleBeginPlaying(playAutomatically);
-        if (debug)
-            console.log("Video: handle playing done...", src, playAutomatically);
-        return true;
-    }
-
-    updateAspect() {
-        if (this.aspectMode === AspectMode.None) return;
-        this.startCoroutine(this.updateAspectImpl());
-    }
-
-    private _overlay: VideoOverlay | null = null;
-
-    /**
-     * If true the video will be rendered in screenspace mode and overlayed on top of the scene.
-     * Alternatively you can also request the video to be played in PictureInPicture mode by calling `requestPictureInPicture()`
-     */
-    get screenspace(): boolean {
-        return this._overlay?.enabled ?? false;
-    }
-    set screenspace(val: boolean) {
-        if (val) {
-            if (!this._videoTexture) return;
-            if (!this._overlay) this._overlay = new VideoOverlay(this.context);
-            this._overlay.add(this._videoTexture);
-        }
-        else this._overlay?.remove(this._videoTexture);
-        if (this._overlay) this._overlay.enabled = val;
-    }
-
-    private _targetObjects: Array<Object3D>;
-
-    private createVideoElement(): HTMLVideoElement {
-        const video = document.createElement("video") as HTMLVideoElement;
-        if (this._crossOrigin)
-            video.setAttribute("crossorigin", this._crossOrigin);
-        if (debug) console.log("created video element", video);
-        return video;
-    }
-
-    private handleBeginPlaying(playAutomatically: boolean) {
-        if (!this.activeAndEnabled) return;
-        if (!this._videoElement) return;
-
-        this._targetObjects.length = 0;
-
-        let target: Object3D | undefined = this.gameObject;
-
-        switch (this.renderMode) {
-            case VideoRenderMode.MaterialOverride:
-                target = this.targetMaterialRenderer?.gameObject;
-                if (!target) target = GameObject.getComponent(this.gameObject, Renderer)?.gameObject;
-                break;
-            case VideoRenderMode.RenderTexture:
-                console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
-                return;
-        }
-
-        if (!target) {
-            console.error("Missing target for video material renderer", this.name, VideoRenderMode[this.renderMode!], this);
-            return;
-        }
-        const mat = target["material"];
-        if (mat) {
-            this._targetObjects.push(target);
-
-            if (mat !== this._videoMaterial) {
-                this._videoMaterial = mat.clone();
-                target["material"] = this._videoMaterial;
-            }
-
-            const fieldName = "map";
-            const videoMaterial = this._videoMaterial as any;
-
-            if (!this.targetMaterialProperty) {
-                videoMaterial[fieldName] = this._videoTexture;
-            }
-            else {
-                switch (this.targetMaterialProperty) {
-                    default:
-                        videoMaterial[fieldName] = this._videoTexture;
-                        break;
-                    // doesnt render:
-                    // case "emissiveTexture":
-                    //     console.log(this.videoMaterial);
-                    //     // (this.videoMaterial as any).map = this.videoTexture;
-                    //     (this.videoMaterial as any).emissive?.set(1,1,1);// = this.videoTexture;
-                    //     (this.videoMaterial as any).emissiveMap = this.videoTexture;
-                    //     break;
-                }
-            }
-        }
-        else {
-            console.warn("Can not play video, no material found, this might be a multimaterial case which is not supported yet");
-            return;
-        }
-        this.updateVideoElementSettings();
-        this.updateVideoElementStyles();
-        if (playAutomatically) {
-            if (this.shouldUseM3U) {
-                this.ensureM3UCanBePlayed();
-            }
-            this.play();
-        }
-    }
-
-    private updateVideoElementSettings() {
-        if (!this._videoElement) return;
-        this._videoElement.loop = this._isLooping;
-        this._videoElement.currentTime = this.currentTime;
-        this._videoElement.playbackRate = this._playbackSpeed;
-        // dont open in fullscreen on ios
-        this._videoElement.playsInline = true;
-        let muted = !this._receivedInput || this.audioOutputMode === VideoAudioOutputMode.None;
-        if (!muted && this._muted) muted = true;
-        this._videoElement.muted = muted;
-        if (this.playOnAwake)
-            this._videoElement.autoplay = true;
-    }
-
-    private updateVideoElementStyles() {
-        if (!this._videoElement) return;
-        // set style here so preview frame is rendered
-        // set display and selectable because otherwise is interfers with input/focus e.g. breaks orbit control
-        this._videoElement.style.userSelect = "none";
-        this._videoElement.style.visibility = "hidden";
-        this._videoElement.style.display = "none";
-        this.updateAspect();
-    }
-
-
-
-
-    private _updateAspectRoutineId: number = -1;
-    private *updateAspectImpl() {
-        const id = ++this._updateAspectRoutineId;
-        const lastAspect: number | undefined = undefined;
-        const stream = this.clip;
-        while (id === this._updateAspectRoutineId && this.aspectMode !== AspectMode.None && this.clip && stream === this.clip && this._isPlaying) {
-            if (!stream || typeof stream === "string") {
-                return;
-            }
-            let aspect: number | undefined = undefined;
-            for (const track of stream.getVideoTracks()) {
-                const settings = track.getSettings();
-                if (settings && settings.width && settings.height) {
-                    aspect = settings.width / settings.height;
-                    break;
-                }
-                // on firefox capture canvas stream works but looks like 
-                // the canvas stream track doesnt contain settings?!!?
-                else {
-                    aspect = this.context.renderer.domElement.clientWidth / this.context.renderer.domElement.clientHeight;
-                }
-            }
-            if (aspect === undefined) {
-                for (let i = 0; i < 10; i++)
-                    yield;
-                if (!this.isPlaying) break;
-                continue;
-            }
-            if (lastAspect === aspect) {
-                yield;
-                continue;
-            }
-            for (const obj of this._targetObjects) {
-                let worldAspect = 1;
-                if (obj.parent) {
-                    const parentScale = getWorldScale(obj.parent);
-                    worldAspect = parentScale.x / parentScale.y;
-                }
-                switch (this.aspectMode) {
-                    case AspectMode.AdjustHeight:
-                        obj.scale.y = 1 / aspect * obj.scale.x * worldAspect;
-                        break;
-                    case AspectMode.AdjustWidth:
-                        obj.scale.x = aspect * obj.scale.y * worldAspect;
-                        break;
-                }
-            }
-            for (let i = 0; i < 3; i++)
-                yield;
-        }
-    }
-
-
-
-
-    private get shouldUseM3U(): boolean { return this.url != undefined && (this.url.endsWith(".m3u8") || this.url.endsWith(".m3u")) && this.source === VideoSource.Url; }
-
-    private ensureM3UCanBePlayed() {
-        if (!this.shouldUseM3U) return;
-        let hls_script = document.head.querySelector("script[data-hls_library]") as HTMLScriptElement;
-        if (!hls_script) {
-            if (debug) console.log("HLS: load script");
-            hls_script = document.createElement("script");
-            hls_script.dataset["hls_library"] = "hls.js";
-            hls_script.src = "https://cdn.jsdelivr.net/npm/hls.js@1";
-            hls_script.addEventListener("load", this.onHlsAvailable);
-            document.head.append(hls_script);
-        }
-        else if (globalThis["Hls"]) {
-            this.onHlsAvailable();
-        }
-        else {
-            hls_script.addEventListener("load", this.onHlsAvailable);
-        }
-    }
-    private _hls?: Hls;
-    private onHlsAvailable = () => {
-        if (debug) console.log("HLS: available", this.clip);
-        if (!this.shouldUseM3U || !this.url) return;
-        if (!this._hls)
-            this._hls = new Hls();
-        this.videoElement!.autoplay = true;
-        this._hls.loadSource(this.url);
-        this._hls.attachMedia(this.videoElement!);
-        this._videoElement?.play();
-        if (debug) console.log("HLS: loaded", this.clip);
-    }
-}
-
-declare class Hls {
-    constructor();
-    loadSource(url: string);
-    attachMedia(videoElement: HTMLVideoElement);
-}
-
-
-
-
-
-
-
-
-class VideoOverlay {
-
-    readonly context: Context;
-
-    constructor(context: Context) {
-        this.context = context;
-        this._input = new VideoOverlayInput(this);
-    }
-
-    get enabled() {
-        return this._isInScreenspaceMode;
-    }
-
-    set enabled(val: boolean) {
-        if (val) this.start();
-        else this.stop();
-    }
-
-
-    add(video: VideoTexture) {
-        if (this._videos.indexOf(video) === -1) {
-            this._videos.push(video);
-        }
-    }
-
-    remove(video: VideoTexture | null | undefined) {
-        if (!video) return;
-        const index = this._videos.indexOf(video);
-        if (index >= 0) {
-            this._videos.splice(index, 1);
-        }
-    }
-
-    start() {
-        if (this._isInScreenspaceMode) return;
-        if (this._videos.length < 0) return;
-        const texture = this._videos[this._videos.length - 1];
-        if (!texture) return;
-
-        this._isInScreenspaceMode = true;
-        if (!this._screenspaceModeQuad) {
-            this._screenspaceModeQuad = ObjectUtils.createPrimitive(PrimitiveType.Quad, {
-                material: new ScreenspaceTexture(texture)
-            });
-            if (!this._screenspaceModeQuad) return;
-            this._screenspaceModeQuad.geometry.scale(2, 2, 2);
-        }
-
-        const quad = this._screenspaceModeQuad;
-        this.context.scene.add(quad);
-        this.updateScreenspaceMaterialUniforms();
-
-        const mat = quad.material as ScreenspaceTexture;
-        mat?.reset();
-
-        this._input?.enable(mat);
-    }
-
-    stop() {
-        this._isInScreenspaceMode = false;
-        if (this._screenspaceModeQuad) {
-            this._input?.disable();
-            this._screenspaceModeQuad.removeFromParent();
-        }
-    }
-
-    updateScreenspaceMaterialUniforms() {
-        const mat = this._screenspaceModeQuad?.material as ScreenspaceTexture;
-        if (!mat) return;
-        // mat.videoAspect = this.videoTexture?.image?.width / this.videoTexture?.image?.height;
-        mat.screenAspect = this.context.domElement.clientWidth / this.context.domElement.clientHeight;
-    }
-
-    private _videos: VideoTexture[] = [];
-    private _screenspaceModeQuad: Mesh | undefined;
-    private _isInScreenspaceMode: boolean = false;
-    private _input: VideoOverlayInput;
-}
-
-
-
-
-class VideoOverlayInput {
-
-    private _onResizeScreenFn?: () => void;
-    private _onKeyUpFn?: (e: KeyboardEvent) => void;
-    private _onMouseWheelFn?: (e: WheelEvent) => void;
-
-    private readonly context: Context;
-    private readonly overlay: VideoOverlay;
-
-    constructor(overlay: VideoOverlay) {
-        this.overlay = overlay;
-        this.context = overlay.context;
-    }
-
-    private _material?: ScreenspaceTexture;
-
-    enable(mat: ScreenspaceTexture) {
-        this._material = mat;
-
-        window.addEventListener("resize", this._onResizeScreenFn = () => {
-            this.overlay.updateScreenspaceMaterialUniforms();
-        });
-        window.addEventListener("keyup", this._onKeyUpFn = (args) => {
-            if (args.key === "Escape")
-                this.overlay.stop();
-        });
-
-        window.addEventListener("wheel", this._onMouseWheelFn = (args) => {
-            if (this.overlay.enabled) {
-                mat.zoom += args.deltaY * .0005;
-                args.preventDefault();
-            }
-        }, { passive: false });
-
-
-        const delta: Vector2 = new Vector2();
-
-        window.addEventListener("mousemove", (args: MouseEvent) => {
-            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
-                const normalizedMovement = new Vector2(args.movementX, args.movementY);
-                normalizedMovement.x /= this.context.domElement.clientWidth;
-                normalizedMovement.y /= this.context.domElement.clientHeight;
-                delta.set(normalizedMovement.x, normalizedMovement.y);
-                delta.multiplyScalar(mat.zoom / -this.context.time.deltaTime * .01);
-                mat.offset = mat.offset.add(delta);
-            }
-        });
-
-        window.addEventListener("pointermove", (args: PointerEvent) => {
-            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
-                const count = this.context.input.getTouchesPressedCount();
-                if (count === 1) {
-                    delta.set(args.movementX, args.movementY);
-                    delta.multiplyScalar(mat.zoom * -this.context.time.deltaTime * .05);
-                    mat.offset = mat.offset.add(delta);
-                }
-            }
-        });
-
-        let lastTouchStartTime = 0;
-        window.addEventListener("touchstart", e => {
-            if (e.touches.length < 2) {
-                if (this.context.time.time - lastTouchStartTime < .3) {
-                    this.overlay.stop();
-                }
-                lastTouchStartTime = this.context.time.time;
-                return;
-            }
-            this._isPinching = true;
-            this._lastPinch = 0;
-        })
-        window.addEventListener("touchmove", e => {
-            if (!this._isPinching || !this._material) return;
-            const touch1 = e.touches[0];
-            const touch2 = e.touches[1];
-            const dx = touch1.clientX - touch2.clientX;
-            const dy = touch1.clientY - touch2.clientY;
-            const distance = Math.sqrt(dx * dx + dy * dy);
-            if (this._lastPinch !== 0) {
-                const delta = distance - this._lastPinch;
-                this._material.zoom -= delta * .004;
-            }
-            this._lastPinch = distance;
-        })
-        window.addEventListener("touchend", () => {
-            this._isPinching = false;
-        })
-    }
-
-    private _isPinching: boolean = false;
-    private _lastPinch = 0;
-
-    disable() {
-
-        if (this._onResizeScreenFn) {
-            window.removeEventListener("resize", this._onResizeScreenFn);
-            this._onResizeScreenFn = undefined;
-        }
-        if (this._onKeyUpFn) {
-            window.removeEventListener("keyup", this._onKeyUpFn);
-            this._onKeyUpFn = undefined;
-        }
-        if (this._onMouseWheelFn) {
-            window.removeEventListener("wheel", this._onMouseWheelFn);
-            this._onMouseWheelFn = undefined;
-        }
-    }
-
-}
-
-class ScreenspaceTexture extends ShaderMaterial {
-
-    set screenAspect(val: number) {
-        this.uniforms["screenAspect"].value = val;
-        this.needsUpdate = true;
-    }
-
-    set offset(vec: Vector2 | { x: number, y: number }) {
-        const val = this.uniforms["offsetScale"].value;
-        val.x = vec.x;
-        val.y = vec.y;
-        // console.log(val);
-        this.uniforms["offsetScale"].value = val;
-        this.needsUpdate = true;
-    }
-
-    private readonly _offset: Vector2 = new Vector2();
-    get offset(): Vector2 {
-        const val = this.uniforms["offsetScale"].value;
-        this._offset.set(val.x, val.y);
-        return this._offset;
-    }
-
-    set zoom(val: number) {
-        const zoom = this.uniforms["offsetScale"].value;
-        if (val < .001) val = .001;
-        zoom.z = val;
-        // zoom.z = this.maxZoom - val;
-        // zoom.z /= this.maxZoom;
-        this.needsUpdate = true;
-    }
-
-    get zoom(): number {
-        return this.uniforms["offsetScale"].value.z;// * this.maxZoom;
-    }
-
-    reset() {
-        this.offset = this.offset.set(0, 0);
-        this.zoom = 1;
-        this.needsUpdate = true;
-    }
-
-    // maxZoom : number = 10
-
-    constructor(tex: Texture) {
-        super();
-
-        this.uniforms = {
-            map: { value: tex },
-            screenAspect: { value: 1 },
-            offsetScale: { value: new Vector4(0, 0, 1, 1) }
-        };
-
-        this.vertexShader = `
-        uniform sampler2D map;
-        uniform float screenAspect;
-        uniform vec4 offsetScale;
-        varying vec2 vUv;
-
-        void main() {
-
-            gl_Position = vec4( position , 1.0 );
-            vUv = uv;
-            vUv.y = 1. - vUv.y;
-
-            // fit into screen
-            ivec2 res = textureSize(map, 0);
-            float videoAspect = float(res.x) / float(res.y);
-            float aspect = videoAspect / screenAspect;
-            if(aspect >= 1.0) 
-            {
-                vUv.y = vUv.y * aspect;
-                float offset = (1. - aspect) * .5;
-                vUv.y = vUv.y + offset;
-            }
-            else
-            {
-                vUv.x = vUv.x / aspect;
-                float offset = (1. - 1. / aspect) * .5;
-                vUv.x = vUv.x + offset;
-            }
-
-            vUv.x -= .5;
-            vUv.y -= .5;
-
-            vUv.x *= offsetScale.z;
-            vUv.y *= offsetScale.z;
-            vUv.x += offsetScale.x;
-            vUv.y += offsetScale.y;
-
-            vUv.x += .5;
-            vUv.y += .5;
-        }
-
-        `
-        this.fragmentShader = `
-        uniform sampler2D map;
-        varying vec2 vUv;
-        void main() {
-            if(vUv.x < 0. || vUv.x > 1. || vUv.y < 0. || vUv.y > 1.)
-                gl_FragColor = vec4(0., 0., 0., 1.);
-            else
-            {
-                vec4 texcolor = texture2D(map, vUv);
-                gl_FragColor = texcolor;
-            }
-        }
-        `
-    }
-}
+import { Material, Mesh, Object3D, ShaderMaterial, SRGBColorSpace, Texture, Vector2, Vector4, VideoTexture } from "three";
+
+import { isDevEnvironment } from "../engine/debug/index.js";
+import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects.js";
+import { awaitInput } from "../engine/engine_input_utils.js";
+import { serializable } from "../engine/engine_serialization_decorator.js";
+import { Context } from "../engine/engine_setup.js";
+import { getWorldScale } from "../engine/engine_three_utils.js";
+import { getParam } from "../engine/engine_utils.js";
+import { Behaviour, GameObject } from "./Component.js";
+import { Renderer } from "./Renderer.js";
+
+const debug = getParam("debugvideo");
+
+
+export enum AspectMode {
+    None = 0,
+    AdjustHeight = 1,
+    AdjustWidth = 2,
+}
+
+export enum VideoSource {
+    /// <summary>
+    ///   <para>Use the current clip as the video content source.</para>
+    /// </summary>
+    VideoClip = 0,
+    /// <summary>
+    ///   <para>Use the current URL as the video content source.</para>
+    /// </summary>
+    Url = 1,
+}
+
+export enum VideoAudioOutputMode {
+    None = 0,
+    AudioSource = 1,
+    Direct = 2,
+    APIOnly = 3,
+}
+
+export enum VideoRenderMode {
+    CameraFarPlane = 0,
+    CameraNearPlane = 1,
+    RenderTexture = 2,
+    MaterialOverride = 3,
+}
+
+/**
+ * The VideoPlayer component can be used to playback video clips from urls, streams or m3u8 playlists (livestreams)
+ * @example Add a video player component to a game object and set the url to a video file. The video will start playing once the object becomes active in your scene
+ * ```typescript
+ * // Add a video player component to a game object and set the url to a video file. The video will start playing once the object becomes active in your scene
+ * const videoPlayer = addComponent(obj, VideoPlayer, {
+ *    url: "https://www.w3schools.com/html/mov_bbb.mp4",
+ *    playOnAwake: true,
+ * });
+ * ```
+ * @category Multimedia
+ * @group Components
+ */
+export class VideoPlayer extends Behaviour {
+
+    /**
+     * When true the video will start playing as soon as the component is enabled
+     */
+    @serializable()
+    playOnAwake: boolean = true;
+
+    /**
+     * The aspect mode to use for the video. If
+     */
+    @serializable()
+    aspectMode: AspectMode = AspectMode.None;
+
+    @serializable(URL)
+    private clip?: string | MediaStream | null = null;
+
+    // set a default src, this should not be undefined
+    @serializable()
+    private source: VideoSource = VideoSource.Url;
+
+    /**
+     * The video clip url to play.
+     */
+    @serializable(URL)
+    get url() { return this._url }
+    /**
+     * The video clip to play. 
+     */
+    set url(val: string | null) {
+        const prev = this._url;
+        const changed = prev !== val;
+        if (this.__didAwake) {
+            if (changed) {
+                this.setClipURL(val ?? "");
+            }
+        }
+        else this._url = val;
+    }
+
+    private _url: string | null = null;
+
+    @serializable()
+    private renderMode?: VideoRenderMode;
+
+    @serializable()
+    private targetMaterialProperty?: string;
+
+    @serializable(Renderer)
+    private targetMaterialRenderer?: Renderer;
+
+    @serializable(Texture)
+    private targetTexture?: Texture;
+
+    @serializable()
+    private time: number = 0;
+
+    private _playbackSpeed: number = 1;
+    /**
+     * Get the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
+     * @default 1
+     */
+    @serializable()
+    get playbackSpeed(): number {
+        return this._videoElement?.playbackRate ?? this._playbackSpeed;
+    }
+    /**
+     * Set the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
+     */
+    set playbackSpeed(val: number) {
+        this._playbackSpeed = val;
+        if (this._videoElement)
+            this._videoElement.playbackRate = val;
+    }
+
+    private _isLooping: boolean = false;
+    get isLooping(): boolean {
+        return this._videoElement?.loop ?? this._isLooping;
+    }
+    @serializable()
+    set isLooping(val: boolean) {
+        this._isLooping = val;
+        if (this._videoElement)
+            this._videoElement.loop = val;
+    }
+
+    /**
+     * @returns the current time of the video in seconds
+     */
+    get currentTime(): number {
+        return this._videoElement?.currentTime ?? this.time;
+    }
+    /**
+     * set the current time of the video in seconds
+     */
+    set currentTime(val: number) {
+        if (this._videoElement) {
+            this._videoElement.currentTime = val;
+        }
+        else this.time = val;
+    }
+
+    /**
+     * @returns true if the video is currently playing
+     */
+    get isPlaying(): boolean {
+        const video = this._videoElement;
+        if (video) {
+            if (video.currentTime > 0 && !video.paused && !video.ended
+                && video.readyState > video.HAVE_CURRENT_DATA)
+                return true;
+            else if (video.srcObject) {
+                const stream = video.srcObject as MediaStream;
+                if (stream.active) return true;
+            }
+        }
+        return false;
+    }
+
+    get crossOrigin(): string | null {
+        return this._videoElement?.crossOrigin ?? this._crossOrigin;
+    }
+    set crossOrigin(val: string | null) {
+        this._crossOrigin = val;
+        if (this._videoElement) {
+            if (val !== null) this._videoElement.setAttribute("crossorigin", val);
+            else this._videoElement.removeAttribute("crossorigin");
+        }
+    }
+
+    /**
+     * the material that is used to render the video
+     */
+    get videoMaterial() {
+        if (!this._videoMaterial) if (!this.create(false)) return null;
+        return this._videoMaterial;
+    }
+
+    /**
+     * the video texture that is used to render the video
+     */
+    get videoTexture() {
+        if (!this._videoTexture) if (!this.create(false)) return null;
+        return this._videoTexture;
+    }
+
+    /**
+     * the HTMLVideoElement that is used to play the video
+     */
+    get videoElement() {
+        if (!this._videoElement) if (!this.create(false)) return null;
+        return this._videoElement!;
+    }
+
+    /**
+     * Request the browser to enter picture in picture mode
+     * @link https://developer.mozilla.org/en-US/docs/Web/API/Picture-in-Picture_API
+     * @returns the promise returned by the browser
+     */
+    requestPictureInPicture() {
+        if (this._videoElement) return this._videoElement.requestPictureInPicture();
+        return null;
+    }
+
+    /**
+     * @returns true if the video is muted
+     */
+    get muted() {
+        return this._videoElement?.muted ?? this._muted;
+    }
+    /**
+     * set the video to be muted
+     */
+    set muted(val: boolean) {
+        this._muted = val;
+        if (this._videoElement) this._videoElement.muted = val;
+    }
+    private _muted: boolean = false;
+
+    /**
+     * The current video clip that is being played
+     */
+    get currentVideo() {
+        return this.clip;
+    }
+
+    @serializable()
+    private set audioOutputMode(mode: VideoAudioOutputMode) {
+        if (mode !== this._audioOutputMode) {
+            if (mode === VideoAudioOutputMode.AudioSource && isDevEnvironment()) console.warn("VideoAudioOutputMode.AudioSource is not yet implemented");
+            this._audioOutputMode = mode;
+            this.updateVideoElementSettings();
+        }
+    }
+    private get audioOutputMode() { return this._audioOutputMode; }
+    private _audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.Direct;
+
+    /** Set this to false to pause video playback while the tab is not active 
+     * @default true
+    */
+    playInBackground: boolean = true;
+
+    private _crossOrigin: string | null = "anonymous";
+
+    private _videoElement: HTMLVideoElement | null = null;
+    private _videoTexture: VideoTexture | null = null;
+    private _videoMaterial: Material | null = null;
+
+    private _isPlaying: boolean = false;
+    private wasPlaying: boolean = false;
+
+    /** ensure's the video element has been created and will start loading the clip */
+    preloadVideo() {
+        if (debug) console.log("Video Preload: " + this.name, this.clip);
+        this.create(false);
+    }
+    /** @deprecated use `preloadVideo()` */
+    preload() { this.preloadVideo(); }
+
+    /** Set a new video stream  
+     * starts to play automatically if the videoplayer hasnt been active before and playOnAwake is true */
+    setVideo(video: MediaStream) {
+        this.clip = video;
+        this.source = VideoSource.VideoClip;
+        if (!this._videoElement) this.create(this.playOnAwake);
+        else {
+            // TODO: how to prevent interruption error when another video is already playing
+            this._videoElement.srcObject = video;
+            if (this._isPlaying)
+                this.play();
+            this.updateAspect();
+        }
+    }
+
+    setClipURL(url: string) {
+        if (this._url === url) return;
+        this._url = url;
+        this.source = VideoSource.Url;
+
+        if (debug) console.log("set url", url);
+        if (!this._videoElement) this.create(this.playOnAwake);
+        else {
+            if (url.endsWith(".m3u8") || url.includes(".m3u")) {
+                this.ensureM3UCanBePlayed();
+            }
+            else {
+                this._videoElement.src = url;
+                if (this._isPlaying) {
+                    this.stop();
+                    this.play();
+                }
+            }
+        }
+    }
+
+    /** @internal */
+    onEnable(): void {
+        if (debug) console.log("VideoPlayer.onEnable", VideoSource[this.source], this.clip, this.url, this)
+        window.addEventListener('visibilitychange', this.visibilityChanged);
+
+        if (this.playOnAwake === true) {
+            this.create(true);
+        }
+        else {
+            this.preloadVideo();
+        }
+
+        if (this.screenspace) {
+            this._overlay?.start();
+        }
+        else this._overlay?.stop();
+    }
+
+    /** @internal */
+    onDisable(): void {
+        window.removeEventListener('visibilitychange', this.visibilityChanged);
+        this._overlay?.stop();
+        this.pause();
+    }
+
+    private visibilityChanged = (_: Event) => {
+        switch (document.visibilityState) {
+            case "hidden":
+                if (!this.playInBackground) {
+                    this.wasPlaying = this._isPlaying;
+                    this.pause();
+                }
+                break;
+            case "visible":
+                if (this.wasPlaying && !this._isPlaying) this.play();
+                break;
+        }
+    }
+
+    /** @internal */
+    onDestroy(): void {
+        if (this._videoElement) {
+            this.videoElement?.remove();
+            this._videoElement = null;
+        }
+        if (this._videoTexture) {
+            this._videoTexture.dispose();
+            this._videoTexture = null;
+        }
+    }
+
+    private _receivedInput: boolean = false;
+
+    /**
+     * @internal
+     */
+    constructor() {
+        super();
+        awaitInput(() => {
+            this._receivedInput = true;
+            this.updateVideoElementSettings();
+        });
+        this._targetObjects = [];
+
+        if (getParam("videoscreenspace")) {
+            window.addEventListener("keydown", evt => {
+                if (evt.key === "f") {
+                    this.screenspace = !this.screenspace;
+                }
+            });
+        }
+    }
+
+    /** start playing the video source */
+    play() {
+        if (!this._videoElement) this.create(false);
+        if (!this._videoElement) {
+            if (debug) console.warn("Can not play: no video element found", this);
+            return
+        }
+        if (this._isPlaying && !this._videoElement?.ended && !this._videoElement?.paused) return;
+        this._isPlaying = true;
+        if (!this._receivedInput) this._videoElement.muted = true;
+        this.handleBeginPlaying(false);
+
+        if (this.shouldUseM3U) {
+            this.ensureM3UCanBePlayed();
+            return;
+        }
+
+        if (debug) console.log("Video Play()", this.clip, this._videoElement, this.time);
+        this._videoElement.currentTime = this.time;
+        this._videoElement.play().catch(err => {
+            console.log(err);
+            // https://developer.chrome.com/blog/play-request-was-interrupted/
+            if (debug)
+                console.error("Error playing video", err, "CODE=" + err.code, this.videoElement?.src, this);
+            setTimeout(() => {
+                if (this._isPlaying && !this.destroyed && this.activeAndEnabled)
+                    this.play();
+            }, 1000);
+        });
+        if (debug) console.log("play", this._videoElement, this.time);
+    }
+
+    /**
+     * Stop the video playback. This will reset the video to the beginning
+     */
+    stop() {
+        this._isPlaying = false;
+        this.time = 0;
+        if (!this._videoElement) return;
+        this._videoElement.currentTime = 0;
+        this._videoElement.pause();
+        if (debug) console.log("STOP", this);
+    }
+
+    /**
+     * Pause the video playback
+     */
+    pause(): void {
+        this.time = this._videoElement?.currentTime ?? 0;
+        this._isPlaying = false;
+        this._videoElement?.pause();
+        if (debug) console.log("PAUSE", this, this.currentTime);
+    }
+
+
+    /** create the video element and assign the video source url or stream */
+    create(playAutomatically: boolean): boolean {
+        let src;
+        switch (this.source) {
+            case VideoSource.VideoClip:
+                src = this.clip;
+                break;
+            case VideoSource.Url:
+                src = this.url;
+                if (!src?.length && typeof this.clip === "string")
+                    src = this.clip;
+                break;
+        }
+
+        if (!src) {
+            if (debug) console.warn("No video source set", this);
+            return false;
+        }
+
+        if (!this._videoElement) {
+            if (debug)
+                console.warn("Create VideoElement", this);
+            this._videoElement = this.createVideoElement();
+            this.context.domElement!.shadowRoot!.prepend(this._videoElement);
+            // hide it because otherwise it would overlay the website with default css
+            this.updateVideoElementStyles();
+        }
+
+        if (typeof src === "string") {
+            if (debug) console.log("Set Video src", src);
+            this._videoElement.src = src;
+            // Nor sure why we did this here, but with this code the video does not restart when being paused / enable toggled
+            // const str = this._videoElement["captureStream"]?.call(this._videoElement);
+            // this.clip = str;
+        }
+        else {
+            if (debug) console.log("Set Video srcObject", src);
+            this._videoElement.srcObject = src;
+        }
+
+
+        if (!this._videoTexture)
+            this._videoTexture = new VideoTexture(this._videoElement);
+        this._videoTexture.flipY = false;
+        this._videoTexture.colorSpace = SRGBColorSpace;
+        if (playAutomatically)
+            this.handleBeginPlaying(playAutomatically);
+        if (debug)
+            console.log("Video: handle playing done...", src, playAutomatically);
+        return true;
+    }
+
+    updateAspect() {
+        if (this.aspectMode === AspectMode.None) return;
+        this.startCoroutine(this.updateAspectImpl());
+    }
+
+    private _overlay: VideoOverlay | null = null;
+
+    /**
+     * If true the video will be rendered in screenspace mode and overlayed on top of the scene.
+     * Alternatively you can also request the video to be played in PictureInPicture mode by calling `requestPictureInPicture()`
+     */
+    get screenspace(): boolean {
+        return this._overlay?.enabled ?? false;
+    }
+    set screenspace(val: boolean) {
+        if (val) {
+            if (!this._videoTexture) return;
+            if (!this._overlay) this._overlay = new VideoOverlay(this.context);
+            this._overlay.add(this._videoTexture);
+        }
+        else this._overlay?.remove(this._videoTexture);
+        if (this._overlay) this._overlay.enabled = val;
+    }
+
+    private _targetObjects: Array<Object3D>;
+
+    private createVideoElement(): HTMLVideoElement {
+        const video = document.createElement("video") as HTMLVideoElement;
+        if (this._crossOrigin)
+            video.setAttribute("crossorigin", this._crossOrigin);
+        if (debug) console.log("created video element", video);
+        return video;
+    }
+
+    private handleBeginPlaying(playAutomatically: boolean) {
+        if (!this.activeAndEnabled) return;
+        if (!this._videoElement) return;
+
+        this._targetObjects.length = 0;
+
+        let target: Object3D | undefined = this.gameObject;
+
+        switch (this.renderMode) {
+            case VideoRenderMode.MaterialOverride:
+                target = this.targetMaterialRenderer?.gameObject;
+                if (!target) target = GameObject.getComponent(this.gameObject, Renderer)?.gameObject;
+                break;
+            case VideoRenderMode.RenderTexture:
+                console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
+                return;
+        }
+
+        if (!target) {
+            console.error("Missing target for video material renderer", this.name, VideoRenderMode[this.renderMode!], this);
+            return;
+        }
+        const mat = target["material"];
+        if (mat) {
+            this._targetObjects.push(target);
+
+            if (mat !== this._videoMaterial) {
+                this._videoMaterial = mat.clone();
+                target["material"] = this._videoMaterial;
+            }
+
+            const fieldName = "map";
+            const videoMaterial = this._videoMaterial as any;
+
+            if (!this.targetMaterialProperty) {
+                videoMaterial[fieldName] = this._videoTexture;
+            }
+            else {
+                switch (this.targetMaterialProperty) {
+                    default:
+                        videoMaterial[fieldName] = this._videoTexture;
+                        break;
+                    // doesnt render:
+                    // case "emissiveTexture":
+                    //     console.log(this.videoMaterial);
+                    //     // (this.videoMaterial as any).map = this.videoTexture;
+                    //     (this.videoMaterial as any).emissive?.set(1,1,1);// = this.videoTexture;
+                    //     (this.videoMaterial as any).emissiveMap = this.videoTexture;
+                    //     break;
+                }
+            }
+        }
+        else {
+            console.warn("Can not play video, no material found, this might be a multimaterial case which is not supported yet");
+            return;
+        }
+        this.updateVideoElementSettings();
+        this.updateVideoElementStyles();
+        if (playAutomatically) {
+            if (this.shouldUseM3U) {
+                this.ensureM3UCanBePlayed();
+            }
+            this.play();
+        }
+    }
+
+    private updateVideoElementSettings() {
+        if (!this._videoElement) return;
+        this._videoElement.loop = this._isLooping;
+        this._videoElement.currentTime = this.currentTime;
+        this._videoElement.playbackRate = this._playbackSpeed;
+        // dont open in fullscreen on ios
+        this._videoElement.playsInline = true;
+        let muted = !this._receivedInput || this.audioOutputMode === VideoAudioOutputMode.None;
+        if (!muted && this._muted) muted = true;
+        this._videoElement.muted = muted;
+        if (this.playOnAwake)
+            this._videoElement.autoplay = true;
+    }
+
+    private updateVideoElementStyles() {
+        if (!this._videoElement) return;
+        // set style here so preview frame is rendered
+        // set display and selectable because otherwise is interfers with input/focus e.g. breaks orbit control
+        this._videoElement.style.userSelect = "none";
+        this._videoElement.style.visibility = "hidden";
+        this._videoElement.style.display = "none";
+        this.updateAspect();
+    }
+
+
+
+
+    private _updateAspectRoutineId: number = -1;
+    private *updateAspectImpl() {
+        const id = ++this._updateAspectRoutineId;
+        const lastAspect: number | undefined = undefined;
+        const stream = this.clip;
+        while (id === this._updateAspectRoutineId && this.aspectMode !== AspectMode.None && this.clip && stream === this.clip && this._isPlaying) {
+            if (!stream || typeof stream === "string") {
+                return;
+            }
+            let aspect: number | undefined = undefined;
+            for (const track of stream.getVideoTracks()) {
+                const settings = track.getSettings();
+                if (settings && settings.width && settings.height) {
+                    aspect = settings.width / settings.height;
+                    break;
+                }
+                // on firefox capture canvas stream works but looks like 
+                // the canvas stream track doesnt contain settings?!!?
+                else {
+                    aspect = this.context.renderer.domElement.clientWidth / this.context.renderer.domElement.clientHeight;
+                }
+            }
+            if (aspect === undefined) {
+                for (let i = 0; i < 10; i++)
+                    yield;
+                if (!this.isPlaying) break;
+                continue;
+            }
+            if (lastAspect === aspect) {
+                yield;
+                continue;
+            }
+            for (const obj of this._targetObjects) {
+                let worldAspect = 1;
+                if (obj.parent) {
+                    const parentScale = getWorldScale(obj.parent);
+                    worldAspect = parentScale.x / parentScale.y;
+                }
+                switch (this.aspectMode) {
+                    case AspectMode.AdjustHeight:
+                        obj.scale.y = 1 / aspect * obj.scale.x * worldAspect;
+                        break;
+                    case AspectMode.AdjustWidth:
+                        obj.scale.x = aspect * obj.scale.y * worldAspect;
+                        break;
+                }
+            }
+            for (let i = 0; i < 3; i++)
+                yield;
+        }
+    }
+
+
+
+
+    private get shouldUseM3U(): boolean { return this.url != undefined && (this.url.endsWith(".m3u8") || this.url.endsWith(".m3u")) && this.source === VideoSource.Url; }
+
+    private ensureM3UCanBePlayed() {
+        if (!this.shouldUseM3U) return;
+        let hls_script = document.head.querySelector("script[data-hls_library]") as HTMLScriptElement;
+        if (!hls_script) {
+            if (debug) console.log("HLS: load script");
+            hls_script = document.createElement("script");
+            hls_script.dataset["hls_library"] = "hls.js";
+            hls_script.src = "https://cdn.jsdelivr.net/npm/hls.js@1";
+            hls_script.addEventListener("load", this.onHlsAvailable);
+            document.head.append(hls_script);
+        }
+        else if (globalThis["Hls"]) {
+            this.onHlsAvailable();
+        }
+        else {
+            hls_script.addEventListener("load", this.onHlsAvailable);
+        }
+    }
+    private _hls?: Hls;
+    private onHlsAvailable = () => {
+        if (debug) console.log("HLS: available", this.clip);
+        if (!this.shouldUseM3U || !this.url) return;
+        if (!this._hls)
+            this._hls = new Hls();
+        this.videoElement!.autoplay = true;
+        this._hls.loadSource(this.url);
+        this._hls.attachMedia(this.videoElement!);
+        this._videoElement?.play();
+        if (debug) console.log("HLS: loaded", this.clip);
+    }
+}
+
+declare class Hls {
+    constructor();
+    loadSource(url: string);
+    attachMedia(videoElement: HTMLVideoElement);
+}
+
+
+
+
+
+
+
+
+class VideoOverlay {
+
+    readonly context: Context;
+
+    constructor(context: Context) {
+        this.context = context;
+        this._input = new VideoOverlayInput(this);
+    }
+
+    get enabled() {
+        return this._isInScreenspaceMode;
+    }
+
+    set enabled(val: boolean) {
+        if (val) this.start();
+        else this.stop();
+    }
+
+
+    add(video: VideoTexture) {
+        if (this._videos.indexOf(video) === -1) {
+            this._videos.push(video);
+        }
+    }
+
+    remove(video: VideoTexture | null | undefined) {
+        if (!video) return;
+        const index = this._videos.indexOf(video);
+        if (index >= 0) {
+            this._videos.splice(index, 1);
+        }
+    }
+
+    start() {
+        if (this._isInScreenspaceMode) return;
+        if (this._videos.length < 0) return;
+        const texture = this._videos[this._videos.length - 1];
+        if (!texture) return;
+
+        this._isInScreenspaceMode = true;
+        if (!this._screenspaceModeQuad) {
+            this._screenspaceModeQuad = ObjectUtils.createPrimitive(PrimitiveType.Quad, {
+                material: new ScreenspaceTexture(texture)
+            });
+            if (!this._screenspaceModeQuad) return;
+            this._screenspaceModeQuad.geometry.scale(2, 2, 2);
+        }
+
+        const quad = this._screenspaceModeQuad;
+        this.context.scene.add(quad);
+        this.updateScreenspaceMaterialUniforms();
+
+        const mat = quad.material as ScreenspaceTexture;
+        mat?.reset();
+
+        this._input?.enable(mat);
+    }
+
+    stop() {
+        this._isInScreenspaceMode = false;
+        if (this._screenspaceModeQuad) {
+            this._input?.disable();
+            this._screenspaceModeQuad.removeFromParent();
+        }
+    }
+
+    updateScreenspaceMaterialUniforms() {
+        const mat = this._screenspaceModeQuad?.material as ScreenspaceTexture;
+        if (!mat) return;
+        // mat.videoAspect = this.videoTexture?.image?.width / this.videoTexture?.image?.height;
+        mat.screenAspect = this.context.domElement.clientWidth / this.context.domElement.clientHeight;
+    }
+
+    private _videos: VideoTexture[] = [];
+    private _screenspaceModeQuad: Mesh | undefined;
+    private _isInScreenspaceMode: boolean = false;
+    private _input: VideoOverlayInput;
+}
+
+
+
+
+class VideoOverlayInput {
+
+    private _onResizeScreenFn?: () => void;
+    private _onKeyUpFn?: (e: KeyboardEvent) => void;
+    private _onMouseWheelFn?: (e: WheelEvent) => void;
+
+    private readonly context: Context;
+    private readonly overlay: VideoOverlay;
+
+    constructor(overlay: VideoOverlay) {
+        this.overlay = overlay;
+        this.context = overlay.context;
+    }
+
+    private _material?: ScreenspaceTexture;
+
+    enable(mat: ScreenspaceTexture) {
+        this._material = mat;
+
+        window.addEventListener("resize", this._onResizeScreenFn = () => {
+            this.overlay.updateScreenspaceMaterialUniforms();
+        });
+        window.addEventListener("keyup", this._onKeyUpFn = (args) => {
+            if (args.key === "Escape")
+                this.overlay.stop();
+        });
+
+        window.addEventListener("wheel", this._onMouseWheelFn = (args) => {
+            if (this.overlay.enabled) {
+                mat.zoom += args.deltaY * .0005;
+                args.preventDefault();
+            }
+        }, { passive: false });
+
+
+        const delta: Vector2 = new Vector2();
+
+        window.addEventListener("mousemove", (args: MouseEvent) => {
+            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
+                const normalizedMovement = new Vector2(args.movementX, args.movementY);
+                normalizedMovement.x /= this.context.domElement.clientWidth;
+                normalizedMovement.y /= this.context.domElement.clientHeight;
+                delta.set(normalizedMovement.x, normalizedMovement.y);
+                delta.multiplyScalar(mat.zoom / -this.context.time.deltaTime * .01);
+                mat.offset = mat.offset.add(delta);
+            }
+        });
+
+        window.addEventListener("pointermove", (args: PointerEvent) => {
+            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
+                const count = this.context.input.getTouchesPressedCount();
+                if (count === 1) {
+                    delta.set(args.movementX, args.movementY);
+                    delta.multiplyScalar(mat.zoom * -this.context.time.deltaTime * .05);
+                    mat.offset = mat.offset.add(delta);
+                }
+            }
+        });
+
+        let lastTouchStartTime = 0;
+        window.addEventListener("touchstart", e => {
+            if (e.touches.length < 2) {
+                if (this.context.time.time - lastTouchStartTime < .3) {
+                    this.overlay.stop();
+                }
+                lastTouchStartTime = this.context.time.time;
+                return;
+            }
+            this._isPinching = true;
+            this._lastPinch = 0;
+        })
+        window.addEventListener("touchmove", e => {
+            if (!this._isPinching || !this._material) return;
+            const touch1 = e.touches[0];
+            const touch2 = e.touches[1];
+            const dx = touch1.clientX - touch2.clientX;
+            const dy = touch1.clientY - touch2.clientY;
+            const distance = Math.sqrt(dx * dx + dy * dy);
+            if (this._lastPinch !== 0) {
+                const delta = distance - this._lastPinch;
+                this._material.zoom -= delta * .004;
+            }
+            this._lastPinch = distance;
+        })
+        window.addEventListener("touchend", () => {
+            this._isPinching = false;
+        })
+    }
+
+    private _isPinching: boolean = false;
+    private _lastPinch = 0;
+
+    disable() {
+
+        if (this._onResizeScreenFn) {
+            window.removeEventListener("resize", this._onResizeScreenFn);
+            this._onResizeScreenFn = undefined;
+        }
+        if (this._onKeyUpFn) {
+            window.removeEventListener("keyup", this._onKeyUpFn);
+            this._onKeyUpFn = undefined;
+        }
+        if (this._onMouseWheelFn) {
+            window.removeEventListener("wheel", this._onMouseWheelFn);
+            this._onMouseWheelFn = undefined;
+        }
+    }
+
+}
+
+class ScreenspaceTexture extends ShaderMaterial {
+
+    set screenAspect(val: number) {
+        this.uniforms["screenAspect"].value = val;
+        this.needsUpdate = true;
+    }
+
+    set offset(vec: Vector2 | { x: number, y: number }) {
+        const val = this.uniforms["offsetScale"].value;
+        val.x = vec.x;
+        val.y = vec.y;
+        // console.log(val);
+        this.uniforms["offsetScale"].value = val;
+        this.needsUpdate = true;
+    }
+
+    private readonly _offset: Vector2 = new Vector2();
+    get offset(): Vector2 {
+        const val = this.uniforms["offsetScale"].value;
+        this._offset.set(val.x, val.y);
+        return this._offset;
+    }
+
+    set zoom(val: number) {
+        const zoom = this.uniforms["offsetScale"].value;
+        if (val < .001) val = .001;
+        zoom.z = val;
+        // zoom.z = this.maxZoom - val;
+        // zoom.z /= this.maxZoom;
+        this.needsUpdate = true;
+    }
+
+    get zoom(): number {
+        return this.uniforms["offsetScale"].value.z;// * this.maxZoom;
+    }
+
+    reset() {
+        this.offset = this.offset.set(0, 0);
+        this.zoom = 1;
+        this.needsUpdate = true;
+    }
+
+    // maxZoom : number = 10
+
+    constructor(tex: Texture) {
+        super();
+
+        this.uniforms = {
+            map: { value: tex },
+            screenAspect: { value: 1 },
+            offsetScale: { value: new Vector4(0, 0, 1, 1) }
+        };
+
+        this.vertexShader = `
+        uniform sampler2D map;
+        uniform float screenAspect;
+        uniform vec4 offsetScale;
+        varying vec2 vUv;
+
+        void main() {
+
+            gl_Position = vec4( position , 1.0 );
+            vUv = uv;
+            vUv.y = 1. - vUv.y;
+
+            // fit into screen
+            ivec2 res = textureSize(map, 0);
+            float videoAspect = float(res.x) / float(res.y);
+            float aspect = videoAspect / screenAspect;
+            if(aspect >= 1.0) 
+            {
+                vUv.y = vUv.y * aspect;
+                float offset = (1. - aspect) * .5;
+                vUv.y = vUv.y + offset;
+            }
+            else
+            {
+                vUv.x = vUv.x / aspect;
+                float offset = (1. - 1. / aspect) * .5;
+                vUv.x = vUv.x + offset;
+            }
+
+            vUv.x -= .5;
+            vUv.y -= .5;
+
+            vUv.x *= offsetScale.z;
+            vUv.y *= offsetScale.z;
+            vUv.x += offsetScale.x;
+            vUv.y += offsetScale.y;
+
+            vUv.x += .5;
+            vUv.y += .5;
+        }
+
+        `
+        this.fragmentShader = `
+        uniform sampler2D map;
+        varying vec2 vUv;
+        void main() {
+            if(vUv.x < 0. || vUv.x > 1. || vUv.y < 0. || vUv.y > 1.)
+                gl_FragColor = vec4(0., 0., 0., 1.);
+            else
+            {
+                vec4 texcolor = texture2D(map, vUv);
+                gl_FragColor = texcolor;
+            }
+        }
+        `
+    }
+}