@realsee/vreo 0.2.0-alpha.8 → 1.0.0-alpha.1

package/resources/Player/index.tsx

@@ -1,4 +1,3 @@
-import * as React from 'react'
 import * as ReactDOM from 'react-dom'
 import { Five, Subscribe } from '@realsee/five'
 
@@ -7,160 +6,209 @@ import { Controller, ControllerContext } from './Controller'
 import { CameraMovementData, VreoKeyframeEnum, VreoKeyframeEvent, VreoUnit } from '../typings/VreoUnit'
 import { reaction } from 'mobx'
 import { Drawer } from './modules/Drawer'
-import { PlayerConfigs } from './typings'
+import { Appearance, PlayerConfigs, WaveAppearance } from './typings'
 import { PopUp } from './modules/PopUp'
 
 export class Player extends Subscribe<VreoKeyframeEvent> {
-  $five: Five
-  private controller: Controller
-  configs: Readonly<PlayerConfigs>
-
-  constructor(five: Five, configs: Partial<PlayerConfigs> = {}) {
-    super()
-    this.controller = new Controller()
-    this.$five = this.controller.$five = five
-
-    if (!configs.containter) {
-      const containter = document.querySelector('#vreo-app') || document.createElement('div')
-      ReactDOM.unmountComponentAtNode(containter)
-      containter.setAttribute('id', 'vreo-app')
-      configs.containter = containter
-      document.body.append(containter)
-    }
+    $five: Five
+    private controller: Controller
+    configs: Readonly<PlayerConfigs>
 
-    this.configs = Object.freeze(
-      Object.assign(
-        {
-          keyframeMap: {},
-        },
-        configs
-      )
-    )
-
-    this.controller.configs = this.configs
-
-    ReactDOM.render(
-      <ControllerContext.Provider value={this.controller}>
-        <App></App>
-        <Drawer />
-        <PopUp />
-        {this.configs.customKeyframes &&
-          this.configs.customKeyframes.map((CustomCmpt, key) => (
-            <CustomCmpt
-              key={key}
-              subscribe={{
-                on: (name, callback) => this.on(name, callback),
-                once: (name, callback) => this.once(name, callback),
-                off: (name, callback) => this.off(name, callback),
-              }}
-              five={five}
-            />
-          ))}
-      </ControllerContext.Provider>,
-      configs.containter
-    )
-
-    // 监听播放情况：抛出触发时机
-    reaction(
-      () => this.controller.playing,
-      (playing) => {
-        this.emit(playing ? 'playing' : 'paused')
-      }
-    )
-  }
-
-  async load(vreoUnit: VreoUnit, currentTime = 0, preload = false) {
-    if (!this.controller.visible) {
-      this.controller.setVisible(true)
-      // 延迟 500ms 规避跟 DOM 动画冲突
-      await new Promise((resolve) => {
-        setTimeout(() => resolve(true), 500)
-      })
-    }
+    constructor(five: Five, configs: Partial<PlayerConfigs> = {}) {
+        super()
+        this.$five = five
 
-    if (this.controller.stopInterval) {
-      this.controller.stopInterval()
-      this.controller.stopInterval = undefined
-    }
 
-    this.controller.vreoUnit = vreoUnit
-    this.controller.videoInstance?.pause()
+        if (!configs.container) {
+            configs.container = configs.containter
+        }
+        if (!configs.container) {
+            const container = document.getElementById('vreo-app') || document.createElement('div')
+            ReactDOM.unmountComponentAtNode(container)
+            container.setAttribute('id', 'vreo-app')
+            configs.container = container
+            const fiveCanvasDomParent = five.getElement()?.parentNode
+            if (fiveCanvasDomParent) {
+                fiveCanvasDomParent.append(container)
+            } else {
+                document.body.append(container)
+            }
+        }
 
-    // 预载逻辑
-    // 是否降低图片分辨率
-    if (this.$five.imageOptions.size && this.configs.imageOptions?.size && this.$five.imageOptions.size > this.configs.imageOptions.size) {
-      this.$five.imageOptions.size = this.configs.imageOptions.size
+        if (!configs.container.classList.contains('vreo-app')) configs.container.classList.add('vreo-app')
+
+        this.configs = Object.freeze( Object.assign({ keyframeMap: {} }, configs) )
+
+        this.controller = new Controller({five, container:configs.container, configs: this.configs})
+
+
+        ReactDOM.render(
+            <ControllerContext.Provider value={this.controller}>
+                <App />
+                <Drawer />
+                <PopUp />
+                {this.configs.customKeyframes &&
+                this.configs.customKeyframes.map((CustomCmpt, key) => (
+                    <CustomCmpt
+                        key={key}
+                        subscribe={{
+                            on: (name, callback) => this.on(name, callback),
+                            once: (name, callback) => this.once(name, callback),
+                            off: (name, callback) => this.off(name, callback),
+                        }}
+                        five={five}
+                    />
+                ))}
+            </ControllerContext.Provider>,
+            configs.container
+        )
+
+        // 监听播放情况：抛出触发时机
+        reaction(
+            () => this.controller.ended,
+            (ended) => {
+                if (ended) {
+                    this.emit('paused', true)
+                }
+            }
+        )
+
+        reaction(
+            () => this.controller.playing,
+            (playing) => {
+                if (!this.controller.ended) {
+                    this.emit(playing ? 'playing' : 'paused')
+                }
+            }
+        )
     }
-    
-    // 预载数据中的点位
-    if (preload || (preload === undefined && this.configs.autoPreload)) {
-      const panoIndexMap = vreoUnit.keyframes.filter((vreoKeyframe) => {
-        if (vreoKeyframe.type !== VreoKeyframeEnum.CameraMovement) {
-          return false
+
+    async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false) {
+        this.controller.setLoading(true)
+        if (force) {
+            vreoUnit = JSON.parse(JSON.stringify(vreoUnit))
+        }
+        if (!this.controller.visible) {
+            this.controller.setVisible(true)
+            // 延迟 500ms 规避跟 DOM 动画冲突
+            await new Promise((resolve) => {
+                setTimeout(() => resolve(true), 500)
+            })
         }
-  
-        const data = vreoKeyframe.data as CameraMovementData
-        if (data.panoIndex === undefined) {
-          return false
+
+        if (this.controller.stopInterval) {
+            this.controller.stopInterval()
+            this.controller.stopInterval = undefined
+        }
+
+        this.controller.vreoUnit = vreoUnit
+        this.controller.mediaInstance?.pause()
+
+        // 预载逻辑
+        // 是否降低图片分辨率
+        if (
+            this.$five.imageOptions.size &&
+            this.configs.imageOptions?.size &&
+            this.$five.imageOptions.size > this.configs.imageOptions.size
+        ) {
+            this.$five.imageOptions.size = this.configs.imageOptions.size
         }
+
+        // 预载数据中的点位
+        if (preload || (preload === undefined && this.configs.autoPreload)) {
+            const panoIndexMap = vreoUnit.keyframes
+                .filter((vreoKeyframe) => {
+                    if (vreoKeyframe.type !== VreoKeyframeEnum.CameraMovement) {
+                        return false
+                    }
+
+                    const data = vreoKeyframe.data as CameraMovementData
+                    if (data.panoIndex === undefined) {
+                        return false
+                    }
+                    return true
+                })
+                .reduce((accu: Record<number, boolean>, curr) => {
+                    const panoIndex = curr.data.panoIndex as number
+                    if (!accu[panoIndex]) {
+                        accu[panoIndex] = true
+                    }
+                    return accu
+                }, {})
+
+            const panoIndexes = Object.keys(panoIndexMap)
+            for (let i = 0; i < panoIndexes.length; i++) {
+                await this.$five.preloadPano(Number(panoIndexes[i]))
+            }
+        }
+
+        // 新数据载入就绪
+        this.emit('loaded', vreoUnit)
+        this.controller.emit('loaded', vreoUnit)
+
+        if (this.controller.videoAgentScene?.videoAgentMesh.mediaInstance) {
+            this.controller.videoAgentScene.videoAgentMesh.mediaInstance.currentTime = currentTime / 1000
+        }
+
+        this.controller.setAvatar(vreoUnit.video.avatar)
+
+        await this.controller.videoAgentScene?.videoAgentMesh.play(
+            vreoUnit.video.url,
+            currentTime / 1000,
+            vreoUnit.video.duration
+        )
+
+        this.controller.setEnded(false)
+        this.play()
+
+        this.controller.run((type, keyframe) => this.emit(type, keyframe))
+        this.controller.setLoading(false)
         return true
-      }).reduce((accu: Record<number, boolean>, curr) => {
-        const panoIndex = curr.data.panoIndex as number
-        if (!accu[panoIndex]) {
-          accu[panoIndex] = true
+    }
+
+    get paused() {
+        return !this.controller.playing
+    }
+
+    play(currentTime?: number) {
+        if (this.controller.playing) return true
+        if (currentTime && this.controller.mediaInstance) {
+            this.controller.mediaInstance.currentTime = currentTime / 1000
         }
-        return accu
-      }, {})
-  
-      const panoIndexs = Object.keys(panoIndexMap)
-      for (let i = 0; i < panoIndexs.length; i++) {
-        await this.$five.preloadPano(Number(panoIndexs[i]))
-      }
+        Object.assign(window, { $vreoController: this.controller })
+
+        this.controller.setPlaying(true)
+        return true
     }
-    
-    // 新数据载入就绪
-    this.emit('loaded', vreoUnit)
-    if (vreoUnit.video.url) {
-      if (this.controller.videoAgentScene?.videoAgentMesh.videoInstance) {
-        this.controller.videoAgentScene.videoAgentMesh.videoInstance.currentTime = currentTime / 1000
-      }
-      await this.controller.videoAgentScene?.videoAgentMesh.play(vreoUnit.video.url)
-      this.play()
+
+    setAppearance(appearance: Appearance) {
+        this.controller.setAppearance(appearance)
     }
-    this.controller.run((type, keyframe) => this.emit(type, keyframe))
-    return true
-  }
-
-  get paused() {
-    return !this.controller.playing
-  }
-
-  play() {
-    if (this.controller.playing) return true
-    this.controller.setPlaying(true)
-    return true
-  }
-
-  pause() {
-    this.controller.setPlaying(false)
-  }
-
-  show() {
-    this.controller.setVisible(true)
-  }
-
-  hide() {
-    this.controller.setVisible(false)
-  }
-
-  dispose() {
-    this.pause()
-    this.controller.dispose()
-
-    if (this.configs.containter) {
-      ReactDOM.unmountComponentAtNode(this.configs.containter as Element)
+
+    pause() {
+        this.controller.setPlaying(false)
+    }
+
+    show() {
+        this.controller.setVisible(true)
+    }
+
+    hide() {
+        this.controller.setVisible(false)
+    }
+
+    getCurrentTime() {
+        return this.controller.currentTime
+    }
+
+    dispose() {
+        this.pause()
+        this.controller.dispose()
+
+        if (this.configs.container) {
+            ReactDOM.unmountComponentAtNode(this.configs.container as Element)
+        }
     }
-  }
 }
 
 console.log(`
@@ -172,6 +220,6 @@ console.log(`
 ┃┃┃┗┓┃┃━┫┃┗┛┗┓┃┗┓┣━━┃┃┃━┫┃┃━┫
 ┗┛┗━┛┗━━┛┗━━━┛┗━┛┗━━┛┗━━┛┗━━┛
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 `)

package/resources/Player/modules/ResizeObserver/index.tsx

@@ -0,0 +1,23 @@
+import React from 'react'
+import { addResizeListener, hasResizeListener } from '../../../shared-utils/addResizeListener'
+import { useController } from '../../hooks'
+
+export function ResizeObserver() {
+  const ref = React.useRef<HTMLDivElement>(null)
+  const controller = useController()
+
+  React.useEffect(() => {
+    if (!ref.current) return
+    const resizeElement = controller.container.parentElement
+    if (hasResizeListener(resizeElement)) return
+    const dispose = addResizeListener(resizeElement, (width, height) => {
+      controller.setContainerSize(width, height)
+    })
+
+    return () => {
+      dispose()
+    }
+  }, [])
+
+  return <div className="resizeObserver-element" ref={ref}></div>
+}

package/resources/Player/modules/VideoAgent/VideoAgentMesh.ts

@@ -1,6 +1,8 @@
 import * as THREE from 'three'
 import { Preloader } from '../../../shared-utils/Preloader'
 import { makeObservable, observable, runInAction } from 'mobx'
+import AudioLike from '../../../shared-utils/AduioLike'
+import { getMediaType } from '../../../shared-utils/getMediaInfo'
 
 const vertexShader = `
 varying vec2 vUv;
@@ -63,7 +65,20 @@ const cacheInstance: {
 } = {}
 
 export interface VideoAgentMeshOptions {
+  /**
+   * 自定义视频实例。
+   */
   videoInstance?: HTMLVideoElement
+  /**
+   * 自定义音频实例。
+   */
+  audioInstance?: HTMLAudioElement
+  /**
+   * 是否开启音频预载能力。**仅对`.mp3`有效**。
+   *
+   * @description 开启预载能力后会通过 `Fetch/XHR` 方式将音频文件下载转成二进制 [`Blob`](https://developer.mozilla.org/zh-CN/docs/Web/API/Blob) 提供给多媒体实例。这样的好处是 **能够保障音视频播放过程中不卡顿**。 但在部分 iOS 系统中会存在兼容问题——播 25s 之后会静音（`muted` 为 `false`，但是没有声音）。
+   */
+  preload?: boolean
 }
 
 /**
@@ -72,17 +87,24 @@ export interface VideoAgentMeshOptions {
 export class VideoAgentMesh extends THREE.Mesh {
   options: VideoAgentMeshOptions
   videoUrl?: string
-  freeze: boolean
+  freeze: boolean 
   paused: boolean
   audioInstance: HTMLAudioElement
+  audioLikeInstance: AudioLike
   $removeEventListener: () => void
 
-  get videoInstance(): HTMLAudioElement {
-    if (this.videoUrl?.endsWith('mp3')) {
+  get mediaInstance(): HTMLAudioElement | HTMLVideoElement | AudioLike {
+    if (!this.videoUrl) {
+      return this.audioLikeInstance
+    }
+
+    if (getMediaType(this.videoUrl) === 'audio') {
       return this.audioInstance
     }
+
     const uniforms = (this.material as THREE.ShaderMaterial).uniforms
     const videoInstance = uniforms.map.value.image as HTMLVideoElement
+
     return videoInstance
   }
 
@@ -99,7 +121,9 @@ export class VideoAgentMesh extends THREE.Mesh {
     height: number,
     widthSegments: number,
     heightSegments: number,
-    options: VideoAgentMeshOptions = {},
+    options: VideoAgentMeshOptions = {
+      preload: true
+    },
   ) {
     if (!options.videoInstance) {
       if (cacheInstance.videoInstance) {
@@ -107,6 +131,7 @@ export class VideoAgentMesh extends THREE.Mesh {
       } else {
         const videoInstance = document.createElement('video')
         videoInstance.style.opacity = '0'
+        videoInstance.style.pointerEvents = 'none'
         videoInstance.style.display = 'none'
         document.body.append(videoInstance)
         options.videoInstance = videoInstance
@@ -114,7 +139,6 @@ export class VideoAgentMesh extends THREE.Mesh {
         videoInstance.playsInline = true
         videoInstance.controls = false
       }
-
     }
 
     const geometry = new THREE.PlaneGeometry(width, height, widthSegments, heightSegments)
@@ -134,89 +158,122 @@ export class VideoAgentMesh extends THREE.Mesh {
     this.freeze = false
     this.paused = true
 
-    {
+    if (!options.audioInstance) {
       if (cacheInstance.audioInstance) {
         this.audioInstance = cacheInstance.audioInstance
-
       } else {
         const audioInstance = document.createElement('audio')
         audioInstance.crossOrigin = ''
         // videoInstance.muted = true
         audioInstance.muted = false
-        audioInstance.setAttribute('playsinline', 'true')
-        audioInstance.setAttribute('webkit-playsinline', 'true')
-        audioInstance.setAttribute('autoplay', 'false')
-        audioInstance.setAttribute('style', 'display: none;')
+        audioInstance.autoplay = false
+        audioInstance.style.display = 'none'
         document.body.appendChild(audioInstance)
         this.audioInstance = audioInstance
         cacheInstance.audioInstance = audioInstance
       }
-
+    } else {
+      this.audioInstance = options.audioInstance
     }
 
+    this.audioLikeInstance = new AudioLike()
+
     makeObservable(this, { paused: observable })
 
     const updatePaused = (paused: boolean) => runInAction(() => (this.paused = paused))
     const onPause = () => updatePaused(true)
     const onPlay = () => updatePaused(false)
+    const onEnded = () => console.log('vreo: video 播放结束')
 
-    this.videoInstance.addEventListener('pause', onPause)
-    this.videoInstance.addEventListener('play', onPlay)
+    this.audioInstance.addEventListener('pause', onPause)
+    this.audioInstance.addEventListener('play', onPlay)
+    this.options.videoInstance?.addEventListener('pause', onPause)
+    this.options.videoInstance?.addEventListener('play', onPlay)
+    this.audioLikeInstance.addEventListener('pause', onPause)
+    this.audioLikeInstance.addEventListener('play', onPlay)
+    this.options.videoInstance?.addEventListener('ended', onEnded)
 
     this.$removeEventListener = () => {
-      this.videoInstance.removeEventListener('pause', onPause)
-      this.videoInstance.removeEventListener('play', onPlay)
+      this.audioInstance.removeEventListener('pause', onPause)
+      this.audioInstance.removeEventListener('play', onPlay)
+      this.options.videoInstance?.removeEventListener('pause', onPause)
+      this.options.videoInstance?.removeEventListener('play', onPlay)
+      this.audioLikeInstance.removeEventListener('pause', onPause)
+      this.audioLikeInstance.removeEventListener('play', onPlay)
+      this.options.videoInstance?.removeEventListener('ended', onEnded)
     }
   }
 
-
   private async update(videoUrl: string) {
+
     if (this.videoUrl === videoUrl) {
       return
     }
 
     this.videoUrl = videoUrl
+
+    // // 兼容非视频场景
+    // if (this.mediaInstance instanceof HTMLAudioElement) {
+    //   this.mediaInstance.style.display = 'none'
+    // } else if (this.mediaInstance instanceof HTMLVideoElement) {
+    //   this.mediaInstance.style.display = 'block'
+    // }
+
     this.freeze = true
-    await this.videoInstance.pause()
+    await this.mediaInstance.pause()
 
     const uniforms = (this.material as THREE.ShaderMaterial).uniforms
-    this.videoInstance.muted = true
-    this.videoInstance.src = await URL.createObjectURL((await Preloader.blob(this.videoUrl)) as unknown as Blob)
-    this.videoInstance.setAttribute('data-src', this.videoUrl)
+    this.mediaInstance.muted = true
+
+    this.mediaInstance.src = (this.options.preload || this.options.preload === undefined || getMediaType(this.videoUrl) === 'video') ?
+      await URL.createObjectURL((await Preloader.blob(this.videoUrl)) as unknown as Blob) : this.videoUrl
+    this.mediaInstance.setAttribute('data-src', this.videoUrl)
 
     const onStart = () => {
-      if (this.videoInstance.currentTime === 0) return
+      if (this.mediaInstance.currentTime === 0) return
       this.freeze = false
-      this.videoInstance.muted = false
-      uniforms.enable.value = this.videoUrl?.endsWith('.mp4') ? 1 : 0
-      this.videoInstance.removeEventListener('timeupdate', onStart, false)
+      this.mediaInstance.muted = false
+      uniforms.enable.value = getMediaType(this.videoUrl) ? 1 : 0
+      this.mediaInstance.removeEventListener('timeupdate', onStart, false)
+
     }
 
-    this.videoInstance.addEventListener('timeupdate', onStart, false)
+    this.mediaInstance.addEventListener('timeupdate', onStart, false)
   }
 
-  async play(videoUrl = '') {
+  async play(videoUrl = '', currentTime = 0, duration?: number) {
     videoUrl = videoUrl || ''
+
+    if (duration && !videoUrl) {
+      if (this.currentTime) {
+        this.mediaInstance.currentTime = currentTime
+      }
+      (this.mediaInstance as AudioLike).duration = duration
+      this.videoUrl = ''
+      this.mediaInstance.play()
+      return true
+    }
+
     if (!videoUrl) {
-      if (this.videoUrl) await this.videoInstance.play()
+      if (this.videoUrl) await this.mediaInstance.play()
       else console.warn('警告：视频资源未初始化。')
       return true
     }
 
     if (videoUrl === this.videoUrl) {
-      this.videoInstance.currentTime = 0
-      await this.videoInstance.play()
+      this.mediaInstance.currentTime = currentTime
+      await this.mediaInstance.play()
       return true
     }
 
     await this.update(videoUrl)
 
-    this.videoInstance.pause()
+    this.mediaInstance.pause()
 
     return await new Promise((resolve) =>
       setTimeout(async () => {
-        this.videoInstance.currentTime = 0
-        await this.videoInstance.play()
+        this.mediaInstance.currentTime = currentTime
+        await this.mediaInstance.play()
         resolve(true)
       }, 20),
     )
@@ -226,7 +283,7 @@ export class VideoAgentMesh extends THREE.Mesh {
    * 转成毫秒，保障精准度
    */
   get currentTime() {
-    return this.videoInstance.currentTime * 1000
+    return this.mediaInstance.currentTime * 1000
   }
 
   dispose() {