@realsee/vreo 0.1.6 → 0.2.0-alpha.11

package/resources/PlayController/index.ts

@@ -0,0 +1,167 @@
+import { Subscribe } from '@realsee/five'
+import { requestAnimationFrameInterval } from '../shared-utils/animationFrame'
+import { VreoKeyframe, VreoKeyframeEvent, VreoUnit } from '../typings/VreoUnit'
+import { createHTMLAudioElement } from './createHTMLAudioElement'
+
+export interface PlayControllerConfig {
+  audio?: HTMLAudioElement
+}
+
+const closures: Record<string, any> = {}
+
+export class PlayController extends Subscribe<VreoKeyframeEvent> {
+
+  vreoUnit?: VreoUnit
+
+  config: PlayControllerConfig = {}
+  stopInterval: () => void
+
+  get paused() {
+    return this.audioInstance.paused
+  }
+
+  get audioInstance() {
+    return this.config.audio!
+  }
+
+  get currentTime() {
+    return this.audioInstance.currentTime * 1000
+  }
+
+  get currentKeyframes() {
+    if (!this.vreoUnit) return []
+    const keyframes = this.vreoUnit.keyframes
+    // 没有被解析过且开始时间低于当前时间戳 100ms
+    return keyframes.filter((keyframe: VreoKeyframe) => {
+      if (keyframe.parsed) return false
+      const dur = this.currentTime - keyframe.start
+      return dur <= 100 && dur >= 0
+    })
+  }
+
+  /**
+   * 音频时长 不一定能获取得到，建议以 VreoUnit 配置的时长为准。
+   */
+  get duration() {
+    return this.audioInstance.duration * 1000
+  }
+
+  constructor(config: PlayControllerConfig = {}) {
+    super()
+
+    if (!config.audio) {
+      config.audio = createHTMLAudioElement()
+    }
+
+    this.config = Object.assign(this.config, config)
+
+    closures.onPlaying = () => this.emit('playing')
+    closures.onPaused = () => this.emit('paused')
+    closures.onEnded = () => this.emit('paused', true)
+
+    this.audioInstance.addEventListener('play', closures.onPlaying)
+    this.audioInstance.addEventListener('pause', closures.onPaused)
+    this.audioInstance.addEventListener('ended', closures.onEnded)
+
+    this.stopInterval = requestAnimationFrameInterval(() => {
+      if (this.audioInstance.paused) {
+        return
+      }
+
+      if (!this.vreoUnit) {
+        return
+      }
+
+      const keyframes = this.vreoUnit.keyframes
+      const currentTime = this.currentTime
+      // 没有被解析过且开始时间低于当前时间戳 100ms
+      const currentKeyframes = keyframes.filter((keyframe: VreoKeyframe) => {
+        if (keyframe.parsed) return false
+
+        const dur = currentTime - keyframe.start
+        return dur <= 60 && dur >= 0
+      })
+  
+      currentKeyframes.forEach((keyframe) => {
+        if (keyframe.parsed) return
+        keyframe.parsed = true
+        this.emit(keyframe.type, keyframe)
+      })
+    })
+
+  }
+
+
+
+  /**
+   * 载入数据
+   */
+  async load(vreoUnit: VreoUnit): Promise<boolean> {
+    this.vreoUnit = vreoUnit
+
+    await this.pause()
+    return await new Promise((resolve, reject) => {
+      if (vreoUnit.video.url === this.audioInstance.src) {
+        return resolve(true)
+      }
+
+      this.audioInstance.src = vreoUnit.video.url
+      const canplaythrough = () => {
+        resolve(true)
+        this.audioInstance.removeEventListener('canplaythrough', canplaythrough)
+      }
+      const error = (err: any) => {
+        reject(err)
+        this.audioInstance.removeEventListener('error', error)
+      }
+      this.audioInstance.addEventListener('canplaythrough', canplaythrough)
+      this.audioInstance.addEventListener('error', error)
+    })
+  }
+
+  /**
+   * 播放
+   */
+  async play() {
+    closures.playPromise = this.audioInstance.play()
+  }
+
+  /**
+   * 暂停
+   */
+  async pause() {
+    if (closures.playPromise) {
+      await closures.playPromise
+    }
+    this.audioInstance.pause()
+  }
+
+  /**
+   * 销毁资源
+   * @notice 一旦销毁，创建的实例将不可用：需重新创建实例。
+   * 
+   * @deprecated **慎重执行**
+   */
+  dispose() {
+    // 清理事件监听
+    this.audioInstance.removeEventListener('play', closures.onPlaying)
+    this.audioInstance.removeEventListener('pasue', closures.onPaused)
+    this.audioInstance.removeEventListener('ended', closures.onEnded)
+    if (this.stopInterval) {
+      this.stopInterval()
+    }
+  }
+}
+
+console.log(`
+
+┏━━━┓━━━━━━━━━┏┓━━━━━━━━━━━━━
+┃┏━┓┃━━━━━━━━━┃┃━━━━━━━━━━━━━
+┃┗━┛┃┏━━┓┏━━┓━┃┃━┏━━┓┏━━┓┏━━┓
+┃┏┓┏┛┃┏┓┃┗━┓┃━┃┃━┃━━┫┃┏┓┃┃┏┓┃
+┃┃┃┗┓┃┃━┫┃┗┛┗┓┃┗┓┣━━┃┃┃━┫┃┃━┫
+┗┛┗━┛┗━━┛┗━━━┛┗━┛┗━━┛┗━━┛┗━━┛
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 
+
+`)

package/resources/Player/App.tsx

@@ -2,7 +2,7 @@ import * as React from 'react'
 import classNames from 'classnames'
 import { observer } from 'mobx-react'
 import { Controller } from './Controller'
-import { useController } from './hooks'
+import { useController, useFiveInstance } from './hooks'
 import { CameraMovement } from './modules/keyframes/CameraMovement'
 import { InfoPanel } from './modules/keyframes/InfoPanel'
 import { ModelVideo } from './modules/keyframes/ModelVideo'
@@ -13,8 +13,10 @@ import { Prompter } from './modules/keyframes/Prompter'
 import { UpdateVRPanorama } from './modules/keyframes/UpdateVRPanorama'
 import { VideoEffect } from './modules/keyframes/VideoEffect'
 import { VideoAgent } from './modules/VideoAgent'
+import { BgMusic } from './modules/keyframes/BgMusic'
 
 const AppView = observer(({ controller }: { controller: Controller }) => {
+  
   return (
     <div
       className={classNames('vreo-panel', {
@@ -33,8 +35,10 @@ const AppView = observer(({ controller }: { controller: Controller }) => {
               controller.setPlaying(true)
             }
           }}
+          options={controller.configs?.videoAgentMeshOptions || {}}
         />
         <Prompter />
+        {controller.configs?.customPanelChildren}
       </div>
     </div>
   )
@@ -53,6 +57,7 @@ export function App() {
         <PanoTextLabel />
         <InfoPanel />
         <PanoEffect />
+        <BgMusic />
       </>
       <AppView controller={controller} />
     </>

package/resources/Player/Controller.ts

@@ -24,6 +24,16 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
     return !this.videoAgentScene?.videoAgentMesh.videoUrl?.endsWith('.mp4')
   }
 
+  popUp: string | JSX.Element | null = null
+
+  openPopUp(popUp: string | JSX.Element | false) {
+    if (!popUp) {
+      this.popUp = null
+      return
+    }
+    this.popUp = popUp
+  }
+
   drawerConfig: {
     content: string | JSX.Element
     height?: number | string
@@ -57,6 +67,8 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
       setVisible: action,
       playing: observable,
       setPlaying: action,
+      popUp: observable.ref,
+      openPopUp: action,
       drawerConfig: observable.ref,
       openDrawer: action,
     })

package/resources/Player/custom/SpatialScenePanel/index.tsx

@@ -0,0 +1,264 @@
+import * as React from 'react'
+import ReactDOM from 'react-dom'
+import { Five } from '@realsee/five'
+import * as THREE from 'three'
+import { Object3D, Raycaster, Vector3 } from 'three'
+import { tweenProgress } from '@realsee/dnalogel/shared-utils/animationFrame/BetterTween'
+import { CSS3DRenderPlugin, CSS3DRenderPluginExportType } from '@realsee/dnalogel/plugins/CSS3DRenderPlugin'
+
+import { VreoKeyframe, VreoKeyframeEnum } from '../../../typings/VreoUnit'
+import { CustomVreoKeyframeProps } from '../../typings'
+
+// __debug__
+// const mockData: SpatialScenePanelData = {
+//   customType: 'SpatialScenePanel',
+//   "stateList": [
+//     {
+//       "icon": "http://vrlab-public.ljcdn.com/release/seesay/tools/___fc42f951076607a22fd5c54f005ba553.png",
+//       "text": "窗帘关闭"
+//     },
+//     {
+//       "icon": "http://vrlab-public.ljcdn.com/release/seesay/tools/___b331e7ad732debc07250a169a3393b13.png",
+//       "text": "摄像头休眠"
+//     },
+//     {
+//       "icon": "http://vrlab-public.ljcdn.com/release/seesay/tools/___2e425a81ac0ad88232d50d4e52a9abfa.png",
+//       "text": "氛围音乐"
+//     },
+//     {
+//       "icon": "http://vrlab-public.ljcdn.com/release/seesay/tools/___bff48bb81cde37aa6a319d6f5d56e3a4.png",
+//       "text": "空调打开"
+//     }
+//   ],
+//   "temperature": "27",
+//   "title": "沉浸式回家"
+// }
+export interface SpatialScenePanelData {
+  customType: 'SpatialScenePanel'
+  title: string
+  temperature: string
+  stateList: {
+    text: string
+    icon: string
+  }[]
+  position?: { x: number; y: number; z: number }
+  quaternion?: [number, number, number, number]
+}
+
+interface SearchParams {
+  five: Five
+  // position: Vector3
+  // rotation?: Vector3
+  // quaternion: [number, number, number, number]
+  dom: HTMLDivElement
+}
+
+// const getSphere = (position: Vector3, color = 0xffff00) => {
+//   const geometry = new SphereGeometry(0.05, 32, 16)
+//   const material = new MeshBasicMaterial({ color })
+//   const sphere = new Mesh(geometry, material)
+//   sphere.position.copy(position)
+//   return sphere
+// }
+
+const searchV1 = ({ five, dom }: SearchParams) => {
+  const cameraDirection = new Vector3()
+  five.camera.getWorldDirection(cameraDirection)
+  const cameraPosition = five.camera.position
+  const [intersect] = five.model.intersectRaycaster(new Raycaster(cameraPosition, cameraDirection))
+  const point = five.project2d(intersect.point)
+
+  const widescreen = window.innerWidth > 560 ? true : false
+  // const width = dom!.clientWidth
+  const height = dom!.clientHeight
+  dom!.style.left = point!.x + 'px'
+  dom!.style.top = point!.y - height / 2 + 'px'
+  const rect = dom!.getBoundingClientRect()
+  const { left, top, bottom, right } = rect
+
+  const leftBottom = new Vector3(left, bottom)
+  const leftTop = new Vector3(left, top)
+  const rightTop = new Vector3(right, top)
+  const rightBottom = new Vector3(right, bottom)
+
+  const points = [leftBottom, rightBottom, rightTop, leftTop].map((point) => {
+    point.setX((point.x / window.innerWidth) * 2 - (widescreen ? 1.25: 1.75))
+    point.setY(-(point.y / window.innerHeight) * 2 + (widescreen ? 1.35: 1.125))
+    point.setZ(0.5)
+    return point.unproject(five.camera).addScaledVector(cameraDirection, 0.2)
+  })
+
+  // __debug__
+  // const colors = [0xdc143c, 0xffff00, 0x0000ff, 0x008000]
+  // points.forEach((p, index) => five.scene.add(getSphere(p, colors[index])))
+
+  // 判断是手机 150
+  const ratio = leftBottom.distanceTo(rightBottom) / (widescreen ? 150 : 120)
+  return { points, ratio }
+}
+
+/**
+ * 空间场景面板
+ * @returns
+ */
+export function SpatialScenePanel(props: CustomVreoKeyframeProps) {
+  const timeoutRef = React.useRef<NodeJS.Timeout | null>(null)
+  const ref = React.useRef<HTMLDivElement>(null)
+  const panelRef = React.useRef<HTMLDivElement | null>(null)
+  const rendererRef = React.useRef<CSS3DRenderPluginExportType>(CSS3DRenderPlugin(props.five))
+
+  React.useEffect(() => {
+    if (!ref.current) {
+      return
+    }
+
+    const callback = (keyframe: VreoKeyframe) => {
+      if (keyframe.data.customType !== 'SpatialScenePanel') {
+        return
+      }
+
+      const data = keyframe.data as SpatialScenePanelData
+
+      // const data = mockData
+      const { points, ratio } = searchV1({
+        five: props.five,
+        dom: ref.current!,
+      })
+
+      const { container, dispose, css3DObject, render } =
+        rendererRef.current.create3DDomContainer(points, { ratio, autoRender: false }) || {}
+
+      Object.assign(window, { $css3DObject: css3DObject })
+      if (!container) return
+
+      ReactDOM.render(<Panel ref={(ref) => (panelRef.current = ref)} data={data} />, container)
+      const startRotateY = -(Math.PI * 2) / 9
+      let lastRotateY = 0
+    
+      tweenProgress(1000)
+        .onUpdate(({ progress }) => {
+          const needRotateY = (Math.PI / 90) * 11 * progress
+          const rotateY = needRotateY - lastRotateY
+          css3DObject?.rotateY(rotateY)
+          // css3DObject?.rotateZ
+          lastRotateY = needRotateY
+        })
+        .onStart(() => {
+          css3DObject?.rotateY(startRotateY)
+          render!()
+        })
+        .onDispose(() => {
+          panelRef.current?.classList.add('show')
+        })
+        .play()
+
+    // Object.assign(window, {  $dispose: () => {
+    //   panelRef.current?.classList.add('hide')
+    //   setTimeout(() => dispose?.(), 1500)
+    // } })
+
+      const { start, end } = keyframe
+
+      timeoutRef.current = setTimeout(() => {
+        panelRef.current?.classList.add('hide')
+        timeoutRef.current = setTimeout(() => {
+          dispose?.()
+          timeoutRef.current = null
+        }, 1500)
+      }, end - start)
+    }
+
+    Object.assign(window, { $callback: callback })
+
+    // setTimeout(() => {callback()}, 8000)
+    props.subscribe.on(VreoKeyframeEnum.Custom, callback)
+
+    return () => {
+      props.subscribe.off(VreoKeyframeEnum.Custom, callback)
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current)
+      }
+    }
+  }, [])
+
+  return <div className="vreo-SpatialScenePanel" ref={ref}></div>
+}
+
+/** 背景动画：控制延迟 */
+function SpatialScenePanelBg() {
+  const ref = React.useRef<HTMLDivElement>(null)
+  const timeoutRef = React.useRef<NodeJS.Timeout | null>(null)
+  React.useEffect(() => {
+    if (!ref.current) return
+
+    const listener = () => {
+      ref.current?.setAttribute('class', 'SpatialScenePanel-bg')
+
+      timeoutRef.current = setTimeout(() => {
+        ref.current?.setAttribute('class', 'SpatialScenePanel-bg SpatialScenePanel-bg--animation')
+        timeoutRef.current = null
+      }, 3000)
+    }
+
+    ref.current.addEventListener('animationend', listener)
+
+    return () => {
+      if (ref.current) ref.current.removeEventListener('animationend', listener)
+      if (timeoutRef.current) clearTimeout(timeoutRef.current)
+    }
+  }, [])
+  return <div ref={ref} className="SpatialScenePanel-bg SpatialScenePanel-bg--animation"></div>
+}
+
+const Panel = React.forwardRef(
+  (props: { data: SpatialScenePanelData }, myRef: React.LegacyRef<HTMLDivElement> | undefined) => (
+    <div className="SpatialScenePanel" ref={myRef}>
+      <div className="SpatialScenePanel-show-area">
+        <SpatialScenePanelBg />
+        <div className="SpatialScenePanel-header">
+          <div className="SpatialScenePanel-title">{props.data?.title}</div>
+          <div className="SpatialScenePanel-temperature">{props.data?.temperature}℃</div>
+        </div>
+        <div className="SpatialScenePanel-body">
+          {props.data?.stateList?.map((state, index) => (
+            <div key={`scene-${state.text}`} className={`SpatialScenePanel-item index_${index}`}>
+              <img className="SpatialScenePanel-icon" src={state.icon}></img>
+              <div className="SpatialScenePanel-text">{state.text}</div>
+              <div className="SpatialScenePanel-dot"></div>
+            </div>
+          ))}
+        </div>
+      </div>
+
+      <div className="SpatialScenePanel-disappear-area"></div>
+    </div>
+  )
+)
+
+
+// export function SpatialScenePanel() {
+//   // const data = mockData
+
+//   return (
+//     <div className="SpatialScenePanel" >
+//     <div className="SpatialScenePanel-show-area">
+//       <SpatialScenePanelBg />
+//       <div className="SpatialScenePanel-header">
+//         <div className="SpatialScenePanel-title">{data?.title}</div>
+//         <div className="SpatialScenePanel-temperature">{data?.temperature}℃</div>
+//       </div>
+//       <div className="SpatialScenePanel-body">
+//         {data?.stateList?.map((state, index) => (
+//           <div key={`scene-${state.text}`} className={`SpatialScenePanel-item index_${index}`}>
+//             <img className="SpatialScenePanel-icon" src={state.icon}></img>
+//             <div className="SpatialScenePanel-text">{state.text}</div>
+//             <div className="SpatialScenePanel-dot"></div>
+//           </div>
+//         ))}
+//       </div>
+//     </div>
+
+//     <div className="SpatialScenePanel-disappear-area"></div>
+//   </div>
+//   )
+// }

package/resources/Player/index.tsx

@@ -4,10 +4,11 @@ import { Five, Subscribe } from '@realsee/five'
 
 import { App } from './App'
 import { Controller, ControllerContext } from './Controller'
-import { VreoKeyframeEvent, VreoKeyframeConfigMap, VreoUnit } from '../typings/VreoUnit'
+import { CameraMovementData, VreoKeyframeEnum, VreoKeyframeEvent, VreoUnit } from '../typings/VreoUnit'
 import { reaction } from 'mobx'
 import { Drawer } from './modules/Drawer'
 import { PlayerConfigs } from './typings'
+import { PopUp } from './modules/PopUp'
 
 export class Player extends Subscribe<VreoKeyframeEvent> {
   $five: Five
@@ -42,6 +43,19 @@ export class Player extends Subscribe<VreoKeyframeEvent> {
       <ControllerContext.Provider value={this.controller}>
         <App></App>
         <Drawer />
+        <PopUp />
+        {this.configs.customKeyframes &&
+          this.configs.customKeyframes.map((CustomCmpt, key) => (
+            <CustomCmpt
+              key={key}
+              subscribe={{
+                on: (name, callback) => this.on(name, callback),
+                once: (name, callback) => this.once(name, callback),
+                off: (name, callback) => this.off(name, callback),
+              }}
+              five={five}
+            />
+          ))}
       </ControllerContext.Provider>,
       configs.containter
     )
@@ -55,9 +69,13 @@ export class Player extends Subscribe<VreoKeyframeEvent> {
     )
   }
 
-  async load(vreoUnit: VreoUnit, currentTime = 0) {
+  async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false) {
+    if (force) {
+      vreoUnit = JSON.parse(JSON.stringify(vreoUnit))
+    }
     if (!this.controller.visible) {
       this.controller.setVisible(true)
+      // 延迟 500ms 规避跟 DOM 动画冲突
       await new Promise((resolve) => {
         setTimeout(() => resolve(true), 500)
       })
@@ -71,6 +89,42 @@ export class Player extends Subscribe<VreoKeyframeEvent> {
     this.controller.vreoUnit = vreoUnit
     this.controller.videoInstance?.pause()
 
+    // 预载逻辑
+    // 是否降低图片分辨率
+    if (this.$five.imageOptions.size && this.configs.imageOptions?.size && this.$five.imageOptions.size > this.configs.imageOptions.size) {
+      this.$five.imageOptions.size = this.configs.imageOptions.size
+    }
+    
+    // 预载数据中的点位
+    if (preload || (preload === undefined && this.configs.autoPreload)) {
+      const panoIndexMap = vreoUnit.keyframes.filter((vreoKeyframe) => {
+        if (vreoKeyframe.type !== VreoKeyframeEnum.CameraMovement) {
+          return false
+        }
+  
+        const data = vreoKeyframe.data as CameraMovementData
+        if (data.panoIndex === undefined) {
+          return false
+        }
+        return true
+      }).reduce((accu: Record<number, boolean>, curr) => {
+        const panoIndex = curr.data.panoIndex as number
+        if (!accu[panoIndex]) {
+          accu[panoIndex] = true
+        }
+        return accu
+      }, {})
+  
+      const panoIndexs = Object.keys(panoIndexMap)
+      for (let i = 0; i < panoIndexs.length; i++) {
+        await this.$five.preloadPano(Number(panoIndexs[i]))
+      }
+    }
+    
+    // 新数据载入就绪
+    this.emit('loaded', vreoUnit)
+    this.controller.emit('loaded', vreoUnit)
+
     if (vreoUnit.video.url) {
       if (this.controller.videoAgentScene?.videoAgentMesh.videoInstance) {
         this.controller.videoAgentScene.videoAgentMesh.videoInstance.currentTime = currentTime / 1000

package/resources/Player/modules/Drawer/index.tsx

@@ -5,7 +5,7 @@ import { Controller } from '../../Controller'
 import { useController } from '../../hooks'
 
 const DrawerView = observer(({ controller }: { controller: Controller }) => {
-  const maxHeight = '540px'
+  const maxHeight = '400px'
   const height = (() => {
     const height = controller.drawerConfig?.height
     if (!height) {

package/resources/Player/modules/PopUp/index.tsx

@@ -0,0 +1,26 @@
+import * as React from 'react'
+import classNames from 'classnames'
+import { observer } from 'mobx-react'
+import { Controller } from '../../Controller'
+import { useController } from '../../hooks'
+
+const PopUpView = observer(({ controller }: { controller: Controller }) => {
+
+  return (
+    <div
+      className={classNames('vreo-PopUp', {
+        'vreo-PopUp-visible': controller.popUp,
+      })}
+      onClick={() => controller.openPopUp(false)}
+    >
+      <div className="vreo-PopUp-inner">
+        {controller.popUp ||  ''}
+      </div>
+    </div>
+  )
+})
+
+export function PopUp() {
+  const controller = useController()
+  return <PopUpView controller={controller} />
+}

package/resources/Player/modules/VideoAgent/VideoAgentMesh.ts

@@ -64,6 +64,7 @@ const cacheInstance: {
 
 export interface VideoAgentMeshOptions {
   videoInstance?: HTMLVideoElement
+  audioInstance?: HTMLAudioElement
 }
 
 /**
@@ -134,10 +135,9 @@ export class VideoAgentMesh extends THREE.Mesh {
     this.freeze = false
     this.paused = true
 
-    {
+    if (!options.audioInstance) {
       if (cacheInstance.audioInstance) {
         this.audioInstance = cacheInstance.audioInstance
-
       } else {
         const audioInstance = document.createElement('audio')
         audioInstance.crossOrigin = ''
@@ -151,7 +151,8 @@ export class VideoAgentMesh extends THREE.Mesh {
         this.audioInstance = audioInstance
         cacheInstance.audioInstance = audioInstance
       }
-
+    } else {
+      this.audioInstance = options.audioInstance
     }
 
     makeObservable(this, { paused: observable })
@@ -196,7 +197,7 @@ export class VideoAgentMesh extends THREE.Mesh {
   }
 
   async play(videoUrl = '') {
-    videoUrl = (videoUrl || '').replace(/https*:/, '')
+    videoUrl = videoUrl || ''
     if (!videoUrl) {
       if (this.videoUrl) await this.videoInstance.play()
       else console.warn('警告：视频资源未初始化。')