@realsee/vreo 0.1.0

package/resources/Player/App.tsx

@@ -0,0 +1,57 @@
+import * as React from 'react'
+import classNames from 'classnames'
+import { observer } from 'mobx-react'
+import { Controller } from './Controller'
+import { useController } from './hooks'
+import { CameraMovement } from './modules/keyframes/CameraMovement'
+import { InfoPanel } from './modules/keyframes/InfoPanel'
+import { ModelVideo } from './modules/keyframes/ModelVideo'
+import { PanoEffect } from './modules/keyframes/PanoEffect'
+import { PanoTag } from './modules/keyframes/PanoTag'
+import { PanoTextLabel } from './modules/keyframes/PanoTextLabel'
+import { Prompter } from './modules/keyframes/Prompter'
+import { UpdateVRPanorama } from './modules/keyframes/UpdateVRPanorama'
+import { VideoEffect } from './modules/keyframes/VideoEffect'
+import { VideoAgent } from './modules/VideoAgent'
+
+const AppView = observer(({ controller }: { controller: Controller }) => {
+  return (
+    <div
+      className={classNames('vreo-panel', {
+        'vreo-panel--hidden': !controller.visible,
+      })}
+    >
+      <div className="vreo-panel-inner">
+        <VideoAgent
+          onClick={() => {
+            if (controller.playing) {
+              controller.setPlaying(false)
+            } else {
+              controller.setPlaying(true)
+            }
+          }}
+        />
+        <Prompter />
+      </div>
+    </div>
+  )
+})
+
+export function App() {
+  const controller = useController()
+  return (
+    <>
+      <>
+        <CameraMovement />
+        <ModelVideo />
+        <UpdateVRPanorama />
+        <VideoEffect />
+        <PanoTag />
+        <PanoTextLabel />
+        <InfoPanel />
+        <PanoEffect />
+      </>
+      <AppView controller={controller} />
+    </>
+  )
+}

package/resources/Player/Controller.ts

@@ -0,0 +1,154 @@
+import { Five, Subscribe } from '@realsee/five'
+import { action, makeObservable, observable, reaction } from 'mobx'
+import * as React from 'react'
+import { requestAnimationFrameInterval } from '../shared-utils/animationFrame'
+import { VreoKeyframe, VreoKeyframeEnum, VreoKeyframeEvent, VreoUnit } from '../typings/VreoUnit'
+import { VideoAgentScene } from './modules/VideoAgent/VideoAgentScene'
+import { PlayerConfigs } from './typings'
+
+/**
+ * 逻辑控制器：内部状态。
+ */
+export class Controller extends Subscribe<VreoKeyframeEvent> {
+
+  $five?: Five
+  configs?: PlayerConfigs
+  videoAgentScene?: VideoAgentScene
+  vreoUnit?: VreoUnit
+  stopInterval?: () => void
+  playing = false
+
+  visible = false
+
+  drawerConfig?: {
+    content: string | JSX.Element
+    height?: number | string
+  }
+
+  setVisible(v: boolean) {
+    this.visible = v
+  }
+
+  setPlaying(playing: boolean) {
+    this.playing = playing
+  }
+
+  openDrawer(drawerConfig?: false | { content: string | JSX.Element; height?: number | string }) {
+    if (!drawerConfig) {
+      this.drawerConfig = {
+        content: '',
+        height: this.drawerConfig?.height
+      }
+      return
+    }
+
+    this.drawerConfig = drawerConfig
+  }
+
+  constructor() {
+    super()
+
+    makeObservable(this, {
+      visible: observable,
+      setVisible: action,
+      playing: observable,
+      setPlaying: action,
+      drawerConfig: observable.ref,
+      openDrawer: action,
+
+    })
+
+    // 监听播放情况：抛出触发时机
+    reaction(
+      () => this.playing,
+      (playing) => {
+        this.emit(playing ? 'playing' : 'paused')
+      }
+    )
+  }
+
+  get ready() {
+    return !!this.videoAgentScene
+  }
+
+  get currentTime() {
+    return this.videoAgentScene?.videoAgentMesh.currentTime || 0
+  }
+
+  get currentKeyframes() {
+    if (!this.vreoUnit) return []
+    const keyframes = this.vreoUnit.keyframes
+    // 没有被解析过且开始时间低于当前时间戳 100ms
+    return keyframes.filter((keyframe: VreoKeyframe) => {
+      if (keyframe.parsed) return false
+      const dur = this.currentTime - keyframe.start
+      return dur <= 100 && dur >= 0
+    })
+  }
+
+  get videoInstance() {
+    return this.videoAgentScene?.videoAgentMesh.videoInstance
+  }
+
+  /**
+   * 逐帧任务
+   */
+  requestAnimationFrameLoop(callback: (type: VreoKeyframeEnum, keyframe: VreoKeyframe) => void) {
+    if (this.videoInstance?.ended) {
+      this.setPlaying(false)
+      this.videoInstance.pause()
+      return
+    }
+    if (!this.playing) {
+      if (!this.videoInstance?.paused) {
+        this.videoInstance?.pause()
+      }
+      return
+    }
+    if (this.videoInstance?.paused && this.playing) {
+      this.videoInstance.play()
+    }
+
+    const currentKeyframes = this.currentKeyframes
+
+    currentKeyframes.forEach((keyframe) => {
+      if (keyframe.parsed) return
+      keyframe.parsed = true
+      this.emit(keyframe.type, keyframe)
+      if (callback) {
+        callback(keyframe.type, keyframe)
+      }
+    })
+  }
+
+  run(callback: (type: VreoKeyframeEnum, keyframe: VreoKeyframe) => void) {
+    if (this.stopInterval) return
+    this.stopInterval = requestAnimationFrameInterval(() => this.requestAnimationFrameLoop(callback))
+  }
+
+  dispose() {
+    this.setPlaying(false)
+
+    if (this.currentKeyframes) {
+      this.currentKeyframes.forEach((keyframe) => (keyframe.parsed = false))
+    }
+
+    /**
+     * 清理掉数据
+     */
+    this.vreoUnit = undefined
+    if (this.videoInstance) {
+      this.videoInstance.currentTime = 0
+    }
+
+    if (this.stopInterval) {
+      this.stopInterval()
+      this.stopInterval = undefined
+    }
+  }
+}
+
+export const ControllerContext = React.createContext<Controller | null>(null)
+
+
+

package/resources/Player/hooks.ts

@@ -0,0 +1,28 @@
+import React from 'react'
+import { Vector3 } from 'three'
+import { ControllerContext } from './Controller'
+
+
+
+export function useController() {
+  const controller = React.useContext(ControllerContext)
+  if (!controller) {
+    throw new Error('没有找到 "ControllerContext"')
+  }
+
+  return controller
+}
+
+export function useFiveInstance() {
+  const controller = useController()
+  if (!controller.$five) {
+    throw new Error('没有找到 "five" 实例')
+  }
+
+  return controller.$five
+}
+
+export function useFiveProject2d() {
+  const five = useFiveInstance()
+  return (vector: Vector3) => five.project2d(vector, false)
+}

package/resources/Player/index.tsx

@@ -0,0 +1,115 @@
+import * as React from 'react'
+import * as ReactDOM from 'react-dom'
+import { Five, Subscribe } from '@realsee/five'
+
+import { App } from './App'
+import { Controller, ControllerContext } from './Controller'
+import { VreoKeyframeEvent, VreoKeyframeConfigMap, VreoUnit } from '../typings/VreoUnit'
+import { reaction } from 'mobx'
+import { Drawer } from './modules/Drawer'
+import { PlayerConfigs } from './typings'
+
+const controller = new Controller()
+
+export class Player extends Subscribe<VreoKeyframeEvent> {
+  $five: Five
+  configs: Readonly<PlayerConfigs>
+
+  constructor(five: Five, configs: Partial<PlayerConfigs> = {}) {
+    super()
+    this.$five = controller.$five = five
+
+    if (!configs.containter) {
+      const containter = document.createElement('div')
+      containter.setAttribute('id', 'vreo-app')
+      configs.containter = containter
+      document.body.append(containter)
+    }
+
+    this.configs = Object.freeze(
+      Object.assign(
+        {
+          keyframeMap: {},
+        },
+        configs
+      )
+    )
+
+    controller.configs = this.configs
+
+    ReactDOM.render(
+      <ControllerContext.Provider value={controller}>
+        <App></App>
+        <Drawer />
+      </ControllerContext.Provider>,
+      configs.containter
+    )
+
+    // 监听播放情况：抛出触发时机
+    reaction(
+      () => controller.playing,
+      (playing) => {
+        this.emit(playing ? 'playing' : 'paused')
+      }
+    )
+  }
+
+  async load(vreoUnit: VreoUnit, currentTime = 0) {
+    if (controller.stopInterval) {
+      controller.stopInterval()
+      controller.stopInterval = undefined
+    }
+
+    controller.vreoUnit = vreoUnit
+    controller.videoInstance?.pause()
+
+    if (vreoUnit.video.url) {
+      if (controller.videoAgentScene?.videoAgentMesh.videoInstance) {
+        controller.videoAgentScene.videoAgentMesh.videoInstance.currentTime = currentTime / 1000
+      }
+      await controller.videoAgentScene?.videoAgentMesh.play(vreoUnit.video.url)
+      this.play()
+    }
+    controller.run((type, keyframe) => this.emit(type, keyframe))
+    return true
+  }
+
+  get paused() {
+    return !controller.playing
+  }
+
+  play() {
+    if (controller.playing) return true
+    controller.setPlaying(true)
+    return true
+  }
+
+  pause() {
+    controller.setPlaying(false)
+  }
+
+  show() {
+    controller.setVisible(true)
+  }
+
+  hide() {
+    controller.setVisible(false)
+  }
+
+  dispose() {
+    controller.dispose()
+  }
+}
+
+console.log(`
+
+┏━━━┓━━━━━━━━━┏┓━━━━━━━━━━━━━
+┃┏━┓┃━━━━━━━━━┃┃━━━━━━━━━━━━━
+┃┗━┛┃┏━━┓┏━━┓━┃┃━┏━━┓┏━━┓┏━━┓
+┃┏┓┏┛┃┏┓┃┗━┓┃━┃┃━┃━━┫┃┏┓┃┃┏┓┃
+┃┃┃┗┓┃┃━┫┃┗┛┗┓┃┗┓┣━━┃┃┃━┫┃┃━┫
+┗┛┗━┛┗━━┛┗━━━┛┗━┛┗━━┛┗━━┛┗━━┛
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 
+
+`)

package/resources/Player/modules/Drawer/index.tsx

@@ -0,0 +1,40 @@
+import * as React from 'react'
+import classNames from 'classnames'
+import { observer } from 'mobx-react'
+import { Controller } from '../../Controller'
+import { useController } from '../../hooks'
+
+const DrawerView = observer(({ controller }: { controller: Controller }) => {
+  const maxHeight = '540px'
+  const height = (() => {
+    const height = controller.drawerConfig?.height
+    if (!height) {
+      return 'max-content'
+    }
+
+    if (typeof height === 'number') {
+      return height + 'px'
+    }
+
+    return height
+  })()
+
+  return (
+    <div
+      className={classNames('vreo-drawer', {
+        'vreo-drawer-visible': controller.drawerConfig && controller.drawerConfig.content,
+      })}
+      onClick={() => controller.openDrawer(false)}
+    >
+      <div className="vreo-drawer-inner" style={{ height, maxHeight }}>
+        {controller.drawerConfig?.content || ''}
+      </div>
+    </div>
+  )
+})
+
+export function Drawer() {
+  const controller = useController()
+
+  return <DrawerView controller={controller} />
+}

package/resources/Player/modules/VideoAgent/VideoAgentMesh.ts

@@ -0,0 +1,202 @@
+import * as THREE from 'three'
+import { Preloader } from '../../../shared-utils/Preloader'
+import { makeObservable, observable, runInAction } from 'mobx'
+
+const vertexShader = `
+varying vec2 vUv;
+varying vec2 vColorUv;
+varying vec2 vMaskUv;
+void main(){
+  vUv = uv;
+  vColorUv = vec2(uv.x / 2.0, uv.y);
+  vMaskUv = vec2(0.5 + uv.x / 2.0, uv.y);
+  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+}
+`
+const fragmentShaderTpl = () => `
+uniform int enable;
+uniform sampler2D map;
+varying vec2 vUv;
+varying vec2 vColorUv;
+varying vec2 vMaskUv;
+
+vec3 hsv2rgb(vec3 c) {
+  const vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);
+  vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
+  return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
+}
+
+vec3 rgb2hsv(vec3 c) {
+  const vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0);
+  vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g));
+  vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r));
+
+  float d = q.x - min(q.w, q.y);
+  return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + 0.001)), d / (q.x + 0.001), q.x);
+}
+
+void main(void) {
+  vec3 color = texture2D(map, vUv).xyz;
+  float amask = 1.0;
+
+  float ref = color.r;
+  if (ref < color.b) ref = color.b;
+  amask = color.g - ref + 0.1;
+  amask = smoothstep(0.1, 0.3, amask);
+  amask = 1.0 - amask;
+
+  vec3 target = rgb2hsv(vec3(0.0 / 255.0, 255.0 / 255.0, 0.0 / 255.0));
+  vec3 hsv = rgb2hsv(color);
+  float distance = abs(hsv.x - target.x);
+  if (distance < 0.15) {
+    hsv.y = 0.0;
+  }
+  color = hsv2rgb(hsv);
+
+  gl_FragColor = vec4(color.rgb, amask * float(enable));
+}
+`
+
+export interface VideoAgentMeshOptions {
+  videoInstance?: HTMLVideoElement
+}
+
+/**
+ * 视频经纪人MESH
+ */
+export class VideoAgentMesh extends THREE.Mesh {
+  options: VideoAgentMeshOptions
+  videoUrl?: string
+  freeze: boolean
+  paused: boolean
+  audioInstance: HTMLAudioElement
+
+  get videoInstance(): HTMLAudioElement {
+    if (this.videoUrl?.endsWith('mp3')) {
+      return this.audioInstance
+    }
+    const uniforms = (this.material as THREE.ShaderMaterial).uniforms
+    const videoInstance = uniforms.map.value.image as HTMLVideoElement
+    return videoInstance
+  }
+
+  /**
+   *
+   * @param width
+   * @param height
+   * @param widthSegments
+   * @param heightSegments
+   * @param options
+   */
+  constructor(
+    width: number,
+    height: number,
+    widthSegments: number,
+    heightSegments: number,
+    options: VideoAgentMeshOptions = {},
+  ) {
+    if (!options.videoInstance) {
+      const videoInstance = document.createElement('video')
+      videoInstance.style.opacity = '0'
+      videoInstance.style.display = 'none'
+      document.body.append(videoInstance)
+      options.videoInstance = videoInstance
+      videoInstance.playsInline = true
+      videoInstance.controls = false
+    }
+
+    const geometry = new THREE.PlaneGeometry(width, height, widthSegments, heightSegments)
+    const videoTexture = new THREE.VideoTexture(options.videoInstance)
+    const material = new THREE.ShaderMaterial({
+      uniforms: {
+        map: { value: videoTexture },
+        enable: { value: 0 },
+      },
+      vertexShader,
+      fragmentShader: fragmentShaderTpl(),
+      transparent: true,
+    })
+
+    super(geometry, material)
+    this.options = options
+    this.freeze = false
+    this.paused = true
+
+    {
+      const audioInstance = document.createElement('audio')
+      audioInstance.crossOrigin = ''
+      // videoInstance.muted = true
+      audioInstance.muted = false
+      audioInstance.setAttribute('playsinline', 'true')
+      audioInstance.setAttribute('webkit-playsinline', 'true')
+      audioInstance.setAttribute('autoplay', 'false')
+      audioInstance.setAttribute('style', 'display: none;')
+      document.body.appendChild(audioInstance)
+      this.audioInstance = audioInstance
+    }
+    makeObservable(this, { paused: observable })
+
+    const updatePaused = (paused: boolean) => runInAction(() => (this.paused = paused))
+    this.videoInstance.addEventListener('pause', () => updatePaused(true))
+    this.videoInstance.addEventListener('play', () => updatePaused(false))
+  }
+
+  private async update(videoUrl: string) {
+    if (this.videoUrl === videoUrl) {
+      return
+    }
+
+    this.videoUrl = videoUrl
+    this.freeze = true
+    await this.videoInstance.pause()
+
+    const uniforms = (this.material as THREE.ShaderMaterial).uniforms
+    this.videoInstance.muted = true
+    this.videoInstance.src = await URL.createObjectURL((await Preloader.blob(this.videoUrl)) as unknown as Blob)
+    this.videoInstance.setAttribute('data-src', this.videoUrl)
+
+    const onStart = () => {
+      if (this.videoInstance.currentTime === 0) return
+      this.freeze = false
+      this.videoInstance.muted = false
+      uniforms.enable.value = 1
+      this.videoInstance.removeEventListener('timeupdate', onStart, false)
+    }
+
+    this.videoInstance.addEventListener('timeupdate', onStart, false)
+  }
+
+  async play(videoUrl = '') {
+    videoUrl = (videoUrl || '').replace(/https*:/, '')
+    if (!videoUrl) {
+      if (this.videoUrl) await this.videoInstance.play()
+      else console.warn('警告：视频资源未初始化。')
+      return true
+    }
+
+    if (videoUrl === this.videoUrl) {
+      this.videoInstance.currentTime = 0
+      await this.videoInstance.play()
+      return true
+    }
+
+    await this.update(videoUrl)
+
+    this.videoInstance.pause()
+
+    return await new Promise((resolve) =>
+      setTimeout(async () => {
+        this.videoInstance.currentTime = 0
+        await this.videoInstance.play()
+        resolve(true)
+      }, 20),
+    )
+  }
+
+  /**
+   * 转成毫秒，保障精准度
+   */
+  get currentTime() {
+    return this.videoInstance.currentTime * 1000
+  }
+}

package/resources/Player/modules/VideoAgent/VideoAgentScene.ts

@@ -0,0 +1,38 @@
+import * as THREE from 'three'
+import { VideoAgentMesh } from './VideoAgentMesh'
+
+export class VideoAgentScene {
+  videoAgentMesh: VideoAgentMesh
+
+  scene = new THREE.Scene()
+  camera = new THREE.OrthographicCamera(-240, 240, 135, -135)
+  renderer = new THREE.WebGLRenderer({ alpha: true })
+
+  constructor(container: HTMLElement, needRender = true) {
+    this.camera.position.set(0, 0, 10)
+    this.camera.lookAt(0, 0, 0)
+    const state = {
+      width: container.offsetWidth,
+      height: container.offsetHeight,
+    }
+    const domElement = this.renderer.domElement
+    domElement.classList.add('VideoAgent-canvas')
+    this.renderer.setSize(state.width * window.devicePixelRatio, state.height * window.devicePixelRatio)
+    domElement.setAttribute('style', `width: ${state.width}px;height: ${state.height}px;`)
+
+    this.videoAgentMesh = new VideoAgentMesh(480, 270, 1, 1)
+    if (needRender) {
+      container.append(domElement)
+      this.scene.add(this.videoAgentMesh)
+      this.run()
+    }
+  }
+
+  run = () => {
+    // 视频开始 1s 不渲染：规避某些设备黑屏闪烁问题。视频前 2s 是最好是静默的。
+    if (!(this.videoAgentMesh.paused || this.videoAgentMesh.freeze) && this.videoAgentMesh.currentTime > 1000) {
+      this.renderer.render(this.scene, this.camera)
+    }
+    requestAnimationFrame(this.run)
+  }
+}

package/resources/Player/modules/VideoAgent/index.tsx

@@ -0,0 +1,31 @@
+import * as React from 'react'
+import { useController } from '../../hooks'
+import { VideoAgentScene } from './VideoAgentScene'
+
+export function VideoAgent(props: { onClick?: () => void }) {
+  const ref = React.useRef<HTMLDivElement>(null)
+  const controller = useController()
+
+  React.useEffect(() => {
+    if (!ref.current) throw new Error('React 渲染异常，请稍后重试 ...')
+    if (controller.videoAgentScene) {
+      console.warn('"VideoAgentScene" 重复初始化，已被过滤 ...')
+      return
+    }
+    const videoAgentScene = new VideoAgentScene(ref.current)
+    controller.videoAgentScene = videoAgentScene
+  }, [])
+
+  return (
+    <div
+      className="VideoAgent"
+      onClick={() => {
+        if (!props.onClick) return
+        props.onClick()
+      }}
+      ref={ref}
+    >
+      <div className="VideoAgent-play"></div>
+    </div>
+  )
+}

package/resources/Player/modules/keyframes/CameraMovement/index.tsx

@@ -0,0 +1,33 @@
+import * as React from 'react'
+import { CameraMovementPlugin } from '../../../../fivePlugins/CameraMovementPlugin'
+import { CameraMovementEffect, RotateArgs } from '../../../../fivePlugins/CameraMovementPlugin/typings'
+import { CameraMovementData, VreoKeyframe, VreoKeyframeEnum } from '../../../../typings/VreoUnit'
+import { useController, useFiveInstance } from '../../../hooks'
+
+export function CameraMovement() {
+  const controller = useController()
+  const five = useFiveInstance()
+  const ref = React.useRef<ReturnType<typeof CameraMovementPlugin>>()
+
+  React.useEffect(() => {
+    const callback = async (keyframe: VreoKeyframe) => {
+      if (!ref.current) {
+        ref.current = CameraMovementPlugin(five, {})
+      }
+      const { start, end, data } = keyframe
+      const cameraMovementData = data as CameraMovementData
+      const effect = cameraMovementData.effect
+      if (effect === CameraMovementEffect.Rotate) {
+        ref.current.rotate(data as RotateArgs, end - start)
+      } else {
+        ref.current.move(data, end - start)
+      }
+    }
+
+    controller.on(VreoKeyframeEnum.CameraMovement, callback)
+    return () => {
+      controller.off(VreoKeyframeEnum.CameraMovement, callback)
+    }
+  }, [controller])
+  return <></>
+}