@realsee/vreo 0.1.0

package/resources/fivePlugins/CameraMovementPlugin/index.ts

@@ -0,0 +1,203 @@
+import { Five } from '@realsee/five'
+import type { FivePlugin } from '@realsee/five'
+import * as TWEEN from '@tweenjs/tween.js'
+
+import {
+  CameraMovementPluginParameterType,
+  CameraMovementPluginExportType,
+  MoveArgs,
+  MoveOpts,
+  RotateArgs,
+  RotateOpts,
+  Rotation,
+} from './typings'
+
+import { Easing, tweenProgress } from '../../shared-utils/animationFrame/tween'
+
+const PI_2 = Math.PI * 2
+
+function formatLatitude(rad: number) {
+  return rad % PI_2
+}
+function formatLongitude(rad: number) {
+  return ((rad % PI_2) + PI_2) % PI_2
+}
+
+const getLongitudeParams = (from: number, to: number, rotation: Rotation): { from: number; to: number } => {
+  const PI = Math.PI
+  const PI_2 = PI * 2
+  from = formatLongitude(from)
+  to = formatLongitude(to)
+  // 逆时针旋转，初始值必须是较大的值
+  if (rotation === Rotation.Anticlockwise && from < to) from += PI_2
+  // 顺时针旋转，结束值必须是较大值
+  if (rotation === Rotation.Clockwise && to < from) to += PI_2
+  // Loop 旋转，找锐角旋转
+  // 如果 to 比 from 大 180°，逆时针转
+  // 如果 from 比 to 大 180°，顺时针转
+  if (rotation === Rotation.Loop && to - from > PI) return getLongitudeParams(from, to, Rotation.Anticlockwise)
+  if (rotation === Rotation.Loop && from - to > PI) return getLongitudeParams(from, to, Rotation.Clockwise)
+  return { from, to }
+}
+
+function progressNumber(from: number, to: number, pst: number) {
+  return from + (to - from) * pst
+}
+
+/**
+ * **运镜插件** 模拟类似于电影运镜效果。
+ */
+export const CameraMovementPlugin: FivePlugin<CameraMovementPluginParameterType, CameraMovementPluginExportType> = (
+  five: Five,
+) => {
+  const move = async (args: Partial<MoveArgs>, duration: number, opts: MoveOpts = { preload: true }) => {
+    if (opts.asyncStartCallback) {
+      opts.asyncStartCallback()
+    }
+
+    if (args.mode && args.mode !== five.currentMode) {
+      await new Promise((resolve) => {
+        if (!args.mode) return
+        five.once('modeChange', (mode) => {
+          five.once('initAnimationEnded', () => {
+            if (mode === args.mode) {
+              resolve(true)
+            }
+          })
+        })
+        five.changeMode(args.mode, {
+          fov: args.fov || undefined,
+          latitude: args.latitude || undefined,
+          longitude: args.longitude || undefined,
+          panoIndex: args.panoIndex || undefined,
+        })
+      })
+    }
+
+    if (args.mode === Five.Mode.Floorplan) {
+      return true
+    }
+
+    if (opts.preload && args.panoIndex !== undefined && args.panoIndex !== five.panoIndex) {
+      await five.preloadPano(args.panoIndex)
+    }
+
+    if (opts.asyncEndCallback) {
+      opts.asyncEndCallback()
+    }
+
+    if (
+      args.panoIndex === undefined &&
+      args.fov === undefined &&
+      args.latitude === undefined &&
+      args.longitude === undefined
+    ) {
+      return true
+    }
+    return await new Promise<boolean>((resolve, reject) => {
+      const panoIndex = args.panoIndex !== undefined ? args.panoIndex : five.panoIndex
+      if (panoIndex !== undefined) {
+        five.moveToPano(
+          panoIndex,
+          Object.assign(
+            {
+              duration, // 移动耗时
+              moveEndCallback: () => resolve(true), // 移动结束
+              moveCancelCallback: () => reject(false), // 移动开始
+            },
+            args,
+          ),
+        )
+      } else {
+        reject(false)
+      }
+    })
+  }
+
+  const getAnimeParams = (args: RotateArgs) => {
+    const currentFiveState = five.state
+    // latitude
+    const fromLatitude = formatLatitude(currentFiveState.latitude)
+    const toLatitude = formatLatitude(args.latitude)
+    // longitude
+    const { from: fromLongitude, to: toLongitude } = getLongitudeParams(
+      currentFiveState.longitude,
+      args.longitude,
+      args.rotation || Rotation.Loop,
+    )
+    // fov
+    const fromFov = currentFiveState.fov
+    const toFov = args.fov
+    return {
+      from: { latitude: fromLatitude, longitude: fromLongitude, fov: fromFov },
+      to: { latitude: toLatitude, longitude: toLongitude, fov: toFov },
+    }
+  }
+
+  const rotate = async (args: RotateArgs, duration: number, opts: RotateOpts = {}) => {
+    if (opts.asyncStartCallback) {
+      opts.asyncStartCallback()
+    }
+
+    if (five.currentMode !== Five.Mode.Panorama) {
+      await five.changeMode(Five.Mode.Panorama)
+    }
+
+    if (args.panoIndex !== undefined && args.panoIndex !== five.panoIndex) {
+      if (opts.preload) await five.preloadPano(args.panoIndex)
+      await new Promise((resolve, reject) => {
+        five.moveToPano(args.panoIndex, {
+          moveEndCallback: () => resolve(true), // 移动结束
+          moveCancelCallback: () => reject(false), // 移动开始
+        })
+      })
+    }
+
+    if (opts.asyncEndCallback) {
+      opts.asyncEndCallback()
+    }
+
+    const { from, to } = getAnimeParams(args)
+    return await new Promise<boolean>((resolve, reject) => {
+      const onUpdate = ({ progress }: { progress: number }) => {
+        const state: Record<string, number> = {}
+        state.longitude = progressNumber(from.longitude, to.longitude, progress)
+        state.latitude = progressNumber(from.latitude, to.latitude, progress)
+        state.fov = progressNumber(from.fov, to.fov, progress)
+        five.setState(state, true)
+      }
+
+      const onComplete = () => {
+        resolve(true)
+      }
+
+      const onDestroy = () => {
+        resolve(false)
+      }
+
+      const animeDuration = args.rotateSpeed
+        ? Math.ceil((Math.abs(to.longitude - from.longitude) / args.rotateSpeed) * 1000)
+        : duration
+
+      type _Tween = TWEEN.Tween<{ progress: number }> & { onDestroy: (p: any) => _Tween; destroy: () => void }
+
+      const tween = (tweenProgress(animeDuration, Easing.Linear.None) as _Tween)
+        .onUpdate(onUpdate)
+        .onComplete(onComplete)
+        .onDestroy(onDestroy)
+      if (args.rotation === Rotation.Loop) {
+        tween.repeat(Infinity).yoyo(true)
+      }
+      setTimeout(() => {
+        // 总时间到了之后强制结束，同时也是 Loop 模式的结束方式
+        tween.destroy()
+        resolve(true)
+      }, duration)
+    })
+  }
+
+  return {
+    move,
+    rotate,
+  }
+}

package/resources/fivePlugins/CameraMovementPlugin/typings.ts

@@ -0,0 +1,57 @@
+/* eslint-disable prettier/prettier */
+import { Mode, Pose } from '@realsee/five'
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface CameraMovementPluginParameterType { }
+
+export interface CameraMovementPluginExportType {
+  /**
+   * 运镜：移动
+   * @param moveArgs
+   * @param duration
+   * @param opts
+   */
+  move(moveArgs: Partial<MoveArgs>, duration: number, opts?: MoveOpts): Promise<boolean>
+  /**
+   * 运镜：旋转
+   * @param args
+   * @param duration
+   * @param opts
+   */
+  rotate(args: RotateArgs, duration: number, opts?: RotateOpts): Promise<boolean>
+}
+
+export enum CameraMovementEffect {
+  Move = 'Move',
+  Rotate = 'Rotate',
+}
+export enum Rotation {
+  Clockwise = 'Clockwise',
+  Anticlockwise = 'Anticlockwise',
+  Loop = 'Loop',
+}
+
+export interface CameraMovementOptsCallback {
+  asyncStartCallback?: () => void
+  asyncEndCallback?: () => void
+}
+
+export type MoveArgs = {
+  mode: Mode
+  panoIndex: number
+} & Pose
+
+export type MoveOpts = {
+  preload?: boolean
+} & CameraMovementOptsCallback
+
+export type RotateArgs = {
+  panoIndex: number
+  loop: boolean
+  rotateSpeed: number
+  rotation: Rotation
+} & Pose
+
+export type RotateOpts = {
+  preload?: boolean
+} & CameraMovementOptsCallback

package/resources/fivePlugins/ModelTVVideoPlugin/index.ts

@@ -0,0 +1,278 @@
+import * as THREE from 'three'
+import type { FivePlugin } from '@realsee/five'
+
+export interface ModelTVVideoPluginData {
+  enable?: boolean
+  video_src: string
+  video_poster_src: string
+  points: { x: number; y: number; z: number }[][]
+}
+
+export interface ModelTVVideoPluginParameterType {
+  videoElement?: HTMLVideoElement
+}
+export interface ModelTVVideoPluginExportType {
+  enable: () => void
+  disable: () => void
+  load: (data: ModelTVVideoPluginData, videoElement?: HTMLVideoElement) => Promise<void>
+}
+
+type ModelTVVideoPluginPoints = THREE.Vector3[][]
+
+type ModelTVVideoPluginState = {
+  videoMeshes: THREE.Mesh<THREE.BufferGeometry, THREE.MeshBasicMaterial>[]
+  videoTextureEnabled: boolean
+  videoSource: string
+  rectPoints: ModelTVVideoPluginPoints
+  enabled: boolean
+  imageTexture?: THREE.Texture
+  videoTexture?: THREE.VideoTexture
+  videoElement?: HTMLVideoElement
+}
+
+export const ModelTVVideoPlugin: FivePlugin<ModelTVVideoPluginParameterType, ModelTVVideoPluginExportType> = (
+  five,
+  { videoElement },
+) => {
+  const state: ModelTVVideoPluginState = {
+    videoMeshes: [],
+    videoTextureEnabled: false,
+    videoSource: '',
+    rectPoints: [],
+    enabled: false,
+    videoElement: videoElement,
+  }
+
+  const setMuted = (muted: boolean) => {
+    if (state.videoTexture) {
+      state.videoTexture.image.muted = muted
+      state.videoTexture.image.play()
+    }
+  }
+  const getMuted = () => {
+    if (state.videoTexture) return state.videoTexture.image.muted
+    else return true
+  }
+
+  const enable = () => {
+    if (state.enabled) return
+    if (!state.videoTexture) return
+
+    state.enabled = true
+    state.videoMeshes = createPanoVideoMeshes()
+    state.videoMeshes.forEach((mesh) => five.scene.add(mesh))
+
+    const play = () => {
+      if (!state.videoTexture) return
+
+      const timeupdate = () => {
+        if (!state.videoTexture) return
+        state.videoTexture?.image.removeEventListener('timeupdate', timeupdate)
+        state.videoTextureEnabled = true
+        state.videoMeshes.forEach((mesh) => {
+          if (mesh.material.map !== state.videoTexture) mesh.material.map = state.videoTexture!
+        })
+        five.needsRender = true
+      }
+      state.videoTexture.image.addEventListener('timeupdate', timeupdate)
+      if (state.videoTexture && state.videoMeshes.length) {
+        state.videoTexture.image.play()
+      }
+    }
+
+    if (five.model.loaded) play()
+    else {
+      return five.once('modelLoaded', () => play())
+    }
+  }
+
+  const disable = () => {
+    if (!state.enabled) return
+    state.enabled = false
+    state.videoMeshes.forEach((mesh) => {
+      mesh.geometry.dispose()
+      mesh.material.dispose()
+      five.scene.remove(mesh)
+      if (state.videoTexture) state.videoTexture.image.pause()
+    })
+    state.videoMeshes = []
+    five.needsRender = true
+  }
+
+  const createPanoVideoMeshes = () => {
+    return state.rectPoints.map((points, index) => {
+      const geometry = new THREE.BufferGeometry()
+      const segments = 128
+
+      const verticesArray: number[] = []
+      verticesArray.push(points[0].x, points[0].y, points[0].z)
+      for (let i = 1; i < segments; i++) {
+        verticesArray.push(
+          points[0].x + ((points[1].x - points[0].x) * i) / segments,
+          points[0].y + ((points[1].y - points[0].y) * i) / segments,
+          points[0].z + ((points[1].z - points[0].z) * i) / segments,
+        )
+      }
+      verticesArray.push(points[1].x, points[1].y, points[1].z)
+      verticesArray.push(points[2].x, points[2].y, points[2].z)
+      for (let i = 1; i < segments; i++) {
+        verticesArray.push(
+          points[2].x + ((points[3].x - points[2].x) * i) / segments,
+          points[2].y + ((points[3].y - points[2].y) * i) / segments,
+          points[2].z + ((points[3].z - points[2].z) * i) / segments,
+        )
+      }
+      verticesArray.push(points[3].x, points[3].y, points[3].z)
+
+      const uvArray: number[] = []
+      uvArray.push(0, 1)
+      for (let i = 1; i < segments; i++) {
+        uvArray.push(0, 1 - i / segments)
+      }
+      uvArray.push(0, 0)
+      uvArray.push(1, 0)
+      for (let i = 1; i < segments; i++) {
+        uvArray.push(1, i / segments)
+      }
+      uvArray.push(1, 1)
+
+      const indicesArray: number[] = []
+      for (let i = 0; i < segments; i++) {
+        indicesArray.push(i, i + 1, segments * 2 - i, i, segments * 2 - i, segments * 2 + 1 - i)
+      }
+      geometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(verticesArray), 3))
+      geometry.setAttribute('uv', new THREE.BufferAttribute(new Float32Array(uvArray), 2))
+      geometry.setIndex(new THREE.BufferAttribute(new Uint32Array(indicesArray), 1))
+      const material = new THREE.MeshBasicMaterial({
+        map: state.videoTextureEnabled ? state.videoTexture : state.imageTexture,
+        side: THREE.DoubleSide,
+      })
+      const mesh = new THREE.Mesh(geometry, material)
+      mesh.renderOrder = 1
+      mesh.name = `video-${index}-${performance.now()}`
+      return mesh
+    })
+  }
+
+  const getImageTexture = (source: string) => {
+    const texture = new THREE.TextureLoader().load(source)
+    texture.minFilter = THREE.LinearFilter
+    texture.magFilter = THREE.LinearFilter
+    texture.format = THREE.RGBFormat
+    return texture
+  }
+
+  const getVideoTexture = (
+    source: string,
+    video?: HTMLVideoElement,
+  ): Promise<THREE.VideoTexture & { videoSource: string }> => {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest()
+      xhr.onreadystatechange = () => {
+        if (xhr.readyState == 4) {
+          if (xhr.status == 200) {
+            const url = window.URL || window.webkitURL
+            video = video || document.createElement('video')
+            video.crossOrigin = 'anonymous'
+            video.autoplay = true
+            video.muted = true
+            video.loop = true
+            video.playsInline = true
+            video.src = url.createObjectURL(xhr.response)
+            const texture = new THREE.VideoTexture(video)
+            texture.minFilter = THREE.LinearFilter
+            texture.magFilter = THREE.LinearFilter
+            texture.format = THREE.RGBFormat
+            resolve(Object.assign(texture, { videoSource: source }))
+          } else {
+            reject(new Error('Video download Error: ' + xhr.status))
+          }
+        }
+      }
+      xhr.onerror = (error) => reject(error)
+      xhr.open('GET', source)
+      xhr.responseType = 'blob'
+      xhr.send()
+    })
+  }
+
+  const load = async (data: ModelTVVideoPluginData, videoElement?: HTMLVideoElement) => {
+    const { video_src, video_poster_src, points } = data
+    state.videoSource = video_src
+    state.rectPoints = points.map((items) => items.map(({ x, y, z }) => new THREE.Vector3(x, y, z)))
+    state.imageTexture = getImageTexture(video_poster_src)
+
+    if (videoElement) {
+      state.videoElement = videoElement
+    }
+
+    state.videoTexture = await getVideoTexture(state.videoSource, state.videoElement)
+
+    state.enabled = !!data.enable
+    if (state.enabled) enable()
+  }
+
+  five.on('modeChange', () => setMuted(true))
+
+  five.on('wantsTapGesture', (raycaster) => {
+    if (!state.enabled) return
+    const [intersect] = raycaster.intersectObjects(five.scene.children, true)
+    if (!!intersect && /^video/.test(intersect.object.name)) {
+      if (state.videoTexture) setMuted(!state.videoTexture.image.muted)
+      return false
+    }
+  })
+
+  five.on('panoArrived', () => {
+    if (!state.enabled) return
+    if (getMuted()) return
+    const cameraPosition = five.camera.position
+
+    const visible = state.rectPoints.find((points) => {
+      const centerVector = points.reduce(
+        (total, curr) =>
+          new THREE.Vector3(
+            total.x + curr.x / points.length,
+            total.y + curr.y / points.length,
+            total.z + curr.z / points.length,
+          ),
+        new THREE.Vector3(),
+      )
+
+      return (
+        points
+          .map(
+            (point) =>
+              new THREE.Vector3(
+                (point.x + centerVector.x) / 2,
+                (point.y + centerVector.y) / 2,
+                (point.z + centerVector.z) / 2,
+              ),
+          )
+          .filter((vector) => {
+            vector = vector.clone().sub(cameraPosition).normalize()
+            const raycaster = new THREE.Raycaster(cameraPosition, vector)
+            const [intersect] = raycaster.intersectObjects(five.scene.children, true)
+            return !!intersect && /^video/.test(intersect.object.name)
+          }).length >= 2
+      )
+    })
+    if (!visible) setMuted(true)
+  })
+
+  five.on('renderFrame', () => {
+    state.videoMeshes.forEach((meshes) => {
+      if (meshes)
+        //@ts-ignore
+        meshes.needsRender = true
+    })
+  })
+
+  five.on('load', (input) => {
+    if (input.modelTVVideoData) {
+      load(input.modelTVVideoData)
+    }
+  })
+
+  return { enable, disable, load }
+}

package/resources/index.ts

@@ -0,0 +1 @@
+export { Player } from './Player'

package/resources/react/index.tsx

@@ -0,0 +1,95 @@
+import { unsafe__useFiveInstance } from '@realsee/five/react'
+import * as React from 'react'
+import { Player } from '../Player'
+import { VreoKeyframeEvent, VreoUnit } from '../typings/VreoUnit'
+
+const VreoContext = React.createContext<Player | null>(null)
+
+export const VreoProvider: React.FC = (props) => {
+  const five = unsafe__useFiveInstance()
+
+  const player = new Player(five)
+  return <VreoContext.Provider value={player}>{props.children}</VreoContext.Provider>
+}
+
+function useVreoInstance() {
+  const context = React.useContext(VreoContext)
+  if (!context) {
+    throw new Error('VreoProvider never found.')
+  }
+
+  return context
+}
+
+/**
+ * VreoPlayer 命令集合。
+ */
+export interface VreoActionCallbacks {
+  /**
+   * 载入剧本数据。
+   */
+  load: (vreoUnit: VreoUnit, currentTime?: number) => Promise<boolean>
+  /**
+   * 播放。
+   */
+  play: () => void
+  /**
+   * 暂停。
+   */
+  pause: () => void
+  /**
+   * 显示 UI 面板。
+   */
+  show: () => void
+  /**
+   * 隐藏 UI 面板。
+   */
+  hide: () => void
+  /**
+   * 销毁数据及部分定时任务。
+   */
+  dispose: () => void
+}
+
+export function useVreoAction() {
+  const player = useVreoInstance()
+
+  const load = React.useCallback(
+    (vreoUnit: VreoUnit, currentTime?: number) => player.load(vreoUnit, currentTime),
+    [player]
+  )
+
+  const play = React.useCallback(() => player.play(), [player])
+  const pause = React.useCallback(() => player.pause(), [player])
+  const show = React.useCallback(() => player.show(), [player])
+  const hide = React.useCallback(() => player.hide(), [player])
+  const dispose = React.useCallback(() => player.dispose(), [player])
+
+  return { load, pause, show, play, hide, dispose }
+}
+
+export function useVreoEventCallback<T extends keyof VreoKeyframeEvent>(
+  name: T,
+  callback: VreoKeyframeEvent[T],
+  deps?: React.DependencyList | undefined
+) {
+  const player = useVreoInstance()
+  let dependencyList: React.DependencyList = [player, name]
+  if (deps !== undefined) {
+    dependencyList = dependencyList.concat(deps)
+  }
+  React.useLayoutEffect(() => {
+    // @ts-ignore
+    return player.on(name, callback)
+  }, dependencyList)
+}
+
+export function useVreoPausedState() {
+  const player = useVreoInstance()
+  const [state, setState] = React.useState(player.paused)
+
+  useVreoEventCallback('paused', () => setState(true))
+  useVreoEventCallback('playing', () => setState(false))
+
+  return state
+}

package/resources/shared-utils/Preloader.ts

@@ -0,0 +1,35 @@
+/**
+ * 预载器
+ */
+export const Preloader = {
+  blob: (url: string) => {
+    const noop = () => null
+    return new Promise<string>((resolve, reject) => {
+      const xhr = new XMLHttpRequest()
+      const clear = () => {
+        xhr.ontimeout = noop
+        xhr.onreadystatechange = noop
+        xhr.onprogress = noop
+        xhr.onabort = noop
+      }
+      xhr.ontimeout = () => {
+        reject(new Error('timeout'))
+        clear()
+      }
+      xhr.onreadystatechange = () => {
+        if (xhr.readyState !== 4) return
+        if (xhr.status === 0 && !xhr.response) return
+        if (xhr.status === 200 || xhr.status === 0) {
+          resolve(xhr.response)
+          clear()
+        } else {
+          reject(new Error('status:' + xhr.status))
+          clear()
+        }
+      }
+      xhr.open('GET', url, true)
+      xhr.responseType = 'blob'
+      xhr.send(null)
+    })
+  },
+}

package/resources/shared-utils/animationFrame/BetterTween.ts

@@ -0,0 +1,33 @@
+import TWEEN from '@tweenjs/tween.js'
+import { requestAnimationFrameInterval } from '.'
+
+export class BetterTween<G> extends TWEEN.Tween<G> {
+  private disposeMethods: (() => void)[] = []
+  private animationFrameDisposer?: () => void
+
+  public onDispose = (callback: () => void) => {
+    this.disposeMethods.push(callback)
+    return this
+  }
+
+  public play = () => {
+    this.start()
+    this.animationFrameDisposer = requestAnimationFrameInterval(() => this.update())
+    return this
+  }
+
+  public dispose = () => {
+    this.stop()
+    this.animationFrameDisposer?.()
+    this.disposeMethods.forEach((fn) => fn())
+    this.disposeMethods = []
+    TWEEN.remove(this)
+  }
+}
+
+export function tweenProgress(duration?: number, easing = TWEEN.Easing.Linear.None) {
+  const tween = new BetterTween({ progress: 0 }).to({ progress: 1 })
+  if (duration !== undefined) tween.duration(duration)
+  if (easing !== undefined) tween.easing(easing)
+  return tween
+}