@realsee/dnalogel 2.0.0-alpha.7 → 2.0.0-beta.36

package/libs/ModelSimulationPlugin/index.js

@@ -0,0 +1,166 @@
+import * as THREE from 'three';
+import vertexShader from './shaders/vertex';
+import fragmentShader from './shaders/fragment';
+import tap from '../shared-utils/tap';
+const houseOutLinePath = [
+    [-3.089, -4.219], [-4.834, -4.219],
+    [-8.197, -4.219], [-10.31, -4.219],
+    [-10.31, -1.002], [-10.31, 0.835],
+    [-10.31, 3.927], [-11.237, 3.927],
+    [-11.237, 5.45], [-8.104, 5.45],
+    [-8.104, 4.557], [-6.914, 4.557],
+    [-6.244, 4.557], [-6.244, 5.436],
+    [-3.78, 5.436], [-0.356, 5.436],
+    [0.286, 5.436], [0.286, 4.073],
+    [0.286, 0.395], [2.31, 0.395],
+    [2.31, -1.323], [2.31, -4.199],
+    [0.468, -4.199], [0.468, -3.547],
+    [-2.969, -3.547], [-3.089, -3.547]
+];
+const defaultKeyframes = 50;
+const defaultBoundingOffset = 0.2;
+let isTexutreReady = false;
+const ModelSimulationPlugin = (five, options) => {
+    let heatTexutres = [];
+    let geometryTexture = null;
+    let simulationMaterial = null;
+    let boundingMaterial = null;
+    let boundingMesh = null;
+    let originMaterial = null;
+    const enable = () => {
+    };
+    const disable = () => {
+        five.model.setMaterial(originMaterial);
+    };
+    const createBounding = (path, material) => {
+        const shape = tap(new THREE.Shape(), shape => {
+            shape.moveTo(path[path.length - 1][0], path[path.length - 1][1]);
+            path.forEach(xz => shape.lineTo(xz[0], xz[1]));
+        });
+        const settings = {
+            depth: options.simulationWorkData.houseHeight + 2 * options.simulationOptions.boundingOffset ?? defaultBoundingOffset,
+            bevelEnabled: false,
+        };
+        const geometry = new THREE.ExtrudeGeometry(shape, settings);
+        boundingMesh = new THREE.Mesh(geometry, material);
+        boundingMesh.rotateX(Math.PI / 2);
+        boundingMesh.translateZ(-options.simulationWorkData.houseHeight - options.simulationOptions.boundingOffset);
+        boundingMesh.updateMatrix();
+        boundingMesh.updateMatrixWorld();
+        return boundingMesh;
+    };
+    const createSimulationMaterial = (isBounding) => {
+        const simulationMaterial = new THREE.RawShaderMaterial({
+            uniforms: {
+                heatMap: { value: new THREE.Texture() },
+                geometryMap: { value: new THREE.Texture() },
+                colorMap: { value: new THREE.TextureLoader().load(options.simulationWorkData.colorMapPath) },
+                bboxMin: { value: new THREE.Vector3().fromArray(options.simulationWorkData.origin) },
+                bboxMax: { value: new THREE.Vector3().fromArray(options.simulationWorkData.origin).add(new THREE.Vector3(options.simulationWorkData.width, options.simulationWorkData.height, options.simulationWorkData.depth).multiplyScalar(options.simulationWorkData.step)) },
+                step: { value: options.simulationWorkData.step },
+                maxT: { value: options.simulationWorkData.maxT },
+                minT: { value: options.simulationWorkData.minT },
+                opacity: { value: isBounding ? options.simulationOptions.boundingOpacity : options.simulationOptions.globalOpacity }
+            },
+            vertexShader,
+            fragmentShader,
+        });
+        if (isBounding)
+            simulationMaterial.transparent = true;
+        return simulationMaterial;
+    };
+    const create3DTextureTask = (url) => {
+        return fetch(url)
+            .then(res => {
+            return res.arrayBuffer();
+        })
+            .then(res => {
+            const texture = new THREE.DataTexture3D(new Float32Array(res), options.simulationWorkData.width, options.simulationWorkData.height, options.simulationWorkData.depth);
+            texture.type = THREE.FloatType;
+            texture.format = THREE.RedFormat;
+            texture.minFilter = THREE.LinearMipmapLinearFilter;
+            texture.magFilter = THREE.LinearFilter;
+            texture.generateMipmaps = true;
+            texture.unpackAlignment = 1;
+            return texture;
+        })
+            .catch(err => console.error("拉取三维纹理数据时发生错误"));
+    };
+    const init = () => {
+        simulationMaterial = createSimulationMaterial(false);
+        boundingMaterial = createSimulationMaterial(true);
+        //加载外bounding
+        five.scene.add(createBounding(houseOutLinePath, boundingMaterial));
+        //缓存原来的材质
+        originMaterial = five.model.getMaterial();
+        five.model.traverse(obj => {
+            if (!(obj instanceof THREE.Mesh))
+                return;
+            if (Array.isArray(obj.material)) {
+                obj.material = obj.material.map(_mtl => simulationMaterial);
+            }
+            else {
+                obj.material = simulationMaterial;
+            }
+        });
+        //温度数据
+        // const geometryTaskUrl = options.resourceUrl + 'geometry.vti'
+        const heatTextureTasks = [];
+        for (let i = 0; i < options.simulationWorkData.maxKeyframes; i++) {
+            const fileUrl = options.simulationWorkData.resourceUrl + i + '.vti';
+            heatTextureTasks.push(create3DTextureTask(fileUrl));
+        }
+        // //有效值数据
+        // create3DTextureTask(geometryTaskUrl)
+        // 	.then(texture => {
+        // 		simulationMaterial.uniforms.geometryMap.value = texture
+        // 		boundingMaterial.uniforms.geometryMap.value = texture
+        // 	})
+        // 	.catch(err => console.error("geometry texture loading failed"))
+        //加载首帧进行初始化
+        create3DTextureTask(options.simulationWorkData.resourceUrl + '0.vti')
+            .then(texture => {
+            simulationMaterial.uniforms.heatMap.value = texture;
+            boundingMaterial.uniforms.heatMap.value = texture;
+            five.needsRender = true;
+        })
+            .catch(err => console.error("loading textures failed"));
+        //加载全部
+        Promise.all(heatTextureTasks)
+            .then(textures => {
+            //缓存结果
+            isTexutreReady = true;
+            console.log("ready");
+            heatTexutres = textures;
+        })
+            .catch(err => console.error("loading textures failed"));
+    };
+    const play = () => {
+        if (!isTexutreReady)
+            throw new Error("textures are not ready yet,plz try again later");
+        let timeCount = 0;
+        const stop = setInterval(() => {
+            if (timeCount < options.simulationWorkData.maxKeyframes) {
+                simulationMaterial.uniforms.heatMap.value = heatTexutres[timeCount];
+                boundingMaterial.uniforms.heatMap.value = heatTexutres[timeCount];
+                timeCount++;
+                five.needsRender = true;
+            }
+        }, 1000 / options.simulationOptions.keyFrameRate ?? defaultKeyframes);
+    };
+    const dispose = () => {
+        simulationMaterial.dispose();
+        boundingMaterial.dispose();
+        boundingMesh.geometry.dispose();
+        geometryTexture.dispose();
+        heatTexutres.forEach(texture => texture.dispose());
+    };
+    return {
+        enable,
+        disable,
+        play,
+        init,
+        dispose
+    };
+};
+export default ModelSimulationPlugin;

package/libs/ModelSimulationPlugin/shaders/fragment.d.ts

@@ -0,0 +1,2 @@
+declare var _default: "#version 300 es\n#define MAX_STEPS 50\nprecision highp float;\nprecision highp sampler3D;\n\nuniform sampler3D heatMap;\nuniform sampler3D geometryMap;\nuniform sampler2D colorMap;\n\nuniform vec3 bboxMin;\nuniform vec3 bboxMax;\nuniform float step;\nuniform float minT;\nuniform float maxT;\nuniform float opacity;\n\nvec4 LinearTosRGB( in vec4 value ) {\n\treturn vec4( mix( pow( value.rgb, vec3( 0.41666 ) ) * 1.055 - vec3( 0.055 ), value.rgb * 12.92, vec3( lessThanEqual( value.rgb, vec3( 0.0031308 ) ) ) ), value.a );\n}\nvec4 linearToOutputTexel( vec4 value ) { return LinearTosRGB( value ); }\n\n\nfloat getHeatByLegalPos(vec3 pos){\n\tfloat geometryFactor = texture(geometryMap,pos).r;\n\tfloat legalFactor = clamp(geometryFactor,0.0,1.0);\n\treturn texture(heatMap,pos).r * legalFactor;\n}\n\nfloat sampleOc(vec3 pos){\n\t//平滑滤波步长\n\t//todo:可以作为define传入的外部常量还是直接传\n\tfloat step_x = step / (bboxMax.x - bboxMin.x) * 0.5;\n\tfloat step_y = step / (bboxMax.y - bboxMin.y) * 0.5;\n\tfloat step_z = step / (bboxMax.z - bboxMin.z) * 0.5;\n\tvec3 p1 = pos + vec3(step_x,step_y,step_z);\n\tvec3 p2 = pos + vec3(step_x,-step_y,step_z);\n\tvec3 p3 = pos + vec3(step_x,step_y,-step_z);\n\tvec3 p4 = pos + vec3(step_x,-step_y,-step_z);\n\tvec3 p5 = pos + vec3(-step_x,step_y,step_z);\n\tvec3 p6 = pos + vec3(-step_x,-step_y,step_z);\n\tvec3 p7 = pos + vec3(-step_x,step_y,-step_z);\n\tvec3 p8 = pos + vec3(-step_x,-step_y,-step_z);\n\n\treturn (getHeatByLegalPos(p1) + getHeatByLegalPos(p2) + getHeatByLegalPos(p3) + getHeatByLegalPos(p4) + getHeatByLegalPos(p5) + getHeatByLegalPos(p6) + getHeatByLegalPos(p7) + getHeatByLegalPos(p8)) * 0.125;\n\n}\n\n//考虑到有效仿真值连通域在三维空间中是封闭流形，这里直接按照normal方向搜索，牺牲一定精度，以保证展示效果\nfloat sampleByNormal(vec3 normal, vec3 pos){\n\tfloat searchStep = step ;\n\tvec3 currentPos = pos;\n\tfor (int iter=0; iter < MAX_STEPS; iter++) {\n\t\tif(iter >= MAX_STEPS){\n\t\t\treturn texture(heatMap,currentPos).r;\n\t\t}\n\t\tif(texture(geometryMap,currentPos).r < 0.5){\n\t\t\tcurrentPos += normal * searchStep;\n\t\t}else{\n\t\t\treturn texture(heatMap,currentPos).r;\n\t\t}\n\t}\n}\n\nfloat sample1(vec3 normal, vec3 pos){\n\treturn texture(heatMap,pos).r;\n}\n\nin vec2 vUv;\nin vec3 vPosition;\nin vec3 vNormal;\nout highp vec4 pc_fragColor;\n\n\nfloat normalizeHeat(float t){\n\tfloat _t = clamp(t,minT,maxT);\n\treturn (_t - minT)/(maxT - minT);\n}\n\nvec3 posToTexCood(vec3 pos){\n\tfloat x = (pos.x - bboxMin.x) / (bboxMax.x - bboxMin.x);\n\tfloat y = (pos.y - bboxMin.y) / (bboxMax.y - bboxMin.y);\n\tfloat z = (pos.z - bboxMin.z) / (bboxMax.z - bboxMin.z);\n\treturn vec3(x,y,z);\n}\n\n\nvoid main() {\n\tvec3 texCood = posToTexCood(vPosition);\n\t// float color = sampleByNormal(vNormal,texCood);\n\tfloat color = sample1(vNormal,texCood);\n\tfloat normalColor= normalizeHeat(color);\n\tvec4 colorMapColor = texture(colorMap,vec2(normalColor,0.2));\n\tpc_fragColor = vec4(colorMapColor.xyz,opacity);\n\t// pc_fragColor = linearToOutputTexel(vec4(colorMapColor.xyz,opacity));\n\t}\n";
+export default _default;

package/libs/ModelSimulationPlugin/shaders/fragment.js

@@ -0,0 +1,96 @@
+export default `#version 300 es
+#define MAX_STEPS 50
+precision highp float;
+precision highp sampler3D;
+
+uniform sampler3D heatMap;
+uniform sampler3D geometryMap;
+uniform sampler2D colorMap;
+
+uniform vec3 bboxMin;
+uniform vec3 bboxMax;
+uniform float step;
+uniform float minT;
+uniform float maxT;
+uniform float opacity;
+
+vec4 LinearTosRGB( in vec4 value ) {
+	return vec4( mix( pow( value.rgb, vec3( 0.41666 ) ) * 1.055 - vec3( 0.055 ), value.rgb * 12.92, vec3( lessThanEqual( value.rgb, vec3( 0.0031308 ) ) ) ), value.a );
+}
+vec4 linearToOutputTexel( vec4 value ) { return LinearTosRGB( value ); }
+
+
+float getHeatByLegalPos(vec3 pos){
+	float geometryFactor = texture(geometryMap,pos).r;
+	float legalFactor = clamp(geometryFactor,0.0,1.0);
+	return texture(heatMap,pos).r * legalFactor;
+}
+
+float sampleOc(vec3 pos){
+	//平滑滤波步长
+	//todo:可以作为define传入的外部常量还是直接传
+	float step_x = step / (bboxMax.x - bboxMin.x) * 0.5;
+	float step_y = step / (bboxMax.y - bboxMin.y) * 0.5;
+	float step_z = step / (bboxMax.z - bboxMin.z) * 0.5;
+	vec3 p1 = pos + vec3(step_x,step_y,step_z);
+	vec3 p2 = pos + vec3(step_x,-step_y,step_z);
+	vec3 p3 = pos + vec3(step_x,step_y,-step_z);
+	vec3 p4 = pos + vec3(step_x,-step_y,-step_z);
+	vec3 p5 = pos + vec3(-step_x,step_y,step_z);
+	vec3 p6 = pos + vec3(-step_x,-step_y,step_z);
+	vec3 p7 = pos + vec3(-step_x,step_y,-step_z);
+	vec3 p8 = pos + vec3(-step_x,-step_y,-step_z);
+
+	return (getHeatByLegalPos(p1) + getHeatByLegalPos(p2) + getHeatByLegalPos(p3) + getHeatByLegalPos(p4) + getHeatByLegalPos(p5) + getHeatByLegalPos(p6) + getHeatByLegalPos(p7) + getHeatByLegalPos(p8)) * 0.125;
+
+}
+
+//考虑到有效仿真值连通域在三维空间中是封闭流形，这里直接按照normal方向搜索，牺牲一定精度，以保证展示效果
+float sampleByNormal(vec3 normal, vec3 pos){
+	float searchStep = step ;
+	vec3 currentPos = pos;
+	for (int iter=0; iter < MAX_STEPS; iter++) {
+		if(iter >= MAX_STEPS){
+			return texture(heatMap,currentPos).r;
+		}
+		if(texture(geometryMap,currentPos).r < 0.5){
+			currentPos += normal * searchStep;
+		}else{
+			return texture(heatMap,currentPos).r;
+		}
+	}
+}
+
+float sample1(vec3 normal, vec3 pos){
+	return texture(heatMap,pos).r;
+}
+
+in vec2 vUv;
+in vec3 vPosition;
+in vec3 vNormal;
+out highp vec4 pc_fragColor;
+
+
+float normalizeHeat(float t){
+	float _t = clamp(t,minT,maxT);
+	return (_t - minT)/(maxT - minT);
+}
+
+vec3 posToTexCood(vec3 pos){
+	float x = (pos.x - bboxMin.x) / (bboxMax.x - bboxMin.x);
+	float y = (pos.y - bboxMin.y) / (bboxMax.y - bboxMin.y);
+	float z = (pos.z - bboxMin.z) / (bboxMax.z - bboxMin.z);
+	return vec3(x,y,z);
+}
+
+
+void main() {
+	vec3 texCood = posToTexCood(vPosition);
+	// float color = sampleByNormal(vNormal,texCood);
+	float color = sample1(vNormal,texCood);
+	float normalColor= normalizeHeat(color);
+	vec4 colorMapColor = texture(colorMap,vec2(normalColor,0.2));
+	pc_fragColor = vec4(colorMapColor.xyz,opacity);
+	// pc_fragColor = linearToOutputTexel(vec4(colorMapColor.xyz,opacity));
+	}
+`;

package/libs/ModelSimulationPlugin/shaders/vertex.d.ts

@@ -0,0 +1,2 @@
+declare var _default: "#version 300 es\n\tin vec3 position;\n\tin vec3 normal;\n\tout vec3 vPosition;\n\tout vec3 vNormal;\n\n\tuniform mat4 modelViewMatrix;\n\tuniform mat4 projectionMatrix;\n\tuniform vec3 cameraPos;\n\t// uniform vec3 normalMatrix;\n\tuniform mat4 modelMatrix;\n\n\tvoid main() {\n\n\t\tvec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );\n\t\tvec4 worldPosition = modelMatrix * vec4(position.xyz,1.0);\n\t\tvPosition = worldPosition.xyz;\n\t\t// vec3 transformedNormal = normalMatrix * normal;\n\t\tvNormal = normalize(normal);\n\n\t\tgl_Position = projectionMatrix * mvPosition;\n\t}\n";
+export default _default;

package/libs/ModelSimulationPlugin/shaders/vertex.js

@@ -0,0 +1,23 @@
+export default `#version 300 es
+	in vec3 position;
+	in vec3 normal;
+	out vec3 vPosition;
+	out vec3 vNormal;
+
+	uniform mat4 modelViewMatrix;
+	uniform mat4 projectionMatrix;
+	uniform vec3 cameraPos;
+	// uniform vec3 normalMatrix;
+	uniform mat4 modelMatrix;
+
+	void main() {
+
+		vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );
+		vec4 worldPosition = modelMatrix * vec4(position.xyz,1.0);
+		vPosition = worldPosition.xyz;
+		// vec3 transformedNormal = normalMatrix * normal;
+		vNormal = normalize(normal);
+
+		gl_Position = projectionMatrix * mvPosition;
+	}
+`;

package/libs/ModelTVVideoPlugin/Plugin.d.ts

@@ -0,0 +1,4 @@
+import type { FivePlugin } from '@realsee/five';
+import type { ModelTVVideoPluginExportType, ModelTVVideoPluginParameterType } from "./typings";
+export declare const ModelTVVideoPlugin: FivePlugin<ModelTVVideoPluginParameterType, ModelTVVideoPluginExportType>;
+export default ModelTVVideoPlugin;

package/libs/ModelTVVideoPlugin/Plugin.js

@@ -0,0 +1,254 @@
+import * as THREE from 'three';
+import { parseModelTVVideoPoints } from "./utils/parseData";
+export const ModelTVVideoPlugin = (five, { videoElement }) => {
+    const state = {
+        videoMeshes: [],
+        videoTextureEnabled: false,
+        videoSource: '',
+        rectPoints: [],
+        enabled: false,
+        videoElement: videoElement,
+    };
+    const setMuted = (muted) => {
+        if (state.videoTexture) {
+            state.videoTexture.image.muted = muted;
+            state.videoTexture.image.play();
+        }
+    };
+    const getMuted = () => {
+        if (state.videoTexture)
+            return state.videoTexture.image.muted;
+        else
+            return true;
+    };
+    const enable = () => {
+        if (state.enabled)
+            return;
+        if (!state.videoTexture)
+            return;
+        state.enabled = true;
+        state.videoMeshes = createPanoVideoMeshes();
+        console.log('state.videoMeshes', state.videoMeshes);
+        state.videoMeshes.forEach((mesh) => five.scene.add(mesh));
+        const play = () => {
+            console.log('play', state.videoTexture);
+            if (!state.videoTexture)
+                return;
+            const timeupdate = () => {
+                var _a;
+                if (!state.videoTexture)
+                    return;
+                (_a = state.videoTexture) === null || _a === void 0 ? void 0 : _a.image.removeEventListener('timeupdate', timeupdate);
+                state.videoTextureEnabled = true;
+                state.videoMeshes.forEach((mesh) => {
+                    if (mesh.material.map !== state.videoTexture)
+                        mesh.material.map = state.videoTexture;
+                });
+                five.needsRender = true;
+            };
+            state.videoTexture.image.addEventListener('timeupdate', timeupdate);
+            if (state.videoTexture && state.videoMeshes.length) {
+                state.videoTexture.image.play();
+            }
+        };
+        if (five.model.loaded)
+            play();
+        else {
+            return five.once('modelLoaded', () => play());
+        }
+    };
+    const disable = () => {
+        if (!state.enabled)
+            return;
+        state.enabled = false;
+        state.videoMeshes.forEach((mesh) => {
+            mesh.geometry.dispose();
+            mesh.material.dispose();
+            five.scene.remove(mesh);
+            if (state.videoTexture)
+                state.videoTexture.image.pause();
+        });
+        state.videoMeshes = [];
+        five.needsRender = true;
+    };
+    const createPanoVideoMeshes = () => {
+        return state.rectPoints.map((points, index) => {
+            const geometry = new THREE.BufferGeometry();
+            const segments = 128;
+            const verticesArray = [];
+            verticesArray.push(points[0].x, points[0].y, points[0].z);
+            for (let i = 1; i < segments; i++) {
+                verticesArray.push(points[0].x + ((points[1].x - points[0].x) * i) / segments, points[0].y + ((points[1].y - points[0].y) * i) / segments, points[0].z + ((points[1].z - points[0].z) * i) / segments);
+            }
+            verticesArray.push(points[1].x, points[1].y, points[1].z);
+            verticesArray.push(points[2].x, points[2].y, points[2].z);
+            for (let i = 1; i < segments; i++) {
+                verticesArray.push(points[2].x + ((points[3].x - points[2].x) * i) / segments, points[2].y + ((points[3].y - points[2].y) * i) / segments, points[2].z + ((points[3].z - points[2].z) * i) / segments);
+            }
+            verticesArray.push(points[3].x, points[3].y, points[3].z);
+            const uvArray = [];
+            uvArray.push(0, 1);
+            for (let i = 1; i < segments; i++) {
+                uvArray.push(0, 1 - i / segments);
+            }
+            uvArray.push(0, 0);
+            uvArray.push(1, 0);
+            for (let i = 1; i < segments; i++) {
+                uvArray.push(1, i / segments);
+            }
+            uvArray.push(1, 1);
+            const indicesArray = [];
+            for (let i = 0; i < segments; i++) {
+                indicesArray.push(i, i + 1, segments * 2 - i, i, segments * 2 - i, segments * 2 + 1 - i);
+            }
+            geometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(verticesArray), 3));
+            geometry.setAttribute('uv', new THREE.BufferAttribute(new Float32Array(uvArray), 2));
+            geometry.setIndex(new THREE.BufferAttribute(new Uint32Array(indicesArray), 1));
+            const material = new THREE.MeshBasicMaterial({
+                map: state.videoTextureEnabled ? state.videoTexture : state.imageTexture,
+                side: THREE.DoubleSide,
+            });
+            const mesh = new THREE.Mesh(geometry, material);
+            mesh.renderOrder = 1;
+            mesh.name = `ModelTVVideoPlugin-${index}-${performance.now()}`;
+            return mesh;
+        });
+    };
+    const getImageTexture = (source) => {
+        const texture = new THREE.TextureLoader().load(source);
+        texture.minFilter = THREE.LinearFilter;
+        texture.magFilter = THREE.LinearFilter;
+        texture.format = THREE.RGBFormat;
+        return texture;
+    };
+    const getVideoTexture = (source, video) => {
+        return new Promise((resolve, reject) => {
+            const xhr = new XMLHttpRequest();
+            xhr.onreadystatechange = () => {
+                if (xhr.readyState == 4) {
+                    if (xhr.status == 200) {
+                        const url = window.URL || window.webkitURL;
+                        video = video || document.createElement('video');
+                        video.crossOrigin = 'anonymous';
+                        video.autoplay = true;
+                        video.muted = true;
+                        video.loop = true;
+                        video.playsInline = true;
+                        video.src = url.createObjectURL(xhr.response);
+                        const texture = new THREE.VideoTexture(video);
+                        texture.minFilter = THREE.LinearFilter;
+                        texture.magFilter = THREE.LinearFilter;
+                        texture.format = THREE.RGBFormat;
+                        resolve(Object.assign(texture, { videoSource: source }));
+                    }
+                    else {
+                        reject(new Error('Video download Error: ' + xhr.status));
+                    }
+                }
+            };
+            xhr.onerror = (error) => reject(error);
+            xhr.open('GET', source);
+            xhr.responseType = 'blob';
+            xhr.send();
+        });
+    };
+    const load = async (data, videoElement) => {
+        const { video_src, video_poster_src, points } = data;
+        state.videoSource = video_src;
+        // TODO 存疑，先处理多一次遍历 o(2n)
+        state.rectPoints = parseModelTVVideoPoints(points)
+            .map((items) => items.map(({ x, y, z }) => new THREE.Vector3(x, y, z)));
+        state.imageTexture = getImageTexture(video_poster_src);
+        if (videoElement) {
+            state.videoElement = videoElement;
+        }
+        state.videoTexture = await getVideoTexture(state.videoSource, state.videoElement);
+        state.enabled = !!data.enable;
+        if (state.enabled)
+            enable();
+    };
+    const dispose = () => {
+        disable();
+        state.videoTexture = undefined;
+        five.off('modeChange', () => setMuted(true));
+        five.off('wantsTapGesture', (raycaster) => {
+            if (!state.enabled)
+                return;
+            const [intersect] = raycaster.intersectObjects(five.scene.children, true);
+            if (!!intersect && /^video/.test(intersect.object.name)) {
+                if (state.videoTexture)
+                    setMuted(!state.videoTexture.image.muted);
+                return false;
+            }
+        });
+        five.off('panoArrived', () => {
+            if (!state.enabled)
+                return;
+            if (getMuted())
+                return;
+            const cameraPosition = five.camera.position;
+            const visible = state.rectPoints.find((points) => {
+                const centerVector = points.reduce((total, curr) => new THREE.Vector3(total.x + curr.x / points.length, total.y + curr.y / points.length, total.z + curr.z / points.length), new THREE.Vector3());
+                return (points.map((point) => new THREE.Vector3((point.x + centerVector.x) / 2, (point.y + centerVector.y) / 2, (point.z + centerVector.z) / 2)).filter((vector) => {
+                    vector = vector.clone().sub(cameraPosition).normalize();
+                    const raycaster = new THREE.Raycaster(cameraPosition, vector);
+                    const [intersect] = raycaster.intersectObjects(five.scene.children, true);
+                    return !!intersect && /^video/.test(intersect.object.name);
+                }).length >= 2);
+            });
+            if (!visible)
+                setMuted(true);
+        });
+        five.off('renderFrame', () => {
+            // console.log('renderFrame', state.videoMeshes)
+            state.videoMeshes.forEach((meshes) => {
+                if (meshes)
+                    //@ts-ignore
+                    meshes.needsRender = true;
+            });
+        });
+    };
+    five.on('modeChange', () => setMuted(true));
+    five.on('wantsTapGesture', (raycaster) => {
+        if (!state.enabled)
+            return;
+        const targetMesh = five.scene.children.find(item => /^ModelTVVideoPlugin/.test(item.name));
+        if (!targetMesh)
+            return;
+        const intersectObjects = raycaster.intersectObjects([targetMesh]);
+        if (intersectObjects.length === 0)
+            return;
+        if (state.videoTexture)
+            setMuted(!state.videoTexture.image.muted); // TODO  toggle video play
+        return false;
+    });
+    five.on('panoArrived', () => {
+        if (!state.enabled)
+            return;
+        if (getMuted())
+            return;
+        const cameraPosition = five.camera.position;
+        const visible = state.rectPoints.find((points) => {
+            const centerVector = points.reduce((total, curr) => new THREE.Vector3(total.x + curr.x / points.length, total.y + curr.y / points.length, total.z + curr.z / points.length), new THREE.Vector3());
+            return (points.map((point) => new THREE.Vector3((point.x + centerVector.x) / 2, (point.y + centerVector.y) / 2, (point.z + centerVector.z) / 2)).filter((vector) => {
+                vector = vector.clone().sub(cameraPosition).normalize();
+                const raycaster = new THREE.Raycaster(cameraPosition, vector);
+                const [intersect] = raycaster.intersectObjects(five.scene.children, true);
+                return !!intersect && /^video/.test(intersect.object.name);
+            }).length >= 2);
+        });
+        if (!visible)
+            setMuted(true);
+    });
+    five.on('renderFrame', () => {
+        // console.log('renderFrame', state.videoMeshes)
+        state.videoMeshes.forEach((meshes) => {
+            if (meshes)
+                //@ts-ignore
+                meshes.needsRender = true;
+        });
+    });
+    Object.assign(window, { $state: state });
+    return { enable, disable, load, dispose };
+};
+export default ModelTVVideoPlugin;

package/libs/ModelTVVideoPlugin/index.d.ts

@@ -0,0 +1,5 @@
+import ModelTVVideoPlugin from './Plugin';
+export { ModelTVVideoPlugin };
+export default ModelTVVideoPlugin;
+export type { ModelTVVideoPluginParameterType, ModelTVVideoPluginData, Point, ModelTVVideoPluginExportType } from './typings';
+export { DIRECTION } from './typings';

package/libs/ModelTVVideoPlugin/index.js

@@ -0,0 +1,4 @@
+import ModelTVVideoPlugin from './Plugin';
+export { ModelTVVideoPlugin };
+export default ModelTVVideoPlugin;
+export { DIRECTION } from './typings';

package/libs/ModelTVVideoPlugin/typings.d.ts

@@ -0,0 +1,25 @@
+export interface ModelTVVideoPluginParameterType {
+    videoElement?: HTMLVideoElement;
+}
+export declare enum DIRECTION {
+    TOP_LEFT = "topLeft",
+    TOP_RIGHT = "topRight",
+    BOTTOM_LEFT = "bottomLeft",
+    BOTTOM_RIGHT = "bottomRight"
+}
+export interface Point {
+    x: number;
+    y: number;
+    z: number;
+}
+export interface ModelTVVideoPluginData {
+    enable?: boolean;
+    video_src: string;
+    video_poster_src: string;
+    points: Point[][] | Record<DIRECTION, Point>[];
+}
+export interface ModelTVVideoPluginExportType {
+    enable: () => void;
+    disable: () => void;
+    load: (data: ModelTVVideoPluginData, videoElement?: HTMLVideoElement) => Promise<void>;
+}

package/libs/ModelTVVideoPlugin/typings.js

@@ -0,0 +1,8 @@
+import "three";
+export var DIRECTION;
+(function (DIRECTION) {
+    DIRECTION["TOP_LEFT"] = "topLeft";
+    DIRECTION["TOP_RIGHT"] = "topRight";
+    DIRECTION["BOTTOM_LEFT"] = "bottomLeft";
+    DIRECTION["BOTTOM_RIGHT"] = "bottomRight";
+})(DIRECTION || (DIRECTION = {}));

package/libs/ModelTVVideoPlugin/utils/parseData.d.ts

@@ -0,0 +1,2 @@
+import { DIRECTION, Point } from "../typings";
+export declare function parseModelTVVideoPoints(points: Point[][] | Record<DIRECTION, Point>[]): Point[][];

package/libs/ModelTVVideoPlugin/utils/parseData.js

@@ -0,0 +1,13 @@
+import { DIRECTION } from "../typings";
+export function parseModelTVVideoPoints(points) {
+    if (points.length === 0)
+        return [];
+    if (Array.isArray(points[0]))
+        return points;
+    let newPoints = [];
+    points.map(point => {
+        const newPoint = [point[DIRECTION.TOP_LEFT], point[DIRECTION.BOTTOM_LEFT], point[DIRECTION.BOTTOM_RIGHT], point[DIRECTION.TOP_RIGHT]];
+        newPoints.push(newPoint);
+    });
+    return newPoints;
+}

package/libs/ModelViewPlugin/Plugin.d.ts

@@ -0,0 +1,17 @@
+import type { FivePlugin } from '@realsee/five';
+export interface ModelViewPluginExportType {
+    appendTo(element: HTMLElement, size?: {
+        width?: number;
+        height?: number;
+    }): void;
+    refresh(size?: {
+        width?: number;
+        height?: number;
+    }): void;
+}
+/**
+ * Five 模型插件
+ * @template ExportType 导出方法
+ */
+export declare const ModelViewPlugin: FivePlugin<void, ModelViewPluginExportType>;
+export default ModelViewPlugin;