@react-three/drei 9.73.3 → 9.74.0

package/README.md

@@ -52,6 +52,7 @@ The `native` route of the library **does not** export `Html` or `Loader`. The de
           <li><a href="#scrollcontrols">ScrollControls</a></li>
           <li><a href="#presentationcontrols">PresentationControls</a></li>
           <li><a href="#keyboardcontrols">KeyboardControls</a></li>
+          <li><a href="#FaceControls">FaceControls</a></li>
         </ul>
         <li><a href="#gizmos">Gizmos</a></li>
         <ul>
@@ -120,6 +121,7 @@ The `native` route of the library **does not** export `Html` or `Loader`. The de
           <li><a href="#useboxprojectedenv">useBoxProjectedEnv</a></li>
           <li><a href="#useTrail">useTrail</a></li>
           <li><a href="#useSurfaceSampler">useSurfaceSampler</a></li>
+          <li><a href="#facelandmarker">FaceLandmarker</a></li>
         </ul>
         <li><a href="#loading">Loaders</a></li>
         <ul>
@@ -359,7 +361,7 @@ If available controls have damping enabled by default, they manage their own upd
 const controls = useThree((state) => state.controls)
 ```
 
-Drei currently exports OrbitControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), MapControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-scene-st), TrackballControls, ArcballControls, FlyControls, DeviceOrientationControls, PointerLockControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), FirstPersonControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) and CameraControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story)
+Drei currently exports OrbitControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), MapControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-scene-st), TrackballControls, ArcballControls, FlyControls, DeviceOrientationControls, PointerLockControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), FirstPersonControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) CameraControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story) and FaceControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-facecontrols)
 
 All controls react to the default camera. If you have a `<PerspectiveCamera makeDefault />` in your scene, they will control it. If you need to inject an imperative camera or one that isn't the default, use the `camera` prop: `<OrbitControls camera={MyCamera} />`.
 
@@ -596,6 +598,82 @@ function Foo() {
 }
 ```
 
+#### FaceControls
+
+The camera follows your face.
+
+<p>
+  <a href="https://codesandbox.io/s/bf01sb"><img width="20%" src="https://github.com/abernier/abernier/assets/76580/2138ef30-48f3-4ae9-b2bb-4f600de0a35e" alt="demo"/></a>
+</p>
+
+Pre-requisite: wrap into a `FaceLandmarker` provider
+
+```tsx
+<FaceLandmarker>...</FaceLandmarker>
+```
+
+```tsx
+<FaceControls />
+```
+
+```tsx
+type FaceControlsProps = {
+  /** The camera to be controlled, default: global state camera */
+  camera?: THREE.Camera
+  /** Whether to autostart the webcam, default: true */
+  autostart?: boolean
+  /** Enable/disable the webcam, default: true */
+  webcam?: boolean
+  /** A custom video URL or mediaStream, default: undefined */
+  webcamVideoTextureSrc?: VideoTextureSrc
+  /** Disable the rAF camera position/rotation update, default: false */
+  manualUpdate?: boolean
+  /** Disable the rVFC face-detection, default: false */
+  manualDetect?: boolean
+  /** Callback function to call on "videoFrame" event, default: undefined */
+  onVideoFrame?: (e: THREE.Event) => void
+  /** Reference this FaceControls instance as state's `controls` */
+  makeDefault?: boolean
+  /** Approximate time to reach the target. A smaller value will reach the target faster. */
+  smoothTime?: number
+  /** Apply position offset extracted from `facialTransformationMatrix` */
+  offset?: boolean
+  /** Offset sensitivity factor, less is more sensible, default: 80 */
+  offsetScalar?: number
+  /** Enable eye-tracking */
+  eyes?: boolean
+  /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */
+  eyesAsOrigin?: boolean
+  /** Constant depth of the Facemesh, default: .15 */
+  depth?: number
+  /** Enable debug mode, default: false */
+  debug?: boolean
+  /** Facemesh options, default: undefined */
+  facemesh?: FacemeshProps
+}
+```
+
+```tsx
+type FaceControlsApi = THREE.EventDispatcher & {
+  /** Detect faces from the video */
+  detect: (video: HTMLVideoElement, time: number) => void
+  /** Compute the target for the camera */
+  computeTarget: () => THREE.Object3D
+  /** Update camera's position/rotation to the `target` */
+  update: (delta: number, target?: THREE.Object3D) => void
+  /** <Facemesh> ref api */
+  facemeshApiRef: RefObject<FacemeshApi>
+  /** <Webcam> ref api */
+  webcamApiRef: RefObject<WebcamApi>
+  /** Play the video */
+  play: () => void
+  /** Pause the video */
+  pause: () => void
+}
+```
+
+> **Note** <br>`FaceControls` uses [`requestVideoFrameCallback`](https://caniuse.com/mdn-api_htmlvideoelement_requestvideoframecallback), you may need [a polyfill](https://github.com/ThaUnknown/rvfc-polyfill) (for Firefox).
+
 # Gizmos
 
 #### GizmoHelper
@@ -938,42 +1016,59 @@ Renders a THREE.Line2 using THREE.CatmullRomCurve3 for interpolation.
 
 [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/shapes-facemesh--facemesh-st)
 
-Renders an oriented [MediaPipe `face` mesh](https://github.com/tensorflow/tfjs-models/tree/master/face-landmarks-detection):
-
-```jsx
-const face = {
-  keypoints: [
-    {x: 406.53152857172876, y: 256.8054528661723, z: 10.2, name: "lips"},
-    {x: 406.544237446397, y: 230.06933367750395, z: 8},
-    ...
-  ],
-  box: {
-    xMin: 304.6476503248806,
-    xMax: 502.5079975897382,
-    yMin: 102.16298762367356,
-    yMax: 349.035215984403,
-    width: 197.86034726485758,
-    height: 246.87222836072945
+Renders an oriented [MediaPipe face mesh](https://developers.google.com/mediapipe/solutions/vision/face_landmarker/web_js#handle_and_display_results):
+
+```jsx
+const faceLandmarkerResult = {
+    "faceLandmarks": [
+      [
+        { "x": 0.5760777592658997, "y": 0.8639070391654968, "z": -0.030997956171631813 },
+        { "x": 0.572094738483429, "y": 0.7886289358139038, "z": -0.07189624011516571 },
+        // ...
+      ],
+      // ...
+    ],
+    "faceBlendshapes": [
+      // ...
+    ],
+    "facialTransformationMatrixes": [
+      // ...
+    ]
   },
 }
+const points = faceLandmarkerResult.faceLandmarks[0]
 
-<Facemesh face={face} />
+<Facemesh points={points} />
 ```
 
 ```tsx
-type FacemeshProps = {
-  /** a MediaPipeFaceMesh object, default: a lambda face */
+export type FacemeshProps = {
+  /** an array of 468+ keypoints as returned by google/mediapipe tasks-vision, default: a sample face */
+  points?: MediaPipePoints
+  /** @deprecated an face object as returned by tensorflow/tfjs-models face-landmarks-detection */
   face?: MediaPipeFaceMesh
-  /** width of the mesh, default: undefined */
+  /** constant width of the mesh, default: undefined */
   width?: number
-  /** or height of the mesh, default: undefined */
+  /** or constant height of the mesh, default: undefined */
   height?: number
-  /** or depth of the mesh, default: 1 */
+  /** or constant depth of the mesh, default: 1 */
   depth?: number
   /** a landmarks tri supposed to be vertical, default: [159, 386, 200] (see: https://github.com/tensorflow/tfjs-models/tree/master/face-landmarks-detection#mediapipe-facemesh-keypoints) */
   verticalTri?: [number, number, number]
-  /** a landmark index to be the origin of the mesh. default: undefined (ie. the bbox center) */
-  origin?: number
+  /** a landmark index (to get the position from) or a vec3 to be the origin of the mesh. default: undefined (ie. the bbox center) */
+  origin?: number | THREE.Vector3
+  /** A facial transformation matrix, as returned by FaceLandmarkerResult.facialTransformationMatrixes (see: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/web_js#handle_and_display_results) */
+  facialTransformationMatrix?: typeof FacemeshDatas.SAMPLE_FACELANDMARKER_RESULT.facialTransformationMatrixes[0]
+  /** Apply position offset extracted from `facialTransformationMatrix` */
+  offset?: boolean
+  /** Offset sensitivity factor, less is more sensible */
+  offsetScalar?: number
+  /** Fface blendshapes, as returned by FaceLandmarkerResult.faceBlendshapes (see: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/web_js#handle_and_display_results) */
+  faceBlendshapes?: typeof FacemeshDatas.SAMPLE_FACELANDMARKER_RESULT.faceBlendshapes[0]
+  /** whether to enable eyes (nb. `faceBlendshapes` is required for), default: true */
+  eyes?: boolean
+  /** Force `origin` to be the middle of the 2 eyes (nb. `eyes` is required for), default: false */
+  eyesAsOrigin?: boolean
   /** debug mode, default: false */
   debug?: boolean
 }
@@ -984,20 +1079,28 @@ Ref-api:
 ```tsx
 const api = useRef<FacemeshApi>()
 
-<Facemesh ref={api} face={face} />
+<Facemesh ref={api} points={points} />
 ```
 
 ```tsx
 type FacemeshApi = {
   meshRef: React.RefObject<THREE.Mesh>
   outerRef: React.RefObject<THREE.Group>
+  eyeRightRef: React.RefObject<FacemeshEyeApi>
+  eyeLeftRef: React.RefObject<FacemeshEyeApi>
 }
 ```
 
-NB: `outerRef` group is oriented as your `face`. You can for example get its world direction:
+You can for example get face mesh world direction:
 
 ```tsx
-meshRef.current.localToWorld(new THREE.Vector3(0, 0, -1))
+api.meshRef.current.localToWorld(new THREE.Vector3(0, 0, -1))
+```
+
+or get L/R iris direction:
+
+```tsx
+api.eyeRightRef.current.irisDirRef.current.localToWorld(new THREE.Vector3(0, 0, -1))
 ```
 
 # Abstractions
@@ -2432,6 +2535,12 @@ const buffer = useSurfaceSampler(
 )
 ```
 
+### FaceLandmarker
+
+![](https://img.shields.io/badge/-suspense-brightgreen)
+
+A @mediapipe/tasks-vision [`FaceLandmarker`](https://developers.google.com/mediapipe/api/solutions/js/tasks-vision.facelandmarker) provider, as well as a `useFaceLandmarker` hook.
+
 # Loading
 
 #### Loader

package/core/FaceControls.cjs.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("@babel/runtime/helpers/extends"),t=require("three"),r=require("react"),a=require("@react-three/fiber"),n=require("maath"),o=require("suspend-react"),c=require("./useVideoTexture.cjs.js"),s=require("./Facemesh.cjs.js"),u=require("./FaceLandmarker.cjs.js");function i(e){return e&&"object"==typeof e&&"default"in e?e:{default:e}}function l(e){if(e&&e.__esModule)return e;var t=Object.create(null);return e&&Object.keys(e).forEach((function(r){if("default"!==r){var a=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(t,r,a.get?a:{enumerable:!0,get:function(){return e[r]}})}})),t.default=e,Object.freeze(t)}require("three/src/math/MathUtils"),require("./Line.cjs.js"),require("three-stdlib"),require("@mediapipe/tasks-vision");var d=i(e),f=l(t),p=l(r);function m(e,t){return e.clone().add(t).multiplyScalar(.5)}function v(e,t,r){const a=e.localToWorld(t);return r.worldToLocal(a)}const b=r.createContext({}),h=r.forwardRef((({camera:e,autostart:t=!0,webcam:o=!0,webcamVideoTextureSrc:c,manualUpdate:i=!1,manualDetect:l=!1,onVideoFrame:h,smoothTime:y=.25,offset:x=!0,offsetScalar:T=80,eyes:E=!1,eyesAsOrigin:j=!0,depth:k=.15,debug:V=!1,facemesh:g,makeDefault:F},R)=>{var q,S;const O=a.useThree((e=>e.scene)),C=a.useThree((e=>e.camera)),M=a.useThree((e=>e.set)),D=a.useThree((e=>e.get)),A=e||C,L=r.useRef(null),P=r.useRef(null),[I]=r.useState((()=>new f.Object3D)),[_]=r.useState((()=>new f.Vector3)),[B]=r.useState((()=>new f.Vector3)),[H]=r.useState((()=>new f.Vector3)),[U]=r.useState((()=>new f.Vector3)),W=r.useCallback((()=>{I.parent=A.parent;const e=P.current;if(e){const{outerRef:t,eyeRightRef:r,eyeLeftRef:a}=e;if(r.current&&a.current){const{irisDirRef:e}=r.current,{irisDirRef:n}=a.current;e.current&&n.current&&t.current&&(_.copy(v(e.current,new f.Vector3(0,0,0),t.current)),B.copy(v(n.current,new f.Vector3(0,0,0),t.current)),I.position.copy(v(t.current,m(_,B),A.parent||O)),H.copy(v(e.current,new f.Vector3(0,0,1),t.current)),U.copy(v(n.current,new f.Vector3(0,0,1),t.current)),I.lookAt(t.current.localToWorld(m(H,U))))}else t.current&&(I.position.copy(v(t.current,new f.Vector3(0,0,0),A.parent||O)),I.lookAt(t.current.localToWorld(new f.Vector3(0,0,1))))}return I}),[A,B,U,_,H,O,I]),[z]=r.useState((()=>new f.Object3D)),G=r.useCallback((function(e,t){if(A){var r;if(null!==(r=t)&&void 0!==r||(t=W()),y>0){const r=1e-9;n.easing.damp3(z.position,t.position,y,e,void 0,void 0,r),n.easing.dampE(z.rotation,t.rotation,y,e,void 0,void 0,r)}else z.position.copy(t.position),z.rotation.copy(t.rotation);A.position.copy(z.position),A.rotation.copy(z.rotation)}}),[A,W,y,z.position,z.rotation]),[J,K]=r.useState(),N=u.useFaceLandmarker(),Q=r.useCallback(((e,t)=>{const r=null==N?void 0:N.detectForVideo(e,t);K(r)}),[N]);a.useFrame(((e,t)=>{i||G(t)}));const X=r.useMemo((()=>Object.assign(Object.create(f.EventDispatcher.prototype),{detect:Q,computeTarget:W,update:G,facemeshApiRef:P,webcamApiRef:L,play:()=>{var e,t;null==(e=L.current)||null==(t=e.videoTextureApiRef.current)||t.texture.source.data.play()},pause:()=>{var e,t;null==(e=L.current)||null==(t=e.videoTextureApiRef.current)||t.texture.source.data.pause()}})),[Q,W,G]);r.useImperativeHandle(R,(()=>X),[X]),r.useEffect((()=>{const e=e=>{l||Q(e.texture.source.data,e.time),h&&h(e)};return X.addEventListener("videoFrame",e),()=>{X.removeEventListener("videoFrame",e)}}),[X,Q,N,l,h]),r.useEffect((()=>{if(F){const e=D().controls;return M({controls:X}),()=>M({controls:e})}}),[F,X,D,M]);const Y=null==J?void 0:J.faceLandmarks[0],Z=null==J||null==(q=J.facialTransformationMatrixes)?void 0:q[0],$=null==J||null==(S=J.faceBlendshapes)?void 0:S[0];return p.createElement(b.Provider,{value:X},o&&p.createElement(r.Suspense,{fallback:null},p.createElement(w,{ref:L,autostart:t,videoTextureSrc:c})),p.createElement(s.Facemesh,d.default({ref:P},g,{points:Y,depth:k,facialTransformationMatrix:Z,faceBlendshapes:$,eyes:E,eyesAsOrigin:j,offset:x,offsetScalar:T,debug:V,"rotation-z":Math.PI,visible:V}),p.createElement("meshBasicMaterial",{side:f.DoubleSide})))})),y=()=>r.useContext(b),w=r.forwardRef((({videoTextureSrc:e,autostart:t=!0},a)=>{const n=r.useRef(null),c=y(),s=o.suspend((async()=>e?Promise.resolve(null):await navigator.mediaDevices.getUserMedia({audio:!1,video:{facingMode:"user"}})),[e]);r.useEffect((()=>(c.dispatchEvent({type:"stream",stream:s}),()=>{null==s||s.getTracks().forEach((e=>e.stop())),o.clear([e])})),[s,c,e]);const u=r.useMemo((()=>({videoTextureApiRef:n})),[]);return r.useImperativeHandle(a,(()=>u),[u]),p.createElement(r.Suspense,{fallback:null},p.createElement(x,{ref:n,src:e||s,start:t}))})),x=r.forwardRef((({src:e,start:t},a)=>{const n=c.useVideoTexture(e,{start:t}),o=n.source.data,s=y(),u=r.useCallback((e=>{s.dispatchEvent({type:"videoFrame",texture:n,time:e})}),[n,s]);T(o,u);const i=r.useMemo((()=>({texture:n})),[n]);return r.useImperativeHandle(a,(()=>i),[i]),p.createElement(p.Fragment,null)})),T=(e,t)=>{r.useEffect((()=>{if(!e||!e.requestVideoFrameCallback)return;let r;return e.requestVideoFrameCallback((function a(...n){t(...n),r=e.requestVideoFrameCallback(a)})),()=>e.cancelVideoFrameCallback(r)}),[e,t])};exports.FaceControls=h,exports.useFaceControls=y;

package/core/FaceControls.d.ts

@@ -0,0 +1,42 @@
+import * as THREE from 'three';
+import * as React from 'react';
+import { RefObject } from 'react';
+import { useVideoTexture } from './useVideoTexture';
+import { FacemeshApi, FacemeshProps } from './Facemesh';
+declare type VideoTextureSrc = Parameters<typeof useVideoTexture>[0];
+export declare type FaceControlsProps = {
+    camera?: THREE.Camera;
+    autostart?: boolean;
+    webcam?: boolean;
+    webcamVideoTextureSrc?: VideoTextureSrc;
+    manualUpdate?: boolean;
+    manualDetect?: boolean;
+    onVideoFrame?: (e: THREE.Event) => void;
+    makeDefault?: boolean;
+    smoothTime?: number;
+    offset?: boolean;
+    offsetScalar?: number;
+    eyes?: boolean;
+    eyesAsOrigin?: boolean;
+    depth?: number;
+    debug?: boolean;
+    facemesh?: FacemeshProps;
+};
+export declare type FaceControlsApi = THREE.EventDispatcher & {
+    detect: (video: HTMLVideoElement, time: number) => void;
+    computeTarget: () => THREE.Object3D;
+    update: (delta: number, target?: THREE.Object3D) => void;
+    facemeshApiRef: RefObject<FacemeshApi>;
+    webcamApiRef: RefObject<WebcamApi>;
+    play: () => void;
+    pause: () => void;
+};
+export declare const FaceControls: React.ForwardRefExoticComponent<FaceControlsProps & React.RefAttributes<FaceControlsApi>>;
+export declare const useFaceControls: () => FaceControlsApi;
+declare type WebcamApi = {
+    videoTextureApiRef: RefObject<VideoTextureApi>;
+};
+declare type VideoTextureApi = {
+    texture: THREE.VideoTexture;
+};
+export {};

package/core/FaceControls.js

@@ -0,0 +1,315 @@
+import _extends from '@babel/runtime/helpers/esm/extends';
+import * as THREE from 'three';
+import * as React from 'react';
+import { forwardRef, useRef, useState, useCallback, useMemo, useImperativeHandle, useEffect, Suspense, useContext, createContext } from 'react';
+import { useThree, useFrame } from '@react-three/fiber';
+import { easing } from 'maath';
+import { suspend, clear } from 'suspend-react';
+import { useVideoTexture } from './useVideoTexture.js';
+import { Facemesh } from './Facemesh.js';
+import { useFaceLandmarker } from './FaceLandmarker.js';
+
+// useVideoTexture 1st arg `src` type
+function mean(v1, v2) {
+  return v1.clone().add(v2).multiplyScalar(0.5);
+}
+
+function localToLocal(objSrc, v, objDst) {
+  // see: https://discourse.threejs.org/t/object3d-localtolocal/51564
+  const v_world = objSrc.localToWorld(v);
+  return objDst.worldToLocal(v_world);
+} //
+//
+//
+
+
+const FaceControlsContext = /*#__PURE__*/createContext({});
+const FaceControls = /*#__PURE__*/forwardRef(({
+  camera,
+  autostart = true,
+  webcam = true,
+  webcamVideoTextureSrc,
+  manualUpdate = false,
+  manualDetect = false,
+  onVideoFrame,
+  smoothTime = 0.25,
+  offset = true,
+  offsetScalar = 80,
+  eyes = false,
+  eyesAsOrigin = true,
+  depth = 0.15,
+  debug = false,
+  facemesh,
+  makeDefault
+}, fref) => {
+  var _faces$facialTransfor, _faces$faceBlendshape;
+
+  const scene = useThree(state => state.scene);
+  const defaultCamera = useThree(state => state.camera);
+  const set = useThree(state => state.set);
+  const get = useThree(state => state.get);
+  const explCamera = camera || defaultCamera;
+  const webcamApiRef = useRef(null);
+  const facemeshApiRef = useRef(null); //
+  // computeTarget()
+  //
+  // Compute `target` position and rotation for the camera (according to <Facemesh>)
+  //
+  //  1. 👀 either following the 2 eyes
+  //  2. 👤 or just the head mesh
+  //
+
+  const [target] = useState(() => new THREE.Object3D());
+  const [irisRightDirPos] = useState(() => new THREE.Vector3());
+  const [irisLeftDirPos] = useState(() => new THREE.Vector3());
+  const [irisRightLookAt] = useState(() => new THREE.Vector3());
+  const [irisLeftLookAt] = useState(() => new THREE.Vector3());
+  const computeTarget = useCallback(() => {
+    // same parent as the camera
+    target.parent = explCamera.parent;
+    const facemeshApi = facemeshApiRef.current;
+
+    if (facemeshApi) {
+      const {
+        outerRef,
+        eyeRightRef,
+        eyeLeftRef
+      } = facemeshApi;
+
+      if (eyeRightRef.current && eyeLeftRef.current) {
+        // 1. 👀
+        const {
+          irisDirRef: irisRightDirRef
+        } = eyeRightRef.current;
+        const {
+          irisDirRef: irisLeftDirRef
+        } = eyeLeftRef.current;
+
+        if (irisRightDirRef.current && irisLeftDirRef.current && outerRef.current) {
+          //
+          // position: mean of irisRightDirPos,irisLeftDirPos
+          //
+          irisRightDirPos.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current));
+          irisLeftDirPos.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current));
+          target.position.copy(localToLocal(outerRef.current, mean(irisRightDirPos, irisLeftDirPos), explCamera.parent || scene)); //
+          // lookAt: mean of irisRightLookAt,irisLeftLookAt
+          //
+
+          irisRightLookAt.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current));
+          irisLeftLookAt.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current));
+          target.lookAt(outerRef.current.localToWorld(mean(irisRightLookAt, irisLeftLookAt)));
+        }
+      } else {
+        // 2. 👤
+        if (outerRef.current) {
+          target.position.copy(localToLocal(outerRef.current, new THREE.Vector3(0, 0, 0), explCamera.parent || scene));
+          target.lookAt(outerRef.current.localToWorld(new THREE.Vector3(0, 0, 1)));
+        }
+      }
+    }
+
+    return target;
+  }, [explCamera, irisLeftDirPos, irisLeftLookAt, irisRightDirPos, irisRightLookAt, scene, target]); //
+  // update()
+  //
+  // Updating the camera `current` position and rotation, following `target`
+  //
+
+  const [current] = useState(() => new THREE.Object3D());
+  const update = useCallback(function (delta, target) {
+    if (explCamera) {
+      var _target;
+
+      (_target = target) !== null && _target !== void 0 ? _target : target = computeTarget();
+
+      if (smoothTime > 0) {
+        // damping current
+        const eps = 1e-9;
+        easing.damp3(current.position, target.position, smoothTime, delta, undefined, undefined, eps);
+        easing.dampE(current.rotation, target.rotation, smoothTime, delta, undefined, undefined, eps);
+      } else {
+        // instant
+        current.position.copy(target.position);
+        current.rotation.copy(target.rotation);
+      }
+
+      explCamera.position.copy(current.position);
+      explCamera.rotation.copy(current.rotation);
+    }
+  }, [explCamera, computeTarget, smoothTime, current.position, current.rotation]); //
+  // detect()
+  //
+
+  const [faces, setFaces] = useState();
+  const faceLandmarker = useFaceLandmarker();
+  const detect = useCallback((video, time) => {
+    const faces = faceLandmarker == null ? void 0 : faceLandmarker.detectForVideo(video, time);
+    setFaces(faces);
+  }, [faceLandmarker]);
+  useFrame((_, delta) => {
+    if (!manualUpdate) {
+      update(delta);
+    }
+  }); // Ref API
+
+  const api = useMemo(() => Object.assign(Object.create(THREE.EventDispatcher.prototype), {
+    detect,
+    computeTarget,
+    update,
+    facemeshApiRef,
+    webcamApiRef,
+    // shorthands
+    play: () => {
+      var _webcamApiRef$current, _webcamApiRef$current2;
+
+      (_webcamApiRef$current = webcamApiRef.current) == null ? void 0 : (_webcamApiRef$current2 = _webcamApiRef$current.videoTextureApiRef.current) == null ? void 0 : _webcamApiRef$current2.texture.source.data.play();
+    },
+    pause: () => {
+      var _webcamApiRef$current3, _webcamApiRef$current4;
+
+      (_webcamApiRef$current3 = webcamApiRef.current) == null ? void 0 : (_webcamApiRef$current4 = _webcamApiRef$current3.videoTextureApiRef.current) == null ? void 0 : _webcamApiRef$current4.texture.source.data.pause();
+    }
+  }), [detect, computeTarget, update]);
+  useImperativeHandle(fref, () => api, [api]); //
+  // events callbacks
+  //
+
+  useEffect(() => {
+    const onVideoFrameCb = e => {
+      if (!manualDetect) detect(e.texture.source.data, e.time);
+      if (onVideoFrame) onVideoFrame(e);
+    };
+
+    api.addEventListener('videoFrame', onVideoFrameCb);
+    return () => {
+      api.removeEventListener('videoFrame', onVideoFrameCb);
+    };
+  }, [api, detect, faceLandmarker, manualDetect, onVideoFrame]); // `controls` global state
+
+  useEffect(() => {
+    if (makeDefault) {
+      const old = get().controls;
+      set({
+        controls: api
+      });
+      return () => set({
+        controls: old
+      });
+    }
+  }, [makeDefault, api, get, set]);
+  const points = faces == null ? void 0 : faces.faceLandmarks[0];
+  const facialTransformationMatrix = faces == null ? void 0 : (_faces$facialTransfor = faces.facialTransformationMatrixes) == null ? void 0 : _faces$facialTransfor[0];
+  const faceBlendshapes = faces == null ? void 0 : (_faces$faceBlendshape = faces.faceBlendshapes) == null ? void 0 : _faces$faceBlendshape[0];
+  return /*#__PURE__*/React.createElement(FaceControlsContext.Provider, {
+    value: api
+  }, webcam && /*#__PURE__*/React.createElement(Suspense, {
+    fallback: null
+  }, /*#__PURE__*/React.createElement(Webcam, {
+    ref: webcamApiRef,
+    autostart: autostart,
+    videoTextureSrc: webcamVideoTextureSrc
+  })), /*#__PURE__*/React.createElement(Facemesh, _extends({
+    ref: facemeshApiRef
+  }, facemesh, {
+    points: points,
+    depth: depth,
+    facialTransformationMatrix: facialTransformationMatrix,
+    faceBlendshapes: faceBlendshapes,
+    eyes: eyes,
+    eyesAsOrigin: eyesAsOrigin,
+    offset: offset,
+    offsetScalar: offsetScalar,
+    debug: debug,
+    "rotation-z": Math.PI,
+    visible: debug
+  }), /*#__PURE__*/React.createElement("meshBasicMaterial", {
+    side: THREE.DoubleSide
+  })));
+});
+const useFaceControls = () => useContext(FaceControlsContext); //
+// Webcam
+//
+
+const Webcam = /*#__PURE__*/forwardRef(({
+  videoTextureSrc,
+  autostart = true
+}, fref) => {
+  const videoTextureApiRef = useRef(null);
+  const faceControls = useFaceControls();
+  const stream = suspend(async () => {
+    return !videoTextureSrc ? await navigator.mediaDevices.getUserMedia({
+      audio: false,
+      video: {
+        facingMode: 'user'
+      }
+    }) : Promise.resolve(null);
+  }, [videoTextureSrc]);
+  useEffect(() => {
+    faceControls.dispatchEvent({
+      type: 'stream',
+      stream
+    });
+    return () => {
+      stream == null ? void 0 : stream.getTracks().forEach(track => track.stop());
+      clear([videoTextureSrc]);
+    };
+  }, [stream, faceControls, videoTextureSrc]); // ref-api
+
+  const api = useMemo(() => ({
+    videoTextureApiRef
+  }), []);
+  useImperativeHandle(fref, () => api, [api]);
+  return /*#__PURE__*/React.createElement(Suspense, {
+    fallback: null
+  }, /*#__PURE__*/React.createElement(VideoTexture, {
+    ref: videoTextureApiRef,
+    src: videoTextureSrc || stream,
+    start: autostart
+  }));
+}); //
+// VideoTexture
+//
+
+const VideoTexture = /*#__PURE__*/forwardRef(({
+  src,
+  start
+}, fref) => {
+  const texture = useVideoTexture(src, {
+    start
+  });
+  const video = texture.source.data;
+  const faceControls = useFaceControls();
+  const onVideoFrame = useCallback(time => {
+    faceControls.dispatchEvent({
+      type: 'videoFrame',
+      texture,
+      time
+    });
+  }, [texture, faceControls]);
+  useVideoFrame(video, onVideoFrame); // ref-api
+
+  const api = useMemo(() => ({
+    texture
+  }), [texture]);
+  useImperativeHandle(fref, () => api, [api]);
+  return /*#__PURE__*/React.createElement(React.Fragment, null);
+});
+
+const useVideoFrame = (video, f) => {
+  // https://web.dev/requestvideoframecallback-rvfc/
+  // https://www.remotion.dev/docs/video-manipulation
+  useEffect(() => {
+    if (!video || !video.requestVideoFrameCallback) return;
+    let handle;
+
+    function callback(...args) {
+      f(...args);
+      handle = video.requestVideoFrameCallback(callback);
+    }
+
+    video.requestVideoFrameCallback(callback);
+    return () => video.cancelVideoFrameCallback(handle);
+  }, [video, f]);
+};
+
+export { FaceControls, useFaceControls };

package/core/FaceLandmarker.cjs.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("react"),t=require("@mediapipe/tasks-vision"),r=require("suspend-react");function a(e){if(e&&e.__esModule)return e;var t=Object.create(null);return e&&Object.keys(e).forEach((function(r){if("default"!==r){var a=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(t,r,a.get?a:{enumerable:!0,get:function(){return e[r]}})}})),t.default=e,Object.freeze(t)}var s=a(e);const n=e.createContext({}),o={basePath:"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm",options:{baseOptions:{modelAssetPath:"https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",delegate:"GPU"},runningMode:"VIDEO",outputFaceBlendshapes:!0,outputFacialTransformationMatrixes:!0}};exports.FaceLandmarker=function({basePath:a=o.basePath,options:i=o.options,children:c}){const u=JSON.stringify(i),l=r.suspend((async()=>await t.FilesetResolver.forVisionTasks(a).then((e=>t.FaceLandmarker.createFromOptions(e,i)))),[a,u]);return e.useEffect((()=>()=>{null==l||l.close(),r.clear([a,u])}),[l,a,u]),s.createElement(n.Provider,{value:l},c)},exports.FaceLandmarkerDefaults=o,exports.useFaceLandmarker=function(){return e.useContext(n)};

package/core/FaceLandmarker.d.ts

@@ -0,0 +1,14 @@
+import { ReactNode } from 'react';
+import { FaceLandmarker as FaceLandmarkerImpl, FaceLandmarkerOptions } from '@mediapipe/tasks-vision';
+declare type FaceLandmarkerProps = {
+    basePath?: string;
+    options?: FaceLandmarkerOptions;
+    children?: ReactNode;
+};
+export declare const FaceLandmarkerDefaults: {
+    basePath: string;
+    options: FaceLandmarkerOptions;
+};
+export declare function FaceLandmarker({ basePath, options, children, }: FaceLandmarkerProps): JSX.Element;
+export declare function useFaceLandmarker(): FaceLandmarkerImpl | undefined;
+export {};

package/core/FaceLandmarker.js

@@ -0,0 +1,43 @@
+import * as React from 'react';
+import { useEffect, useContext, createContext } from 'react';
+import { FilesetResolver, FaceLandmarker as FaceLandmarker$1 } from '@mediapipe/tasks-vision';
+import { suspend, clear } from 'suspend-react';
+
+/* eslint react-hooks/exhaustive-deps: 1 */
+const FaceLandmarkerContext = /*#__PURE__*/createContext({});
+const FaceLandmarkerDefaults = {
+  basePath: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm',
+  options: {
+    baseOptions: {
+      modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+      delegate: 'GPU'
+    },
+    runningMode: 'VIDEO',
+    outputFaceBlendshapes: true,
+    outputFacialTransformationMatrixes: true
+  }
+};
+function FaceLandmarker({
+  basePath = FaceLandmarkerDefaults.basePath,
+  options = FaceLandmarkerDefaults.options,
+  children
+}) {
+  const opts = JSON.stringify(options);
+  const faceLandmarker = suspend(async () => {
+    return await FilesetResolver.forVisionTasks(basePath).then(vision => FaceLandmarker$1.createFromOptions(vision, options));
+  }, [basePath, opts]);
+  useEffect(() => {
+    return () => {
+      faceLandmarker == null ? void 0 : faceLandmarker.close();
+      clear([basePath, opts]);
+    };
+  }, [faceLandmarker, basePath, opts]);
+  return /*#__PURE__*/React.createElement(FaceLandmarkerContext.Provider, {
+    value: faceLandmarker
+  }, children);
+}
+function useFaceLandmarker() {
+  return useContext(FaceLandmarkerContext);
+}
+
+export { FaceLandmarker, FaceLandmarkerDefaults, useFaceLandmarker };