@octanejs/drei 0.0.1

package/src/web/CycleRaycast.three.tsrx

@@ -0,0 +1,81 @@
+import { type RefObject, useThree } from '@octanejs/three';
+import { useEffect, useRef } from '@octanejs/three/renderer';
+import * as THREE from 'three';
+
+type CycleRaycastProps = {
+	onChanged?: (hits: THREE.Intersection[], cycle: number) => null;
+	preventDefault?: boolean;
+	scroll?: boolean;
+	keyCode?: number;
+	portal?: RefObject<HTMLElement>;
+};
+
+export function CycleRaycast({
+	onChanged,
+	portal,
+	preventDefault = true,
+	scroll = true,
+	keyCode = 9,
+}: CycleRaycastProps) @{
+	const cycle = useRef(0);
+	const setEvents = useThree((state) => state.setEvents);
+	const get = useThree((state) => state.get);
+	const renderer = useThree((state) => state.gl);
+	useEffect(() => {
+		let hits: THREE.Intersection[] = [];
+		let lastEvent: PointerEvent = undefined!;
+		const previous = get().events.filter;
+		const target = portal?.current ?? (renderer.domElement as Node).parentNode;
+		const renderStatus = () =>
+			target && onChanged && onChanged(hits, Math.round(cycle.current) % hits.length);
+		setEvents({
+			filter: (intersections, state) => {
+				let clone = [...intersections];
+				if (
+					clone.length !== hits.length ||
+						!hits.every((hit) => clone.map((entry) => entry.object.uuid).includes(hit.object.uuid))
+				) {
+					cycle.current = 0;
+					hits = clone;
+					renderStatus();
+				}
+				if (previous) clone = previous(clone, state);
+				for (let index = 0; index < Math.round(cycle.current) % clone.length; index++) {
+					const first = clone.shift() as THREE.Intersection;
+					clone = [...clone, first];
+				}
+				return clone;
+			},
+		});
+		const refresh = (update: (current: number) => number) => {
+			cycle.current = update(cycle.current);
+			get().events.handlers?.onPointerCancel(undefined as any);
+			get().events.handlers?.onPointerMove(lastEvent);
+			renderStatus();
+		};
+		const tabEvent = (event: KeyboardEvent) => {
+			if ((event.keyCode || event.which) === keyCode) {
+				if (preventDefault) event.preventDefault();
+				if (hits.length > 1) refresh((current) => current + 1);
+			}
+		};
+		const wheelEvent = (event: WheelEvent) => {
+			if (preventDefault) event.preventDefault();
+			let delta = 0;
+			if ((event as any).wheelDelta) delta = (event as any).wheelDelta / 120;
+			else if (event.detail) delta = -event.detail / 3;
+			if (hits.length > 1) refresh((current) => Math.abs(current - delta));
+		};
+		const moveEvent = (event: PointerEvent) => (lastEvent = event);
+		document.addEventListener('pointermove', moveEvent, { passive: true });
+		if (scroll) document.addEventListener('wheel', wheelEvent);
+		if (keyCode !== undefined) document.addEventListener('keydown', tabEvent);
+		return () => {
+			setEvents({ filter: previous });
+			if (keyCode !== undefined) document.removeEventListener('keydown', tabEvent);
+			if (scroll) document.removeEventListener('wheel', wheelEvent);
+			document.removeEventListener('pointermove', moveEvent);
+		};
+	}, [renderer, get, setEvents, preventDefault, scroll, keyCode]);
+	null;
+}

package/src/web/DragControls.three.tsrx

@@ -0,0 +1,82 @@
+/** @jsxImportSource @octanejs/three/intrinsics */
+import { type ThreeElements, type ThreeRef, useThree } from '@octanejs/three';
+import { useImperativeHandle, useLayoutEffect, useRef } from '@octanejs/three/renderer';
+import type { DragConfig } from '@use-gesture/core/types';
+import type { OctaneNode } from 'octane';
+import * as THREE from 'three';
+import { useGesture } from './useGesture.js';
+
+const initialModelPosition = new THREE.Vector3();
+const mousePosition2D = new THREE.Vector2();
+const mousePosition3D = new THREE.Vector3();
+const dragOffset = new THREE.Vector3();
+const dragPlaneNormal = new THREE.Vector3();
+const dragPlane = new THREE.Plane();
+
+type ControlsProto = { enabled: boolean };
+
+export type DragControlsProps = Omit<ThreeElements['group'], 'ref' | 'children' | 'matrix'> & {
+	autoTransform?: boolean;
+	matrix?: THREE.Matrix4;
+	axisLock?: 'x' | 'y' | 'z';
+	dragLimits?: [[number, number] | undefined, [number, number] | undefined, [number, number] | undefined];
+	onHover?: (hovering: boolean) => void;
+	onDragStart?: (origin: THREE.Vector3) => void;
+	onDrag?: (localMatrix: THREE.Matrix4, deltaLocalMatrix: THREE.Matrix4, worldMatrix: THREE.Matrix4, deltaWorldMatrix: THREE.Matrix4) => void;
+	onDragEnd?: () => void;
+	children: OctaneNode;
+	dragConfig?: DragConfig;
+};
+
+export function DragControls({
+	autoTransform = true, matrix, axisLock, dragLimits, onHover, onDragStart, onDrag, onDragEnd, children, dragConfig, ref: forwardedRef, ...props
+}: DragControlsProps & { ref?: ThreeRef<THREE.Group> }) @{
+	const defaultControls = useThree((state) => (state as any).controls) as ControlsProto | undefined;
+	const { camera, size, raycaster, invalidate } = useThree();
+	const ref = useRef<THREE.Group | null>(null);
+	const bind = useGesture({
+		onHover: ({ hovering }: any) => onHover?.(hovering ?? false),
+		onDragStart: ({ event }: any) => {
+			if (defaultControls) defaultControls.enabled = false;
+			const { point } = event;
+			ref.current!.matrix.decompose(initialModelPosition, new THREE.Quaternion(), new THREE.Vector3());
+			mousePosition3D.copy(point);
+			dragOffset.copy(mousePosition3D).sub(initialModelPosition);
+			onDragStart?.(initialModelPosition);
+			invalidate();
+		},
+		onDrag: ({ xy: [dragX, dragY], intentional }: any) => {
+			if (!intentional) return;
+			const normalizedMouseX = ((dragX - size.left) / size.width) * 2 - 1;
+			const normalizedMouseY = -((dragY - size.top) / size.height) * 2 + 1;
+			mousePosition2D.set(normalizedMouseX, normalizedMouseY);
+			raycaster.setFromCamera(mousePosition2D, camera);
+			if (!axisLock) camera.getWorldDirection(dragPlaneNormal).negate();
+			else dragPlaneNormal.set(axisLock === 'x' ? 1 : 0, axisLock === 'y' ? 1 : 0, axisLock === 'z' ? 1 : 0);
+			dragPlane.setFromNormalAndCoplanarPoint(dragPlaneNormal, mousePosition3D);
+			raycaster.ray.intersectPlane(dragPlane, mousePosition3D);
+			const previousLocalMatrix = ref.current!.matrix.clone();
+			const previousWorldMatrix = ref.current!.matrixWorld.clone();
+			const intendedNewPosition = new THREE.Vector3(mousePosition3D.x - dragOffset.x, mousePosition3D.y - dragOffset.y, mousePosition3D.z - dragOffset.z);
+			if (dragLimits) {
+				intendedNewPosition.x = dragLimits[0] ? THREE.MathUtils.clamp(intendedNewPosition.x, ...dragLimits[0]) : intendedNewPosition.x;
+				intendedNewPosition.y = dragLimits[1] ? THREE.MathUtils.clamp(intendedNewPosition.y, ...dragLimits[1]) : intendedNewPosition.y;
+				intendedNewPosition.z = dragLimits[2] ? THREE.MathUtils.clamp(intendedNewPosition.z, ...dragLimits[2]) : intendedNewPosition.z;
+			}
+			const localMatrix = autoTransform ? ref.current!.matrix : new THREE.Matrix4().copy(ref.current!.matrix);
+			localMatrix.setPosition(intendedNewPosition);
+			const deltaLocalMatrix = localMatrix.clone().multiply(previousLocalMatrix.invert());
+			const deltaWorldMatrix = localMatrix.clone().multiply(previousWorldMatrix.invert());
+			onDrag?.(localMatrix, deltaLocalMatrix, ref.current!.matrixWorld, deltaWorldMatrix);
+			invalidate();
+		},
+		onDragEnd: () => {
+			if (defaultControls) defaultControls.enabled = true;
+			onDragEnd?.();
+			invalidate();
+		},
+	}, { drag: { filterTaps: true, threshold: 1, ...(typeof dragConfig === 'object' ? dragConfig : {}) } });
+	useImperativeHandle((forwardedRef ?? null) as any, () => ref.current!, []);
+	useLayoutEffect(() => { if (matrix) ref.current!.matrix = matrix; }, [matrix]);
+	<group ref={ref} {...((bind?.() ?? {}) as any)} matrix={matrix as any} matrixAutoUpdate={false} {...props}>{children}</group>
+}

package/src/web/FaceControls.three.tsrx

@@ -0,0 +1,178 @@
+/** @jsxImportSource @octanejs/three/intrinsics */
+import type { FaceLandmarker, FaceLandmarkerResult } from '@mediapipe/tasks-vision';
+import { type ThreeRef, useFrame, useThree } from '@octanejs/three';
+import {
+	type RefObject,
+	useCallback,
+	useEffect,
+	useImperativeHandle,
+	useMemo,
+	useRef,
+	useState,
+} from '@octanejs/three/renderer';
+import { createContext, useContext } from 'octane';
+import { easing } from 'maath';
+import * as THREE from 'three';
+import { VideoTexture, type VideoTextureProps } from '../core/VideoTexture.three.tsrx';
+import { Facemesh, type FacemeshApi, type FacemeshProps } from './Facemesh.three.tsrx';
+import { useFaceLandmarker } from './FaceLandmarker.three.tsrx';
+import { WebcamVideoTexture } from './media-textures.three.tsrx';
+
+type VideoFrame = Parameters<FaceLandmarker['detectForVideo']>[0];
+
+function mean(v1: THREE.Vector3, v2: THREE.Vector3): THREE.Vector3 {
+	return v1.clone().add(v2).multiplyScalar(0.5);
+}
+
+function localToLocal(objSrc: THREE.Object3D, value: THREE.Vector3, objDst: THREE.Object3D): THREE.Vector3 {
+	return objDst.worldToLocal(objSrc.localToWorld(value));
+}
+
+export type FaceControlsProps = {
+	camera?: THREE.Camera;
+	videoTexture?: VideoTextureProps;
+	manualDetect?: boolean;
+	faceLandmarkerResult?: FaceLandmarkerResult;
+	manualUpdate?: boolean;
+	makeDefault?: boolean;
+	smoothTime?: number;
+	offset?: boolean;
+	offsetScalar?: number;
+	eyes?: boolean;
+	eyesAsOrigin?: boolean;
+	depth?: number;
+	debug?: boolean;
+	facemesh?: FacemeshProps;
+};
+
+export type FaceControlsApi = THREE.EventDispatcher & {
+	computeTarget: () => THREE.Object3D;
+	update: (delta: number, target?: THREE.Object3D) => void;
+	facemeshApiRef: RefObject<FacemeshApi | null>;
+};
+
+const FaceControlsContext = createContext({} as FaceControlsApi);
+
+export function FaceControls({
+	camera,
+	videoTexture = { start: true } as VideoTextureProps,
+	manualDetect = false,
+	faceLandmarkerResult,
+	manualUpdate = false,
+	makeDefault,
+	smoothTime = 0.25,
+	offset = true,
+	offsetScalar = 80,
+	eyes = false,
+	eyesAsOrigin = true,
+	depth = 0.15,
+	debug = false,
+	facemesh,
+	ref,
+}: FaceControlsProps & { ref?: ThreeRef<FaceControlsApi> }) @{
+	const scene = useThree((state) => state.scene);
+	const defaultCamera = useThree((state) => state.camera);
+	const set = useThree((state) => state.set);
+	const get = useThree((state) => state.get);
+	const explCamera = camera || defaultCamera;
+	const facemeshApiRef = useRef<FacemeshApi | null>(null);
+	const [target] = useState(() => new THREE.Object3D());
+	const [irisRightDirPos] = useState(() => new THREE.Vector3());
+	const [irisLeftDirPos] = useState(() => new THREE.Vector3());
+	const [irisRightLookAt] = useState(() => new THREE.Vector3());
+	const [irisLeftLookAt] = useState(() => new THREE.Vector3());
+	const computeTarget = useCallback<FaceControlsApi['computeTarget']>(() => {
+		target.parent = explCamera.parent;
+		const facemeshApi = facemeshApiRef.current;
+		if (facemeshApi) {
+			const { outerRef, eyeRightRef, eyeLeftRef } = facemeshApi;
+			if (eyeRightRef.current && eyeLeftRef.current) {
+				const { irisDirRef: irisRightDirRef } = eyeRightRef.current;
+				const { irisDirRef: irisLeftDirRef } = eyeLeftRef.current;
+				if (irisRightDirRef.current && irisLeftDirRef.current && outerRef.current) {
+					irisRightDirPos.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current));
+					irisLeftDirPos.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current));
+					target.position.copy(localToLocal(outerRef.current, mean(irisRightDirPos, irisLeftDirPos), explCamera.parent || scene));
+					irisRightLookAt.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current));
+					irisLeftLookAt.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current));
+					target.lookAt(outerRef.current.localToWorld(mean(irisRightLookAt, irisLeftLookAt)));
+				}
+			} else if (outerRef.current) {
+				target.position.copy(localToLocal(outerRef.current, new THREE.Vector3(0, 0, 0), explCamera.parent || scene));
+				target.lookAt(outerRef.current.localToWorld(new THREE.Vector3(0, 0, 1)));
+			}
+		}
+		return target;
+	}, [explCamera, irisLeftDirPos, irisLeftLookAt, irisRightDirPos, irisRightLookAt, scene, target]);
+
+	const [current] = useState(() => new THREE.Object3D());
+	const update = useCallback<FaceControlsApi['update']>((delta, nextTarget) => {
+		if (explCamera) {
+			nextTarget ??= computeTarget();
+			if (smoothTime > 0) {
+				const eps = 1e-9;
+				easing.damp3(current.position, nextTarget.position, smoothTime, delta, undefined, undefined, eps);
+				easing.dampE(current.rotation, nextTarget.rotation, smoothTime, delta, undefined, undefined, eps);
+			} else {
+				current.position.copy(nextTarget.position);
+				current.rotation.copy(nextTarget.rotation);
+			}
+			explCamera.position.copy(current.position);
+			explCamera.rotation.copy(current.rotation);
+		}
+	}, [explCamera, computeTarget, smoothTime, current.position, current.rotation]);
+	useFrame((_, delta) => { if (!manualUpdate) update(delta); });
+
+	const videoTextureRef = useRef<THREE.VideoTexture | null>(null);
+	const [_faceLandmarkerResult, setFaceLandmarkerResult] = useState<FaceLandmarkerResult | undefined>(undefined);
+	const faceLandmarker = useFaceLandmarker();
+	const onVideoFrame = useCallback<NonNullable<VideoTextureProps['onVideoFrame']>>((now) => {
+		const texture = videoTextureRef.current;
+		if (!texture) return;
+		const videoFrame = texture.source.data as VideoFrame;
+		setFaceLandmarkerResult(faceLandmarker?.detectForVideo(videoFrame, now));
+	}, [faceLandmarker]);
+
+	const api = useMemo<FaceControlsApi>(() => Object.assign(Object.create(THREE.EventDispatcher.prototype), {
+		computeTarget, update, facemeshApiRef,
+	}), [computeTarget, update]);
+	useImperativeHandle((ref ?? null) as any, () => api, [api]);
+	useEffect(() => {
+		if (makeDefault) {
+			const old = get().controls;
+			set({ controls: api });
+			return () => set({ controls: old });
+		}
+	}, [makeDefault, api, get, set]);
+
+	const result = faceLandmarkerResult ?? _faceLandmarkerResult;
+	const points = result?.faceLandmarks[0];
+	const facialTransformationMatrix = result?.facialTransformationMatrixes?.[0];
+	const faceBlendshapes = result?.faceBlendshapes?.[0];
+	const videoTextureProps = { onVideoFrame, ...videoTexture };
+	const facemeshChildren = facemesh?.children ?? <meshNormalMaterial side={THREE.DoubleSide} />;
+	<FaceControlsContext.Provider value={api}>
+		{!manualDetect &&
+			('src' in videoTextureProps
+				? <VideoTexture ref={videoTextureRef} {...videoTextureProps as VideoTextureProps} />
+				: <WebcamVideoTexture ref={videoTextureRef} {...videoTextureProps as any} />)}
+		<Facemesh
+			ref={facemeshApiRef}
+			{...facemesh as any}
+			points={points}
+			depth={depth}
+			facialTransformationMatrix={facialTransformationMatrix}
+			faceBlendshapes={faceBlendshapes}
+			eyes={eyes}
+			eyesAsOrigin={eyesAsOrigin}
+			offset={offset}
+			offsetScalar={offsetScalar}
+			debug={debug}
+			rotation-z={Math.PI}
+			visible={debug}
+			children={facemeshChildren}
+		/>
+	</FaceControlsContext.Provider>
+}
+
+export const useFaceControls = (): FaceControlsApi => useContext(FaceControlsContext);

package/src/web/FaceLandmarker.three.tsrx

@@ -0,0 +1,48 @@
+/** @jsxImportSource @octanejs/three/intrinsics */
+import type { FaceLandmarker as FaceLandmarkerImpl, FaceLandmarkerOptions } from '@mediapipe/tasks-vision';
+import { use, useEffect, useImperativeHandle } from '@octanejs/three/renderer';
+import { createContext, type OctaneNode, useContext } from 'octane';
+import { clearResource, resourcePromise } from '../core/resource.js';
+import { loadFaceLandmarker } from './face-landmarker-resource.js';
+
+const FaceLandmarkerContext = createContext<FaceLandmarkerImpl | undefined>(undefined);
+
+export type FaceLandmarkerProps = {
+	basePath?: string;
+	options?: FaceLandmarkerOptions;
+	children?: OctaneNode;
+};
+
+export const FaceLandmarkerDefaults = {
+	basePath: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17/wasm',
+	options: {
+		baseOptions: {
+			modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+			delegate: 'GPU',
+		},
+		runningMode: 'VIDEO',
+		outputFaceBlendshapes: true,
+		outputFacialTransformationMatrixes: true,
+	} as FaceLandmarkerOptions,
+};
+
+export function FaceLandmarker({
+	basePath = FaceLandmarkerDefaults.basePath,
+	options = FaceLandmarkerDefaults.options,
+	children,
+	ref,
+}: FaceLandmarkerProps & { ref?: unknown }) @{
+	const serializedOptions = JSON.stringify(options);
+	const keys = ['face-landmarker', basePath, serializedOptions] as const;
+	const faceLandmarker = use(resourcePromise(loadFaceLandmarker, keys));
+	useEffect(() => () => {
+		faceLandmarker?.close();
+		clearResource(keys);
+	}, [faceLandmarker, basePath, serializedOptions]);
+	useImperativeHandle((ref ?? null) as any, () => faceLandmarker, [faceLandmarker]);
+	<FaceLandmarkerContext.Provider value={faceLandmarker}>{children}</FaceLandmarkerContext.Provider>
+}
+
+export function useFaceLandmarker(): FaceLandmarkerImpl | undefined {
+	return useContext(FaceLandmarkerContext);
+}