@viamrobotics/motion-tools 0.16.4 → 0.17.0

package/dist/components/Geometry.svelte

@@ -10,7 +10,14 @@
 	import type { WorldObject } from '../WorldObject.svelte'
 	import { PLYLoader } from 'three/addons/loaders/PLYLoader.js'
 
+	import { WEBLABS_EXPERIMENTS } from '../hooks/useWeblabs.svelte'
+	import { useSettings } from '../hooks/useSettings.svelte'
+	import { useWeblabs } from '../hooks/useWeblabs.svelte'
+	import { use3DModels } from '../hooks/use3DModels.svelte'
+	const settings = useSettings()
 	const plyLoader = new PLYLoader()
+	const weblabs = useWeblabs()
+	const componentModels = use3DModels()
 
 	interface Props extends ThrelteProps<Group> {
 		uuid: string
@@ -33,9 +40,28 @@
 		...rest
 	}: Props = $props()
 
+	const gltfModel = $derived.by(() => {
+		const [componentName, id] = name.split(':')
+		if (!componentName || !id) {
+			return undefined
+		}
+		return componentModels.current?.[componentName]?.[id]
+	})
+
 	const type = $derived(geometry?.geometryType?.case)
 	const color = $derived(overrideColor ?? metadata.color ?? colors.default)
 
+	const renderModels = $derived(
+		(settings.current.renderArmModels === 'model' ||
+			settings.current.renderArmModels === 'colliders+model') &&
+			gltfModel
+	)
+	const renderPrimitives = $derived(
+		settings.current.renderArmModels === 'colliders' ||
+			settings.current.renderArmModels === 'colliders+model' ||
+			!gltfModel
+	)
+
 	const group = new Group()
 	const mesh = $derived.by(() => {
 		if (type === undefined) {
@@ -50,6 +76,16 @@
 		return result
 	})
 
+	$effect.pre(() => {
+		if (
+			weblabs.isActive(WEBLABS_EXPERIMENTS.MOTION_TOOLS_RENDER_ARM_MODELS) &&
+			renderModels &&
+			!renderPrimitives
+		) {
+			geo = undefined
+		}
+	})
+
 	$effect.pre(() => {
 		if (geometry?.center && mesh) {
 			poseToObject3d(geometry.center, mesh)
@@ -103,39 +139,45 @@
 			{uuid}
 			bvh={{ enabled: false }}
 		>
-			{#if geometry.geometryType.case === 'bufferGeometry'}
-				<T
-					is={geometry.geometryType.value}
-					{oncreate}
-				/>
-			{:else if geometry.geometryType.case === 'mesh'}
-				{@const mesh = geometry.geometryType.value.mesh}
-				{@const meshGeometry = parsePlyInput(mesh)}
-				<T
-					is={meshGeometry}
-					{oncreate}
-				/>
-			{:else if geometry.geometryType.case === 'line' && metadata.points}
-				<MeshLineGeometry points={metadata.points} />
-			{:else if geometry.geometryType.case === 'box'}
-				{@const dimsMm = geometry.geometryType.value.dimsMm ?? { x: 0, y: 0, z: 0 }}
-				<T.BoxGeometry
-					args={[dimsMm.x * 0.001, dimsMm.y * 0.001, dimsMm.z * 0.001]}
-					{oncreate}
-				/>
-			{:else if geometry.geometryType.case === 'sphere'}
-				{@const radiusMm = geometry.geometryType.value.radiusMm ?? 0}
-				<T.SphereGeometry
-					args={[radiusMm * 0.001]}
-					{oncreate}
-				/>
-			{:else if geometry.geometryType.case === 'capsule'}
-				{@const { lengthMm, radiusMm } = geometry.geometryType.value}
-				<T
-					is={CapsuleGeometry}
-					args={[radiusMm * 0.001, lengthMm * 0.001]}
-					{oncreate}
-				/>
+			{#if weblabs.isActive(WEBLABS_EXPERIMENTS.MOTION_TOOLS_RENDER_ARM_MODELS) && renderModels}
+				<T is={gltfModel} />
+			{/if}
+
+			{#if !weblabs.isActive(WEBLABS_EXPERIMENTS.MOTION_TOOLS_RENDER_ARM_MODELS) || renderPrimitives}
+				{#if geometry.geometryType.case === 'bufferGeometry'}
+					<T
+						is={geometry.geometryType.value}
+						{oncreate}
+					/>
+				{:else if geometry.geometryType.case === 'mesh'}
+					{@const mesh = geometry.geometryType.value.mesh}
+					{@const meshGeometry = parsePlyInput(mesh)}
+					<T
+						is={meshGeometry}
+						{oncreate}
+					/>
+				{:else if geometry.geometryType.case === 'line' && metadata.points}
+					<MeshLineGeometry points={metadata.points} />
+				{:else if geometry.geometryType.case === 'box'}
+					{@const dimsMm = geometry.geometryType.value.dimsMm ?? { x: 0, y: 0, z: 0 }}
+					<T.BoxGeometry
+						args={[dimsMm.x * 0.001, dimsMm.y * 0.001, dimsMm.z * 0.001]}
+						{oncreate}
+					/>
+				{:else if geometry.geometryType.case === 'sphere'}
+					{@const radiusMm = geometry.geometryType.value.radiusMm ?? 0}
+					<T.SphereGeometry
+						args={[radiusMm * 0.001]}
+						{oncreate}
+					/>
+				{:else if geometry.geometryType.case === 'capsule'}
+					{@const { lengthMm, radiusMm } = geometry.geometryType.value}
+					<T
+						is={CapsuleGeometry}
+						args={[radiusMm * 0.001, lengthMm * 0.001]}
+						{oncreate}
+					/>
+				{/if}
 			{/if}
 
 			{#if geometry.geometryType.case === 'line'}

package/dist/components/SceneProviders.svelte

@@ -23,6 +23,7 @@
 	import { provideArrows } from '../hooks/useArrows.svelte'
 	import { provideFramelessComponents } from '../hooks/useFramelessComponents.svelte'
 	import { provideResourceByName } from '../hooks/useResourceByName.svelte'
+	import { provide3DModels } from '../hooks/use3DModels.svelte'
 
 	interface Props {
 		cameraPose?: CameraPose
@@ -47,6 +48,7 @@
 	provideResourceByName(() => partID.current)
 	provideFrames(() => partID.current)
 	provideGeometries(() => partID.current)
+	provide3DModels(() => partID.current)
 	providePointclouds(() => partID.current)
 	provideMotionClient(() => partID.current)
 	provideArmClient(() => partID.current)

package/dist/components/Tree/Settings.svelte

@@ -7,6 +7,8 @@
 	import { useResourceNames } from '@viamrobotics/svelte-sdk'
 	import { usePartID } from '../../hooks/usePartID.svelte'
 	import { RefreshRates, useMachineSettings } from '../../hooks/useMachineSettings.svelte'
+	import WeblabActive from '../weblab/WeblabActive.svelte'
+	import { WEBLABS_EXPERIMENTS } from '../../hooks/useWeblabs.svelte'
 
 	const partID = usePartID()
 	const cameras = useResourceNames(() => partID.current, 'camera')
@@ -167,6 +169,26 @@
 			<label class="flex items-center justify-between gap-2">
 				Render stats <Switch bind:on={settings.current.renderStats} />
 			</label>
+			<WeblabActive experiment={WEBLABS_EXPERIMENTS.MOTION_TOOLS_RENDER_ARM_MODELS}>
+				<label class="flex items-center justify-between gap-2">
+					Render Arm Models
+					<Select
+						value={settings.current.renderArmModels}
+						onchange={(event: InputEvent) => {
+							if (event.target instanceof HTMLSelectElement) {
+								settings.current.renderArmModels = event.target.value as
+									| 'colliders'
+									| 'colliders+model'
+									| 'model'
+							}
+						}}
+					>
+						<option value="colliders">Colliders</option>
+						<option value="colliders+model">Colliders + Model</option>
+						<option value="model">Model</option>
+					</Select>
+				</label>
+			</WeblabActive>
 		</div>
 	</div>
 </Drawer>

package/dist/components/xr/XR.svelte.d.ts

@@ -1,3 +1,4 @@
-declare const Xr: import("svelte").Component<Record<string, any>, {}, "">;
-type Xr = ReturnType<typeof Xr>;
-export default Xr;
+import { XR } from '@threlte/xr';
+declare const XR: import("svelte").Component<Record<string, any>, {}, "">;
+type XR = ReturnType<typeof XR>;
+export default XR;

package/dist/hooks/use3DModels.svelte.d.ts

@@ -0,0 +1,7 @@
+import type { Group } from 'three';
+interface Context {
+    current: Record<string, Record<string, Group>>;
+}
+export declare const provide3DModels: (partID: () => string) => void;
+export declare const use3DModels: () => Context;
+export {};

package/dist/hooks/use3DModels.svelte.js

@@ -0,0 +1,59 @@
+import { ArmClient } from '@viamrobotics/sdk';
+import { createResourceClient, useResourceNames } from '@viamrobotics/svelte-sdk';
+import { getContext, setContext } from 'svelte';
+import { useWeblabs, WEBLABS_EXPERIMENTS } from './useWeblabs.svelte';
+import { useSettings } from './useSettings.svelte';
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { DRACOLoader } from 'three/addons/loaders/DRACOLoader.js';
+const gltfLoader = new GLTFLoader();
+const dracoLoader = new DRACOLoader();
+dracoLoader.setDecoderPath('https://www.gstatic.com/draco/versioned/decoders/1.5.6/');
+gltfLoader.setDRACOLoader(dracoLoader);
+const key = Symbol('3d-models-context');
+export const provide3DModels = (partID) => {
+    const weblabs = useWeblabs();
+    const settings = useSettings();
+    const current = $state.raw({});
+    const arms = useResourceNames(partID, 'arm');
+    const armClients = $derived(arms.current.map((arm) => createResourceClient(ArmClient, partID, () => arm.name)));
+    const clients = $derived(armClients.filter((client) => {
+        return arms.current.some((arm) => arm.name === client.current?.name);
+    }));
+    $effect(() => {
+        const fetch3DModels = async () => {
+            for (const client of clients) {
+                if (!client.current)
+                    continue;
+                try {
+                    const models = await client.current.get3DModels();
+                    if (!(client.current.name in current)) {
+                        current[client.current.name] = {};
+                    }
+                    for (const [id, model] of Object.entries(models)) {
+                        const arrayBuffer = model.mesh.buffer.slice(model.mesh.byteOffset, model.mesh.byteOffset + model.mesh.byteLength);
+                        const gltfModel = await gltfLoader.parseAsync(arrayBuffer, '');
+                        current[client.current.name][id] = gltfModel.scene;
+                    }
+                }
+                catch (error) {
+                    // some arms may not implement this api yet
+                    console.warn(`${client.current.name} returned an error: ${error} when getting 3D models`);
+                }
+            }
+        };
+        const shouldFetchModels = settings.current.isLoaded &&
+            (settings.current.renderArmModels === 'model' ||
+                settings.current.renderArmModels === 'colliders+model');
+        if (weblabs.isActive(WEBLABS_EXPERIMENTS.MOTION_TOOLS_RENDER_ARM_MODELS) && shouldFetchModels) {
+            fetch3DModels();
+        }
+    });
+    setContext(key, {
+        get current() {
+            return current;
+        },
+    });
+};
+export const use3DModels = () => {
+    return getContext(key);
+};

package/dist/hooks/useSettings.svelte.d.ts

@@ -1,4 +1,5 @@
 interface Settings {
+    isLoaded: boolean;
     cameraMode: 'orthographic' | 'perspective';
     transforming: boolean;
     snapping: boolean;
@@ -18,6 +19,7 @@ interface Settings {
     enableXR: boolean;
     enableArmPositionsWidget: boolean;
     renderStats: boolean;
+    renderArmModels: 'colliders' | 'colliders+model' | 'model';
 }
 interface Context {
     current: Settings;

package/dist/hooks/useSettings.svelte.js

@@ -2,6 +2,7 @@ import { get, set } from 'idb-keyval';
 import { getContext, setContext } from 'svelte';
 const key = Symbol('dashboard-context');
 const defaults = () => ({
+    isLoaded: false,
     cameraMode: 'perspective',
     transforming: false,
     snapping: false,
@@ -21,6 +22,7 @@ const defaults = () => ({
     enableXR: false,
     enableArmPositionsWidget: false,
     renderStats: false,
+    renderArmModels: 'colliders+model',
 });
 export const provideSettings = () => {
     let settings = $state(defaults());
@@ -30,6 +32,7 @@ export const provideSettings = () => {
             settings = { ...settings, ...response };
         }
         settingsLoaded = true;
+        settings.isLoaded = true;
     });
     $effect(() => {
         if (settingsLoaded) {

package/dist/hooks/useWeblabs.svelte.d.ts

@@ -1,5 +1,6 @@
 export declare const WEBLABS_EXPERIMENTS: {
     readonly MOTION_TOOLS_EDIT_FRAME: "MOTION_TOOLS_EDIT_FRAME";
+    readonly MOTION_TOOLS_RENDER_ARM_MODELS: "MOTION_TOOLS_RENDER_ARM_MODELS";
 };
 export declare const WEBLABS_CONTEXT_KEY: unique symbol;
 interface Context {

package/dist/hooks/useWeblabs.svelte.js

@@ -2,6 +2,7 @@ import { getContext, setContext } from 'svelte';
 import { SvelteSet } from 'svelte/reactivity';
 export const WEBLABS_EXPERIMENTS = {
     MOTION_TOOLS_EDIT_FRAME: 'MOTION_TOOLS_EDIT_FRAME',
+    MOTION_TOOLS_RENDER_ARM_MODELS: 'MOTION_TOOLS_RENDER_ARM_MODELS',
 };
 export const WEBLABS_CONTEXT_KEY = Symbol('weblabs-context');
 const getCookie = (name) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viamrobotics/motion-tools",
-  "version": "0.16.4",
+  "version": "0.17.0",
   "description": "Motion visualization with Viam",
   "license": "Apache-2.0",
   "type": "module",
@@ -37,7 +37,7 @@
     "@typescript-eslint/eslint-plugin": "8.42.0",
     "@typescript-eslint/parser": "8.42.0",
     "@viamrobotics/prime-core": "0.1.5",
-    "@viamrobotics/sdk": "0.52.0",
+    "@viamrobotics/sdk": "0.55.0",
     "@viamrobotics/svelte-sdk": "0.7.1",
     "@vitejs/plugin-basic-ssl": "2.1.0",
     "@zag-js/svelte": "1.22.1",

package/dist/WorldObject.svelte.d.ts

@@ -1,65 +0,0 @@
-import type { Geometry, PlainMessage, Pose, Struct, TransformWithUUID } from '@viamrobotics/sdk';
-import { BatchedMesh, Color, Object3D, Vector3, type BufferGeometry } from 'three';
-import type { ValueOf } from 'type-fest';
-import type { OBB } from 'three/addons/math/OBB.js';
-export type PointsGeometry = {
-    center: undefined;
-    geometryType: {
-        case: 'points';
-        value: Float32Array<ArrayBuffer>;
-    };
-};
-export type LinesGeometry = {
-    center: undefined;
-    geometryType: {
-        case: 'line';
-        value: Float32Array;
-    };
-};
-export type ThreeBufferGeometry = {
-    center: undefined;
-    geometryType: {
-        case: 'bufferGeometry';
-        value: BufferGeometry;
-    };
-};
-export type Geometries = Geometry | PointsGeometry | LinesGeometry | ThreeBufferGeometry;
-export declare const SupportedShapes: {
-    readonly points: "points";
-    readonly line: "line";
-    readonly arrow: "arrow";
-};
-export type Metadata = {
-    colors?: Float32Array;
-    color?: Color;
-    opacity?: number;
-    gltf?: {
-        scene: Object3D;
-    };
-    points?: Vector3[];
-    pointSize?: number;
-    lineWidth?: number;
-    lineDotColor?: Color;
-    batched?: {
-        id: number;
-        object: BatchedMesh;
-    };
-    shape?: ValueOf<typeof SupportedShapes>;
-    getBoundingBoxAt?: (box: OBB) => void;
-};
-export declare const isMetadataKey: (key: string) => key is keyof Metadata;
-export declare class WorldObject<T extends Geometries = Geometries> {
-    uuid: string;
-    name: string;
-    referenceFrame: string | undefined;
-    pose: PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Pose>;
-    geometry?: T;
-    metadata: Metadata;
-    localEditedPose: PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Pose>;
-    constructor(name?: string, pose?: Pose, parent?: string, geometry?: T, metadata?: Metadata);
-    toJSON(): Omit<WorldObject, 'toJSON' | 'fromJSON' | 'metadata'>;
-    fromJSON(json: WorldObject): this;
-}
-export declare const parseMetadata: (fields?: PlainMessage<Struct>["fields"]) => Metadata;
-export declare const fromTransform: (transform: TransformWithUUID) => WorldObject<PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Geometry>>;
-export declare const determinePose: (object: WorldObject, pose: Pose | undefined) => Pose;

package/dist/geometry.d.ts

@@ -1,4 +0,0 @@
-import type { Geometry } from '@viamrobotics/sdk';
-import type { Frame } from './frame';
-export declare const createGeometry: (geometryType?: Geometry["geometryType"], label?: string) => Geometry;
-export declare const createGeometryFromFrame: (frame: Frame) => import("@viamrobotics/sdk").PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Geometry> | undefined;

package/dist/hooks/usePose.svelte.d.ts

@@ -1,3 +0,0 @@
-export declare const usePose: (name: () => string, parent: () => string | undefined) => {
-    readonly current: import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Pose | undefined;
-};

package/dist/hooks/useWorldState.svelte.d.ts

@@ -1,22 +0,0 @@
-import { type TransformWithUUID, ResourceName } from '@viamrobotics/sdk';
-interface Context {
-    names: ResourceName[];
-    current: Record<string, ReturnType<typeof createWorldState>>;
-}
-export declare const provideWorldStates: () => void;
-export declare const useWorldStates: () => Context;
-export declare const useWorldState: (resourceName: () => string) => {
-    readonly name: string;
-    readonly transforms: TransformWithUUID[];
-    readonly worldObjects: import("../WorldObject.svelte").WorldObject<import("@viamrobotics/sdk").PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Geometry>>[];
-    readonly listUUIDs: import("@tanstack/svelte-query").QueryObserverResult<string[]>;
-    readonly getTransforms: import("@tanstack/svelte-query").QueryObserverResult<TransformWithUUID>[] | undefined;
-};
-declare const createWorldState: (partID: () => string, resourceName: () => string) => {
-    readonly name: string;
-    readonly transforms: TransformWithUUID[];
-    readonly worldObjects: import("../WorldObject.svelte").WorldObject<import("@viamrobotics/sdk").PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Geometry>>[];
-    readonly listUUIDs: import("@tanstack/svelte-query").QueryObserverResult<string[]>;
-    readonly getTransforms: import("@tanstack/svelte-query").QueryObserverResult<TransformWithUUID>[] | undefined;
-};
-export {};

package/dist/transform.d.ts

@@ -1,15 +0,0 @@
-import type { Geometry, Pose } from '@viamrobotics/sdk';
-import { type Object3D, Matrix4, Quaternion, Vector3 } from 'three';
-import type { Frame } from './frame';
-export declare const createPose: (pose?: Partial<Pose>) => Pose;
-export declare const createPoseFromFrame: (frame: Partial<Frame>) => Pose;
-export declare const quaternionToPose: (quaternion: Quaternion, pose: Partial<Pose>) => void;
-export declare const vector3ToPose: (vec3: Vector3, pose: Partial<Pose>) => void;
-export declare const object3dToPose: (object3d: Object3D, pose: Partial<Pose>) => Partial<import("@viamrobotics/sdk").PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Pose>>;
-export declare const poseToQuaternion: (pose: Partial<Pose>, quaternion: Quaternion) => void;
-export declare const poseToVector3: (pose: Partial<Pose>, vec3: Vector3) => void;
-export declare const poseToObject3d: (pose: Partial<Pose>, object3d: Object3D) => void;
-export declare const poseToDirection: (pose: Pose) => Vector3;
-export declare const scaleToDimensions: (scale: Vector3, geometry: Geometry["geometryType"]) => void;
-export declare const poseToMatrix: (pose: Pose) => Matrix4;
-export declare const matrixToPose: (matrix: Matrix4) => import("@viamrobotics/sdk").PlainMessage<import("@viamrobotics/sdk/dist/gen/common/v1/common_pb").Pose>;