@viamrobotics/motion-tools 1.10.0 → 1.11.0

package/dist/components/Geometry.svelte

@@ -1,57 +1,84 @@
-<!--
-
-This component is consumed as a library export
-and should remain pure, i.e. no hooks should be used.
-
--->
 <script lang="ts">
-	import { T, type Props as ThrelteProps } from '@threlte/core'
+	import { T, useThrelte, type Props as ThrelteProps } from '@threlte/core'
 	import { type Snippet } from 'svelte'
-	import { meshBounds, MeshLineGeometry, MeshLineMaterial } from '@threlte/extras'
-	import { BufferGeometry, DoubleSide, FrontSide, Group, Mesh } from 'three'
+	import { meshBounds } from '@threlte/extras'
+	import { BufferGeometry, Color, DoubleSide, FrontSide, Group, Mesh } from 'three'
+	import { Line2, LineGeometry, LineMaterial } from 'three/examples/jsm/Addons.js'
 	import { CapsuleGeometry } from '../three/CapsuleGeometry'
-	import { poseToObject3d } from '../transform'
 	import { colors, darkenColor } from '../color'
 	import AxesHelper from './AxesHelper.svelte'
-	import type { WorldObject } from '../WorldObject.svelte'
-	import { parsePlyInput } from '../ply'
+	import type { Entity } from 'koota'
+	import { traits, useTrait } from '../ecs'
+	import { poseToObject3d } from '../transform'
+	import type { Pose } from '@viamrobotics/sdk'
 
 	interface Props extends ThrelteProps<Group> {
-		uuid: string
-		name: string
-		geometry?: WorldObject['geometry']
-		pose: WorldObject['pose']
-		metadata: WorldObject['metadata']
+		entity: Entity
 		color?: string
 		model?: Group
+		pose?: Pose
 		renderMode?: 'model' | 'colliders' | 'colliders+model'
+		ref?: Group
 		children?: Snippet<[{ ref: Group }]>
 	}
 
 	let {
-		uuid,
-		name,
-		geometry,
-		metadata,
-		pose,
+		entity,
 		color: overrideColor,
 		model,
 		renderMode = 'colliders',
+		pose,
+		ref = $bindable(),
 		children,
 		...rest
 	}: Props = $props()
 
-	const type = $derived(geometry?.geometryType?.case)
-	const color = $derived(overrideColor ?? metadata.color ?? colors.default)
+	const colorUtil = new Color()
+
+	const { invalidate } = useThrelte()
+	const name = useTrait(() => entity, traits.Name)
+	const entityColor = useTrait(() => entity, traits.Color)
+	const opacity = useTrait(() => entity, traits.Opacity)
+	const box = useTrait(() => entity, traits.Box)
+	const capsule = useTrait(() => entity, traits.Capsule)
+	const sphere = useTrait(() => entity, traits.Sphere)
+	const bufferGeometry = useTrait(() => entity, traits.BufferGeometry)
+	const linePositions = useTrait(() => entity, traits.LinePositions)
+	const lineWidth = useTrait(() => entity, traits.LineWidth)
+	const center = useTrait(() => entity, traits.Center)
+	const showAxesHelper = useTrait(() => entity, traits.ShowAxesHelper)
+
+	const geometryType = $derived.by(() => {
+		if (box.current) return 'box'
+		if (capsule.current) return 'capsule'
+		if (sphere.current) return 'sphere'
+		if (bufferGeometry.current) return 'buffer'
+		if (linePositions.current) return 'line'
+	})
+
+	const color = $derived.by(() => {
+		if (overrideColor) {
+			return overrideColor
+		}
+		if (entityColor.current) {
+			return colorUtil
+				.setRGB(entityColor.current.r, entityColor.current.g, entityColor.current.b)
+				.getHexString()
+		}
+		return colors.default
+	})
 
 	const group = new Group()
+	ref = group
+
 	const mesh = $derived.by(() => {
-		if (type === undefined) {
+		if (geometryType === undefined) {
 			return
 		}
-		const result = new Mesh()
 
-		if (type === 'mesh' || type === 'points' || type === 'line') {
+		const result = geometryType === 'line' ? new Line2() : new Mesh()
+
+		if (geometryType === 'line') {
 			result.raycast = meshBounds
 		}
 
@@ -59,90 +86,106 @@ and should remain pure, i.e. no hooks should be used.
 	})
 
 	$effect.pre(() => {
-		if (geometry?.center && mesh) {
-			poseToObject3d(geometry.center, mesh)
+		if (mesh && center.current) {
+			poseToObject3d(center.current, mesh)
+			invalidate()
 		}
 	})
 
+	const entityPose = useTrait(() => entity, traits.Pose)
+	const resolvedPose = $derived(pose ?? entityPose.current)
 	$effect.pre(() => {
-		poseToObject3d(pose, group)
+		if (resolvedPose) {
+			poseToObject3d(resolvedPose, group)
+			invalidate()
+		}
 	})
 
 	let geo = $state.raw<BufferGeometry>()
 
-	const oncreate = (ref: BufferGeometry) => {
-		geo = ref
+	const oncreate = (bufferGeometry: BufferGeometry) => {
+		geo = bufferGeometry
 	}
+
+	$effect.pre(() => {
+		if (mesh && bufferGeometry.current) {
+			mesh.geometry = bufferGeometry.current
+			oncreate(bufferGeometry.current)
+
+			return () => {
+				geo = undefined
+				mesh?.geometry?.dispose()
+			}
+		}
+	})
 </script>
 
 <T
 	is={group}
 	{...rest}
 >
-	{#if geometry?.geometryType}
-		<AxesHelper
-			width={3}
-			length={0.1}
-		/>
+	{#if geometryType}
+		{#if showAxesHelper.current}
+			<AxesHelper
+				width={3}
+				length={0.1}
+			/>
+		{/if}
 
 		<T
 			is={mesh}
-			{name}
-			{uuid}
-			bvh={{ enabled: false }}
+			name={entity}
+			userData.name={name}
+			bvh={{ enabled: geometryType === 'buffer' }}
 		>
 			{#if model && renderMode.includes('model')}
 				<T is={model} />
 			{/if}
 
-			{#if renderMode.includes('colliders')}
-				{#if geometry.geometryType.case === 'bufferGeometry'}
-					<T
-						is={geometry.geometryType.value}
-						{oncreate}
-					/>
-				{:else if geometry.geometryType.case === 'mesh'}
-					{@const mesh = geometry.geometryType.value.mesh}
-					{@const meshGeometry = parsePlyInput(mesh)}
+			{#if !model || renderMode.includes('colliders')}
+				{#if linePositions.current}
 					<T
-						is={meshGeometry}
-						{oncreate}
+						is={LineGeometry}
+						oncreate={(ref) => {
+							if (linePositions.current) {
+								ref.setPositions(linePositions.current)
+							}
+						}}
 					/>
-				{:else if geometry.geometryType.case === 'line' && metadata.points}
-					<MeshLineGeometry points={metadata.points} />
-				{:else if geometry.geometryType.case === 'box'}
-					{@const dimsMm = geometry.geometryType.value.dimsMm ?? { x: 0, y: 0, z: 0 }}
+				{:else if box.current}
+					{@const { x, y, z } = box.current ?? { x: 0, y: 0, z: 0 }}
 					<T.BoxGeometry
-						args={[dimsMm.x * 0.001, dimsMm.y * 0.001, dimsMm.z * 0.001]}
+						args={[x * 0.001, y * 0.001, z * 0.001]}
 						{oncreate}
 					/>
-				{:else if geometry.geometryType.case === 'sphere'}
-					{@const radiusMm = geometry.geometryType.value.radiusMm ?? 0}
+				{:else if sphere.current}
+					{@const { r } = sphere.current ?? { r: 0 }}
 					<T.SphereGeometry
-						args={[radiusMm * 0.001]}
+						args={[r * 0.001]}
 						{oncreate}
 					/>
-				{:else if geometry.geometryType.case === 'capsule'}
-					{@const { lengthMm, radiusMm } = geometry.geometryType.value}
+				{:else if capsule.current}
+					{@const { r, l } = capsule.current ?? { r: 0, l: 0 }}
 					<T
 						is={CapsuleGeometry}
-						args={[radiusMm * 0.001, lengthMm * 0.001]}
+						args={[r * 0.001, l * 0.001]}
 						{oncreate}
 					/>
 				{/if}
 			{/if}
 
-			{#if geometry.geometryType.case === 'line'}
-				<MeshLineMaterial
+			{#if linePositions.current}
+				<T
+					is={LineMaterial}
 					{color}
-					width={metadata.lineWidth ?? 0.005}
+					width={lineWidth.current ? lineWidth.current * 0.001 : 0.5}
 				/>
 			{:else}
 				<T.MeshToonMaterial
 					{color}
-					side={geometry.geometryType.case === 'mesh' ? DoubleSide : FrontSide}
-					transparent
-					opacity={metadata.opacity ?? 0.7}
+					side={geometryType === 'buffer' ? DoubleSide : FrontSide}
+					transparent={(opacity.current ?? 0.7) < 1}
+					opacity={opacity.current ?? 0.7}
 				/>
 
 				{#if geo && renderMode.includes('colliders')}
@@ -156,10 +199,9 @@ and should remain pure, i.e. no hooks should be used.
 				{/if}
 			{/if}
 		</T>
-	{:else}
+	{:else if showAxesHelper.current}
 		<AxesHelper
-			{name}
-			{uuid}
+			name={name.current}
 			width={3}
 			length={0.1}
 		/>

package/dist/components/Geometry.svelte.d.ts

@@ -1,20 +1,19 @@
 import { type Props as ThrelteProps } from '@threlte/core';
 import { type Snippet } from 'svelte';
 import { Group } from 'three';
-import type { WorldObject } from '../WorldObject.svelte';
+import type { Entity } from 'koota';
+import type { Pose } from '@viamrobotics/sdk';
 interface Props extends ThrelteProps<Group> {
-    uuid: string;
-    name: string;
-    geometry?: WorldObject['geometry'];
-    pose: WorldObject['pose'];
-    metadata: WorldObject['metadata'];
+    entity: Entity;
     color?: string;
     model?: Group;
+    pose?: Pose;
     renderMode?: 'model' | 'colliders' | 'colliders+model';
+    ref?: Group;
     children?: Snippet<[{
         ref: Group;
     }]>;
 }
-declare const Geometry: import("svelte").Component<Props, {}, "">;
+declare const Geometry: import("svelte").Component<Props, {}, "ref">;
 type Geometry = ReturnType<typeof Geometry>;
 export default Geometry;

package/dist/components/SceneProviders.svelte

@@ -17,6 +17,7 @@
 	import { provideOrigin } from './xr/useOrigin.svelte'
 	import { provideWorldStates } from '../hooks/useWorldState.svelte'
 	import { provideArmClient } from '../hooks/useArmClient.svelte'
+	import { provideArmKinematics } from '../hooks/useArmKinematics.svelte'
 	import { provideFramelessComponents } from '../hooks/useFramelessComponents.svelte'
 	import { provideResourceByName } from '../hooks/useResourceByName.svelte'
 	import { provide3DModels } from '../hooks/use3DModels.svelte'
@@ -48,6 +49,7 @@
 	providePointclouds(() => partID.current)
 	providePointcloudObjects(() => partID.current)
 	provideArmClient(() => partID.current)
+	provideArmKinematics(() => partID.current)
 	provideWorldStates()
 	provideFramelessComponents()
 

package/dist/components/Snapshot.svelte

@@ -14,7 +14,7 @@ Renders a Snapshot protobuf by spawning its transforms and drawings as entities
 ```
 -->
 <script lang="ts">
-	import type { Snapshot as SnapshotProto } from '../draw/v1/snapshot_pb'
+	import type { Snapshot as SnapshotProto } from '../buf/draw/v1/snapshot_pb'
 	import { useWorld } from '../ecs'
 	import { useSettings } from '../hooks/useSettings.svelte'
 	import { spawnSnapshotEntities, destroyEntities, applySceneMetadata } from '../snapshot'

package/dist/components/Snapshot.svelte.d.ts

@@ -1,4 +1,4 @@
-import type { Snapshot as SnapshotProto } from '../draw/v1/snapshot_pb';
+import type { Snapshot as SnapshotProto } from '../buf/draw/v1/snapshot_pb';
 interface Props {
     snapshot: SnapshotProto;
 }

package/dist/components/overlay/Details.svelte

@@ -30,6 +30,7 @@
 	import { useCameraControls } from '../../hooks/useControls.svelte'
 	import { useLinkedEntities } from '../../hooks/useLinked.svelte'
 	import AddRelationship from './AddRelationship.svelte'
+	import { createPose } from '../../transform'
 
 	const { ...rest } = $props()
 
@@ -124,6 +125,25 @@
 		}
 	})
 
+	$effect(() => {
+		if (entity) {
+			const worldPose = createPose({
+				x: worldPosition.x,
+				y: worldPosition.y,
+				z: worldPosition.z,
+				oX: worldOrientation.x,
+				oY: worldOrientation.y,
+				oZ: worldOrientation.z,
+				theta: MathUtils.radToDeg(worldOrientation.th),
+			})
+			if (entity.has(traits.WorldPose)) {
+				entity.set(traits.WorldPose, worldPose)
+			} else {
+				entity.add(traits.WorldPose(worldPose))
+			}
+		}
+	})
+
 	const getCopyClipboardText = () => {
 		return JSON.stringify(
 			{

package/dist/components/overlay/left-pane/TreeContainer.svelte

@@ -4,7 +4,6 @@
 	import { useSelectedEntity } from '../../../hooks/useSelection.svelte'
 	import { provideTreeExpandedContext } from './useExpanded.svelte'
 	import Logs from './Logs.svelte'
-	import Widgets from './Widgets.svelte'
 	import AddFrames from './AddFrames.svelte'
 	import { useEnvironment } from '../../../hooks/useEnvironment.svelte'
 	import { usePartID } from '../../../hooks/usePartID.svelte'
@@ -85,5 +84,4 @@
 	{/if}
 
 	<Logs />
-	<Widgets />
 </div>

package/dist/components/overlay/settings/Settings.svelte

@@ -15,6 +15,7 @@
 	import Tabs from './Tabs.svelte'
 	import { PersistedState } from 'runed'
 	import ToggleGroup from '../ToggleGroup.svelte'
+	import XRControllerSettings from '../../xr/XRControllerSettings.svelte'
 
 	const { invalidate } = useThrelte()
 	const partID = usePartID()
@@ -36,6 +37,10 @@
 		invalidate()
 	})
 
+	const currentRobotCameraWidgets = $derived(
+		settings.current.openCameraWidgets[partID.current] || []
+	)
+
 	const isOpen = new PersistedState('settings-is-open', false)
 	const activeTab = new PersistedState('settings-active-tab', 'Connection')
 </script>
@@ -258,6 +263,50 @@
 	</div>
 {/snippet}
 
+{#snippet XR()}
+	<div class="flex flex-col gap-2.5 text-xs">
+		<XRControllerSettings />
+	</div>
+{/snippet}
+
+{#snippet Widgets()}
+	<div class="text-gray-9 flex flex-col gap-1 text-xs">
+		<label class="flex items-center justify-between gap-2 py-1">
+			Arm positions
+			<Switch bind:on={settings.current.enableArmPositionsWidget} />
+		</label>
+
+		{@render SectionTitle('Camera widgets')}
+
+		{#each cameras.current as camera (camera)}
+			{@const isWidgetOpen = currentRobotCameraWidgets.includes(camera.name)}
+			<div class="flex items-center justify-between gap-2 py-0.5">
+				<span class="min-w-0 truncate">{camera.name}</span>
+				<Switch
+					on={isWidgetOpen}
+					on:change={(event) => {
+						if (event.detail) {
+							settings.current.openCameraWidgets = {
+								...settings.current.openCameraWidgets,
+								[partID.current]: [...currentRobotCameraWidgets, camera.name],
+							}
+						} else {
+							settings.current.openCameraWidgets = {
+								...settings.current.openCameraWidgets,
+								[partID.current]: currentRobotCameraWidgets.filter(
+									(widget) => widget !== camera.name
+								),
+							}
+						}
+					}}
+				/>
+			</div>
+		{:else}
+			No cameras detected
+		{/each}
+	</div>
+{/snippet}
+
 <FloatingPanel
 	title="Settings"
 	bind:isOpen={isOpen.current}
@@ -270,7 +319,9 @@
 			{ label: 'Scene', content: Scene },
 			{ label: 'Pointclouds', content: Pointclouds },
 			{ label: 'Vision', content: Vision },
+			{ label: 'Widgets', content: Widgets },
 			{ label: 'Stats', content: Stats },
+			...('xr' in navigator ? [{ label: 'VR / AR', content: XR }] : []),
 		]}
 		onValueChange={(value) => {
 			activeTab.current = value

package/dist/components/overlay/widgets/Camera.svelte

@@ -1,8 +1,8 @@
 <script lang="ts">
 	import { draggable } from '@neodrag/svelte'
 	import { Icon, Select } from '@viamrobotics/prime-core'
-	import { CameraStream, useRobotClient } from '@viamrobotics/svelte-sdk'
-	import { StreamClient } from '@viamrobotics/sdk'
+	import { CameraStream, useRobotClient, useConnectionStatus } from '@viamrobotics/svelte-sdk'
+	import { StreamClient, MachineConnectionEvent } from '@viamrobotics/sdk'
 	import { useSettings } from '../../../hooks/useSettings.svelte'
 	import { usePartID } from '../../../hooks/usePartID.svelte'
 	import { useEnvironment } from '../../../hooks/useEnvironment.svelte'
@@ -17,6 +17,7 @@
 	const settings = useSettings()
 	const partID = usePartID()
 	const client = useRobotClient(() => partID.current)
+	const connectionStatus = useConnectionStatus(() => partID.current)
 	const environment = useEnvironment()
 
 	let dragElement = $state.raw<HTMLElement>()
@@ -70,13 +71,18 @@
 		}
 	}
 
-	// Create a single StreamClient instance per robot client
-	let streamClient = $derived(client.current ? new StreamClient(client.current) : undefined)
+	// Only create StreamClient when connection is fully established
+	let streamClient = $derived(
+		client.current && connectionStatus.current === MachineConnectionEvent.CONNECTED
+			? new StreamClient(client.current)
+			: undefined
+	)
 
 	$effect(() => {
 		if (streamClient) {
 			isLoading = true
 			error = undefined
+
 			streamClient
 				.getOptions(name)
 				.then((options) => {
@@ -164,14 +170,16 @@
 			class="relative min-h-0 w-full flex-1 overflow-hidden bg-black [&_img]:h-full [&_img]:w-full [&_img]:object-fill [&_video]:h-full [&_video]:w-full [&_video]:object-fill"
 			style:aspect-ratio={aspectRatio}
 		>
-			{#key environment.current.viewerMode === 'monitor'}
-				<CameraStream
-					{name}
-					partID={partID.current}
-					onloadedmetadata={onMediaLoad}
-					onload={onMediaLoad}
-				/>
-			{/key}
+			{#if connectionStatus.current === MachineConnectionEvent.CONNECTED}
+				{#key environment.current.viewerMode === 'monitor'}
+					<CameraStream
+						{name}
+						partID={partID.current}
+						onloadedmetadata={onMediaLoad}
+						onload={onMediaLoad}
+					/>
+				{/key}
+			{/if}
 
 			<!-- FPS Pill -->
 			{#if fps > 0}