@viamrobotics/test-widgets 0.0.1

package/dist/components/widgets/vision-service/detection.svelte

@@ -0,0 +1,101 @@
+<script lang="ts">
+	import { useThrelte } from '@threlte/core'
+	import { useInteractivity } from '@threlte/extras'
+	import { onMount } from 'svelte'
+	import { Vector2 } from 'three'
+	import { Container, Text } from 'threlte-uikit'
+
+	import { lightTextColors } from './color'
+	import { type Detection, useDetections } from './context.svelte'
+
+	const { size } = useThrelte()
+
+	interface Props {
+		detection: Detection
+		factor?: number
+	}
+
+	const { detection, factor = 0 }: Props = $props()
+
+	const normalizedDeviceCoordsToPixel = (
+		ndcCoords: Vector2,
+		canvasWidth: number,
+		canvasHeight: number
+	) => {
+		const screenX = (ndcCoords.x + 1) / 2
+		const screenY = (ndcCoords.y + 1) / 2
+		const pixelX = Math.round(screenX * canvasWidth)
+		const pixelY = Math.round((1 - screenY) * canvasHeight)
+		return new Vector2(pixelX, pixelY)
+	}
+
+	const { pointer } = useInteractivity()
+	const context = useDetections()
+
+	const factoredxMin = $derived(Number(detection.xMin) * factor)
+	const factoredxMax = $derived(Number(detection.xMax) * factor)
+	const factoredyMin = $derived(Number(detection.yMin) * factor)
+	const factoredyMax = $derived(Number(detection.yMax) * factor)
+
+	/**
+	 * Check preexisting mouse coords for hover state
+	 */
+	$effect.pre(() => {
+		const pixelCoords = normalizedDeviceCoordsToPixel($pointer, $size.width, $size.height)
+
+		if (
+			factoredxMin < pixelCoords.x &&
+			factoredxMax > pixelCoords.x &&
+			factoredyMin < pixelCoords.y &&
+			factoredyMax > pixelCoords.y
+		) {
+			context.hovered.add(detection.id)
+		} else {
+			context.hovered.delete(detection.id)
+		}
+	})
+
+	const hovering = $derived(context.hovered.has(detection.id))
+
+	// These are ballparking, but seem good enough for nearly all cases
+	const isDetectionNearTop = $derived(factoredyMin < 30)
+	const isDetectionNearRight = $derived(factoredxMin > $size.width * 0.66)
+
+	onMount(() => {
+		return () => {
+			context.hovered.delete(detection.id)
+		}
+	})
+</script>
+
+<Container
+	name="Detection {detection.id}"
+	positionType="absolute"
+	positionLeft={factoredxMin}
+	positionTop={factoredyMin}
+	borderColor={hovering ? detection.color : '#aaa'}
+	borderWidth={2}
+	width={factoredxMax - factoredxMin}
+	height={factoredyMax - factoredyMin}
+	onpointerleave={() => context.hovered.delete(detection.id)}
+>
+	{#if hovering}
+		<Text
+			positionType="absolute"
+			positionBottom={isDetectionNearTop ? undefined : '100%'}
+			positionTop={isDetectionNearTop ? -1 : undefined}
+			positionLeft={isDetectionNearRight ? undefined : -2}
+			positionRight={isDetectionNearRight ? -2 : undefined}
+			borderColor={detection.color}
+			backgroundColor={detection.color}
+			color={lightTextColors.includes(detection.color) ? 'white' : 'black'}
+			borderWidth={2}
+			paddingX={8}
+			paddingY={4}
+			fontSize={11}
+			fontWeight="medium"
+			fontFamily="publicSans"
+			text={`${detection.className} (${detection.confidence.toString()}%)`}
+		/>
+	{/if}
+</Container>

package/dist/components/widgets/vision-service/detection.svelte.d.ts

@@ -0,0 +1,8 @@
+import { type Detection } from './context.svelte';
+interface Props {
+    detection: Detection;
+    factor?: number;
+}
+declare const Detection: import("svelte").Component<Props, {}, "">;
+type Detection = ReturnType<typeof Detection>;
+export default Detection;

package/dist/components/widgets/vision-service/geometry-details.svelte

@@ -0,0 +1,116 @@
+<script lang="ts">
+	import type { GeometriesInFrame, Geometry } from '@viamrobotics/sdk'
+
+	import { Expandable, Icon } from '@viamrobotics/prime-core'
+
+	interface Props {
+		geometries: GeometriesInFrame
+	}
+
+	const { geometries }: Props = $props()
+
+	const formatNumber = (value: number | undefined): string => {
+		if (value === undefined) return '—'
+		return value.toFixed(3)
+	}
+
+	const getGeometryLabel = (geometry: Geometry, index: number): string => {
+		const typeCase = geometry.geometryType.case ?? 'unknown'
+		if (geometry.label) {
+			return `${geometry.label} (${typeCase})`
+		}
+		return `${typeCase} ${index}`
+	}
+
+	const getDimensionsDisplay = (geometry: Geometry): { label: string; value: string[] }[] => {
+		const { geometryType } = geometry
+
+		switch (geometryType.case) {
+			case 'box': {
+				const box = geometryType.value
+				return [
+					{
+						label: 'Dimensions',
+						value: [
+							`x: ${formatNumber(box.dimsMm?.x)}mm`,
+							`y: ${formatNumber(box.dimsMm?.y)}mm`,
+							`z: ${formatNumber(box.dimsMm?.z)}mm`,
+						],
+					},
+				]
+			}
+			case 'sphere': {
+				const sphere = geometryType.value
+				return [{ label: 'Radius', value: [`${formatNumber(sphere.radiusMm)}mm`] }]
+			}
+			case 'capsule': {
+				const capsule = geometryType.value
+				return [
+					{ label: 'Radius', value: [`${formatNumber(capsule.radiusMm)}mm`] },
+					{ label: 'Length', value: [`${formatNumber(capsule.lengthMm)}mm`] },
+				]
+			}
+			default: {
+				return []
+			}
+		}
+	}
+</script>
+
+<div class="flex flex-col gap-1 p-2">
+	{#each geometries.geometries as geometry, index (`${index}:${geometry.label}`)}
+		<Expandable
+			class="border-light border"
+			triggerClass="flex w-full items-center gap-2 px-3 py-2 text-left text-sm font-medium hover:bg-gray-50"
+			contentClass="px-3 pb-3"
+		>
+			{#snippet trigger({ isOpen })}
+				<Icon
+					name={isOpen ? 'chevron-down' : 'chevron-right'}
+					cx="h-4 w-4 shrink-0 text-gray-500"
+				/>
+				<span class="truncate">{getGeometryLabel(geometry, index)}</span>
+			{/snippet}
+			{#snippet content()}
+				<dl class="text-subtle-1 flex flex-col gap-1 text-sm">
+					{#each getDimensionsDisplay(geometry) as { label, value } (label)}
+						<div class="flex flex-col gap-0.5">
+							<dt class="font-bold">{label}</dt>
+							<dd class="flex flex-col pl-2 font-mono">
+								{#each value as v, i (i)}
+									<span>{v}</span>
+								{/each}
+							</dd>
+						</div>
+					{/each}
+
+					{#if geometry.center}
+						<div class="flex flex-col gap-0.5">
+							<dt class="font-bold">Position</dt>
+							<dd class="flex flex-col pl-2 font-mono">
+								<span>x: {formatNumber(geometry.center.x)}</span>
+								<span>y: {formatNumber(geometry.center.y)}</span>
+								<span>z: {formatNumber(geometry.center.z)}</span>
+							</dd>
+						</div>
+					{/if}
+
+					{#if geometry.center?.theta}
+						<div class="flex flex-col gap-0.5">
+							<dt class="font-bold">Rotation</dt>
+							<dd class="pl-2 font-mono">
+								θ: {formatNumber(geometry.center.theta)}°
+							</dd>
+						</div>
+					{/if}
+				</dl>
+			{/snippet}
+		</Expandable>
+	{/each}
+
+	{#if geometries.referenceFrame}
+		<div class="text-subtle-2 mt-1 px-1 text-xs">
+			Reference frame: <span class="font-mono">{geometries.referenceFrame}</span>
+		</div>
+	{/if}
+</div>

package/dist/components/widgets/vision-service/geometry-details.svelte.d.ts

@@ -0,0 +1,7 @@
+import type { GeometriesInFrame } from '@viamrobotics/sdk';
+interface Props {
+    geometries: GeometriesInFrame;
+}
+declare const GeometryDetails: import("svelte").Component<Props, {}, "">;
+type GeometryDetails = ReturnType<typeof GeometryDetails>;
+export default GeometryDetails;

package/dist/components/widgets/vision-service/get-image-size.d.ts

@@ -0,0 +1,14 @@
+export interface Size {
+    /** The image width, resized to the natural width with a max of 66% width of the test card */
+    width: number;
+    /** The image height, resized by the aspect ratio */
+    height: number;
+    /** The resizing factor. For example, from 1000px to 800px would be 0.8 */
+    factor: number;
+}
+interface ContainerSizeOptions {
+    maxHeight: number;
+    maxWidth: number;
+}
+export declare const getImageSize: (img: HTMLImageElement, container: HTMLElement | ContainerSizeOptions) => Size;
+export {};

package/dist/components/widgets/vision-service/get-image-size.js

@@ -0,0 +1,28 @@
+const twoThirds = 2 / 3;
+export const getImageSize = (img, container) => {
+    const aspectRatio = img.naturalWidth / img.naturalHeight;
+    /**
+     * Portrait images
+     */
+    if (img.naturalHeight > img.naturalWidth) {
+        // Allow a maximum height of 2/3 of the browser window or up to an arbitrarily specified height
+        const maxHeight = 'maxHeight' in container ? container.maxHeight : window.innerHeight * twoThirds;
+        const heightNumber = Math.min(maxHeight, img.naturalHeight);
+        return {
+            width: heightNumber * aspectRatio,
+            height: heightNumber,
+            factor: 1 - (img.naturalHeight - heightNumber) / img.naturalHeight,
+        };
+    }
+    /**
+     * Landscape images
+     */
+    // Allow a maximum width of 2/3 of the container's width or up to an arbitrarily specified width
+    const maxWidth = 'maxWidth' in container ? container.maxWidth : container.clientWidth * twoThirds;
+    const widthNumber = Math.min(maxWidth, img.naturalWidth);
+    return {
+        width: widthNumber,
+        height: widthNumber * (1 / aspectRatio),
+        factor: 1 - (img.naturalWidth - widthNumber) / img.naturalWidth,
+    };
+};

package/dist/components/widgets/vision-service/image.svelte

@@ -0,0 +1,133 @@
+<script lang="ts">
+	import type { cameraApi, Classification, Detection } from '@viamrobotics/sdk'
+
+	import { resize } from '@svelte-put/resize'
+	import { Canvas } from '@threlte/core'
+	import { Progress } from '@viamrobotics/prime-core'
+	import { provideFontFamilies } from 'threlte-uikit'
+
+	import { useMeasureFps } from '../../../fps.svelte'
+
+	import { provideDetectionsContext } from './context.svelte'
+	import { getImageSize, type Size } from './get-image-size'
+	import Legend from './legend.svelte'
+	import Scene from './scene.svelte'
+
+	interface Props {
+		data?: {
+			detections: Detection[]
+			image: ({ image: Uint8Array } & Exclude<cameraApi.Image, 'image'>) | undefined
+			classifications: Classification[]
+		}
+		detectionsSupported: boolean
+		classificationsSupported: boolean
+	}
+
+	const { data, detectionsSupported, classificationsSupported }: Props = $props()
+
+	provideDetectionsContext(() => data?.detections ?? [])
+
+	const img = document.createElement('img')
+
+	let src = $state('')
+
+	const handleImageLoad = () => {
+		setSize()
+	}
+
+	$effect(() => {
+		img.addEventListener('load', handleImageLoad)
+		return () => img.removeEventListener('load', handleImageLoad)
+	})
+
+	let url = ''
+
+	const getImageSrc = (image: string | Uint8Array | undefined) => {
+		if (typeof image === 'string') {
+			return `data:image/jpeg;base64,${image}`
+		} else if (image !== undefined) {
+			URL.revokeObjectURL(url)
+			url = URL.createObjectURL(new Blob([image as BlobPart], { type: 'image/jpeg' }))
+			return url
+		}
+
+		return ''
+	}
+
+	$effect.pre(() => {
+		img.src = getImageSrc(data?.image?.image)
+		src = img.src
+	})
+
+	provideFontFamilies({
+		publicSans: {
+			// This must remain `/static/*` while the frontend directory still exist
+			medium: '/static/fonts/public-sans.json',
+		},
+	})
+
+	let container = $state<HTMLElement>()
+	let size = $state<Size>()
+
+	const setSize = () => {
+		if (container) {
+			size = getImageSize(img, container)
+		}
+	}
+
+	$effect(() => {
+		void container
+		setSize()
+	})
+
+	const fps = useMeasureFps()
+
+	let lastSrc = ''
+	$effect(() => {
+		if (src !== lastSrc) {
+			fps.measure()
+			lastSrc = src
+		}
+	})
+</script>
+
+<div
+	class="flex h-full min-h-0 w-full min-w-0 gap-2 py-2 pl-2"
+	bind:this={container}
+	use:resize
+	onresize={setSize}
+>
+	{#if !size}
+		<div class="grid w-full place-content-center">
+			<Progress size="large" />
+		</div>
+	{:else}
+		<div
+			class="relative max-h-full min-h-0 max-w-full"
+			style:width={`${size.width.toString()}px`}
+			style:height={`${size.height.toString()}px`}
+		>
+			<div
+				class="pointer-events-none absolute bottom-2 left-2 z-10 rounded-[3px] bg-black/30 px-1 py-0.5 text-right font-mono text-xs text-white"
+			>
+				{fps.current.toFixed(1)}fps
+			</div>
+			<Canvas>
+				<Scene
+					{src}
+					factor={size.factor}
+				/>
+			</Canvas>
+		</div>
+		<div
+			class="flex grow flex-col pr-2"
+			style:height={`${size.height.toString()}px`}
+		>
+			<Legend
+				classifications={data?.classifications}
+				{detectionsSupported}
+				{classificationsSupported}
+			/>
+		</div>
+	{/if}
+</div>

package/dist/components/widgets/vision-service/image.svelte.d.ts

@@ -0,0 +1,15 @@
+import type { cameraApi, Classification, Detection } from '@viamrobotics/sdk';
+interface Props {
+    data?: {
+        detections: Detection[];
+        image: ({
+            image: Uint8Array;
+        } & Exclude<cameraApi.Image, 'image'>) | undefined;
+        classifications: Classification[];
+    };
+    detectionsSupported: boolean;
+    classificationsSupported: boolean;
+}
+declare const Image: import("svelte").Component<Props, {}, "">;
+type Image = ReturnType<typeof Image>;
+export default Image;

package/dist/components/widgets/vision-service/legend.svelte

@@ -0,0 +1,135 @@
+<script lang="ts">
+	import type { Classification } from '@viamrobotics/sdk'
+
+	import { Icon } from '@viamrobotics/prime-core'
+	import { SvelteSet } from 'svelte/reactivity'
+
+	import { useDetections } from './context.svelte'
+
+	interface Props {
+		classifications?: Classification[]
+		detectionsSupported: boolean
+		classificationsSupported: boolean
+	}
+
+	const { classifications = [], detectionsSupported, classificationsSupported }: Props = $props()
+
+	const context = useDetections()
+
+	let hoveredLabel = $state<keyof typeof context.byLabel>()
+
+	const expandedLabels = new SvelteSet<string>()
+
+	let selectedTab = $derived<'detections' | 'classifications'>(
+		detectionsSupported ? 'detections' : 'classifications'
+	)
+
+	const toggleExpand = (label: string) => {
+		if (expandedLabels.has(label)) {
+			expandedLabels.delete(label)
+		} else {
+			expandedLabels.add(label)
+		}
+	}
+
+	$effect.pre(() => {
+		if (hoveredLabel === undefined) {
+			context.hovered.clear()
+		} else {
+			const data = context.byLabel[hoveredLabel]
+
+			if (data) {
+				for (const detection of data.detections) {
+					context.hovered.add(detection.id)
+				}
+			}
+		}
+	})
+
+	const compareConfidence = (a: Classification, b: Classification) => {
+		return b.confidence - a.confidence
+	}
+
+	const sortedClassifications = $derived(classifications.toSorted(compareConfidence))
+</script>
+
+<div class="flex">
+	{#if detectionsSupported}
+		<button
+			class="grow border-b {selectedTab === 'detections' ? 'border-black' : ''}"
+			onclick={() => (selectedTab = 'detections')}>Labels</button
+		>
+	{/if}
+
+	{#if classificationsSupported}
+		<button
+			class="grow border-b px-4 py-1 {selectedTab === 'classifications' ? 'border-black' : ''}"
+			onclick={() => (selectedTab = 'classifications')}
+		>
+			Classifications
+		</button>
+	{/if}
+</div>
+
+<div class="overflow-x-hidden overflow-y-auto py-2 text-sm">
+	{#if selectedTab === 'classifications'}
+		<ul class="flex flex-col gap-2">
+			{#each sortedClassifications.slice(0, 100) as classification (classification)}
+				<li class="bg-light flex justify-between gap-2 rounded px-2 py-1">
+					<span class="text-ellipsis">{classification.className}</span>
+					<span>{(classification.confidence * 100).toFixed(2)}%</span>
+				</li>
+			{/each}
+		</ul>
+	{:else if selectedTab === 'detections'}
+		<ul>
+			{#each Object.entries(context.byLabel) as [label, { detections, color }] (label)}
+				<li>
+					<button
+						aria-label={label}
+						class="hover:bg-light flex w-full items-center justify-between gap-2 py-1 pr-1"
+						onpointerover={() => (hoveredLabel = label)}
+						onpointerleave={() => (hoveredLabel = undefined)}
+						onclick={() => toggleExpand(label)}
+					>
+						<div class="flex items-center gap-2">
+							<Icon
+								name="chevron-right"
+								cx="text-gray-6 {expandedLabels.has(label) ? 'rotate-90' : ''}"
+							/>
+							<div
+								class="h-3 w-3"
+								style:background-color={color}
+							></div>
+							{label}
+						</div>
+
+						<div class="bg-medium rounded px-2 py-1 text-xs">
+							{detections.length}
+						</div>
+					</button>
+				</li>
+				{#if expandedLabels.has(label)}
+					<ul>
+						{#each detections as detection (detection.id)}
+							<li>
+								<button
+									class="hover:bg-light w-full py-1 pl-11 text-left {context.hovered.has(
+										detection.id
+									)
+										? 'bg-light'
+										: ''}"
+									onpointerenter={() => context.hovered.add(detection.id)}
+									onpointerleave={() => context.hovered.delete(detection.id)}
+								>
+									{label}
+									<span class="text-subtle-2 pl-1">{detection.confidence}%</span>
+								</button>
+							</li>
+						{/each}
+					</ul>
+				{/if}
+			{/each}
+		</ul>
+	{/if}
+</div>

package/dist/components/widgets/vision-service/legend.svelte.d.ts

@@ -0,0 +1,9 @@
+import type { Classification } from '@viamrobotics/sdk';
+interface Props {
+    classifications?: Classification[];
+    detectionsSupported: boolean;
+    classificationsSupported: boolean;
+}
+declare const Legend: import("svelte").Component<Props, {}, "">;
+type Legend = ReturnType<typeof Legend>;
+export default Legend;

package/dist/components/widgets/vision-service/object-point-cloud-scene.svelte

@@ -0,0 +1,74 @@
+<script lang="ts">
+	import type { PointCloudObject } from '@viamrobotics/sdk'
+	import type { OrbitControls as OrbitControlsType } from 'three/examples/jsm/controls/OrbitControls.js'
+
+	import { T, useThrelte } from '@threlte/core'
+	import { OrbitControls } from '@threlte/extras'
+	import { Color } from 'three'
+	import { PCDLoader } from 'three/examples/jsm/loaders/PCDLoader.js'
+
+	interface Props {
+		object: PointCloudObject
+	}
+
+	const { object }: Props = $props()
+
+	const { scene, invalidate } = useThrelte()
+	scene.background = new Color(0xe0_e0_e0)
+
+	const loader = new PCDLoader()
+
+	let controlsRef = $state<OrbitControlsType>()
+
+	const points = $derived.by(() => {
+		const dataCopy = new Uint8Array(object.pointCloud)
+		return loader.parse(dataCopy.buffer)
+	})
+
+	const boundingInfo = $derived.by(() => {
+		if (!points) return undefined
+
+		points.geometry.computeBoundingSphere()
+		const sphere = points.geometry.boundingSphere
+		if (!sphere) return undefined
+
+		return {
+			center: sphere.center.clone(),
+			distance: sphere.radius * 2.5,
+		}
+	})
+
+	const cameraPosition = $derived.by((): [number, number, number] => {
+		if (!boundingInfo) return [0, 0, 2]
+		const { center, distance } = boundingInfo
+		return [center.x, center.y, center.z + distance]
+	})
+
+	$effect(() => {
+		if (controlsRef && boundingInfo) {
+			controlsRef.target.copy(boundingInfo.center)
+			controlsRef.update()
+			invalidate()
+		}
+	})
+</script>
+
+<T.PerspectiveCamera
+	makeDefault
+	position={cameraPosition}
+	fov={50}
+	near={0.001}
+	far={1000}
+>
+	<OrbitControls bind:ref={controlsRef} />
+</T.PerspectiveCamera>
+
+<T.HemisphereLight args={[0xaa_aa_aa, 0x44_44_44, 3]} />
+<T.DirectionalLight
+	args={[0xff_ff_ff, 1.5]}
+	position={[1, 1, 1]}
+/>
+
+{#if points}
+	<T is={points} />
+{/if}

package/dist/components/widgets/vision-service/object-point-cloud-scene.svelte.d.ts

@@ -0,0 +1,7 @@
+import type { PointCloudObject } from '@viamrobotics/sdk';
+interface Props {
+    object: PointCloudObject;
+}
+declare const ObjectPointCloudScene: import("svelte").Component<Props, {}, "">;
+type ObjectPointCloudScene = ReturnType<typeof ObjectPointCloudScene>;
+export default ObjectPointCloudScene;