@viamrobotics/test-widgets 0.0.1

package/dist/components/widgets/camera/live-or-polling-video.svelte

@@ -0,0 +1,419 @@
+<script lang="ts">
+	import type { ResizeDetail } from '@svelte-put/resize'
+
+	import * as Sentry from '@sentry/svelte'
+	import { resize } from '@svelte-put/resize'
+	import { createMutation, type QueryObserverResult } from '@tanstack/svelte-query'
+	import { Button, Label, Select } from '@viamrobotics/prime-core'
+	import { CameraClient, type RobotClient, streamApi, StreamClient } from '@viamrobotics/sdk'
+	import { useRobotClient } from '@viamrobotics/svelte-sdk'
+	import { untrack } from 'svelte'
+
+	import { assertExists } from '../../../assert'
+	import ContentRect from '../../content-rect.svelte'
+	import ErrorDisplay from '../../error.svelte'
+	import MutationSection from '../../mutation-section.svelte'
+	import Progress from '../../progress.svelte'
+	import { formatNumeric } from '../../../format'
+	import { useMeasureFps } from '../../../fps.svelte'
+
+	interface Props {
+		resourceName: string
+		partID: string
+		showResolutionOptions?: boolean
+		isLive: boolean
+		data: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['data']
+		error?: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['error']
+		isLoading: boolean
+		videoClass?: string
+		showMousePositionTooltip?: boolean
+		setMediaStream?: ((mediaStream: MediaStream | null) => void) | undefined
+		refetch: () => Promise<unknown>
+	}
+
+	const {
+		resourceName,
+		partID,
+		showResolutionOptions,
+		isLive,
+		data,
+		error,
+		isLoading,
+		videoClass = '',
+		showMousePositionTooltip = false,
+		setMediaStream: setPipMediaStream,
+		refetch,
+	}: Props = $props()
+
+	let videoElement = $state.raw<HTMLVideoElement>()
+	let isStreamLoading = $state(false)
+	let streamError: Error | null
+	let vfcID = 0
+	let liveStreamStart: DOMHighResTimeStamp | undefined
+
+	const setMediaStream = (mediaStream: MediaStream | null) => {
+		if (videoElement) {
+			// TODO(2025-01-31): Remove these function guards.
+			// Sentry has some error traces where "cancelVideoFrameCallback is not a function".
+			// Firefox only added this a few versions ago, so this guard can be removed eventually.
+			// https://caniuse.com/?search=cancelvideoframecallback
+			if (typeof videoElement.cancelVideoFrameCallback === 'function') {
+				videoElement.cancelVideoFrameCallback(vfcID)
+			}
+			if (typeof videoElement.requestVideoFrameCallback === 'function' && mediaStream !== null) {
+				vfcID = videoElement.requestVideoFrameCallback((now) => {
+					if (liveStreamStart) {
+						Sentry.setMeasurement(
+							'control.camera.timeToVideoFrame',
+							now - liveStreamStart,
+							'millisecond'
+						)
+					}
+					isStreamLoading = false
+				})
+			}
+			videoElement.srcObject = mediaStream
+		}
+		setPipMediaStream?.(mediaStream)
+	}
+
+	// For live streams, set the media stream when there is a new video track.
+	const client = useRobotClient(() => partID)
+
+	const handleTrack = (event: unknown) => {
+		if (liveStreamStart) {
+			Sentry.setMeasurement(
+				'control.camera.timeToTrack',
+				performance.now() - liveStreamStart,
+				'millisecond'
+			)
+		}
+		const [eventStream] = (event as { streams: MediaStream[] }).streams
+
+		if (!eventStream || eventStream.id !== resourceName) {
+			return
+		}
+
+		setMediaStream(eventStream)
+	}
+
+	// For polling, create the query and render the result to an image
+	// tag. Then draw the image onto a canvas. Finally, capture the
+	// canvas in a media stream
+	const img = document.createElement('img')
+	const canvas = document.createElement('canvas')
+	const canvasCtx = canvas.getContext('2d')
+	const drawImage = () => {
+		canvas.width = img.naturalWidth
+		canvas.height = img.naturalHeight
+		canvasCtx?.drawImage(img, 0, 0)
+	}
+
+	let streamClient: StreamClient | undefined
+	let resolutionOptions = $state<streamApi.Resolution[]>([])
+	let selectedResolution: streamApi.Resolution | undefined
+
+	const disableStream = async () => {
+		try {
+			isStreamLoading = false
+			streamError = null
+			const last = streamClient
+			streamClient = undefined
+			last?.off('track', handleTrack)
+			await last?.remove(resourceName)
+			resolutionOptions = []
+			selectedResolution = undefined
+			liveStreamStart = undefined
+		} catch (nextError: unknown) {
+			lastError = nextError as Error
+		}
+	}
+
+	const handleResolutionChange = (event: Event) => {
+		if (event.target instanceof HTMLSelectElement) {
+			const [width, height] = event.target.value.split('x').map(Number) as [number, number]
+			selectedResolution = new streamApi.Resolution({ width, height })
+		}
+	}
+
+	// NOTE(ELP): I don't know that this will be used anywhere else, so I'm
+	// opting to leave it here for now. This could be moved to a shared
+	// createMutation function.
+	const mutationOptions = $derived({
+		mutationKey: [partID, resourceName, 'setOptions'],
+		mutationFn: async (resolution?: streamApi.Resolution) => {
+			assertExists(streamClient, 'Expected stream client')
+			if (resolution) {
+				return streamClient.setOptions(resourceName, resolution.width, resolution.height)
+			}
+			return streamClient.resetOptions(resourceName)
+		},
+	})
+
+	const resolutionMutation = createMutation(() => mutationOptions)
+
+	const startStream = async (robotClient: RobotClient, name: string, live: boolean) => {
+		try {
+			await disableStream()
+			// Don't setMediaStream(null) here or else the PiP thrashes.
+
+			if (live) {
+				// Live streams
+
+				isStreamLoading = true
+				liveStreamStart = performance.now()
+				streamClient = new StreamClient(robotClient)
+				streamClient.on('track', handleTrack)
+				await streamClient.getStream(name)
+				resolutionOptions = await streamClient.getOptions(name)
+				selectedResolution = resolutionOptions[0]
+			} else {
+				// Polling
+
+				// Trigger a refetch on initial change.
+				await refetch()
+
+				setMediaStream(canvas.captureStream())
+			}
+		} catch (nextError) {
+			streamError = nextError as Error
+		}
+	}
+
+	$effect(() => {
+		if (client.current && !lastError) {
+			void startStream(client.current, resourceName, isLive)
+		}
+	})
+
+	$effect(() => {
+		return untrack(() => {
+			disableStream().then(() => {
+				setMediaStream(null)
+			})
+		})
+	})
+
+	$effect(() => {
+		if (lastError) {
+			disableStream()
+				.then(() => {
+					setMediaStream(null)
+				})
+				.catch((nextError: unknown) => {
+					lastError = nextError as Error
+				})
+		}
+	})
+
+	$effect(() => {
+		img.addEventListener('load', drawImage)
+
+		return () => {
+			img.removeEventListener('load', drawImage)
+		}
+	})
+
+	$effect(() => {
+		if (!data?.images?.[0]?.image) {
+			return
+		}
+
+		const imageBlob = new Blob([new Uint8Array(data.images[0].image)], {
+			type: data.images[0].mimeType || 'image/jpeg',
+		})
+
+		img.src = URL.createObjectURL(imageBlob)
+		return () => URL.revokeObjectURL(img.src)
+	})
+
+	let contentRect = $state.raw<DOMRect>()
+	const handleResize = (event: CustomEvent<ResizeDetail>) => {
+		contentRect = event.detail.entry.contentRect
+	}
+
+	let hoverTooltipOpen = $state(false)
+	let mouseHoverImagePosition = $state({ pctX: 0, pctY: 0, absX: 0, absY: 0 })
+	let mouseHoverClientPosition = $state({ x: 0, y: 0 })
+
+	const handleMouseMoveWithinVideo = (event: MouseEvent | WheelEvent) => {
+		if (!videoElement || !contentRect || !showMousePositionTooltip) {
+			return
+		}
+
+		mouseHoverClientPosition = { x: event.clientX, y: event.clientY }
+
+		const rect = videoElement.getBoundingClientRect()
+		const dx = event.pageX - (rect.x ?? 0)
+		const dy = event.pageY - (rect.y ?? 0)
+
+		let pctX = dx / (rect.width ?? 1)
+		let pctY = dy / (rect.height ?? 1)
+
+		pctX = Math.min(Math.max(pctX, 0), 1)
+		pctY = Math.min(Math.max(pctY, 0), 1)
+
+		let absX = 0
+		let absY = 0
+
+		if (isLive && selectedResolution) {
+			absX = pctX * selectedResolution.width
+			absY = pctY * selectedResolution.height
+		} else {
+			absX = pctX * canvas.width
+			absY = pctY * canvas.height
+		}
+		mouseHoverImagePosition = { pctX, pctY, absX, absY }
+	}
+	const handleMouseEnterVideo = () => {
+		if (showMousePositionTooltip) {
+			hoverTooltipOpen = true
+		}
+	}
+	const handleMouseLeaveVideo = () => {
+		hoverTooltipOpen = false
+	}
+
+	const fps = useMeasureFps()
+
+	$effect(() => {
+		let id: number | undefined
+
+		const measure = (now: DOMHighResTimeStamp) => {
+			fps.measure(now)
+			id = videoElement?.requestVideoFrameCallback?.(measure)
+		}
+
+		id = videoElement?.requestVideoFrameCallback?.(measure)
+		return () => id && videoElement?.cancelVideoFrameCallback(id)
+	})
+
+	// Errors are null during loading, so keep the latest errors during polling.
+	let lastError = $state.raw<Error>()
+	$effect(() => {
+		if (!isLive && error) {
+			lastError = error
+		} else if (isLive && streamError) {
+			lastError = streamError
+		} else {
+			lastError = undefined
+		}
+	})
+</script>
+
+<!-- This breaks out of using the <Query> component
+  to merge live stream and TanStack query errors -->
+{#if lastError}
+	<ContentRect
+		{contentRect}
+		cx="bg-medium/50 absolute flex h-64 w-80 items-center justify-center"
+	>
+		<ErrorDisplay
+			class="pb-4"
+			{lastError}
+		/>
+	</ContentRect>
+{:else if isLive ? isStreamLoading : isLoading}
+	<ContentRect
+		{contentRect}
+		cx="absolute h-64 w-80"
+	>
+		<Progress />
+	</ContentRect>
+{/if}
+
+<div class="relative">
+	<div
+		class="absolute bottom-2 left-2 z-10 rounded-[3px] bg-black/30 px-1 py-0.5 text-right font-mono text-xs text-white"
+	>
+		{fps.current.toFixed(1)}fps
+	</div>
+	<video
+		bind:this={videoElement}
+		class={videoClass}
+		onmouseenter={handleMouseEnterVideo}
+		onmouseleave={handleMouseLeaveVideo}
+		onmousemove={handleMouseMoveWithinVideo}
+		onwheel={handleMouseMoveWithinVideo}
+		muted
+		autoplay
+		controls={false}
+		playsinline
+		aria-label={`${resourceName} stream`}
+		use:resize
+		onresized={handleResize}
+	>
+	</video>
+</div>
+
+{#if hoverTooltipOpen}
+	<div
+		role="tooltip"
+		class="bg-extralight fixed z-10 rounded-sm border-0 p-1 text-xs"
+		style:left="{mouseHoverClientPosition.x + 10}px"
+		style:top="{mouseHoverClientPosition.y + 10}px"
+		onmouseenter={handleMouseEnterVideo}
+		onmouseleave={handleMouseLeaveVideo}
+		onmousemove={handleMouseMoveWithinVideo}
+		onwheel={handleMouseMoveWithinVideo}
+	>
+		{Math.round(mouseHoverImagePosition.absX)}x{Math.round(mouseHoverImagePosition.absY)}
+		<br />
+		({formatNumeric(mouseHoverImagePosition.pctX * 100, 1)}%x{formatNumeric(
+			mouseHoverImagePosition.pctY * 100,
+			1
+		)}%)
+	</div>
+{/if}
+
+{#if showResolutionOptions && isLive}
+	<MutationSection
+		title="SetStreamOptions"
+		description="Change the resolution of the live stream video feed"
+		lastError={resolutionMutation.error}
+		class="-m-4 mt-4"
+	>
+		<div class="flex w-full flex-row justify-between">
+			<Label cx="max-w-[96px] gap-1 text-xs">
+				<span class="flex gap-1 whitespace-nowrap">
+					Resolution
+					<abbr class="text-disabled">(width x height)</abbr>
+				</span>
+				<Select
+					slot="input"
+					on:change={handleResolutionChange}
+				>
+					{#each resolutionOptions as { width, height } (`${width}x${height}`)}
+						<option>
+							{width}x{height}
+						</option>
+					{/each}
+					{#if resolutionOptions.length === 0}
+						<option disabled>Camera did not return resolution options</option>
+					{/if}
+				</Select>
+			</Label>
+			<div class="flex flex-row items-end gap-2">
+				<Button
+					onclick={() => {
+						resolutionMutation.mutate(undefined)
+					}}
+				>
+					Reset
+				</Button>
+				<Button
+					icon="play-circle-outline"
+					variant="dark"
+					disabled={resolutionOptions.length === 0}
+					onclick={() => {
+						if (selectedResolution) {
+							resolutionMutation.mutate(selectedResolution)
+						}
+					}}
+				>
+					Execute
+				</Button>
+			</div>
+		</div>
+	</MutationSection>
+{/if}

package/dist/components/widgets/camera/live-or-polling-video.svelte.d.ts

@@ -0,0 +1,18 @@
+import { type QueryObserverResult } from '@tanstack/svelte-query';
+import { CameraClient } from '@viamrobotics/sdk';
+interface Props {
+    resourceName: string;
+    partID: string;
+    showResolutionOptions?: boolean;
+    isLive: boolean;
+    data: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['data'];
+    error?: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['error'];
+    isLoading: boolean;
+    videoClass?: string;
+    showMousePositionTooltip?: boolean;
+    setMediaStream?: ((mediaStream: MediaStream | null) => void) | undefined;
+    refetch: () => Promise<unknown>;
+}
+declare const LiveOrPollingVideo: import("svelte").Component<Props, {}, "">;
+type LiveOrPollingVideo = ReturnType<typeof LiveOrPollingVideo>;
+export default LiveOrPollingVideo;

package/dist/components/widgets/camera/picture-in-picture-button.svelte

@@ -0,0 +1,81 @@
+<script lang="ts">
+	import type { Writable } from 'svelte/store'
+
+	import { Button } from '@viamrobotics/prime-core'
+	import { onMount } from 'svelte'
+
+	import ErrorDisplay from '../../error.svelte'
+
+	interface Props {
+		resourceName: string
+		mediaStream: MediaStream | null
+		isShowingPip: Writable<boolean>
+	}
+
+	const { resourceName, mediaStream, isShowingPip }: Props = $props()
+
+	let videoElement = $state.raw<HTMLVideoElement>()
+	let lastErr = $state.raw<Error>()
+
+	const onEnter = () => {
+		$isShowingPip = true
+	}
+	const onLeave = () => {
+		$isShowingPip = false
+	}
+
+	onMount(() => {
+		videoElement?.addEventListener('enterpictureinpicture', onEnter)
+		videoElement?.addEventListener('leavepictureinpicture', onLeave)
+
+		return () => {
+			videoElement?.removeEventListener('enterpictureinpicture', onEnter)
+			videoElement?.removeEventListener('leavepictureinpicture', onLeave)
+			if ($isShowingPip) {
+				document.exitPictureInPicture()
+			}
+		}
+	})
+
+	$effect.pre(() => {
+		if (videoElement) {
+			videoElement.srcObject = mediaStream
+		}
+	})
+
+	const togglePictureInPicture = async () => {
+		lastErr = undefined
+
+		try {
+			await ($isShowingPip
+				? document.exitPictureInPicture()
+				: videoElement?.requestPictureInPicture())
+		} catch (error) {
+			lastErr = error as Error
+		}
+	}
+</script>
+
+<Button
+	icon="picture-in-picture-top-right"
+	onclick={togglePictureInPicture}
+>
+	Toggle picture-in-picture
+</Button>
+
+<!-- Display a fixed video element on the page but essentially hide it.
+  There must always be a video element visible for PiP. -->
+<video
+	class="fixed right-0 bottom-0 h-px w-px opacity-[0.01]"
+	bind:this={videoElement}
+	muted
+	autoplay
+	controls={false}
+	playsinline
+	aria-label={`${resourceName} picture-in-picture stream`}
+></video>
+
+<ErrorDisplay
+	class="max-w-50"
+	lastError={lastErr}
+/>

package/dist/components/widgets/camera/picture-in-picture-button.svelte.d.ts

@@ -0,0 +1,9 @@
+import type { Writable } from 'svelte/store';
+interface Props {
+    resourceName: string;
+    mediaStream: MediaStream | null;
+    isShowingPip: Writable<boolean>;
+}
+declare const PictureInPictureButton: import("svelte").Component<Props, {}, "">;
+type PictureInPictureButton = ReturnType<typeof PictureInPictureButton>;
+export default PictureInPictureButton;

package/dist/components/widgets/discovery/component-preview.d.ts

@@ -0,0 +1,9 @@
+import type { appRobotApi } from '@viamrobotics/sdk';
+import type { Snippet } from 'svelte';
+export interface ComponentPreview {
+    component: appRobotApi.ComponentConfig;
+    preview: string | undefined;
+    loading: boolean;
+}
+export type ComponentPreviewSnippet = Snippet<[ComponentPreview]>;
+export type ComponentPreviews = Record<string, ComponentPreview>;

package/dist/components/widgets/discovery/component-preview.js

@@ -0,0 +1 @@
+export {};

package/dist/components/widgets/discovery/discovery.svelte

@@ -0,0 +1,155 @@
+<script lang="ts">
+	import { appRobotApi, DiscoveryClient, Struct } from '@viamrobotics/sdk'
+	import {
+		createResourceClient,
+		createResourceMutation,
+		createResourceQuery,
+	} from '@viamrobotics/svelte-sdk'
+	import { isEqual } from 'lodash-es'
+
+	import ApiSection from '../../api-section.svelte'
+	import ConnectionStatus from '../../connection-status.svelte'
+	import Query from '../../query.svelte'
+	import RefetchController from '../../refetch-controller.svelte'
+	import {
+		createRefetchIntervalStore,
+		RefetchIntervals,
+	} from '../../refetch-interval-store.svelte'
+
+	import type { ComponentPreviews, ComponentPreviewSnippet } from './component-preview'
+
+	import ResourcesList from './resources-list.svelte'
+
+	interface Props {
+		partID: string
+		resourceName: string
+		/**  A handler for adding a component to the robot. */
+		onAddComponent?: ((component: appRobotApi.ComponentConfig) => void) | undefined
+		/** A snippet for creating component previews. Requires a `preview` DoCommand to be implemented. */
+		componentPreview?: ComponentPreviewSnippet
+	}
+
+	const { partID, resourceName, onAddComponent, componentPreview }: Props = $props()
+
+	const refetchInterval = createRefetchIntervalStore(
+		() => partID,
+		() => resourceName,
+		'discovery-card',
+		RefetchIntervals.MANUAL
+	)
+
+	const previews = $state<ComponentPreviews>({})
+	let discoveries = $state.raw<appRobotApi.ComponentConfig[]>([])
+	let discoveriesUpdated = $state(false)
+
+	const client = createResourceClient(
+		DiscoveryClient,
+		() => partID,
+		() => resourceName
+	)
+
+	const discoveryQuery = createResourceQuery(client, 'discoverResources', () => ({
+		refetchInterval: refetchInterval.current,
+	}))
+
+	const doCommandMutation = createResourceMutation(client, 'doCommand')
+
+	const makeDoCommandMutation = async (component: appRobotApi.ComponentConfig) => {
+		const parsedInput = Struct.fromJson({
+			command: 'preview',
+			attributes: component.attributes?.toJson() ?? {},
+		})
+
+		try {
+			const result = await doCommandMutation.mutateAsync([parsedInput])
+
+			const data = JSON.parse(JSON.stringify(result) ?? '{}') as {
+				preview: string
+			}
+			previews[component.name] = {
+				component,
+				preview: data.preview,
+				loading: false,
+			}
+		} catch {
+			previews[component.name] = {
+				component,
+				preview: undefined,
+				loading: false,
+			}
+		}
+	}
+
+	const makeDoCommandMutations = async (components: appRobotApi.ComponentConfig[]) => {
+		const [currentComponent, ...remainingComponents] = components
+		if (currentComponent === undefined) {
+			return
+		}
+
+		await makeDoCommandMutation(currentComponent)
+
+		if (remainingComponents.length > 0) {
+			await makeDoCommandMutations(remainingComponents)
+		}
+	}
+
+	$effect(() => {
+		if (discoveryQuery.isFetching) {
+			discoveriesUpdated = true
+		}
+	})
+
+	$effect(() => {
+		if (discoveryQuery.isSuccess) {
+			if (isEqual(discoveries, discoveryQuery.data)) {
+				discoveriesUpdated = false
+			} else {
+				discoveries = discoveryQuery.data
+			}
+		}
+	})
+
+	$effect(() => {
+		if (discoveries && discoveriesUpdated && componentPreview !== undefined) {
+			for (const discovery of discoveries) {
+				previews[discovery.name] = {
+					component: discovery,
+					preview: undefined,
+					loading: true,
+				}
+			}
+
+			void makeDoCommandMutations(discoveries)
+		}
+	})
+</script>
+
+<ConnectionStatus {partID}>
+	{#snippet connected()}
+		<div class="p-4 pb-3">
+			<RefetchController
+				{refetchInterval}
+				queries={[discoveryQuery]}
+			/>
+		</div>
+
+		<ApiSection
+			title="DiscoverResources"
+			class="relative"
+		>
+			<Query
+				query={discoveryQuery}
+				contentCx="h-6"
+			>
+				{#if discoveryQuery.data !== undefined}
+					<ResourcesList
+						data={discoveryQuery.data}
+						{previews}
+						{componentPreview}
+						{onAddComponent}
+					/>
+				{/if}
+			</Query>
+		</ApiSection>
+	{/snippet}
+</ConnectionStatus>

package/dist/components/widgets/discovery/discovery.svelte.d.ts

@@ -0,0 +1,13 @@
+import { appRobotApi } from '@viamrobotics/sdk';
+import type { ComponentPreviewSnippet } from './component-preview';
+interface Props {
+    partID: string;
+    resourceName: string;
+    /**  A handler for adding a component to the robot. */
+    onAddComponent?: ((component: appRobotApi.ComponentConfig) => void) | undefined;
+    /** A snippet for creating component previews. Requires a `preview` DoCommand to be implemented. */
+    componentPreview?: ComponentPreviewSnippet;
+}
+declare const Discovery: import("svelte").Component<Props, {}, "">;
+type Discovery = ReturnType<typeof Discovery>;
+export default Discovery;