@viamrobotics/test-widgets 0.6.0 → 0.7.0

package/dist/components/widgets/camera/camera.svelte

@@ -12,12 +12,15 @@
 		createRefetchIntervalStore,
 		RefetchIntervals,
 	} from '../../refetch-interval-store.svelte'
+	import SectionTitle from '../../section-title.svelte'
 
 	import PCDWidget from '../pcd/pcd-widget.svelte'
 	import ExportScreenshot from './export-screenshot.svelte'
+	import { getPanoCoverageFromXmp } from './get-pano-coverage-from-xmp'
 	import { getSourceNames } from './get-source-names'
 	import { getXmpJsonFromImageBytes, type XmpJson } from './get-xmp-json-from-image'
 	import LiveOrPollingVideo from './live-or-polling-video.svelte'
+	import LiveThreeSixtyCameraView from './live-three-sixty-camera-view.svelte'
 	import { pickImageForSource } from './pick-image-for-source'
 	import PictureInPictureButton from './picture-in-picture-button.svelte'
 	import ThreeSixtyCameraView from './three-sixty-camera-view.svelte'
@@ -58,16 +61,30 @@
 		() => resourceName
 	)
 
+	const isLive = $derived(refetchInterval.current === RefetchIntervals.LIVE)
+
 	const imageQuery = createResourceQuery(
 		client,
 		'getImages',
 		() => (selectedSource ? ([[selectedSource]] as [string[]]) : ([] as [])),
 		() => ({
-			enabled: isPlaying && refetchInterval.current !== RefetchIntervals.LIVE,
+			enabled: isPlaying && !isLive,
 			refetchInterval: refetchInterval.current,
 		})
 	)
 
+	// The live WebRTC stream carries no XMP, so when live we fetch a single frame
+	// to learn the GPano crop geometry needed to render the stream on the right band.
+	const liveCoverageProbe = createResourceQuery(
+		client,
+		'getImages',
+		() => (selectedSource ? ([[selectedSource]] as [string[]]) : ([] as [])),
+		() => ({
+			enabled: isPlaying && isLive,
+			refetchInterval: false,
+		})
+	)
+
 	$effect(() => {
 		if (sourceNames.length === 0 && imageQuery.data?.images) {
 			const names = getSourceNames(imageQuery.data.images)
@@ -79,7 +96,7 @@
 	})
 
 	const xmpJson = $derived.by((): XmpJson | null => {
-		const imageRecord = imageQuery.data?.images?.[0]
+		const imageRecord = isLive ? liveCoverageProbe.data?.images?.[0] : imageQuery.data?.images?.[0]
 		const image = imageRecord?.image
 		if (!image) {
 			return null
@@ -88,9 +105,14 @@
 		return getXmpJsonFromImageBytes(new Uint8Array(image), imageRecord.mimeType)
 	})
 
-	const is360EnabledImage = $derived.by((): boolean => {
-		return xmpJson?.['viam:is360'] === 'true'
-	})
+	// Covers both the full-sphere `viam:equirectangular` tag and GPano
+	// (Google Photo Sphere) cropped-area metadata. Non-null means the image
+	// can be rendered on a viewing sphere.
+	const panoCoverage = $derived.by(() => getPanoCoverageFromXmp(xmpJson))
+	// A GPano frame is a partial band, not a full 360° sphere.
+	const panoToggleLabel = $derived(
+		panoCoverage?.kind === 'gpano' ? 'Display as panorama' : 'Display as 360°'
+	)
 
 	const pointcloudQuery = createResourceQuery(client, 'getPointCloud', () => ({
 		enabled: isShowingPointcloud,
@@ -129,152 +151,179 @@
 	}
 
 	const headingID = $props.id()
+	const getImagesHeadingID = `${headingID}-images`
+	const getPointCloudHeadingID = `${headingID}-point-cloud`
 </script>
 
 <ConnectionStatus {partID}>
 	{#snippet connected()}
-		{#if isPlaying}
-			<div class="flex gap-4 p-4 pb-3">
-				<RefetchController
-					{refetchInterval}
-					allowLive
-					queries={[imageQuery, pointcloudQuery]}
+		<section
+			class="flex flex-col"
+			aria-labelledby={getImagesHeadingID}
+		>
+			<div class="flex flex-col gap-0.5 p-4 pb-0">
+				<SectionTitle
+					title="GetImages"
+					api="rdk:component:camera"
+					headingId={getImagesHeadingID}
 				/>
-				{#if sourceNames.length > 0 && refetchInterval.current !== RefetchIntervals.LIVE}
-					<Label position="left">
-						Source
-						<Select
-							value={selectedSource}
-							on:change={onSourceSelect}
-							slot="input"
-						>
-							{#each sourceNames as name (name)}
-								<option value={name}>{name}</option>
-							{/each}
-						</Select>
-					</Label>
-				{/if}
-				{#if is360EnabledImage}
-					<Label>
-						Display as 360°
-						<ToggleButtons
-							slot="input"
-							options={['On', 'Off']}
-							selected={displayAs360 ? 'On' : 'Off'}
-							on:input={(event) => {
-								displayAs360 = event.detail === 'On'
-							}}
-						/>
-					</Label>
-				{/if}
-			</div>
-		{/if}
-		<div class="flex h-full w-full gap-4 p-4">
-			<div class="grow">
-				{#if isPlaying}
-					{#if displayAs360}
-						<Canvas>
-							<ThreeSixtyCameraView data={imageQuery.data} />
-						</Canvas>
-					{:else}
-						<LiveOrPollingVideo
-							{partID}
-							{resourceName}
-							showResolutionOptions
-							isLive={refetchInterval.current === RefetchIntervals.LIVE}
-							data={imageQuery.data}
-							error={imageQuery.error}
-							isLoading={imageQuery.isLoading}
-							refetch={imageQuery.refetch}
-							showMousePositionTooltip={mousePostionTooltip === 'On'}
-							sourceName={selectedSource}
-						/>
-					{/if}
-				{:else}
-					<div class="bg-medium flex h-64 w-80 items-center justify-center">
-						<Button
-							icon="play-circle-outline"
-							variant="dark"
-							onclick={() => {
-								isPlaying = true
-							}}
-						>
-							Start feed
-						</Button>
-					</div>
-				{/if}
 			</div>
 			{#if isPlaying}
-				<div class="flex flex-col items-start gap-2">
-					<ExportScreenshot
-						name={client.current?.name ?? ''}
-						sourceName={selectedSource}
-						getImage={exportScreenshotQuery.refetch}
+				<div class="flex gap-4 p-4 pb-3">
+					<RefetchController
+						{refetchInterval}
+						allowLive
+						queries={[imageQuery, pointcloudQuery]}
 					/>
-					{#if addImageToDataset}
-						<Button
-							icon="layers-triple-outline"
-							onclick={async () => {
-								let imgData
-								if (refetchInterval.current === RefetchIntervals.LIVE) {
-									const { isSuccess, data } = await imageQuery.refetch()
-									if (!isSuccess) return
-									imgData = data
-								} else {
-									imgData = imageQuery.data
-								}
-
-								const matchingImage = pickImageForSource(imgData?.images, selectedSource)
-								const image = matchingImage?.image
-								const mimeType = matchingImage?.mimeType
-
-								if (image) {
-									addImageToDataset({
-										binaryData: new Uint8Array(image),
-										partID,
-										componentType: 'camera',
-										componentName: resourceName,
-										methodName: 'captureAllFromCamera',
-										mimeType: mimeType ?? 'image/png',
-										dataRequestTimes: [new Date(), new Date()],
-									})
-								}
-							}}
-						>
-							Add current image to dataset
-						</Button>
+					{#if sourceNames.length > 0 && refetchInterval.current !== RefetchIntervals.LIVE}
+						<Label position="left">
+							Source
+							<Select
+								value={selectedSource}
+								on:change={onSourceSelect}
+								slot="input"
+							>
+								{#each sourceNames as name (name)}
+									<option value={name}>{name}</option>
+								{/each}
+							</Select>
+						</Label>
 					{/if}
-					{#if !isFirefox}
-						<PictureInPictureButton
-							{resourceName}
-							rate={refetchInterval.current}
-						/>
+					{#if panoCoverage}
+						<Label>
+							{panoToggleLabel}
+							<ToggleButtons
+								slot="input"
+								options={['On', 'Off']}
+								selected={displayAs360 ? 'On' : 'Off'}
+								on:input={(event) => {
+									displayAs360 = event.detail === 'On'
+								}}
+							/>
+						</Label>
 					{/if}
-					<Label>
-						Mouse Position Tooltip
-
-						<ToggleButtons
-							slot="input"
-							options={['On', 'Off']}
-							selected={mousePostionTooltip}
-							on:input={setMousePostionTooltip}
-						/>
-					</Label>
 				</div>
 			{/if}
-		</div>
+			<div class="flex h-full w-full gap-4 p-4">
+				<div class="grow">
+					{#if isPlaying}
+						{#if displayAs360}
+							{#if isLive}
+								<!-- renderMode="always" keeps the live VideoTexture advancing each frame -->
+								<Canvas renderMode="always">
+									<LiveThreeSixtyCameraView
+										{partID}
+										{resourceName}
+										coverage={panoCoverage}
+									/>
+								</Canvas>
+							{:else}
+								<Canvas>
+									<ThreeSixtyCameraView
+										data={imageQuery.data}
+										coverage={panoCoverage}
+									/>
+								</Canvas>
+							{/if}
+						{:else}
+							<LiveOrPollingVideo
+								{partID}
+								{resourceName}
+								showResolutionOptions
+								{isLive}
+								data={imageQuery.data}
+								error={imageQuery.error}
+								isLoading={imageQuery.isLoading}
+								refetch={imageQuery.refetch}
+								showMousePositionTooltip={mousePostionTooltip === 'On'}
+								sourceName={selectedSource}
+							/>
+						{/if}
+					{:else}
+						<div class="bg-medium flex h-64 w-80 items-center justify-center">
+							<Button
+								icon="play-circle-outline"
+								variant="dark"
+								onclick={() => {
+									isPlaying = true
+								}}
+							>
+								Start feed
+							</Button>
+						</div>
+					{/if}
+				</div>
+				{#if isPlaying}
+					<div class="flex flex-col items-start gap-2">
+						<ExportScreenshot
+							name={client.current?.name ?? ''}
+							sourceName={selectedSource}
+							getImage={exportScreenshotQuery.refetch}
+						/>
+						{#if addImageToDataset}
+							<Button
+								icon="layers-triple-outline"
+								onclick={async () => {
+									let imgData
+									if (refetchInterval.current === RefetchIntervals.LIVE) {
+										const { isSuccess, data } = await imageQuery.refetch()
+										if (!isSuccess) return
+										imgData = data
+									} else {
+										imgData = imageQuery.data
+									}
+
+									const matchingImage = pickImageForSource(imgData?.images, selectedSource)
+									const image = matchingImage?.image
+									const mimeType = matchingImage?.mimeType
+
+									if (image) {
+										addImageToDataset({
+											binaryData: new Uint8Array(image),
+											partID,
+											componentType: 'camera',
+											componentName: resourceName,
+											methodName: 'captureAllFromCamera',
+											mimeType: mimeType ?? 'image/png',
+											dataRequestTimes: [new Date(), new Date()],
+										})
+									}
+								}}
+							>
+								Add current image to dataset
+							</Button>
+						{/if}
+						{#if !isFirefox}
+							<PictureInPictureButton
+								{resourceName}
+								rate={refetchInterval.current}
+							/>
+						{/if}
+						<Label>
+							Mouse Position Tooltip
+
+							<ToggleButtons
+								slot="input"
+								options={['On', 'Off']}
+								selected={mousePostionTooltip}
+								on:input={setMousePostionTooltip}
+							/>
+						</Label>
+					</div>
+				{/if}
+			</div>
+		</section>
 
 		<section
 			class="flex flex-col gap-4 p-4"
-			aria-labelledby={headingID}
+			aria-labelledby={getPointCloudHeadingID}
 		>
-			<div>
-				<h3
-					id={headingID}
-					class="pb-1.5 font-semibold"
-				>
-					GetPointCloud
-				</h3>
+			<div class="flex flex-col gap-0.5">
+				<SectionTitle
+					title="GetPointCloud"
+					api="rdk:component:camera"
+					headingId={getPointCloudHeadingID}
+				/>
 				<p class="text-subtle-2 text-xs">
 					Get depth measurement data from cameras with depth sensing support
 				</p>

package/dist/components/widgets/camera/get-pano-coverage-from-xmp.d.ts

@@ -0,0 +1,31 @@
+import type { XmpJson } from './get-xmp-json-from-image';
+/**
+ * Angular extent of a (partial) sphere, in Three.js `SphereGeometry` terms. The
+ * `kind` discriminator records which metadata form produced it so the UI can pick
+ * a matching label; the renderer only consumes the angles.
+ */
+export interface PanoCoverage {
+    kind: 'equirectangular' | 'gpano';
+    /** Azimuth (longitude) start, radians. */
+    phiStart: number;
+    /** Azimuth span, radians (full sphere = 2π). */
+    phiLength: number;
+    /** Polar start from the +Y north pole, radians. */
+    thetaStart: number;
+    /** Polar span, radians (full sphere = π). */
+    thetaLength: number;
+}
+/**
+ * Resolve how an image's pixels map onto a viewing sphere from its XMP metadata.
+ *
+ * - `viam:equirectangular="true"` → a full 360°×180° sphere.
+ * - `GPano:ProjectionType="equirectangular"` → the cropped sub-rectangle described
+ *   by the GPano `FullPano*`/`CroppedArea*` pixel fields, mapped to a partial sphere
+ *   (e.g. an equatorial band with empty poles). Malformed/zero fields fall back to
+ *   a full sphere rather than producing NaN geometry.
+ *
+ * @param xmp - Parsed XMP attributes, or `null` when the image has none.
+ * @returns The sphere coverage, or `null` if the image is not a sphere-displayable
+ *   equirectangular panorama.
+ */
+export declare const getPanoCoverageFromXmp: (xmp: XmpJson | null) => PanoCoverage | null;

package/dist/components/widgets/camera/get-pano-coverage-from-xmp.js

@@ -0,0 +1,56 @@
+const TAU = Math.PI * 2;
+const FULL_ANGLES = {
+    phiStart: 0,
+    phiLength: TAU,
+    thetaStart: 0,
+    thetaLength: Math.PI,
+};
+/**
+ * Resolve how an image's pixels map onto a viewing sphere from its XMP metadata.
+ *
+ * - `viam:equirectangular="true"` → a full 360°×180° sphere.
+ * - `GPano:ProjectionType="equirectangular"` → the cropped sub-rectangle described
+ *   by the GPano `FullPano*`/`CroppedArea*` pixel fields, mapped to a partial sphere
+ *   (e.g. an equatorial band with empty poles). Malformed/zero fields fall back to
+ *   a full sphere rather than producing NaN geometry.
+ *
+ * @param xmp - Parsed XMP attributes, or `null` when the image has none.
+ * @returns The sphere coverage, or `null` if the image is not a sphere-displayable
+ *   equirectangular panorama.
+ */
+export const getPanoCoverageFromXmp = (xmp) => {
+    if (!xmp) {
+        return null;
+    }
+    if (xmp['viam:equirectangular'] === 'true') {
+        return { kind: 'equirectangular', ...FULL_ANGLES };
+    }
+    if (xmp['GPano:ProjectionType'] !== 'equirectangular') {
+        return null;
+    }
+    const num = (key) => Number(xmp[key]);
+    const fullWidth = num('GPano:FullPanoWidthPixels');
+    const fullHeight = num('GPano:FullPanoHeightPixels');
+    // Malformed/zero dims → full sphere rather than NaN geometry.
+    if (!(fullWidth > 0) || !(fullHeight > 0)) {
+        return { kind: 'gpano', ...FULL_ANGLES };
+    }
+    const croppedWidth = num('GPano:CroppedAreaImageWidthPixels');
+    const croppedHeight = num('GPano:CroppedAreaImageHeightPixels');
+    // Malformed/zero cropped dims → full sphere rather than zero-area geometry.
+    if (!(croppedWidth > 0) || !(croppedHeight > 0)) {
+        return { kind: 'gpano', ...FULL_ANGLES };
+    }
+    const coverage = {
+        kind: 'gpano',
+        phiStart: TAU * (num('GPano:CroppedAreaLeftPixels') / fullWidth),
+        phiLength: TAU * (croppedWidth / fullWidth),
+        thetaStart: Math.PI * (num('GPano:CroppedAreaTopPixels') / fullHeight),
+        thetaLength: Math.PI * (croppedHeight / fullHeight),
+    };
+    // Any non-finite cropped field → full sphere fallback.
+    const angles = [coverage.phiStart, coverage.phiLength, coverage.thetaStart, coverage.thetaLength];
+    return angles.every((element) => Number.isFinite(element))
+        ? coverage
+        : { kind: 'gpano', ...FULL_ANGLES };
+};

package/dist/components/widgets/camera/live-or-polling-video.svelte

@@ -393,6 +393,7 @@
 {#if showResolutionOptions && isLive}
 	<MutationSection
 		title="SetStreamOptions"
+		api="rdk:component:camera"
 		description="Change the resolution of the live stream video feed"
 		lastError={resolutionMutation.error}
 		class="-m-4 mt-4"

package/dist/components/widgets/camera/live-three-sixty-camera-view.svelte

@@ -0,0 +1,61 @@
+<script lang="ts">
+	import { useThrelte } from '@threlte/core'
+	import { createStreamClient } from '@viamrobotics/svelte-sdk'
+	import { VideoTexture } from 'three'
+
+	import type { PanoCoverage } from './get-pano-coverage-from-xmp'
+
+	import PanoSphereScene from './pano-sphere-scene.svelte'
+
+	interface Props {
+		partID: string
+		resourceName: string
+		coverage?: PanoCoverage | null
+	}
+
+	const { partID, resourceName, coverage = null }: Props = $props()
+
+	const stream = createStreamClient(
+		() => partID,
+		() => resourceName
+	)
+	const { renderer } = useThrelte()
+
+	// A muted, inline, autoplaying <video> backs the VideoTexture. Mirrors the
+	// element setup in live-or-polling-video.svelte; created up front like the
+	// other camera widgets create their <img>/<canvas> elements.
+	const video = document.createElement('video')
+	video.muted = true
+	video.autoplay = true
+	video.playsInline = true
+
+	let map = $state.raw<VideoTexture>()
+
+	$effect(() => {
+		const mediaStream = stream.mediaStream
+		if (!mediaStream) {
+			video.srcObject = null
+			map = undefined
+			return
+		}
+
+		video.srcObject = mediaStream
+		void video.play()
+
+		const texture = new VideoTexture(video)
+		texture.colorSpace = renderer.outputColorSpace
+		map = texture
+
+		return () => {
+			texture.dispose()
+			video.pause()
+			video.srcObject = null
+			map = undefined
+		}
+	})
+</script>
+
+<PanoSphereScene
+	{coverage}
+	{map}
+/>

package/dist/components/widgets/camera/live-three-sixty-camera-view.svelte.d.ts

@@ -0,0 +1,9 @@
+import type { PanoCoverage } from './get-pano-coverage-from-xmp';
+interface Props {
+    partID: string;
+    resourceName: string;
+    coverage?: PanoCoverage | null;
+}
+declare const LiveThreeSixtyCameraView: import("svelte").Component<Props, {}, "">;
+type LiveThreeSixtyCameraView = ReturnType<typeof LiveThreeSixtyCameraView>;
+export default LiveThreeSixtyCameraView;

package/dist/components/widgets/camera/pano-sphere-scene.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+	import { T } from '@threlte/core'
+	import { OrbitControls } from '@threlte/extras'
+	import { BackSide, type Texture } from 'three'
+
+	import type { PanoCoverage } from './get-pano-coverage-from-xmp'
+
+	interface Props {
+		coverage?: PanoCoverage | null
+		/** Texture to paint on the inside of the sphere (image frame or live video). */
+		map?: Texture
+	}
+
+	const { coverage = null, map }: Props = $props()
+
+	// Default to a full sphere so a missing coverage renders the original 360° view.
+	// `kind` is irrelevant to geometry; only the angles are consumed below.
+	const sphere = $derived(
+		coverage ?? { phiStart: 0, phiLength: Math.PI * 2, thetaStart: 0, thetaLength: Math.PI }
+	)
+
+	// A band that covers less than the full pole-to-pole sweep is a horizontal-only
+	// view: fitting the camera's vertical FOV to the band removes the empty sphere
+	// above and below it, and locking the vertical tilt keeps the user from swinging
+	// toward those empty caps. A full sphere keeps the original free 75° view.
+	const isPartialBand = $derived(sphere.thetaLength < Math.PI - 1e-3)
+	const verticalCenter = $derived(sphere.thetaStart + sphere.thetaLength / 2)
+	const fov = $derived(isPartialBand ? (sphere.thetaLength * 180) / Math.PI : 75)
+</script>
+
+<T.PerspectiveCamera
+	makeDefault
+	position={[0, 0, 0.1]}
+	{fov}
+>
+	<OrbitControls
+		enableZoom={true}
+		enablePan={false}
+		minPolarAngle={isPartialBand ? verticalCenter : 0}
+		maxPolarAngle={isPartialBand ? verticalCenter : Math.PI}
+	/>
+</T.PerspectiveCamera>
+
+{#if map}
+	<T.Mesh scale={[-1, 1, 1]}>
+		<T.SphereGeometry
+			args={[500, 60, 40, sphere.phiStart, sphere.phiLength, sphere.thetaStart, sphere.thetaLength]}
+		/>
+		<T.MeshBasicMaterial
+			{map}
+			side={BackSide}
+		/>
+	</T.Mesh>
+{/if}

package/dist/components/widgets/camera/pano-sphere-scene.svelte.d.ts

@@ -0,0 +1,10 @@
+import { type Texture } from 'three';
+import type { PanoCoverage } from './get-pano-coverage-from-xmp';
+interface Props {
+    coverage?: PanoCoverage | null;
+    /** Texture to paint on the inside of the sphere (image frame or live video). */
+    map?: Texture;
+}
+declare const PanoSphereScene: import("svelte").Component<Props, {}, "">;
+type PanoSphereScene = ReturnType<typeof PanoSphereScene>;
+export default PanoSphereScene;