@viamrobotics/test-widgets 0.6.0 → 0.6.1

package/dist/components/widgets/camera/camera.svelte

@@ -15,9 +15,11 @@
 
 	import PCDWidget from '../pcd/pcd-widget.svelte'
 	import ExportScreenshot from './export-screenshot.svelte'
+	import { getPanoCoverageFromXmp } from './get-pano-coverage-from-xmp'
 	import { getSourceNames } from './get-source-names'
 	import { getXmpJsonFromImageBytes, type XmpJson } from './get-xmp-json-from-image'
 	import LiveOrPollingVideo from './live-or-polling-video.svelte'
+	import LiveThreeSixtyCameraView from './live-three-sixty-camera-view.svelte'
 	import { pickImageForSource } from './pick-image-for-source'
 	import PictureInPictureButton from './picture-in-picture-button.svelte'
 	import ThreeSixtyCameraView from './three-sixty-camera-view.svelte'
@@ -58,16 +60,30 @@
 		() => resourceName
 	)
 
+	const isLive = $derived(refetchInterval.current === RefetchIntervals.LIVE)
+
 	const imageQuery = createResourceQuery(
 		client,
 		'getImages',
 		() => (selectedSource ? ([[selectedSource]] as [string[]]) : ([] as [])),
 		() => ({
-			enabled: isPlaying && refetchInterval.current !== RefetchIntervals.LIVE,
+			enabled: isPlaying && !isLive,
 			refetchInterval: refetchInterval.current,
 		})
 	)
 
+	// The live WebRTC stream carries no XMP, so when live we fetch a single frame
+	// to learn the GPano crop geometry needed to render the stream on the right band.
+	const liveCoverageProbe = createResourceQuery(
+		client,
+		'getImages',
+		() => (selectedSource ? ([[selectedSource]] as [string[]]) : ([] as [])),
+		() => ({
+			enabled: isPlaying && isLive,
+			refetchInterval: false,
+		})
+	)
+
 	$effect(() => {
 		if (sourceNames.length === 0 && imageQuery.data?.images) {
 			const names = getSourceNames(imageQuery.data.images)
@@ -79,7 +95,7 @@
 	})
 
 	const xmpJson = $derived.by((): XmpJson | null => {
-		const imageRecord = imageQuery.data?.images?.[0]
+		const imageRecord = isLive ? liveCoverageProbe.data?.images?.[0] : imageQuery.data?.images?.[0]
 		const image = imageRecord?.image
 		if (!image) {
 			return null
@@ -88,9 +104,14 @@
 		return getXmpJsonFromImageBytes(new Uint8Array(image), imageRecord.mimeType)
 	})
 
-	const is360EnabledImage = $derived.by((): boolean => {
-		return xmpJson?.['viam:is360'] === 'true'
-	})
+	// Covers both the full-sphere `viam:equirectangular` tag and GPano
+	// (Google Photo Sphere) cropped-area metadata. Non-null means the image
+	// can be rendered on a viewing sphere.
+	const panoCoverage = $derived.by(() => getPanoCoverageFromXmp(xmpJson))
+	// A GPano frame is a partial band, not a full 360° sphere.
+	const panoToggleLabel = $derived(
+		panoCoverage?.kind === 'gpano' ? 'Display as panorama' : 'Display as 360°'
+	)
 
 	const pointcloudQuery = createResourceQuery(client, 'getPointCloud', () => ({
 		enabled: isShowingPointcloud,
@@ -154,9 +175,9 @@
 						</Select>
 					</Label>
 				{/if}
-				{#if is360EnabledImage}
+				{#if panoCoverage}
 					<Label>
-						Display as 360°
+						{panoToggleLabel}
 						<ToggleButtons
 							slot="input"
 							options={['On', 'Off']}
@@ -173,15 +194,29 @@
 			<div class="grow">
 				{#if isPlaying}
 					{#if displayAs360}
-						<Canvas>
-							<ThreeSixtyCameraView data={imageQuery.data} />
-						</Canvas>
+						{#if isLive}
+							<!-- renderMode="always" keeps the live VideoTexture advancing each frame -->
+							<Canvas renderMode="always">
+								<LiveThreeSixtyCameraView
+									{partID}
+									{resourceName}
+									coverage={panoCoverage}
+								/>
+							</Canvas>
+						{:else}
+							<Canvas>
+								<ThreeSixtyCameraView
+									data={imageQuery.data}
+									coverage={panoCoverage}
+								/>
+							</Canvas>
+						{/if}
 					{:else}
 						<LiveOrPollingVideo
 							{partID}
 							{resourceName}
 							showResolutionOptions
-							isLive={refetchInterval.current === RefetchIntervals.LIVE}
+							{isLive}
 							data={imageQuery.data}
 							error={imageQuery.error}
 							isLoading={imageQuery.isLoading}

package/dist/components/widgets/camera/get-pano-coverage-from-xmp.d.ts

@@ -0,0 +1,31 @@
+import type { XmpJson } from './get-xmp-json-from-image';
+/**
+ * Angular extent of a (partial) sphere, in Three.js `SphereGeometry` terms. The
+ * `kind` discriminator records which metadata form produced it so the UI can pick
+ * a matching label; the renderer only consumes the angles.
+ */
+export interface PanoCoverage {
+    kind: 'equirectangular' | 'gpano';
+    /** Azimuth (longitude) start, radians. */
+    phiStart: number;
+    /** Azimuth span, radians (full sphere = 2π). */
+    phiLength: number;
+    /** Polar start from the +Y north pole, radians. */
+    thetaStart: number;
+    /** Polar span, radians (full sphere = π). */
+    thetaLength: number;
+}
+/**
+ * Resolve how an image's pixels map onto a viewing sphere from its XMP metadata.
+ *
+ * - `viam:equirectangular="true"` → a full 360°×180° sphere.
+ * - `GPano:ProjectionType="equirectangular"` → the cropped sub-rectangle described
+ *   by the GPano `FullPano*`/`CroppedArea*` pixel fields, mapped to a partial sphere
+ *   (e.g. an equatorial band with empty poles). Malformed/zero fields fall back to
+ *   a full sphere rather than producing NaN geometry.
+ *
+ * @param xmp - Parsed XMP attributes, or `null` when the image has none.
+ * @returns The sphere coverage, or `null` if the image is not a sphere-displayable
+ *   equirectangular panorama.
+ */
+export declare const getPanoCoverageFromXmp: (xmp: XmpJson | null) => PanoCoverage | null;

package/dist/components/widgets/camera/get-pano-coverage-from-xmp.js

@@ -0,0 +1,56 @@
+const TAU = Math.PI * 2;
+const FULL_ANGLES = {
+    phiStart: 0,
+    phiLength: TAU,
+    thetaStart: 0,
+    thetaLength: Math.PI,
+};
+/**
+ * Resolve how an image's pixels map onto a viewing sphere from its XMP metadata.
+ *
+ * - `viam:equirectangular="true"` → a full 360°×180° sphere.
+ * - `GPano:ProjectionType="equirectangular"` → the cropped sub-rectangle described
+ *   by the GPano `FullPano*`/`CroppedArea*` pixel fields, mapped to a partial sphere
+ *   (e.g. an equatorial band with empty poles). Malformed/zero fields fall back to
+ *   a full sphere rather than producing NaN geometry.
+ *
+ * @param xmp - Parsed XMP attributes, or `null` when the image has none.
+ * @returns The sphere coverage, or `null` if the image is not a sphere-displayable
+ *   equirectangular panorama.
+ */
+export const getPanoCoverageFromXmp = (xmp) => {
+    if (!xmp) {
+        return null;
+    }
+    if (xmp['viam:equirectangular'] === 'true') {
+        return { kind: 'equirectangular', ...FULL_ANGLES };
+    }
+    if (xmp['GPano:ProjectionType'] !== 'equirectangular') {
+        return null;
+    }
+    const num = (key) => Number(xmp[key]);
+    const fullWidth = num('GPano:FullPanoWidthPixels');
+    const fullHeight = num('GPano:FullPanoHeightPixels');
+    // Malformed/zero dims → full sphere rather than NaN geometry.
+    if (!(fullWidth > 0) || !(fullHeight > 0)) {
+        return { kind: 'gpano', ...FULL_ANGLES };
+    }
+    const croppedWidth = num('GPano:CroppedAreaImageWidthPixels');
+    const croppedHeight = num('GPano:CroppedAreaImageHeightPixels');
+    // Malformed/zero cropped dims → full sphere rather than zero-area geometry.
+    if (!(croppedWidth > 0) || !(croppedHeight > 0)) {
+        return { kind: 'gpano', ...FULL_ANGLES };
+    }
+    const coverage = {
+        kind: 'gpano',
+        phiStart: TAU * (num('GPano:CroppedAreaLeftPixels') / fullWidth),
+        phiLength: TAU * (croppedWidth / fullWidth),
+        thetaStart: Math.PI * (num('GPano:CroppedAreaTopPixels') / fullHeight),
+        thetaLength: Math.PI * (croppedHeight / fullHeight),
+    };
+    // Any non-finite cropped field → full sphere fallback.
+    const angles = [coverage.phiStart, coverage.phiLength, coverage.thetaStart, coverage.thetaLength];
+    return angles.every((element) => Number.isFinite(element))
+        ? coverage
+        : { kind: 'gpano', ...FULL_ANGLES };
+};

package/dist/components/widgets/camera/live-three-sixty-camera-view.svelte

@@ -0,0 +1,61 @@
+<script lang="ts">
+	import { useThrelte } from '@threlte/core'
+	import { createStreamClient } from '@viamrobotics/svelte-sdk'
+	import { VideoTexture } from 'three'
+
+	import type { PanoCoverage } from './get-pano-coverage-from-xmp'
+
+	import PanoSphereScene from './pano-sphere-scene.svelte'
+
+	interface Props {
+		partID: string
+		resourceName: string
+		coverage?: PanoCoverage | null
+	}
+
+	const { partID, resourceName, coverage = null }: Props = $props()
+
+	const stream = createStreamClient(
+		() => partID,
+		() => resourceName
+	)
+	const { renderer } = useThrelte()
+
+	// A muted, inline, autoplaying <video> backs the VideoTexture. Mirrors the
+	// element setup in live-or-polling-video.svelte; created up front like the
+	// other camera widgets create their <img>/<canvas> elements.
+	const video = document.createElement('video')
+	video.muted = true
+	video.autoplay = true
+	video.playsInline = true
+
+	let map = $state.raw<VideoTexture>()
+
+	$effect(() => {
+		const mediaStream = stream.mediaStream
+		if (!mediaStream) {
+			video.srcObject = null
+			map = undefined
+			return
+		}
+
+		video.srcObject = mediaStream
+		void video.play()
+
+		const texture = new VideoTexture(video)
+		texture.colorSpace = renderer.outputColorSpace
+		map = texture
+
+		return () => {
+			texture.dispose()
+			video.pause()
+			video.srcObject = null
+			map = undefined
+		}
+	})
+</script>
+
+<PanoSphereScene
+	{coverage}
+	{map}
+/>

package/dist/components/widgets/camera/live-three-sixty-camera-view.svelte.d.ts

@@ -0,0 +1,9 @@
+import type { PanoCoverage } from './get-pano-coverage-from-xmp';
+interface Props {
+    partID: string;
+    resourceName: string;
+    coverage?: PanoCoverage | null;
+}
+declare const LiveThreeSixtyCameraView: import("svelte").Component<Props, {}, "">;
+type LiveThreeSixtyCameraView = ReturnType<typeof LiveThreeSixtyCameraView>;
+export default LiveThreeSixtyCameraView;

package/dist/components/widgets/camera/pano-sphere-scene.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+	import { T } from '@threlte/core'
+	import { OrbitControls } from '@threlte/extras'
+	import { BackSide, type Texture } from 'three'
+
+	import type { PanoCoverage } from './get-pano-coverage-from-xmp'
+
+	interface Props {
+		coverage?: PanoCoverage | null
+		/** Texture to paint on the inside of the sphere (image frame or live video). */
+		map?: Texture
+	}
+
+	const { coverage = null, map }: Props = $props()
+
+	// Default to a full sphere so a missing coverage renders the original 360° view.
+	// `kind` is irrelevant to geometry; only the angles are consumed below.
+	const sphere = $derived(
+		coverage ?? { phiStart: 0, phiLength: Math.PI * 2, thetaStart: 0, thetaLength: Math.PI }
+	)
+
+	// A band that covers less than the full pole-to-pole sweep is a horizontal-only
+	// view: fitting the camera's vertical FOV to the band removes the empty sphere
+	// above and below it, and locking the vertical tilt keeps the user from swinging
+	// toward those empty caps. A full sphere keeps the original free 75° view.
+	const isPartialBand = $derived(sphere.thetaLength < Math.PI - 1e-3)
+	const verticalCenter = $derived(sphere.thetaStart + sphere.thetaLength / 2)
+	const fov = $derived(isPartialBand ? (sphere.thetaLength * 180) / Math.PI : 75)
+</script>
+
+<T.PerspectiveCamera
+	makeDefault
+	position={[0, 0, 0.1]}
+	{fov}
+>
+	<OrbitControls
+		enableZoom={true}
+		enablePan={false}
+		minPolarAngle={isPartialBand ? verticalCenter : 0}
+		maxPolarAngle={isPartialBand ? verticalCenter : Math.PI}
+	/>
+</T.PerspectiveCamera>
+
+{#if map}
+	<T.Mesh scale={[-1, 1, 1]}>
+		<T.SphereGeometry
+			args={[500, 60, 40, sphere.phiStart, sphere.phiLength, sphere.thetaStart, sphere.thetaLength]}
+		/>
+		<T.MeshBasicMaterial
+			{map}
+			side={BackSide}
+		/>
+	</T.Mesh>
+{/if}

package/dist/components/widgets/camera/pano-sphere-scene.svelte.d.ts

@@ -0,0 +1,10 @@
+import { type Texture } from 'three';
+import type { PanoCoverage } from './get-pano-coverage-from-xmp';
+interface Props {
+    coverage?: PanoCoverage | null;
+    /** Texture to paint on the inside of the sphere (image frame or live video). */
+    map?: Texture;
+}
+declare const PanoSphereScene: import("svelte").Component<Props, {}, "">;
+type PanoSphereScene = ReturnType<typeof PanoSphereScene>;
+export default PanoSphereScene;

package/dist/components/widgets/camera/three-sixty-camera-view.svelte

@@ -1,19 +1,22 @@
 <script lang="ts">
 	import type { QueryObserverResult } from '@tanstack/svelte-query'
-	import type { OrbitControls as OrbitControlsType } from 'three/examples/jsm/controls/OrbitControls.js'
+	import type { Texture } from 'three'
 
-	import { T } from '@threlte/core'
-	import { OrbitControls, useTexture } from '@threlte/extras'
+	import { useTexture } from '@threlte/extras'
 	import { CameraClient } from '@viamrobotics/sdk'
-	import { BackSide } from 'three'
 
-	const {
-		data,
-	}: { data: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['data'] } =
-		$props()
+	import type { PanoCoverage } from './get-pano-coverage-from-xmp'
+
+	import PanoSphereScene from './pano-sphere-scene.svelte'
+
+	interface Props {
+		data: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['data']
+		coverage?: PanoCoverage | null
+	}
+
+	const { data, coverage = null }: Props = $props()
 
 	let imageUrl = $state.raw('')
-	let controlsRef = $state<OrbitControlsType>()
 
 	$effect(() => {
 		const imageRecord = data?.images?.[0]
@@ -23,47 +26,34 @@
 			return
 		}
 
-		const imageBytes = new Uint8Array(image)
-		const imageBlob = new Blob([imageBytes], {
+		const imageBlob = new Blob([new Uint8Array(image)], {
 			type: imageRecord.mimeType || 'image/jpeg',
 		})
 		const url = URL.createObjectURL(imageBlob)
 		imageUrl = url
 
-		return () => {
-			URL.revokeObjectURL(url)
-		}
+		return () => URL.revokeObjectURL(url)
 	})
 
-	const texture = $derived.by(() => (imageUrl ? useTexture(imageUrl) : null))
-</script>
-
-<T.PerspectiveCamera
-	makeDefault
-	position={[0, 0, 0.1]}
-	fov={75}
->
-	<OrbitControls
-		bind:ref={controlsRef}
-		enableZoom={true}
-		enablePan={false}
-	/>
-</T.PerspectiveCamera>
+	// `useTexture` returns a store that resolves to the decoded texture (with the
+	// renderer's color space already applied). Mirror its value into reactive state so
+	// the sphere scene can render the camera even before the first frame loads, and so
+	// `map` clears when the source goes away.
+	const textureStore = $derived(imageUrl ? useTexture(imageUrl) : undefined)
+	let map = $state.raw<Texture>()
+	$effect(() => {
+		if (!textureStore) {
+			map = undefined
+			return
+		}
 
-{#if $texture}
-	{#await texture then map}
-		<T.Mesh scale={[-1, 1, 1]}>
-			<T.SphereGeometry args={[500, 60, 40]} />
-			<T.MeshBasicMaterial
-				{map}
-				side={BackSide}
-			/>
-		</T.Mesh>
-	{/await}
-{/if}
+		return textureStore.subscribe((value) => {
+			map = value ?? undefined
+		})
+	})
+</script>
 
-<T.AmbientLight intensity={0.5} />
-<T.DirectionalLight
-	position={[5, 5, 5]}
-	intensity={1}
+<PanoSphereScene
+	{coverage}
+	{map}
 />

package/dist/components/widgets/camera/three-sixty-camera-view.svelte.d.ts

@@ -1,8 +1,10 @@
 import type { QueryObserverResult } from '@tanstack/svelte-query';
 import { CameraClient } from '@viamrobotics/sdk';
-type $$ComponentProps = {
+import type { PanoCoverage } from './get-pano-coverage-from-xmp';
+interface Props {
     data: QueryObserverResult<Awaited<ReturnType<CameraClient['getImages']>>>['data'];
-};
-declare const ThreeSixtyCameraView: import("svelte").Component<$$ComponentProps, {}, "">;
+    coverage?: PanoCoverage | null;
+}
+declare const ThreeSixtyCameraView: import("svelte").Component<Props, {}, "">;
 type ThreeSixtyCameraView = ReturnType<typeof ThreeSixtyCameraView>;
 export default ThreeSixtyCameraView;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viamrobotics/test-widgets",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "license": "Apache-2.0",
   "type": "module",
   "files": [