@viamrobotics/motion-tools 1.34.5 → 1.34.6

package/dist/hooks/useFramelessComponents.svelte.js

@@ -1,9 +1,11 @@
 import { getContext, setContext } from 'svelte';
+import { useFragmentInfo } from './useFragmentInfo.svelte';
 import { useFrames } from './useFrames.svelte';
 import { usePartConfig } from './usePartConfig.svelte';
 const key = Symbol('frameless-components-context');
 export const provideFramelessComponents = () => {
     const partConfig = usePartConfig();
+    const fragmentInfo = useFragmentInfo();
     const frames = useFrames();
     const current = $derived.by(() => {
         const { components } = partConfig.current;
@@ -11,7 +13,7 @@ export const provideFramelessComponents = () => {
             ?.filter((component) => component.frame === undefined)
             .map((component) => component.name) ?? [];
         const fragmentComponentsWithNoFrame = new Set(partComponentsWIthNoFrame);
-        for (const fragmentComponentName of Object.keys(partConfig.componentNameToFragmentInfo)) {
+        for (const fragmentComponentName of Object.keys(fragmentInfo.current)) {
             if (frames.current.some((frame) => frame.referenceFrame === fragmentComponentName)) {
                 continue;
             }

package/dist/hooks/usePartConfig.svelte.d.ts

@@ -11,16 +11,11 @@ export interface PartConfig {
         mods: any[];
     }[];
 }
-export type FragmentInfo = {
-    id: string;
-    variables: Record<string, string>;
-};
 interface PartConfigContext {
     current: PartConfig;
     isDirty: boolean;
     hasPendingSave: boolean;
     hasEditPermissions: boolean;
-    componentNameToFragmentInfo: Record<string, FragmentInfo>;
     updateFrame: (componentName: string, referenceFrame: string, pose: Pose, geometry?: Frame['geometry']) => void;
     deleteFrame: (componentName: string) => void;
     createFrame: (componentName: string) => void;
@@ -34,7 +29,6 @@ export declare const usePartConfig: () => PartConfigContext;
 interface AppEmbeddedPartConfigProps {
     current: Struct;
     isDirty: boolean;
-    componentNameToFragmentInfo: Record<string, FragmentInfo>;
     setLocalPartConfig: (config: Struct) => void;
 }
 export {};

package/dist/hooks/usePartConfig.svelte.js

@@ -2,11 +2,13 @@ import { Pose, Struct } from '@viamrobotics/sdk';
 import { createAppMutation, createAppQuery } from '@viamrobotics/svelte-sdk';
 import { getContext, setContext } from 'svelte';
 import { createFrame } from '../frame';
+import { useFragmentInfo } from './useFragmentInfo.svelte';
 import { createPoseFromFrame } from '../transform';
 const key = Symbol('part-config-context');
 export const providePartConfig = (partID, params) => {
     const props = $derived(params());
     const config = $derived(props ? useEmbeddedPartConfig(props) : useStandalonePartConfig(partID));
+    const fragmentInfo = useFragmentInfo();
     const getCurrent = () => {
         return (config.current?.toJson?.() ?? { components: [] });
     };
@@ -157,9 +159,6 @@ export const providePartConfig = (partID, params) => {
         get current() {
             return current;
         },
-        get componentNameToFragmentInfo() {
-            return config.componentNameToFragmentInfo;
-        },
         get isDirty() {
             return config.isDirty;
         },
@@ -170,7 +169,7 @@ export const providePartConfig = (partID, params) => {
             return config.hasEditPermissions;
         },
         updateFrame: (componentName, referenceFrame, framePosition, frameGeometry) => {
-            const fragmentId = config.componentNameToFragmentInfo[componentName]?.id;
+            const fragmentId = fragmentInfo.current[componentName]?.id;
             if (fragmentId === undefined) {
                 updatePartFrame(componentName, referenceFrame, framePosition, frameGeometry);
             }
@@ -179,7 +178,7 @@ export const providePartConfig = (partID, params) => {
             }
         },
         deleteFrame: (componentName) => {
-            const fragmentId = config.componentNameToFragmentInfo[componentName]?.id;
+            const fragmentId = fragmentInfo.current[componentName]?.id;
             if (fragmentId === undefined) {
                 deletePartFrame(componentName);
             }
@@ -188,7 +187,7 @@ export const providePartConfig = (partID, params) => {
             }
         },
         createFrame: (componentName) => {
-            const fragmentId = config.componentNameToFragmentInfo[componentName]?.id;
+            const fragmentId = fragmentInfo.current[componentName]?.id;
             if (fragmentId === undefined) {
                 createPartFrame(componentName);
             }
@@ -246,9 +245,6 @@ const useEmbeddedPartConfig = (props) => {
         get current() {
             return props.current ?? new Struct();
         },
-        get componentNameToFragmentInfo() {
-            return props.componentNameToFragmentInfo;
-        },
         set(config) {
             const struct = Struct.fromJson(config);
             return props.setLocalPartConfig(struct);
@@ -274,54 +270,6 @@ const useStandalonePartConfig = (partID) => {
     let isDirty = $state(false);
     let hasPendingSave = $state(false);
     const hasEditPermissions = $derived(networkPartConfig !== undefined);
-    const configJSON = $derived.by(() => {
-        if (!networkPartConfig) {
-            return undefined;
-        }
-        try {
-            return networkPartConfig.toJson();
-        }
-        catch {
-            return undefined;
-        }
-    });
-    const fragmentQueries = $derived((configJSON?.fragments ?? []).map((fragmentId) => {
-        const id = typeof fragmentId === 'string' ? fragmentId : fragmentId.id;
-        return createAppQuery('getFragment', () => [id], { refetchInterval: false });
-    }));
-    const fragmentIdToVariables = $derived.by(() => {
-        const results = {};
-        for (const fragment of configJSON?.fragments ?? []) {
-            const id = typeof fragment === 'string' ? fragment : fragment.id;
-            const variables = typeof fragment === 'string' ? {} : fragment.variables;
-            results[id] = variables;
-        }
-        return results;
-    });
-    const componentNameToFragmentInfo = $derived.by(() => {
-        const results = {};
-        for (const query of fragmentQueries) {
-            if (!query.data) {
-                continue;
-            }
-            const fragmentId = query.data.id;
-            const components = query.data?.fragment?.fields['components']?.kind;
-            if (components?.case === 'listValue') {
-                for (const component of components.value.values) {
-                    if (component.kind.case === 'structValue') {
-                        const componentName = component.kind.value.fields['name']?.kind;
-                        if (componentName.case === 'stringValue') {
-                            results[componentName.value] = {
-                                id: fragmentId,
-                                variables: fragmentIdToVariables[fragmentId] ?? {},
-                            };
-                        }
-                    }
-                }
-            }
-        }
-        return results;
-    });
     let lastPartID;
     $effect.pre(() => {
         const id = partID();
@@ -354,9 +302,6 @@ const useStandalonePartConfig = (partID) => {
         get hasEditPermissions() {
             return hasEditPermissions;
         },
-        get componentNameToFragmentInfo() {
-            return componentNameToFragmentInfo;
-        },
         set(config) {
             current = Struct.fromJson(config);
             isDirty = true;

package/dist/plugins/Focus/FocusBox.svelte

@@ -2,11 +2,13 @@
 	import { T, useThrelte } from '@threlte/core'
 	import { Gizmo, TrackballControls } from '@threlte/extras'
 	import { untrack } from 'svelte'
-	import { Box3, Vector3 } from 'three'
+	import { Box3, Matrix4, Vector3 } from 'three'
 
 	import Camera from '../../components/Camera.svelte'
+	import { composeBoxMatrix } from '../../components/Entities/composeBoxMatrix'
 	import { traits, useQuery } from '../../ecs'
 	import { useCameraControls } from '../../hooks/useControls.svelte'
+	import { expandBoxByTransformedBox } from '../../three/OBBHelper'
 
 	const { scene } = useThrelte()
 	const cameraControls = useCameraControls()
@@ -35,6 +37,8 @@
 
 	const box = new Box3()
 	const vec = new Vector3()
+	const unitBox = new Box3(new Vector3(-0.5, -0.5, -0.5), new Vector3(0.5, 0.5, 0.5))
+	const boxMatrix = new Matrix4()
 
 	let center = $state.raw<[number, number, number]>([0, 0, 0])
 	let size = $state.raw<[number, number, number]>([0, 0, 0])
@@ -48,6 +52,13 @@
 	$effect(() => {
 		box.makeEmpty()
 		for (const entity of untrack(() => selected.current)) {
+			// Boxes render instanced, so the entity's named scene object
+			// carries no geometry — frame them from traits instead.
+			if (composeBoxMatrix(entity, boxMatrix)) {
+				expandBoxByTransformedBox(box, unitBox, boxMatrix)
+				continue
+			}
+
 			const object3d = scene.getObjectByName(entity as unknown as string)
 			if (object3d) {
 				box.expandByObject(object3d)

package/dist/plugins/LLMSceneBuilder/frameDeltaAdapter.d.ts

@@ -1,6 +1,13 @@
-import type { Pose } from '@viamrobotics/sdk';
+import type { Pose, Transform } from '@viamrobotics/sdk';
 import type { Frame } from '../../frame';
+import type { FragmentInfo } from '../../hooks/useFragmentInfo.svelte';
 import type { PartConfig } from '../../hooks/usePartConfig.svelte';
+/**
+ * Resolves current frames for fragment-defined components from live framesystem
+ * data and any config $set overrides. Returns a FragmentInfo map suitable for
+ * validateProposedFrameDeltas and LLM inference.
+ */
+export declare function resolveFragmentCurrentFrames(fragmentNames: string[], fragmentInfo: Record<string, FragmentInfo>, liveFrames: Transform[], configFrames: Record<string, Transform>): Record<string, FragmentInfo>;
 export interface FrameDelta {
     componentName: string;
     translation?: {
@@ -34,7 +41,7 @@ export interface UpdateError {
  * applying them. Each PreparedUpdate carries old and new values so the caller
  * can render a diff and confirm via useSceneBuilder's confirm().
  */
-export declare function validateProposedFrameDeltas(deltas: FrameDelta[], config: PartConfig): {
+export declare function validateProposedFrameDeltas(deltas: FrameDelta[], config: PartConfig, fragmentFrames?: Record<string, FragmentInfo>): {
     errors: UpdateError[];
     prepared: PreparedUpdate[];
 };

package/dist/plugins/LLMSceneBuilder/frameDeltaAdapter.js

@@ -1,4 +1,47 @@
-import { applyEulerDeltaToPose, createPoseFromFrame } from '../../transform';
+import { applyEulerDeltaToPose, createPose, createPoseFromFrame } from '../../transform';
+/**
+ * Resolves current frames for fragment-defined components from live framesystem
+ * data and any config $set overrides. Returns a FragmentInfo map suitable for
+ * validateProposedFrameDeltas and LLM inference.
+ */
+export function resolveFragmentCurrentFrames(fragmentNames, fragmentInfo, liveFrames, configFrames) {
+    const liveByName = {};
+    for (const frame of liveFrames) {
+        liveByName[frame.referenceFrame] = frame;
+    }
+    const result = {};
+    for (const name of fragmentNames) {
+        const meta = fragmentInfo[name];
+        if (!meta)
+            continue;
+        const observed = (configFrames[name] ?? liveByName[name])?.poseInObserverFrame;
+        if (!observed)
+            continue;
+        const pose = createPose(observed.pose);
+        result[name] = {
+            id: meta.id,
+            variables: meta.variables,
+            frame: {
+                parent: observed.referenceFrame,
+                translation: {
+                    x: pose.x,
+                    y: pose.y,
+                    z: pose.z,
+                },
+                orientation: {
+                    type: 'ov_degrees',
+                    value: {
+                        x: pose.oX,
+                        y: pose.oY,
+                        z: pose.oZ,
+                        th: pose.theta,
+                    },
+                },
+            },
+        };
+    }
+    return result;
+}
 function mergeTranslation(a, b) {
     return a || b ? { x: b?.x ?? a?.x, y: b?.y ?? a?.y, z: b?.z ?? a?.z } : undefined;
 }
@@ -12,10 +55,13 @@ function mergeOrientation(a, b) {
  * applying them. Each PreparedUpdate carries old and new values so the caller
  * can render a diff and confirm via useSceneBuilder's confirm().
  */
-export function validateProposedFrameDeltas(deltas, config) {
+export function validateProposedFrameDeltas(deltas, config, fragmentFrames = {}) {
     const errors = [];
     const prepared = [];
-    const knownNames = new Set(config.components.map((c) => c.name));
+    const knownNames = new Set([
+        ...config.components.map((c) => c.name),
+        ...Object.keys(fragmentFrames),
+    ]);
     // Merge multiple deltas for the same component — the LLM sometimes splits
     // translation and orientation into separate entries despite the schema saying one per component.
     const mergedDeltas = new Map();
@@ -35,13 +81,21 @@ export function validateProposedFrameDeltas(deltas, config) {
         }
     }
     for (const delta of mergedDeltas.values()) {
-        const component = config.components.find((c) => c.name === delta.componentName);
-        if (!component) {
+        const partComponent = config.components.find((c) => c.name === delta.componentName);
+        // Fragment-defined components aren't in config.components; their current
+        // frame comes from `fragmentFrames` (resolved from the live framesystem).
+        // Part config wins when the same name exists in both.
+        const fragmentEntry = partComponent ? undefined : fragmentFrames[delta.componentName];
+        const frame = partComponent?.frame ?? fragmentEntry?.frame;
+        if (!partComponent && !fragmentEntry) {
             errors.push({ componentName: delta.componentName, reason: 'Component not found in config' });
             continue;
         }
-        if (!component.frame) {
-            errors.push({ componentName: delta.componentName, reason: 'Component has no frame' });
+        if (!frame) {
+            errors.push({
+                componentName: delta.componentName,
+                reason: fragmentEntry ? 'Fragment has no frame' : 'Component has no frame',
+            });
             continue;
         }
         if (delta.parent !== undefined &&
@@ -55,8 +109,9 @@ export function validateProposedFrameDeltas(deltas, config) {
             });
             continue;
         }
-        const previousPose = createPoseFromFrame(component.frame);
-        const previousParent = component.frame.parent;
+        const previousPose = createPoseFromFrame(frame);
+        const previousParent = frame.parent;
+        const geometry = frame.geometry;
         const newParent = delta.parent ?? previousParent;
         const newPose = {
             x: delta.translation?.x ?? previousPose.x,
@@ -83,7 +138,7 @@ export function validateProposedFrameDeltas(deltas, config) {
             previousParent,
             pose: newPose,
             previousPose,
-            geometry: component.frame.geometry,
+            geometry,
             explanation: delta.explanation,
         });
     }

package/dist/plugins/LLMSceneBuilder/useSceneBuilder.svelte.js

@@ -1,10 +1,17 @@
 import { getContext, setContext } from 'svelte';
+import { useConfigFrames } from '../../hooks/useConfigFrames.svelte';
+import { useFragmentInfo } from '../../hooks/useFragmentInfo.svelte';
+import { useFrames } from '../../hooks/useFrames.svelte';
 import { usePartConfig } from '../../hooks/usePartConfig.svelte';
 import { createPoseFromFrame, poseToEulerDegrees } from '../../transform';
-import { validateProposedFrameDeltas } from './frameDeltaAdapter';
+import { resolveFragmentCurrentFrames, validateProposedFrameDeltas, } from './frameDeltaAdapter';
 const key = Symbol('scene-builder-context');
 export const provideSceneBuilder = (onInfer) => {
     const partConfig = usePartConfig();
+    const fragmentInfo = useFragmentInfo();
+    const frames = useFrames();
+    const configFrames = useConfigFrames();
+    const fragmentFrames = $derived(resolveFragmentCurrentFrames(Object.keys(fragmentInfo.current), fragmentInfo.current, frames.current ?? [], configFrames.current ?? {}));
     let uiState = $state('idle');
     let deltas = $state([]);
     let explanation = $state('');
@@ -13,7 +20,7 @@ export const provideSceneBuilder = (onInfer) => {
     // confirm() therefore always applies the LLM's intent against the latest config —
     // drag changes to unspecified axes are preserved, not overwritten.
     const validation = $derived.by(() => deltas.length > 0
-        ? validateProposedFrameDeltas(deltas, partConfig.current)
+        ? validateProposedFrameDeltas(deltas, partConfig.current, fragmentFrames)
         : { prepared: [], errors: [] });
     const updateErrors = $derived(validation.errors);
     const diffGroups = $derived(validation.prepared.flatMap((u) => {
@@ -87,20 +94,37 @@ export const provideSceneBuilder = (onInfer) => {
         },
         async submit(prompt) {
             uiState = 'loading';
-            const components = partConfig.current.components
+            const partComponents = partConfig.current.components
                 .filter((c) => c.frame !== undefined)
                 .map(({ name, frame }) => {
                 const pose = createPoseFromFrame(frame);
-                const { roll, pitch, yaw } = poseToEulerDegrees(pose);
+                const orientation = poseToEulerDegrees(pose);
                 return {
                     name,
                     frame: {
                         parent: frame.parent,
                         translation: frame.translation,
-                        orientation: { roll, pitch, yaw },
+                        orientation,
                     },
                 };
             });
+            // Fragment components never appear in partConfig.components, so there's
+            // no name overlap with partComponents.
+            const fragmentComponents = Object.entries(fragmentFrames)
+                .filter(([, current]) => current.frame !== undefined)
+                .map(([name, current]) => {
+                const pose = createPoseFromFrame(current.frame);
+                const orientation = poseToEulerDegrees(pose);
+                return {
+                    name,
+                    frame: {
+                        parent: current.frame.parent,
+                        translation: current.frame.translation,
+                        orientation,
+                    },
+                };
+            });
+            const components = [...partComponents, ...fragmentComponents];
             try {
                 const data = await onInfer(prompt.trim(), components);
                 deltas = data.updates;

package/dist/three/OBBHelper.d.ts

@@ -1,7 +1,14 @@
-import { Object3D, Vector3 } from 'three';
+import { Box3, Matrix4, Object3D, Vector3 } from 'three';
 import { LineSegments2 } from 'three/examples/jsm/lines/LineSegments2.js';
+export declare const expandBoxByTransformedBox: (box: Box3, childBox: Box3, matrix: Matrix4) => void;
 export declare class OBBHelper extends LineSegments2 {
     constructor(color?: number, linewidth?: number);
+    /**
+     * Drive the helper with a composed world transform whose scale holds the
+     * box extents in meters — the helper's unit-box edges span ±0.5, so the
+     * matrix maps them onto the box's faces directly.
+     */
+    setFromMatrix4(matrix: Matrix4): this;
     setFromOBB(obb: {
         center: Vector3;
         halfSize: Vector3;

package/dist/three/OBBHelper.js

@@ -22,7 +22,7 @@ const corners = [
     new Vector3(),
     new Vector3(),
 ];
-const expandBoxByTransformedBox = (box, childBox, matrix) => {
+export const expandBoxByTransformedBox = (box, childBox, matrix) => {
     const min = childBox.min;
     const max = childBox.max;
     corners[0].set(min.x, min.y, min.z).applyMatrix4(matrix);
@@ -58,6 +58,16 @@ export class OBBHelper extends LineSegments2 {
         this.frustumCulled = false;
         this.renderOrder = 999;
     }
+    /**
+     * Drive the helper with a composed world transform whose scale holds the
+     * box extents in meters — the helper's unit-box edges span ±0.5, so the
+     * matrix maps them onto the box's faces directly.
+     */
+    setFromMatrix4(matrix) {
+        this.matrix.copy(matrix);
+        this.matrixWorld.copy(this.matrix);
+        return this;
+    }
     setFromOBB(obb) {
         basis.setFromMatrix3(obb.rotation);
         quaternion.setFromRotationMatrix(basis);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viamrobotics/motion-tools",
-  "version": "1.34.5",
+  "version": "1.34.6",
   "description": "Motion visualization with Viam",
   "license": "Apache-2.0",
   "type": "module",
@@ -29,6 +29,7 @@
     "@testing-library/jest-dom": "6.8.0",
     "@testing-library/svelte": "5.2.8",
     "@testing-library/user-event": "^14.6.1",
+    "@three.ez/instanced-mesh": "^0.3.15",
     "@threlte/core": "8.5.14",
     "@threlte/extras": "9.21.0",
     "@threlte/rapier": "3.4.1",