@viamrobotics/motion-tools 1.33.2 → 1.34.0

package/dist/hooks/useSettings.svelte.d.ts

@@ -1,5 +1,6 @@
 import type { ColorRepresentation } from 'three';
 export interface Settings {
+    anthropicKey: string;
     cameraMode: 'orthographic' | 'perspective';
     interactionMode: 'navigate' | 'measure' | 'select' | 'gizmo';
     refreshRates: {

package/dist/hooks/useSettings.svelte.js

@@ -7,6 +7,7 @@ export const RefreshRates = {
     vision: 'vision',
 };
 const defaults = () => ({
+    anthropicKey: '',
     cameraMode: 'perspective',
     refreshRates: {
         poses: 1000,

package/dist/hooks/useStandaloneLLM.svelte.d.ts

@@ -0,0 +1,7 @@
+import { type InferCallback } from '../plugins';
+interface StandaloneLLMContext {
+    current: InferCallback;
+}
+export declare const provideStandaloneLLM: () => StandaloneLLMContext;
+export declare const useStandaloneLLMContext: () => StandaloneLLMContext;
+export {};

package/dist/hooks/useStandaloneLLM.svelte.js

@@ -0,0 +1,32 @@
+import { getContext, setContext } from 'svelte';
+import { backendIP, websocketPort } from '../defines';
+import { useSettings } from './useSettings.svelte';
+import {} from '../plugins';
+const key = Symbol('standalone-llm-context');
+export const provideStandaloneLLM = () => {
+    const settings = useSettings();
+    const standaloneInfer = async (prompt, components) => {
+        const res = await fetch(`http://${backendIP}:${websocketPort}/scene-builder`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                prompt,
+                components,
+                anthropicKey: settings.current.anthropicKey || undefined,
+            }),
+        });
+        if (!res.ok) {
+            throw new Error(`${res.status}: ${await res.text()}`);
+        }
+        return res.json();
+    };
+    const context = setContext(key, {
+        get current() {
+            return standaloneInfer;
+        },
+    });
+    return context;
+};
+export const useStandaloneLLMContext = () => {
+    return getContext(key);
+};

package/dist/plugins/LLMSceneBuilder/AISettings.svelte

@@ -0,0 +1,20 @@
+<script lang="ts">
+	import { Input } from '@viamrobotics/prime-core'
+
+	import { useSettings } from '../../hooks/useSettings.svelte'
+
+	const settings = useSettings()
+</script>
+
+<div class="flex flex-col gap-2.5 text-xs">
+	<h3 class="border-gray-3 border-b py-1 text-sm"><strong>Anthropic</strong></h3>
+	<label class="flex flex-col gap-1">
+		API key
+		<Input
+			type="password"
+			bind:value={settings.current.anthropicKey}
+			placeholder="sk-ant-..."
+		/>
+	</label>
+	<p class="text-gray-5">Used by the Scene Builder AI feature. Stored locally in your browser.</p>
+</div>

package/dist/plugins/LLMSceneBuilder/AISettings.svelte.d.ts

@@ -0,0 +1,18 @@
+interface $$__sveltets_2_IsomorphicComponent<Props extends Record<string, any> = any, Events extends Record<string, any> = any, Slots extends Record<string, any> = any, Exports = {}, Bindings = string> {
+    new (options: import('svelte').ComponentConstructorOptions<Props>): import('svelte').SvelteComponent<Props, Events, Slots> & {
+        $$bindings?: Bindings;
+    } & Exports;
+    (internal: unknown, props: {
+        $$events?: Events;
+        $$slots?: Slots;
+    }): Exports & {
+        $set?: any;
+        $on?: any;
+    };
+    z_$$bindings?: Bindings;
+}
+declare const AISettings: $$__sveltets_2_IsomorphicComponent<Record<string, never>, {
+    [evt: string]: CustomEvent<any>;
+}, {}, {}, string>;
+type AISettings = InstanceType<typeof AISettings>;
+export default AISettings;

package/dist/plugins/LLMSceneBuilder/LLMSceneBuilder.svelte

@@ -0,0 +1,14 @@
+<script lang="ts">
+	import SceneBuilder from './SceneBuilder.svelte'
+	import { type InferCallback, provideSceneBuilder } from './useSceneBuilder.svelte'
+
+	interface Props {
+		onInfer: InferCallback
+	}
+
+	const { onInfer }: Props = $props()
+
+	provideSceneBuilder((prompt, components) => onInfer(prompt, components))
+</script>
+
+<SceneBuilder />

package/dist/plugins/LLMSceneBuilder/LLMSceneBuilder.svelte.d.ts

@@ -0,0 +1,7 @@
+import { type InferCallback } from './useSceneBuilder.svelte';
+interface Props {
+    onInfer: InferCallback;
+}
+declare const LLMSceneBuilder: import("svelte").Component<Props, {}, "">;
+type LLMSceneBuilder = ReturnType<typeof LLMSceneBuilder>;
+export default LLMSceneBuilder;

package/dist/plugins/LLMSceneBuilder/SceneBuilder.svelte

@@ -0,0 +1,149 @@
+<script lang="ts">
+	import { Portal } from '@threlte/extras'
+
+	import DashboardButton from '../../components/overlay/dashboard/Button.svelte'
+	import FloatingPanel from '../../components/overlay/FloatingPanel.svelte'
+
+	import { useSceneBuilder } from './useSceneBuilder.svelte'
+
+	const sceneBuilder = useSceneBuilder()
+
+	let isOpen = $state(false)
+	let prompt = $state('')
+
+	const canSubmit = $derived(prompt.trim().length > 0 && sceneBuilder.uiState === 'idle')
+</script>
+
+<Portal id="dashboard">
+	<fieldset>
+		<DashboardButton
+			active={isOpen}
+			icon="robot-outline"
+			description="Frame Builder"
+			onclick={() => (isOpen = !isOpen)}
+		/>
+	</fieldset>
+</Portal>
+
+<Portal id="dom">
+	<FloatingPanel
+		bind:isOpen
+		title="Frame Builder"
+		defaultSize={{ width: 480, height: 420 }}
+		resizable
+	>
+		<div class="flex h-full flex-col gap-3 p-3 text-xs">
+			<!-- prompt input -->
+			<div class="flex gap-2">
+				<textarea
+					class="flex-1 resize-none rounded border border-gray-300 p-2 text-xs focus:ring-1 focus:ring-gray-400 focus:outline-none disabled:bg-gray-50 disabled:text-gray-400"
+					placeholder="Describe the frame change, e.g. 'Move the arm 200mm forward along X'"
+					rows={3}
+					disabled={sceneBuilder.uiState === 'loading' || sceneBuilder.uiState === 'diff'}
+					bind:value={prompt}
+					onkeydown={(e) => {
+						e.stopImmediatePropagation()
+						if (e.key === 'Enter' && !e.shiftKey && canSubmit) {
+							e.preventDefault()
+							sceneBuilder.submit(prompt)
+							prompt = ''
+						}
+					}}
+				></textarea>
+				<button
+					class="self-end rounded bg-gray-800 px-3 py-1.5 text-xs text-white hover:bg-gray-700 disabled:cursor-not-allowed disabled:opacity-40"
+					disabled={!canSubmit}
+					onclick={() => {
+						sceneBuilder.submit(prompt)
+						prompt = ''
+					}}
+				>
+					Submit
+				</button>
+			</div>
+
+			<!-- loading -->
+			{#if sceneBuilder.uiState === 'loading'}
+				<p class="text-gray-5 text-center">Thinking…</p>
+			{/if}
+
+			<!-- diff ready -->
+			{#if sceneBuilder.uiState === 'diff'}
+				{#if sceneBuilder.explanation}
+					<p class="text-gray-6 italic">{sceneBuilder.explanation}</p>
+				{/if}
+
+				{#if sceneBuilder.diffGroups.length > 0}
+					<div class="flex flex-col gap-2 overflow-auto">
+						{#each sceneBuilder.diffGroups as group (group.componentName)}
+							<div class="rounded border border-gray-200">
+								<div
+									class="flex items-baseline gap-2 border-b border-gray-200 bg-gray-50 px-2 py-1"
+								>
+									<span class="font-mono font-medium">{group.componentName}</span>
+									{#if group.explanation}
+										<span class="text-gray-5 truncate italic">{group.explanation}</span>
+									{/if}
+								</div>
+								<table class="w-full text-left">
+									<thead class="sr-only">
+										<tr>
+											<th>Field</th>
+											<th>Before</th>
+											<th>After</th>
+										</tr>
+									</thead>
+									<tbody>
+										{#each group.changes as change (change.field)}
+											<tr class="border-t border-gray-100 first:border-t-0">
+												<td class="px-2 py-1 font-mono">{change.field}</td>
+												<td class="text-red-6 px-2 py-1 font-mono">{change.oldValue}</td>
+												<td class="text-green-6 px-2 py-1 font-mono">{change.newValue}</td>
+											</tr>
+										{/each}
+									</tbody>
+								</table>
+							</div>
+						{/each}
+					</div>
+				{:else}
+					<p class="text-gray-5 text-center">No frame changes proposed.</p>
+				{/if}
+
+				{#if sceneBuilder.updateErrors.length > 0}
+					<ul class="text-red-6 space-y-0.5">
+						{#each sceneBuilder.updateErrors as err (err.componentName)}
+							<li><span class="font-mono">{err.componentName}</span>: {err.reason}</li>
+						{/each}
+					</ul>
+				{/if}
+
+				<div class="mt-auto flex gap-2">
+					<button
+						class="rounded bg-gray-800 px-3 py-1.5 text-white hover:bg-gray-700"
+						onclick={sceneBuilder.confirm}
+					>
+						Confirm
+					</button>
+					<button
+						class="rounded border border-gray-300 px-3 py-1.5 hover:bg-gray-50"
+						onclick={sceneBuilder.cancel}
+					>
+						Cancel
+					</button>
+				</div>
+			{/if}
+
+			<!-- error -->
+			{#if sceneBuilder.uiState === 'error'}
+				<p class="text-red-6">{sceneBuilder.errorMessage}</p>
+				<button
+					class="self-start rounded border border-gray-300 px-3 py-1.5 hover:bg-gray-50"
+					onclick={sceneBuilder.resetError}
+				>
+					Try again
+				</button>
+			{/if}
+		</div>
+	</FloatingPanel>
+</Portal>

package/dist/plugins/LLMSceneBuilder/SceneBuilder.svelte.d.ts

@@ -0,0 +1,3 @@
+declare const SceneBuilder: import("svelte").Component<Record<string, never>, {}, "">;
+type SceneBuilder = ReturnType<typeof SceneBuilder>;
+export default SceneBuilder;

package/dist/plugins/LLMSceneBuilder/frameDeltaAdapter.d.ts

@@ -0,0 +1,40 @@
+import type { Pose } from '@viamrobotics/sdk';
+import type { Frame } from '../../frame';
+import type { PartConfig } from '../../hooks/usePartConfig.svelte';
+export interface FrameDelta {
+    componentName: string;
+    translation?: {
+        x?: number;
+        y?: number;
+        z?: number;
+    };
+    orientation?: {
+        roll?: number;
+        pitch?: number;
+        yaw?: number;
+    };
+    parent?: string;
+    explanation?: string;
+}
+export interface PreparedUpdate {
+    componentName: string;
+    parent: string;
+    previousParent: string;
+    pose: Pose;
+    previousPose: Pose;
+    geometry?: Frame['geometry'];
+    explanation?: string;
+}
+export interface UpdateError {
+    componentName: string;
+    reason: string;
+}
+/**
+ * Validates LLM-proposed frame deltas and computes the resulting changes without
+ * applying them. Each PreparedUpdate carries old and new values so the caller
+ * can render a diff and confirm via useSceneBuilder's confirm().
+ */
+export declare function validateProposedFrameDeltas(deltas: FrameDelta[], config: PartConfig): {
+    errors: UpdateError[];
+    prepared: PreparedUpdate[];
+};

package/dist/plugins/LLMSceneBuilder/frameDeltaAdapter.js

@@ -0,0 +1,65 @@
+import { applyEulerDeltaToPose, createPoseFromFrame } from '../../transform';
+/**
+ * Validates LLM-proposed frame deltas and computes the resulting changes without
+ * applying them. Each PreparedUpdate carries old and new values so the caller
+ * can render a diff and confirm via useSceneBuilder's confirm().
+ */
+export function validateProposedFrameDeltas(deltas, config) {
+    const errors = [];
+    const prepared = [];
+    const knownNames = new Set(config.components.map((c) => c.name));
+    for (const delta of deltas) {
+        const component = config.components.find((c) => c.name === delta.componentName);
+        if (!component) {
+            errors.push({ componentName: delta.componentName, reason: 'Component not found in config' });
+            continue;
+        }
+        if (!component.frame) {
+            errors.push({ componentName: delta.componentName, reason: 'Component has no frame' });
+            continue;
+        }
+        if (delta.parent !== undefined &&
+            delta.parent !== 'world' &&
+            (!knownNames.has(delta.parent) || delta.parent === delta.componentName)) {
+            errors.push({
+                componentName: delta.componentName,
+                reason: delta.parent === delta.componentName
+                    ? `Component cannot be its own parent`
+                    : `Parent '${delta.parent}' not found in config`,
+            });
+            continue;
+        }
+        const previousPose = createPoseFromFrame(component.frame);
+        const previousParent = component.frame.parent;
+        const newParent = delta.parent ?? previousParent;
+        const newPose = {
+            x: delta.translation?.x ?? previousPose.x,
+            y: delta.translation?.y ?? previousPose.y,
+            z: delta.translation?.z ?? previousPose.z,
+            oX: previousPose.oX,
+            oY: previousPose.oY,
+            oZ: previousPose.oZ,
+            theta: previousPose.theta,
+        };
+        if (delta.orientation) {
+            applyEulerDeltaToPose(previousPose, delta.orientation, newPose);
+        }
+        if ([newPose.x, newPose.y, newPose.z, newPose.oX, newPose.oY, newPose.oZ, newPose.theta].some((v) => !Number.isFinite(v))) {
+            errors.push({
+                componentName: delta.componentName,
+                reason: 'Proposed values contain non-finite numbers',
+            });
+            continue;
+        }
+        prepared.push({
+            componentName: delta.componentName,
+            parent: newParent,
+            previousParent,
+            pose: newPose,
+            previousPose,
+            geometry: component.frame.geometry,
+            explanation: delta.explanation,
+        });
+    }
+    return { errors, prepared };
+}

package/dist/plugins/LLMSceneBuilder/useSceneBuilder.svelte.d.ts

@@ -0,0 +1,46 @@
+import { type FrameDelta, type UpdateError } from './frameDeltaAdapter';
+type UIState = 'idle' | 'loading' | 'diff' | 'error';
+interface FieldChange {
+    field: string;
+    oldValue: string;
+    newValue: string;
+}
+interface DiffGroup {
+    componentName: string;
+    explanation?: string;
+    changes: FieldChange[];
+}
+interface SceneBuilderContext {
+    readonly uiState: UIState;
+    readonly updateErrors: UpdateError[];
+    readonly explanation: string;
+    readonly errorMessage: string;
+    readonly diffGroups: DiffGroup[];
+    submit(prompt: string): Promise<void>;
+    confirm(): void;
+    cancel(): void;
+    resetError(): void;
+}
+export interface ComponentFrameInfo {
+    name: string;
+    frame: {
+        parent: string | undefined;
+        translation: {
+            x?: number;
+            y?: number;
+            z?: number;
+        } | undefined;
+        orientation: {
+            roll: number;
+            pitch: number;
+            yaw: number;
+        };
+    };
+}
+export type InferCallback = (prompt: string, components: ComponentFrameInfo[]) => Promise<{
+    updates: FrameDelta[];
+    explanation: string;
+}>;
+export declare const provideSceneBuilder: (onInfer: InferCallback) => void;
+export declare const useSceneBuilder: () => SceneBuilderContext;
+export {};

package/dist/plugins/LLMSceneBuilder/useSceneBuilder.svelte.js

@@ -0,0 +1,134 @@
+import { getContext, setContext } from 'svelte';
+import { usePartConfig } from '../../hooks/usePartConfig.svelte';
+import { createPoseFromFrame, poseToEulerDegrees } from '../../transform';
+import { validateProposedFrameDeltas } from './frameDeltaAdapter';
+const key = Symbol('scene-builder-context');
+export const provideSceneBuilder = (onInfer) => {
+    const partConfig = usePartConfig();
+    let uiState = $state('idle');
+    let deltas = $state([]);
+    let explanation = $state('');
+    let errorMessage = $state('');
+    // Re-derived whenever deltas or the current config changes (e.g. from a drag).
+    // confirm() therefore always applies the LLM's intent against the latest config —
+    // drag changes to unspecified axes are preserved, not overwritten.
+    const validation = $derived.by(() => deltas.length > 0
+        ? validateProposedFrameDeltas(deltas, partConfig.current)
+        : { prepared: [], errors: [] });
+    const updateErrors = $derived(validation.errors);
+    const diffGroups = $derived(validation.prepared.flatMap((u) => {
+        const changes = [];
+        const roundMm = (v) => `${Math.round(v * 100) / 100}mm`;
+        const roundDeg = (v) => `${Math.round(v * 100) / 100}°`;
+        if (u.parent !== u.previousParent) {
+            changes.push({ field: 'parent', oldValue: u.previousParent, newValue: u.parent });
+        }
+        if (u.pose.x !== u.previousPose.x) {
+            changes.push({
+                field: 'translation.x',
+                oldValue: roundMm(u.previousPose.x),
+                newValue: roundMm(u.pose.x),
+            });
+        }
+        if (u.pose.y !== u.previousPose.y) {
+            changes.push({
+                field: 'translation.y',
+                oldValue: roundMm(u.previousPose.y),
+                newValue: roundMm(u.pose.y),
+            });
+        }
+        if (u.pose.z !== u.previousPose.z) {
+            changes.push({
+                field: 'translation.z',
+                oldValue: roundMm(u.previousPose.z),
+                newValue: roundMm(u.pose.z),
+            });
+        }
+        if (u.pose.oX !== u.previousPose.oX ||
+            u.pose.oY !== u.previousPose.oY ||
+            u.pose.oZ !== u.previousPose.oZ ||
+            u.pose.theta !== u.previousPose.theta) {
+            const prev = poseToEulerDegrees(u.previousPose);
+            const next = poseToEulerDegrees(u.pose);
+            for (const axis of ['yaw', 'pitch', 'roll']) {
+                if (Math.abs(next[axis] - prev[axis]) > 0.01) {
+                    changes.push({
+                        field: axis,
+                        oldValue: roundDeg(prev[axis]),
+                        newValue: roundDeg(next[axis]),
+                    });
+                }
+            }
+        }
+        return changes.length > 0
+            ? [{ componentName: u.componentName, explanation: u.explanation, changes }]
+            : [];
+    }));
+    const clear = () => {
+        deltas = [];
+        explanation = '';
+        errorMessage = '';
+    };
+    setContext(key, {
+        get uiState() {
+            return uiState;
+        },
+        get updateErrors() {
+            return updateErrors;
+        },
+        get explanation() {
+            return explanation;
+        },
+        get errorMessage() {
+            return errorMessage;
+        },
+        get diffGroups() {
+            return diffGroups;
+        },
+        async submit(prompt) {
+            uiState = 'loading';
+            const components = partConfig.current.components
+                .filter((c) => c.frame !== undefined)
+                .map(({ name, frame }) => {
+                const pose = createPoseFromFrame(frame);
+                const { roll, pitch, yaw } = poseToEulerDegrees(pose);
+                return {
+                    name,
+                    frame: {
+                        parent: frame.parent,
+                        translation: frame.translation,
+                        orientation: { roll, pitch, yaw },
+                    },
+                };
+            });
+            try {
+                const data = await onInfer(prompt.trim(), components);
+                deltas = data.updates;
+                explanation = data.explanation;
+                uiState = 'diff';
+            }
+            catch (error) {
+                errorMessage = error instanceof Error ? error.message : String(error);
+                uiState = 'error';
+            }
+        },
+        confirm() {
+            for (const update of validation.prepared) {
+                partConfig.updateFrame(update.componentName, update.parent, update.pose, update.geometry);
+            }
+            clear();
+            uiState = 'idle';
+        },
+        cancel() {
+            clear();
+            uiState = 'idle';
+        },
+        resetError() {
+            errorMessage = '';
+            uiState = 'idle';
+        },
+    });
+};
+export const useSceneBuilder = () => {
+    return getContext(key);
+};

package/dist/plugins/index.d.ts

@@ -7,5 +7,9 @@ export { default as DrawService } from './DrawService/DrawService.svelte';
 export { default as Skybox } from './Skybox/Skybox.svelte';
 export { default as Debug } from './Debug/Debug.svelte';
 export { default as Focus } from './Focus/Focus.svelte';
+export { default as LLMSceneBuilder } from './LLMSceneBuilder/LLMSceneBuilder.svelte';
+export { default as AISettings } from './LLMSceneBuilder/AISettings.svelte';
+export type { InferCallback, ComponentFrameInfo } from './LLMSceneBuilder/useSceneBuilder.svelte';
+export type { FrameDelta } from './LLMSceneBuilder/frameDeltaAdapter';
 export { default as XR } from './XR/XR.svelte';
 export { default as XRSettings } from './XR/XRSettings.svelte';

package/dist/plugins/index.js

@@ -11,5 +11,8 @@ export { default as Skybox } from './Skybox/Skybox.svelte';
 // Debug
 export { default as Debug } from './Debug/Debug.svelte';
 export { default as Focus } from './Focus/Focus.svelte';
+// LLMSceneBuilder
+export { default as LLMSceneBuilder } from './LLMSceneBuilder/LLMSceneBuilder.svelte';
+export { default as AISettings } from './LLMSceneBuilder/AISettings.svelte';
 export { default as XR } from './XR/XR.svelte';
 export { default as XRSettings } from './XR/XRSettings.svelte';

package/dist/transform.js

@@ -136,6 +136,27 @@ export const matrixToPose = (matrix, pose) => {
  * `Frame.svelte` uses to blend live kinematics with user-staged edits;
  * `worldMatrix.ts` premultiplies the result by the parent's `WorldMatrix`.
  */
+export const poseToEulerDegrees = (pose) => {
+    poseToQuaternion(pose, quaternion);
+    euler.setFromQuaternion(quaternion, 'ZYX');
+    return {
+        roll: MathUtils.radToDeg(euler.x),
+        pitch: MathUtils.radToDeg(euler.y),
+        yaw: MathUtils.radToDeg(euler.z),
+    };
+};
+export const applyEulerDeltaToPose = (pose, delta, out) => {
+    poseToQuaternion(pose, quaternion);
+    euler.setFromQuaternion(quaternion, 'ZYX');
+    if (delta.roll !== undefined)
+        euler.x = MathUtils.degToRad(delta.roll);
+    if (delta.pitch !== undefined)
+        euler.y = MathUtils.degToRad(delta.pitch);
+    if (delta.yaw !== undefined)
+        euler.z = MathUtils.degToRad(delta.yaw);
+    quaternion.setFromEuler(euler);
+    quaternionToPose(quaternion, out);
+};
 export const composeLocalMatrix = (live, baseline, edited, out) => {
     matA.copy(baseline).invert();
     out.copy(live).multiply(matA).multiply(edited);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viamrobotics/motion-tools",
-  "version": "1.33.2",
+  "version": "1.34.0",
   "description": "Motion visualization with Viam",
   "license": "Apache-2.0",
   "type": "module",
@@ -163,13 +163,17 @@
   ],
   "dependencies": {
     "@bufbuild/protobuf": "1.10.1",
+    "@connectrpc/connect": "1.7.0",
+    "@connectrpc/connect-web": "1.7.0",
+    "@langchain/anthropic": "^1.4.0",
     "@neodrag/svelte": "^2.3.3",
     "d3-force": "^3.0.0",
     "filtrex": "^3.1.0",
     "koota": "0.6.5",
     "lodash-es": "4.18.1",
     "three-mesh-bvh": "^0.9.8",
-    "uuid-tool": "^2.0.3"
+    "uuid-tool": "^2.0.3",
+    "zod": "^4.4.3"
   },
   "scripts": {
     "dev": "concurrently \"pnpm dev:bun\" \"go run cmd/draw-server/main.go -port 3030\"",