@gradio/model3d 0.7.0 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @gradio/model3d
 
+## 0.8.0
+
+### Features
+
+- [#7406](https://github.com/gradio-app/gradio/pull/7406) [`3e886d8`](https://github.com/gradio-app/gradio/commit/3e886d8f0ac55c416dae51c1c2661e16eb34718e) - Model3D Gaussian Splatting.  Thanks [@dylanebert](https://github.com/dylanebert)!
+
 ## 0.7.0
 
 ### Features

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gradio/model3d",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Gradio UI packages",
   "type": "module",
   "author": "",
@@ -11,12 +11,13 @@
     "babylonjs": "^4.2.1",
     "babylonjs-loaders": "^4.2.1",
     "dequal": "^2.0.2",
-    "@gradio/client": "^0.12.0",
-    "@gradio/atoms": "^0.5.1",
+    "gsplat": "^1.0.5",
+    "@gradio/atoms": "^0.5.2",
+    "@gradio/client": "^0.12.1",
     "@gradio/icons": "^0.3.2",
-    "@gradio/upload": "^0.7.2",
-    "@gradio/utils": "^0.2.2",
-    "@gradio/statustracker": "^0.4.5",
+    "@gradio/statustracker": "^0.4.6",
+    "@gradio/upload": "^0.7.3",
+    "@gradio/utils": "^0.3.0",
     "@gradio/wasm": "^0.6.0"
   },
   "main_changeset": true,

package/shared/Canvas3DGS.svelte

@@ -0,0 +1,130 @@
+<script lang="ts">
+	import { onMount } from "svelte";
+	import * as SPLAT from "gsplat";
+	import type { FileData } from "@gradio/client";
+	import { resolve_wasm_src } from "@gradio/wasm/svelte";
+
+	export let value: FileData;
+	export let zoom_speed: number;
+	export let pan_speed: number;
+
+	$: url = value.url;
+
+	/* URL resolution for the Wasm mode. */
+	export let resolved_url: typeof url = undefined; // Exposed to be bound to the download link in the parent component.
+	// The prop can be updated before the Promise from `resolve_wasm_src` is resolved.
+	// In such a case, the resolved url for the old `url` has to be discarded,
+	// This variable `latest_url` is used to pick up only the value resolved for the latest `url`.
+	let latest_url: typeof url;
+	$: {
+		// In normal (non-Wasm) Gradio, the original `url` should be used immediately
+		// without waiting for `resolve_wasm_src()` to resolve.
+		// If it waits, a blank element is displayed until the async task finishes
+		// and it leads to undesirable flickering.
+		// So set `resolved_url` immediately above, and update it with the resolved values below later.
+		resolved_url = url;
+
+		if (url) {
+			latest_url = url;
+			const resolving_url = url;
+			resolve_wasm_src(url).then((resolved) => {
+				if (latest_url === resolving_url) {
+					resolved_url = resolved ?? undefined;
+				} else {
+					resolved && URL.revokeObjectURL(resolved);
+				}
+			});
+		}
+	}
+
+	let canvas: HTMLCanvasElement;
+	let scene: SPLAT.Scene;
+	let camera: SPLAT.Camera;
+	let renderer: SPLAT.WebGLRenderer | null = null;
+	let controls: SPLAT.OrbitControls;
+	let mounted = false;
+	let frameId: number | null = null;
+
+	function reset_scene(): void {
+		if (frameId !== null) {
+			cancelAnimationFrame(frameId);
+			frameId = null;
+		}
+
+		if (renderer !== null) {
+			renderer.dispose();
+			renderer = null;
+		}
+
+		scene = new SPLAT.Scene();
+		camera = new SPLAT.Camera();
+		renderer = new SPLAT.WebGLRenderer(canvas);
+		controls = new SPLAT.OrbitControls(camera, canvas);
+		controls.zoomSpeed = zoom_speed;
+		controls.panSpeed = pan_speed;
+
+		if (!value) {
+			return;
+		}
+
+		let loading = false;
+		const load = async (): Promise<void> => {
+			if (loading) {
+				console.error("Already loading");
+				return;
+			}
+			if (!resolved_url) {
+				throw new Error("No resolved URL");
+			}
+			loading = true;
+			if (resolved_url.endsWith(".ply")) {
+				await SPLAT.PLYLoader.LoadAsync(resolved_url, scene, undefined);
+			} else if (resolved_url.endsWith(".splat")) {
+				await SPLAT.Loader.LoadAsync(resolved_url, scene, undefined);
+			} else {
+				throw new Error("Unsupported file type");
+			}
+			loading = false;
+		};
+
+		const frame = (): void => {
+			if (!renderer) {
+				return;
+			}
+
+			if (loading) {
+				frameId = requestAnimationFrame(frame);
+				return;
+			}
+
+			controls.update();
+			renderer.render(scene, camera);
+
+			frameId = requestAnimationFrame(frame);
+		};
+
+		load();
+		frameId = requestAnimationFrame(frame);
+	}
+
+	onMount(() => {
+		if (value != null) {
+			reset_scene();
+		}
+		mounted = true;
+
+		return () => {
+			if (renderer) {
+				renderer.dispose();
+			}
+		};
+	});
+
+	$: ({ path } = value || {
+		path: undefined
+	});
+
+	$: canvas && mounted && path && reset_scene();
+</script>
+
+<canvas bind:this={canvas}></canvas>

package/shared/Model3D.svelte

@@ -2,7 +2,6 @@
 	import type { FileData } from "@gradio/client";
 	import { BlockLabel, IconButton } from "@gradio/atoms";
 	import { File, Download, Undo } from "@gradio/icons";
-	import Canvas3D from "./Canvas3D.svelte";
 	import type { I18nFormatter } from "@gradio/utils";
 	import { dequal } from "dequal";
 
@@ -22,9 +21,21 @@
 
 	let current_settings = { camera_position, zoom_speed, pan_speed };
 
-	let canvas3d: Canvas3D;
+	let canvas3dgs: any;
+	let canvas3d: any;
+	let use_3dgs = false;
 	let resolved_url: string | undefined;
 
+	async function loadCanvas3D(): Promise<any> {
+		const module = await import("./Canvas3D.svelte");
+		return module.default;
+	}
+
+	async function loadCanvas3DGS(): Promise<any> {
+		const module = await import("./Canvas3DGS.svelte");
+		return module.default;
+	}
+
 	function handle_undo(): void {
 		canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
 	}
@@ -39,6 +50,21 @@
 			current_settings = { camera_position, zoom_speed, pan_speed };
 		}
 	}
+
+	$: {
+		if (value) {
+			use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply");
+			if (use_3dgs) {
+				loadCanvas3DGS().then((module) => {
+					canvas3dgs = module;
+				});
+			} else {
+				loadCanvas3D().then((module) => {
+					canvas3d = module;
+				});
+			}
+		}
+	}
 </script>
 
 <BlockLabel
@@ -59,15 +85,25 @@
 			</a>
 		</div>
 
-		<Canvas3D
-			bind:this={canvas3d}
-			bind:resolved_url
-			{value}
-			{clear_color}
-			{camera_position}
-			{zoom_speed}
-			{pan_speed}
-		/>
+		{#if use_3dgs}
+			<svelte:component
+				this={canvas3dgs}
+				bind:resolved_url
+				{value}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{:else}
+			<svelte:component
+				this={canvas3d}
+				bind:resolved_url
+				{value}
+				{clear_color}
+				{camera_position}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{/if}
 	</div>
 {/if}
 

package/shared/Model3DUpload.svelte

@@ -5,7 +5,6 @@
 	import { BlockLabel } from "@gradio/atoms";
 	import { File } from "@gradio/icons";
 	import type { I18nFormatter } from "@gradio/utils";
-	import Canvas3D from "./Canvas3D.svelte";
 
 	export let value: null | FileData;
 	export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
@@ -39,9 +38,22 @@
 		dispatch("change");
 	}
 
-	let canvas3D: Canvas3D;
+	let canvas3d: any;
+	let canvas3dgs: any;
+	let use_3dgs = false;
+
+	async function loadCanvas3D(): Promise<any> {
+		const module = await import("./Canvas3D.svelte");
+		return module.default;
+	}
+
+	async function loadCanvas3DGS(): Promise<any> {
+		const module = await import("./Canvas3DGS.svelte");
+		return module.default;
+	}
+
 	async function handle_undo(): Promise<void> {
-		canvas3D.reset_camera_position(camera_position, zoom_speed, pan_speed);
+		canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
 	}
 
 	const dispatch = createEventDispatcher<{
@@ -54,6 +66,21 @@
 	let dragging = false;
 
 	$: dispatch("drag", dragging);
+
+	$: {
+		if (value) {
+			use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply");
+			if (use_3dgs) {
+				loadCanvas3DGS().then((module) => {
+					canvas3dgs = module;
+				});
+			} else {
+				loadCanvas3D().then((module) => {
+					canvas3d = module;
+				});
+			}
+		}
+	}
 </script>
 
 <BlockLabel {show_label} Icon={File} label={label || "3D Model"} />
@@ -62,7 +89,7 @@
 	<Upload
 		on:load={handle_upload}
 		{root}
-		filetype={[".stl", ".obj", ".gltf", ".glb", "model/obj"]}
+		filetype={[".stl", ".obj", ".gltf", ".glb", "model/obj", ".splat", ".ply"]}
 		bind:dragging
 	>
 		<slot />
@@ -76,14 +103,19 @@
 			on:undo={handle_undo}
 			absolute
 		/>
-		<Canvas3D
-			bind:this={canvas3D}
-			{value}
-			{clear_color}
-			{camera_position}
-			{zoom_speed}
-			{pan_speed}
-		></Canvas3D>
+
+		{#if use_3dgs}
+			<svelte:component this={canvas3dgs} {value} {zoom_speed} {pan_speed} />
+		{:else}
+			<svelte:component
+				this={canvas3d}
+				{value}
+				{clear_color}
+				{camera_position}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{/if}
 	</div>
 {/if}