@gradio/multimodaltextbox 0.14.1 → 0.14.2

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @gradio/multimodaltextbox
 
+## 0.14.2
+
+### Fixes
+
+- [#13802](https://github.com/gradio-app/gradio/pull/13802) [`d857094`](https://github.com/gradio-app/gradio/commit/d857094868c85167dd332b6093a3914dee960840) - Handle a paste in `gr.MultimodalTextbox` as text or as an image, not both.  Thanks @hysts!
+
+### Dependency updates
+
+- @gradio/client@2.6.0
+- @gradio/atoms@0.26.2
+- @gradio/utils@0.14.1
+- @gradio/upload@0.18.3
+- @gradio/image@0.28.3
+- @gradio/audio@0.24.3
+- @gradio/video@0.23.1
+
+## 0.14.1
+
+### Dependency updates
+
+- @gradio/client@2.5.1
+
+## 0.14.1
+
+### Dependency updates
+
+- @gradio/client@2.5.0
+
 ## 0.14.1
 
 ### Dependency updates

package/Index.svelte

@@ -1,6 +1,4 @@
-<svelte:options accessors={true} />
-
-<script context="module" lang="ts">
+<script module lang="ts">
 	export { default as BaseMultimodalTextbox } from "./shared/MultimodalTextbox.svelte";
 	export { default as BaseExample } from "./Example.svelte";
 </script>

package/MultimodalTextbox.test.ts

@@ -1,6 +1,7 @@
-import { test, describe, assert, afterEach } from "vitest";
-import { cleanup, render } from "@self/tootils/render";
+import { test, describe, assert, afterEach, expect, vi } from "vitest";
+import { cleanup, render, mock_client } from "@self/tootils/render";
 import event from "@testing-library/user-event";
+import { tick } from "svelte";
 
 import MultimodalTextbox from "./Index.svelte";
 import type { ILoadingStatus as LoadingStatus } from "@gradio/statustracker";
@@ -83,4 +84,144 @@ describe("MultimodalTextbox", () => {
 		await event.click(submitButton);
 		assert.equal(mock.callCount, 1);
 	});
+
+	function paste_event(
+		text: string | null,
+		image_name?: string,
+		with_html = true
+	): ClipboardEvent {
+		const data = new DataTransfer();
+		if (text) {
+			data.setData("text/plain", text);
+			if (with_html) {
+				data.setData("text/html", `<table><tr><td>${text}</td></tr></table>`);
+			}
+		}
+		if (image_name) {
+			data.items.add(
+				new File([new Uint8Array([1, 2, 3])], image_name, {
+					type: "image/png"
+				})
+			);
+		}
+		return new ClipboardEvent("paste", {
+			clipboardData: data,
+			bubbles: true,
+			cancelable: true
+		});
+	}
+
+	test("a paste is handled as text or as an image, never both", async () => {
+		const { getByTestId, listen } = await render(MultimodalTextbox, {
+			show_label: true,
+			max_lines: 1,
+			loading_status,
+			lines: 1,
+			value: { text: "", files: [] },
+			label: "MultimodalTextbox",
+			interactive: true,
+			root: "http://localhost:7860",
+			sources: ["upload"],
+			client: mock_client()
+		});
+
+		const upload = listen("upload");
+		const textbox = getByTestId("textbox");
+
+		const excel_paste = paste_event("1\t2\n3\t4", "cells.png");
+		textbox.dispatchEvent(excel_paste);
+		// an image alone is still attached. Waiting for that upload also bounds
+		// the assertion that the paste above uploaded nothing.
+		const image_paste = paste_event(null, "screenshot.png");
+		textbox.dispatchEvent(image_paste);
+
+		const uploaded = (): string[] =>
+			upload.mock.calls.flat(2).map((file) => file.orig_name);
+
+		// wait for the screenshot itself, so a regression that uploads
+		// cells.png first cannot satisfy the wait and slip past
+		await vi.waitFor(() => expect(uploaded()).toContain("screenshot.png"));
+		await tick();
+
+		assert.deepEqual(uploaded(), ["screenshot.png"]);
+		assert.isFalse(excel_paste.defaultPrevented);
+		assert.isTrue(image_paste.defaultPrevented);
+	});
+
+	test("with nowhere to upload, only a text paste falls back to the browser", async () => {
+		const { getByTestId, listen } = await render(MultimodalTextbox, {
+			show_label: true,
+			max_lines: 1,
+			loading_status,
+			lines: 1,
+			// file_count "single" with a file already attached unmounts Upload
+			file_count: "single",
+			value: {
+				text: "",
+				files: [
+					{
+						path: "cats.jpg",
+						orig_name: "cats.jpg",
+						mime_type: "image/jpeg",
+						meta: { _type: "gradio.FileData" }
+					}
+				]
+			},
+			label: "MultimodalTextbox",
+			interactive: true,
+			root: "http://localhost:7860",
+			max_plain_text_length: 1000,
+			sources: ["upload"],
+			client: mock_client()
+		});
+
+		const upload = listen("upload");
+		const textbox = getByTestId("textbox");
+
+		const image_paste = paste_event(null, "screenshot.png");
+		textbox.dispatchEvent(image_paste);
+		const long_paste = paste_event("x".repeat(2000));
+		textbox.dispatchEvent(long_paste);
+		const path_paste = paste_event("file:///cats.jpg", "cats.jpg", false);
+		textbox.dispatchEvent(path_paste);
+
+		await tick();
+		await tick();
+
+		expect(upload).not.toHaveBeenCalled();
+		// the browser would paste the image's file name, which is worth nothing
+		assert.isTrue(image_paste.defaultPrevented);
+		// text is its own fallback, so it must not be swallowed either way
+		assert.isFalse(long_paste.defaultPrevented);
+		assert.isFalse(path_paste.defaultPrevented);
+	});
+
+	test("text with no HTML flavor leaves the image alone", async () => {
+		const { getByTestId, listen } = await render(MultimodalTextbox, {
+			show_label: true,
+			max_lines: 1,
+			loading_status,
+			lines: 1,
+			value: { text: "", files: [] },
+			label: "MultimodalTextbox",
+			interactive: true,
+			root: "http://localhost:7860",
+			sources: ["upload"],
+			client: mock_client()
+		});
+
+		const upload = listen("upload");
+		// a file manager copying an image can put its path in text/plain with
+		// no text/html, and there the image is the point of the paste
+		getByTestId("textbox").dispatchEvent(
+			paste_event("file:///cats.jpg", "cats.jpg", false)
+		);
+
+		await vi.waitFor(() => expect(upload).toHaveBeenCalled());
+
+		assert.deepEqual(
+			upload.mock.calls.flat(2).map((file) => file.orig_name),
+			["cats.jpg"]
+		);
+	});
 });

package/dist/Index.svelte

@@ -1,6 +1,4 @@
-<svelte:options accessors={true} />
-
-<script context="module" lang="ts">
+<script module lang="ts">
 	export { default as BaseMultimodalTextbox } from "./shared/MultimodalTextbox.svelte";
 	export { default as BaseExample } from "./Example.svelte";
 </script>

package/dist/shared/MultimodalTextbox.svelte

@@ -312,23 +312,31 @@
 		const items = event.clipboardData.items;
 		const text = event.clipboardData.getData("text");
 
-		if (text && text.length > max_plain_text_length) {
+		if (text && text.length > max_plain_text_length && upload_component) {
 			event.preventDefault();
 			const file = new window.File([text], "pasted_text.txt", {
 				type: "text/plain",
 				lastModified: Date.now()
 			});
-			if (upload_component) {
-				upload_component.load_files([file]);
-			}
+			upload_component.load_files([file]);
 			return;
 		}
 
+		// text plus HTML means the image is a rendering of that content. Text
+		// alone may be an image's own file path on platforms we have not
+		// measured, so leave those pastes to the image branch (#10910)
+		if (text && event.clipboardData.types.includes("text/html")) return;
+
 		for (let index in items) {
 			const item = items[index];
 			if (item.kind === "file" && item.type.includes("image")) {
 				const blob = item.getAsFile();
-				if (blob) upload_component.load_files([blob]);
+				if (blob) {
+					// claim the paste so the browser cannot insert the file name,
+					// unless that would leave text with nothing to replace it
+					if (upload_component || !text) event.preventDefault();
+					upload_component?.load_files([blob]);
+				}
 			}
 		}
 	}
@@ -376,10 +384,10 @@
 				}
 
 				if (valid_files.length > 0) {
-					upload_component.load_files(valid_files);
+					upload_component?.load_files(valid_files);
 				}
 			} else {
-				upload_component.load_files(files);
+				upload_component?.load_files(files);
 			}
 		}
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gradio/multimodaltextbox",
-  "version": "0.14.1",
+  "version": "0.14.2",
   "description": "Gradio UI packages",
   "type": "module",
   "author": "",
@@ -22,15 +22,15 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@gradio/statustracker": "^0.15.2",
-    "@gradio/atoms": "^0.26.1",
-    "@gradio/upload": "^0.18.2",
     "@gradio/icons": "^0.16.0",
-    "@gradio/utils": "^0.14.0",
-    "@gradio/video": "^0.23.0",
-    "@gradio/image": "^0.28.1",
-    "@gradio/audio": "^0.24.2",
-    "@gradio/client": "^2.4.0"
+    "@gradio/atoms": "^0.26.2",
+    "@gradio/statustracker": "^0.15.3",
+    "@gradio/utils": "^0.14.1",
+    "@gradio/image": "^0.28.3",
+    "@gradio/upload": "^0.18.3",
+    "@gradio/video": "^0.23.1",
+    "@gradio/client": "^2.6.0",
+    "@gradio/audio": "^0.24.3"
   },
   "devDependencies": {
     "@gradio/preview": "^0.17.0"

package/shared/MultimodalTextbox.svelte

@@ -312,23 +312,31 @@
 		const items = event.clipboardData.items;
 		const text = event.clipboardData.getData("text");
 
-		if (text && text.length > max_plain_text_length) {
+		if (text && text.length > max_plain_text_length && upload_component) {
 			event.preventDefault();
 			const file = new window.File([text], "pasted_text.txt", {
 				type: "text/plain",
 				lastModified: Date.now()
 			});
-			if (upload_component) {
-				upload_component.load_files([file]);
-			}
+			upload_component.load_files([file]);
 			return;
 		}
 
+		// text plus HTML means the image is a rendering of that content. Text
+		// alone may be an image's own file path on platforms we have not
+		// measured, so leave those pastes to the image branch (#10910)
+		if (text && event.clipboardData.types.includes("text/html")) return;
+
 		for (let index in items) {
 			const item = items[index];
 			if (item.kind === "file" && item.type.includes("image")) {
 				const blob = item.getAsFile();
-				if (blob) upload_component.load_files([blob]);
+				if (blob) {
+					// claim the paste so the browser cannot insert the file name,
+					// unless that would leave text with nothing to replace it
+					if (upload_component || !text) event.preventDefault();
+					upload_component?.load_files([blob]);
+				}
 			}
 		}
 	}
@@ -376,10 +384,10 @@
 				}
 
 				if (valid_files.length > 0) {
-					upload_component.load_files(valid_files);
+					upload_component?.load_files(valid_files);
 				}
 			} else {
-				upload_component.load_files(files);
+				upload_component?.load_files(files);
 			}
 		}
 	}