@huggingface/tasks 0.19.65 → 0.19.67

package/src/tasks/image-text-to-image/inference.ts

@@ -0,0 +1,75 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters)
+ * must be provided, or both.
+ */
+export interface ImageTextToImageInput {
+	/**
+	 * The input image data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the image data as a raw bytes payload. Either this or prompt must be
+	 * provided.
+	 */
+	inputs?: Blob;
+	/**
+	 * Additional inference parameters for Image Text To Image
+	 */
+	parameters?: ImageTextToImageParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters for Image Text To Image
+ */
+export interface ImageTextToImageParameters {
+	/**
+	 * For diffusion models. A higher guidance scale value encourages the model to generate
+	 * images closely linked to the text prompt at the expense of lower image quality.
+	 */
+	guidance_scale?: number;
+	/**
+	 * One prompt to guide what NOT to include in image generation.
+	 */
+	negative_prompt?: string;
+	/**
+	 * For diffusion models. The number of denoising steps. More denoising steps usually lead to
+	 * a higher quality image at the expense of slower inference.
+	 */
+	num_inference_steps?: number;
+	/**
+	 * The text prompt to guide the image generation. Either this or inputs (image) must be
+	 * provided.
+	 */
+	prompt?: string;
+	/**
+	 * Seed for the random number generator.
+	 */
+	seed?: number;
+	/**
+	 * The size in pixels of the output image. This parameter is only supported by some
+	 * providers and for specific models. It will be ignored when unsupported.
+	 */
+	target_size?: TargetSize;
+	[property: string]: unknown;
+}
+/**
+ * The size in pixels of the output image. This parameter is only supported by some
+ * providers and for specific models. It will be ignored when unsupported.
+ */
+export interface TargetSize {
+	height: number;
+	width: number;
+	[property: string]: unknown;
+}
+/**
+ * Outputs of inference for the Image Text To Image task
+ */
+export interface ImageTextToImageOutput {
+	/**
+	 * The generated image returned as raw bytes in the payload.
+	 */
+	image: unknown;
+	[property: string]: unknown;
+}

package/src/tasks/image-text-to-image/spec/input.json

@@ -0,0 +1,59 @@
+{
+	"$id": "/inference/schemas/image-text-to-image/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) must be provided, or both.",
+	"title": "ImageTextToImageInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"type": "string",
+			"description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload. Either this or prompt must be provided.",
+			"comment": "type=binary"
+		},
+		"parameters": {
+			"description": "Additional inference parameters for Image Text To Image",
+			"$ref": "#/$defs/ImageTextToImageParameters"
+		}
+	},
+	"$defs": {
+		"ImageTextToImageParameters": {
+			"title": "ImageTextToImageParameters",
+			"type": "object",
+			"properties": {
+				"prompt": {
+					"type": "string",
+					"description": "The text prompt to guide the image generation. Either this or inputs (image) must be provided."
+				},
+				"guidance_scale": {
+					"type": "number",
+					"description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality."
+				},
+				"negative_prompt": {
+					"type": "string",
+					"description": "One prompt to guide what NOT to include in image generation."
+				},
+				"num_inference_steps": {
+					"type": "integer",
+					"description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference."
+				},
+				"target_size": {
+					"type": "object",
+					"description": "The size in pixels of the output image. This parameter is only supported by some providers and for specific models. It will be ignored when unsupported.",
+					"properties": {
+						"width": {
+							"type": "integer"
+						},
+						"height": {
+							"type": "integer"
+						}
+					},
+					"required": ["width", "height"]
+				},
+				"seed": {
+					"type": "integer",
+					"description": "Seed for the random number generator."
+				}
+			}
+		}
+	}
+}

package/src/tasks/image-text-to-image/spec/output.json

@@ -0,0 +1,13 @@
+{
+	"$id": "/inference/schemas/image-text-to-image/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image Text To Image task",
+	"title": "ImageTextToImageOutput",
+	"type": "object",
+	"properties": {
+		"image": {
+			"description": "The generated image returned as raw bytes in the payload."
+		}
+	},
+	"required": ["image"]
+}

package/src/tasks/image-text-to-video/about.md

@@ -0,0 +1,71 @@
+## Use Cases
+
+### Image Animation
+
+Image-text-to-video models can be used to animate still images based on text descriptions. For example, you can provide a landscape photo and the instruction "A camera pan from left to right" to create a video with camera movement.
+
+### Dynamic Content Creation
+
+Transform images into video by adding motion, transformations, or effects described in text prompts. This is useful for creating engaging social media content, presentations, or marketing materials.
+
+### Guided Video Generation
+
+Use a reference image with text prompts to guide the video generation process. This provides more control over the visual style and composition compared to text-to-video models alone.
+
+### Story Visualization
+
+Create video sequences from storyboards or concept art by providing scene descriptions. This can help filmmakers and animators visualize scenes before production.
+
+### Motion Control
+
+Generate videos with specific camera movements, object motions, or scene transitions by combining reference images with detailed motion descriptions.
+
+## Task Variants
+
+### Image-to-Video with Motion Control
+
+Models that generate videos from images while following specific motion instructions, such as camera movements, object animations, or scene dynamics.
+
+### Reference-guided Video Generation
+
+Models that use a reference image to guide the visual style and composition of the generated video while incorporating text prompts for motion and transformation control.
+
+### Conditional Video Synthesis
+
+Models that perform specific video transformations based on text conditions, such as adding weather effects, time-of-day changes, or environmental animations.
+
+## Inference
+
+You can use the Diffusers library to interact with image-text-to-video models. Here's example snippet to use `LTXImageToVideoPipeline`.
+
+```python
+import torch
+from diffusers import LTXImageToVideoPipeline
+from diffusers.utils import export_to_video, load_image
+
+pipe = LTXImageToVideoPipeline.from_pretrained("Lightricks/LTX-Video", torch_dtype=torch.bfloat16)
+pipe.to("cuda")
+
+image = load_image(
+    "https://huggingface.co/datasets/a-r-r-o-w/tiny-meme-dataset-captioned/resolve/main/images/8.png"
+)
+prompt = "A young girl stands calmly in the foreground, looking directly at the camera, as a house fire rages in the background. Flames engulf the structure, with smoke billowing into the air. Firefighters in protective gear rush to the scene, a fire truck labeled '38' visible behind them. The girl's neutral expression contrasts sharply with the chaos of the fire, creating a poignant and emotionally charged scene."
+negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
+
+video = pipe(
+    image=image,
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    width=704,
+    height=480,
+    num_frames=161,
+    num_inference_steps=50,
+).frames[0]
+export_to_video(video, "output.mp4", fps=24)
+```
+
+## Useful Resources
+
+- [LTX-Video Model Card](https://huggingface.co/Lightricks/LTX-Video)
+- [Text-to-Video: The Task, Challenges and the Current State](https://huggingface.co/blog/text-to-video)
+- [Diffusers documentation on Video Generation](https://huggingface.co/docs/diffusers/using-diffusers/text-img2vid)

package/src/tasks/image-text-to-video/data.ts

@@ -0,0 +1,54 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [],
+	demo: {
+		inputs: [
+			{
+				filename: "image-text-to-video-input.jpg",
+				type: "img",
+			},
+			{
+				label: "Input",
+				content: "Darth Vader is surfing on the waves.",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "image-text-to-video-output.gif",
+				type: "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.",
+			id: "fvd",
+		},
+		{
+			description:
+				"CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.",
+			id: "clipsim",
+		},
+	],
+	models: [
+		{
+			description: "A powerful model for image-text-to-video generation.",
+			id: "Lightricks/LTX-Video",
+		},
+	],
+	spaces: [
+		{
+			description: "An application for image-text-to-video generation.",
+			id: "Lightricks/ltx-video-distilled",
+		},
+	],
+	summary:
+		"Image-text-to-video models take an reference image and a text instructions as and generate a video based on them. These models are useful for animating still images, creating dynamic content from static references, and generating videos with specific motion or transformation guidance.",
+	widgetModels: ["Lightricks/LTX-Video"],
+	youtubeId: undefined,
+};
+
+export default taskData;

package/src/tasks/image-text-to-video/inference.ts

@@ -0,0 +1,77 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters)
+ * must be provided, or both.
+ */
+export interface ImageTextToVideoInput {
+	/**
+	 * The input image data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the image data as a raw bytes payload. Either this or prompt must be
+	 * provided.
+	 */
+	inputs?: Blob;
+	/**
+	 * Additional inference parameters for Image Text To Video
+	 */
+	parameters?: ImageTextToVideoParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters for Image Text To Video
+ */
+export interface ImageTextToVideoParameters {
+	/**
+	 * For diffusion models. A higher guidance scale value encourages the model to generate
+	 * videos closely linked to the text prompt at the expense of lower image quality.
+	 */
+	guidance_scale?: number;
+	/**
+	 * One prompt to guide what NOT to include in video generation.
+	 */
+	negative_prompt?: string;
+	/**
+	 * The num_frames parameter determines how many video frames are generated.
+	 */
+	num_frames?: number;
+	/**
+	 * The number of denoising steps. More denoising steps usually lead to a higher quality
+	 * video at the expense of slower inference.
+	 */
+	num_inference_steps?: number;
+	/**
+	 * The text prompt to guide the video generation. Either this or inputs (image) must be
+	 * provided.
+	 */
+	prompt?: string;
+	/**
+	 * Seed for the random number generator.
+	 */
+	seed?: number;
+	/**
+	 * The size in pixel of the output video frames.
+	 */
+	target_size?: TargetSize;
+	[property: string]: unknown;
+}
+/**
+ * The size in pixel of the output video frames.
+ */
+export interface TargetSize {
+	height: number;
+	width: number;
+	[property: string]: unknown;
+}
+/**
+ * Outputs of inference for the Image Text To Video task
+ */
+export interface ImageTextToVideoOutput {
+	/**
+	 * The generated video returned as raw bytes in the payload.
+	 */
+	video: unknown;
+	[property: string]: unknown;
+}

package/src/tasks/image-text-to-video/spec/input.json

@@ -0,0 +1,63 @@
+{
+	"$id": "/inference/schemas/image-text-to-video/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) must be provided, or both.",
+	"title": "ImageTextToVideoInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"type": "string",
+			"description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload. Either this or prompt must be provided.",
+			"comment": "type=binary"
+		},
+		"parameters": {
+			"description": "Additional inference parameters for Image Text To Video",
+			"$ref": "#/$defs/ImageTextToVideoParameters"
+		}
+	},
+	"$defs": {
+		"ImageTextToVideoParameters": {
+			"title": "ImageTextToVideoParameters",
+			"type": "object",
+			"properties": {
+				"prompt": {
+					"type": "string",
+					"description": "The text prompt to guide the video generation. Either this or inputs (image) must be provided."
+				},
+				"guidance_scale": {
+					"type": "number",
+					"description": "For diffusion models. A higher guidance scale value encourages the model to generate videos closely linked to the text prompt at the expense of lower image quality."
+				},
+				"negative_prompt": {
+					"type": "string",
+					"description": "One prompt to guide what NOT to include in video generation."
+				},
+				"num_inference_steps": {
+					"type": "integer",
+					"description": "The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference."
+				},
+				"num_frames": {
+					"type": "number",
+					"description": "The num_frames parameter determines how many video frames are generated."
+				},
+				"target_size": {
+					"type": "object",
+					"description": "The size in pixel of the output video frames.",
+					"properties": {
+						"width": {
+							"type": "integer"
+						},
+						"height": {
+							"type": "integer"
+						}
+					},
+					"required": ["width", "height"]
+				},
+				"seed": {
+					"type": "integer",
+					"description": "Seed for the random number generator."
+				}
+			}
+		}
+	}
+}

package/src/tasks/image-text-to-video/spec/output.json

@@ -0,0 +1,13 @@
+{
+	"$id": "/inference/schemas/image-text-to-video/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image Text To Video task",
+	"title": "ImageTextToVideoOutput",
+	"type": "object",
+	"properties": {
+		"video": {
+			"description": "The generated video returned as raw bytes in the payload."
+		}
+	},
+	"required": ["video"]
+}

package/src/tasks/index.ts

@@ -14,6 +14,8 @@ import imageFeatureExtraction from "./image-feature-extraction/data.js";
 import imageToImage from "./image-to-image/data.js";
 import imageToText from "./image-to-text/data.js";
 import imageTextToText from "./image-text-to-text/data.js";
+import imageTextToImage from "./image-text-to-image/data.js";
+import imageTextToVideo from "./image-text-to-video/data.js";
 import imageSegmentation from "./image-segmentation/data.js";
 import imageToVideo from "./image-to-video/data.js";
 import maskGeneration from "./mask-generation/data.js";
@@ -74,6 +76,16 @@ export type * from "./image-to-image/inference.js";
 export type { ImageToTextInput, ImageToTextOutput, ImageToTextParameters } from "./image-to-text/inference.js";
 export type * from "./image-segmentation/inference.js";
 export type { ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters } from "./image-to-video/inference.js";
+export type {
+	ImageTextToImageInput,
+	ImageTextToImageOutput,
+	ImageTextToImageParameters,
+} from "./image-text-to-image/inference.js";
+export type {
+	ImageTextToVideoInput,
+	ImageTextToVideoOutput,
+	ImageTextToVideoParameters,
+} from "./image-text-to-video/inference.js";
 export type * from "./object-detection/inference.js";
 export type * from "./depth-estimation/inference.js";
 export type * from "./question-answering/inference.js";
@@ -133,6 +145,8 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"image-feature-extraction": ["timm", "transformers"],
 	"image-segmentation": ["transformers", "transformers.js"],
 	"image-text-to-text": ["transformers"],
+	"image-text-to-image": ["diffusers"],
+	"image-text-to-video": ["diffusers"],
 	"image-to-image": ["diffusers", "transformers", "transformers.js"],
 	"image-to-text": ["transformers", "transformers.js"],
 	"image-to-video": ["diffusers"],
@@ -220,6 +234,8 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"image-segmentation": getData("image-segmentation", imageSegmentation),
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-text-to-text": getData("image-text-to-text", imageTextToText),
+	"image-text-to-image": getData("image-text-to-image", imageTextToImage),
+	"image-text-to-video": getData("image-text-to-video", imageTextToVideo),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": getData("image-to-video", imageToVideo),
 	"keypoint-detection": getData("keypoint-detection", keypointDetection),