@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/text-to-3d/about.md

@@ -0,0 +1,62 @@
+## Use Cases
+
+Text-to-3D models can be used in a wide variety of applications that require 3D, such as games, animation, design, architecture, engineering, marketing, and more.
+
+![Text-to-3D Thumbnail](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/text-to-3d/text-to-3d-thumbnail.png)
+
+This task is similar to the [image-to-3d](https://huggingface.co/tasks/image-to-3d) task, but takes text input instead of image input. In practice, this is often equivalent to a combination of [text-to-image](https://huggingface.co/tasks/text-to-image) and [image-to-3d](https://huggingface.co/tasks/image-to-3d). That is, the text is first converted to an image, then the image is converted to 3D.
+
+### Generating Meshes
+
+Meshes are the standard representation of 3D in industry.
+
+### Generating Gaussian Splats
+
+[Gaussian Splatting](https://huggingface.co/blog/gaussian-splatting) is a rendering technique that represents scenes as fuzzy points.
+
+### Inference
+
+Inference for this task typically leverages the [Diffusers](https://huggingface.co/docs/diffusers/index) library for inference, using [Custom Pipelines](https://huggingface.co/docs/diffusers/v0.6.0/en/using-diffusers/custom_pipelines).
+
+These are unstandardized and depend on the model. More details can be found in each model repository.
+
+```python
+import torch
+import requests
+import numpy as np
+from io import BytesIO
+from diffusers import DiffusionPipeline
+from PIL import Image
+
+pipeline = DiffusionPipeline.from_pretrained(
+    "dylanebert/LGM-full",
+    custom_pipeline="dylanebert/LGM-full",
+    torch_dtype=torch.float16,
+    trust_remote_code=True,
+).to("cuda")
+
+input_prompt = "a cat statue"
+result = pipeline(input_prompt, None)
+result_path = "/tmp/output.ply"
+pipeline.save_ply(result, result_path)
+```
+
+In the code above, we:
+
+1. Import the necessary libraries
+2. Load the `LGM-full` model and custom pipeline
+3. Define the input prompt
+4. Run the pipeline on the input prompt
+5. Save the output to a file
+
+### Output Formats
+
+Meshes can be in `.obj`, `.glb`, `.stl`, or `.gltf` format. Other formats are allowed, but won't be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component.
+
+Splats can be in `.ply` or `.splat` format. They can be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component using the [gsplat.js](https://github.com/huggingface/gsplat.js) library.
+
+## Useful Resources
+
+- [ML for 3D Course](https://huggingface.co/learn/ml-for-3d-course)
+- [3D Arena Leaderboard](https://huggingface.co/spaces/dylanebert/3d-arena)
+- [gsplat.js](https://github.com/huggingface/gsplat.js)

package/src/tasks/text-to-3d/data.ts

@@ -0,0 +1,56 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A large dataset of over 10 million 3D objects.",
+			id: "allenai/objaverse-xl",
+		},
+		{
+			description: "Descriptive captions for 3D objects in Objaverse.",
+			id: "tiange/Cap3D",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Prompt",
+				content: "a cat statue",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				label: "Result",
+				content: "text-to-3d-3d-output-filename.glb",
+				type: "text",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "Text-to-3D mesh model by OpenAI",
+			id: "openai/shap-e",
+		},
+		{
+			description: "Generative 3D gaussian splatting model.",
+			id: "ashawkey/LGM",
+		},
+	],
+	spaces: [
+		{
+			description: "Text-to-3D demo with mesh outputs.",
+			id: "hysts/Shap-E",
+		},
+		{
+			description: "Text/image-to-3D demo with splat outputs.",
+			id: "ashawkey/LGM",
+		},
+	],
+	summary: "Text-to-3D models take in text input and produce 3D output.",
+	widgetModels: [],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/text-to-audio/inference.ts

@@ -0,0 +1,143 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Text To Audio inference
+ */
+export interface TextToAudioInput {
+	/**
+	 * The input text data
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: TextToAudioParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Text To Audio
+ */
+export interface TextToAudioParameters {
+	/**
+	 * Parametrization of the text generation process
+	 */
+	generation_parameters?: GenerationParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Parametrization of the text generation process
+ *
+ * Ad-hoc parametrization of the text generation process
+ */
+export interface GenerationParameters {
+	/**
+	 * Whether to use sampling instead of greedy decoding when generating new tokens.
+	 */
+	do_sample?: boolean;
+	/**
+	 * Controls the stopping condition for beam-based methods.
+	 */
+	early_stopping?: EarlyStoppingUnion;
+	/**
+	 * If set to float strictly between 0 and 1, only tokens with a conditional probability
+	 * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from
+	 * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language
+	 * Model Desmoothing](https://hf.co/papers/2210.15191) for more details.
+	 */
+	epsilon_cutoff?: number;
+	/**
+	 * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to
+	 * float strictly between 0 and 1, a token is only considered if it is greater than either
+	 * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter
+	 * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In
+	 * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
+	 * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191)
+	 * for more details.
+	 */
+	eta_cutoff?: number;
+	/**
+	 * The maximum length (in tokens) of the generated text, including the input.
+	 */
+	max_length?: number;
+	/**
+	 * The maximum number of tokens to generate. Takes precedence over max_length.
+	 */
+	max_new_tokens?: number;
+	/**
+	 * The minimum length (in tokens) of the generated text, including the input.
+	 */
+	min_length?: number;
+	/**
+	 * The minimum number of tokens to generate. Takes precedence over min_length.
+	 */
+	min_new_tokens?: number;
+	/**
+	 * Number of groups to divide num_beams into in order to ensure diversity among different
+	 * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details.
+	 */
+	num_beam_groups?: number;
+	/**
+	 * Number of beams to use for beam search.
+	 */
+	num_beams?: number;
+	/**
+	 * The value balances the model confidence and the degeneration penalty in contrastive
+	 * search decoding.
+	 */
+	penalty_alpha?: number;
+	/**
+	 * The value used to modulate the next token probabilities.
+	 */
+	temperature?: number;
+	/**
+	 * The number of highest probability vocabulary tokens to keep for top-k-filtering.
+	 */
+	top_k?: number;
+	/**
+	 * If set to float < 1, only the smallest set of most probable tokens with probabilities
+	 * that add up to top_p or higher are kept for generation.
+	 */
+	top_p?: number;
+	/**
+	 * Local typicality measures how similar the conditional probability of predicting a target
+	 * token next is to the expected conditional probability of predicting a random token next,
+	 * given the partial text already generated. If set to float < 1, the smallest set of the
+	 * most locally typical tokens with probabilities that add up to typical_p or higher are
+	 * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details.
+	 */
+	typical_p?: number;
+	/**
+	 * Whether the model should use the past last key/values attentions to speed up decoding
+	 */
+	use_cache?: boolean;
+	[property: string]: unknown;
+}
+
+/**
+ * Controls the stopping condition for beam-based methods.
+ */
+export type EarlyStoppingUnion = boolean | "never";
+
+/**
+ * Outputs of inference for the Text To Audio task
+ */
+export interface TextToAudioOutput {
+	/**
+	 * The generated audio waveform.
+	 */
+	audio: unknown;
+	samplingRate: unknown;
+	/**
+	 * The sampling rate of the generated audio waveform.
+	 */
+	sampling_rate?: number;
+	[property: string]: unknown;
+}

package/src/tasks/text-to-audio/spec/input.json

@@ -0,0 +1,31 @@
+{
+	"$id": "/inference/schemas/text-to-audio/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Text To Audio inference",
+	"title": "TextToAudioInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/TextToAudioParameters"
+		}
+	},
+	"$defs": {
+		"TextToAudioParameters": {
+			"title": "TextToAudioParameters",
+			"description": "Additional inference parameters for Text To Audio",
+			"type": "object",
+			"properties": {
+				"generation_parameters": {
+					"description": "Parametrization of the text generation process",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/text-to-audio/spec/output.json

@@ -0,0 +1,17 @@
+{
+	"$id": "/inference/schemas/text-to-audio/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Text To Audio task",
+	"title": "TextToAudioOutput",
+	"type": "object",
+	"properties": {
+		"audio": {
+			"description": "The generated audio waveform."
+		},
+		"sampling_rate": {
+			"type": "number",
+			"description": "The sampling rate of the generated audio waveform."
+		}
+	},
+	"required": ["audio", "samplingRate"]
+}

package/src/tasks/text-to-image/about.md

@@ -0,0 +1,96 @@
+## Use Cases
+
+### Data Generation
+
+Businesses can generate data for their use cases by inputting text and getting image outputs.
+
+### Immersive Conversational Chatbots
+
+Chatbots can be made more immersive if they provide contextual images based on the input provided by the user.
+
+### Creative Ideas for Fashion Industry
+
+Different patterns can be generated to obtain unique pieces of fashion. Text-to-image models make creations easier for designers to conceptualize their design before actually implementing it.
+
+### Architecture Industry
+
+Architects can utilise the models to construct an environment based out on the requirements of the floor plan. This can also include the furniture that has to be placed in that environment.
+
+## Task Variants
+
+### Image Editing
+
+Image editing with text-to-image models involves modifying an image following edit instructions provided in a text prompt.
+
+- **Synthetic image editing**: Adjusting images that were initially created using an input prompt while preserving the overall meaning or context of the original image.
+
+  ![Examples](https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/edit_p2p.png)
+  _Figure taken from ["InstructPix2Pix: Learning to Follow Image Editing Instructions"](https://www.timothybrooks.com/instruct-pix2pix)_
+
+- **Real image editing**: Similar to synthetic image editing, except we're using real photos/images. This task is usually more complex.
+
+  ![Examples](https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/pix2pix.jpeg)
+  _Figure taken from ["Prompt-to-Prompt Image Editing with Cross-Attention Control"](https://prompt-to-prompt.github.io)_
+
+### Personalization
+
+Personalization refers to techniques used to customize text-to-image models. We introduce new subjects or concepts to the model, which the model can then generate when we refer to them with a text prompt.
+
+For example, you can use these techniques to generate images of your dog in imaginary settings, after you have taught the model using a few reference images of the subject (or just one in some cases). Teaching the model a new concept can be achieved through fine-tuning, or by using training-free techniques.
+
+## Inference
+
+You can use diffusers pipelines to infer with `text-to-image` models.
+
+```python
+from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+
+model_id = "stabilityai/stable-diffusion-2"
+scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
+pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, torch_dtype=torch.float16)
+pipe = pipe.to("cuda")
+
+prompt = "a photo of an astronaut riding a horse on mars"
+image = pipe(prompt).images[0]
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer text-to-image models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.textToImage({
+	model: "stabilityai/stable-diffusion-2",
+	inputs: "award winning high resolution photo of a giant tortoise/((ladybird)) hybrid, [trending on artstation]",
+	parameters: {
+		negative_prompt: "blurry",
+	},
+});
+```
+
+## Useful Resources
+
+### Model Inference
+
+- [Hugging Face Diffusion Models Course](https://github.com/huggingface/diffusion-models-class)
+- [Getting Started with Diffusers](https://huggingface.co/docs/diffusers/index)
+- [Text-to-Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation)
+- [Using Stable Diffusion with Core ML on Apple Silicon](https://huggingface.co/blog/diffusers-coreml)
+- [A guide on Vector Quantized Diffusion](https://huggingface.co/blog/vq-diffusion)
+- [🧨 Stable Diffusion in JAX/Flax](https://huggingface.co/blog/stable_diffusion_jax)
+- [Running IF with 🧨 diffusers on a Free Tier Google Colab](https://huggingface.co/blog/if)
+- [Introducing Würstchen: Fast Diffusion for Image Generation](https://huggingface.co/blog/wuerstchen)
+- [Efficient Controllable Generation for SDXL with T2I-Adapters](https://huggingface.co/blog/t2i-sdxl-adapters)
+- [Welcome aMUSEd: Efficient Text-to-Image Generation](https://huggingface.co/blog/amused)
+- Image Editing Demos: [LEDITS++](https://huggingface.co/spaces/editing-images/leditsplusplus), [Turbo Edit](https://huggingface.co/spaces/turboedit/turbo_edit), [InstructPix2Pix](https://huggingface.co/spaces/timbrooks/instruct-pix2pix), [CosXL](https://huggingface.co/spaces/multimodalart/cosxl)
+- Training free Personalization Demos: [Face-to-All](https://huggingface.co/spaces/multimodalart/face-to-all), [InstantStyle](https://huggingface.co/spaces/InstantX/InstantStyle), [RB-modulation](https://huggingface.co/spaces/fffiloni/RB-Modulation), [Photomaker v2](https://huggingface.co/spaces/TencentARC/PhotoMaker-V2)
+
+### Model Fine-tuning
+
+- [Finetune Stable Diffusion Models with DDPO via TRL](https://huggingface.co/blog/pref-tuning)
+- [LoRA training scripts of the world, unite!](https://huggingface.co/blog/sdxl_lora_advanced_script)
+- [Using LoRA for Efficient Stable Diffusion Fine-Tuning](https://huggingface.co/blog/lora)
+- LoRA fine tuning Spaces: [FLUX.1 finetuning](https://huggingface.co/spaces/autotrain-projects/train-flux-lora-ease), [SDXL finetuning](https://huggingface.co/spaces/multimodalart/lora-ease)
+
+This page was made possible thanks to the efforts of [Ishan Dutta](https://huggingface.co/ishandutta), [Enrique Elias Ubaldo](https://huggingface.co/herrius) and [Oğuz Akif](https://huggingface.co/oguzakif).

package/src/tasks/text-to-image/data.ts

@@ -0,0 +1,100 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "RedCaps is a large-scale dataset of 12M image-text pairs collected from Reddit.",
+			id: "red_caps",
+		},
+		{
+			description: "Conceptual Captions is a dataset consisting of ~3.3M images annotated with captions.",
+			id: "conceptual_captions",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Input",
+				content: "A city above clouds, pastel colors, Victorian style",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "image.jpeg",
+				type: "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"The Inception Score (IS) measure assesses diversity and meaningfulness. It uses a generated image sample to predict its label. A higher score signifies more diverse and meaningful images.",
+			id: "IS",
+		},
+		{
+			description:
+				"The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.",
+			id: "FID",
+		},
+		{
+			description:
+				"R-precision assesses how the generated image aligns with the provided text description. It uses the generated images as queries to retrieve relevant text descriptions. The top 'r' relevant descriptions are selected and used to calculate R-precision as r/R, where 'R' is the number of ground truth descriptions associated with the generated images. A higher R-precision value indicates a better model.",
+			id: "R-Precision",
+		},
+	],
+	models: [
+		{
+			description: "One of the most powerful image generation models that can generate realistic outputs.",
+			id: "black-forest-labs/FLUX.1-dev",
+		},
+		{
+			description: "A powerful yet fast image generation model.",
+			id: "latent-consistency/lcm-lora-sdxl",
+		},
+		{
+			description: "Text-to-image model for photorealistic generation.",
+			id: "Kwai-Kolors/Kolors",
+		},
+		{
+			description: "A powerful text-to-image model.",
+			id: "stabilityai/stable-diffusion-3-medium-diffusers",
+		},
+	],
+	spaces: [
+		{
+			description: "A powerful text-to-image application.",
+			id: "stabilityai/stable-diffusion-3-medium",
+		},
+		{
+			description: "A text-to-image application to generate comics.",
+			id: "jbilcke-hf/ai-comic-factory",
+		},
+		{
+			description: "An application to match multiple custom image generation models.",
+			id: "multimodalart/flux-lora-lab",
+		},
+		{
+			description: "A powerful yet very fast image generation application.",
+			id: "latent-consistency/lcm-lora-for-sdxl",
+		},
+		{
+			description: "A gallery to explore various text-to-image models.",
+			id: "multimodalart/LoraTheExplorer",
+		},
+		{
+			description: "An application for `text-to-image`, `image-to-image` and image inpainting.",
+			id: "ArtGAN/Stable-Diffusion-ControlNet-WebUI",
+		},
+		{
+			description: "An application to generate realistic images given photos of a person and a prompt.",
+			id: "InstantX/InstantID",
+		},
+	],
+	summary:
+		"Text-to-image is the task of generating images from input text. These pipelines can also be used to modify and edit images based on text prompts.",
+	widgetModels: ["black-forest-labs/FLUX.1-dev"],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/text-to-image/inference.ts

@@ -0,0 +1,75 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Text To Image inference
+ */
+export interface TextToImageInput {
+	/**
+	 * The input text data (sometimes called "prompt")
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: TextToImageParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Text To Image
+ */
+export interface TextToImageParameters {
+	/**
+	 * A higher guidance scale value encourages the model to generate images closely linked to
+	 * the text prompt, but values too high may cause saturation and other artifacts.
+	 */
+	guidance_scale?: number;
+	/**
+	 * One or several prompt to guide what NOT to include in image generation.
+	 */
+	negative_prompt?: string[];
+	/**
+	 * The number of denoising steps. More denoising steps usually lead to a higher quality
+	 * image at the expense of slower inference.
+	 */
+	num_inference_steps?: number;
+	/**
+	 * Override the scheduler with a compatible one.
+	 */
+	scheduler?: string;
+	/**
+	 * Seed for the random number generator.
+	 */
+	seed?: number;
+	/**
+	 * The size in pixel of the output image
+	 */
+	target_size?: TargetSize;
+	[property: string]: unknown;
+}
+
+/**
+ * The size in pixel of the output image
+ */
+export interface TargetSize {
+	height: number;
+	width: number;
+	[property: string]: unknown;
+}
+
+/**
+ * Outputs of inference for the Text To Image task
+ */
+export interface TextToImageOutput {
+	/**
+	 * The generated image returned as raw bytes in the payload.
+	 */
+	image: unknown;
+	[property: string]: unknown;
+}

package/src/tasks/text-to-image/spec/input.json

@@ -0,0 +1,63 @@
+{
+	"$id": "/inference/schemas/text-to-image/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Text To Image inference",
+	"title": "TextToImageInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data (sometimes called \"prompt\")",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/TextToImageParameters"
+		}
+	},
+	"$defs": {
+		"TextToImageParameters": {
+			"title": "TextToImageParameters",
+			"description": "Additional inference parameters for Text To Image",
+			"type": "object",
+			"properties": {
+				"guidance_scale": {
+					"type": "number",
+					"description": "A higher guidance scale value encourages the model to generate images closely linked to the text prompt, but values too high may cause saturation and other artifacts."
+				},
+				"negative_prompt": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					},
+					"description": "One or several prompt to guide what NOT to include in image generation."
+				},
+				"num_inference_steps": {
+					"type": "integer",
+					"description": "The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference."
+				},
+				"target_size": {
+					"type": "object",
+					"description": "The size in pixel of the output image",
+					"properties": {
+						"width": {
+							"type": "integer"
+						},
+						"height": {
+							"type": "integer"
+						}
+					},
+					"required": ["width", "height"]
+				},
+				"scheduler": {
+					"type": "string",
+					"description": "Override the scheduler with a compatible one."
+				},
+				"seed": {
+					"type": "integer",
+					"description": "Seed for the random number generator."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/text-to-image/spec/output.json

@@ -0,0 +1,13 @@
+{
+	"$id": "/inference/schemas/text-to-image/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Text To Image task",
+	"title": "TextToImageOutput",
+	"type": "object",
+	"properties": {
+		"image": {
+			"description": "The generated image returned as raw bytes in the payload."
+		}
+	},
+	"required": ["image"]
+}