@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/text-to-speech/about.md

@@ -0,0 +1,63 @@
+## Use Cases
+
+Text-to-Speech (TTS) models can be used in any speech-enabled application that requires converting text to speech imitating human voice.
+
+### Voice Assistants
+
+TTS models are used to create voice assistants on smart devices. These models are a better alternative compared to concatenative methods where the assistant is built by recording sounds and mapping them, since the outputs in TTS models contain elements in natural speech such as emphasis.
+
+### Announcement Systems
+
+TTS models are widely used in airport and public transportation announcement systems to convert the announcement of a given text into speech.
+
+## Inference Endpoints
+
+The Hub contains over [1500 TTS models](https://huggingface.co/models?pipeline_tag=text-to-speech&sort=downloads) that you can use right away by trying out the widgets directly in the browser or calling the models as a service using Inference Endpoints. Here is a simple code snippet to get you started:
+
+```python
+import json
+import requests
+
+headers = {"Authorization": f"Bearer {API_TOKEN}"}
+API_URL = "https://api-inference.huggingface.co/models/microsoft/speecht5_tts"
+
+def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response
+
+output = query({"text_inputs": "Max is the best doggo."})
+```
+
+You can also use libraries such as [espnet](https://huggingface.co/models?library=espnet&pipeline_tag=text-to-speech&sort=downloads) or [transformers](https://huggingface.co/models?pipeline_tag=text-to-speech&library=transformers&sort=trending) if you want to handle the Inference directly.
+
+## Direct Inference
+
+Now, you can also use the Text-to-Speech pipeline in Transformers to synthesise high quality voice.
+
+```python
+from transformers import pipeline
+
+synthesizer = pipeline("text-to-speech", "suno/bark")
+
+synthesizer("Look I am generating speech in three lines of code!")
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer summarization models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.textToSpeech({
+	model: "facebook/mms-tts",
+	inputs: "text to generate speech from",
+});
+```
+
+## Useful Resources
+
+- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter6/introduction)
+- [ML for Audio Study Group - Text to Speech Deep Dive](https://www.youtube.com/watch?v=aLBedWj-5CQ)
+- [Speech Synthesis, Recognition, and More With SpeechT5](https://huggingface.co/blog/speecht5)
+- [Optimizing a Text-To-Speech model using 🤗 Transformers](https://huggingface.co/blog/optimizing-bark)
+- [Train your own TTS models with Parler-TTS](https://github.com/huggingface/parler-tts)

package/src/tasks/text-to-speech/data.ts

@@ -0,0 +1,79 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	canonicalId: "text-to-audio",
+	datasets: [
+		{
+			description: "10K hours of multi-speaker English dataset.",
+			id: "parler-tts/mls_eng_10k",
+		},
+		{
+			description: "Multi-speaker English dataset.",
+			id: "mythicinfinity/libritts_r",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Input",
+				content: "I love audio models on the Hub!",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "audio.wav",
+				type: "audio",
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "The Mel Cepstral Distortion (MCD) metric is used to calculate the quality of generated speech.",
+			id: "mel cepstral distortion",
+		},
+	],
+	models: [
+		{
+			description: "A powerful TTS model.",
+			id: "parler-tts/parler-tts-large-v1",
+		},
+		{
+			description: "A massively multi-lingual TTS model.",
+			id: "coqui/XTTS-v2",
+		},
+		{
+			description: "Robust TTS model.",
+			id: "metavoiceio/metavoice-1B-v0.1",
+		},
+		{
+			description: "A prompt based, powerful TTS model.",
+			id: "parler-tts/parler_tts_mini_v0.1",
+		},
+	],
+	spaces: [
+		{
+			description: "An application for generate highly realistic, multilingual speech.",
+			id: "suno/bark",
+		},
+		{
+			description:
+				"An application on XTTS, a voice generation model that lets you clone voices into different languages.",
+			id: "coqui/xtts",
+		},
+		{
+			description: "An application that generates speech in different styles in English and Chinese.",
+			id: "mrfakename/E2-F5-TTS",
+		},
+		{
+			description: "An application that synthesizes speech for diverse speaker prompts.",
+			id: "parler-tts/parler_tts_mini",
+		},
+	],
+	summary:
+		"Text-to-Speech (TTS) is the task of generating natural sounding speech given text input. TTS models can be extended to have a single model that generates speech for multiple speakers and multiple languages.",
+	widgetModels: ["suno/bark"],
+	youtubeId: "NW62DpzJ274",
+};
+
+export default taskData;

package/src/tasks/text-to-speech/inference.ts

@@ -0,0 +1,145 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Text To Speech inference
+ */
+export interface TextToSpeechInput {
+	/**
+	 * The input text data
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: TextToSpeechParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Text To Speech
+ */
+export interface TextToSpeechParameters {
+	/**
+	 * Parametrization of the text generation process
+	 */
+	generation_parameters?: GenerationParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Parametrization of the text generation process
+ *
+ * Ad-hoc parametrization of the text generation process
+ */
+export interface GenerationParameters {
+	/**
+	 * Whether to use sampling instead of greedy decoding when generating new tokens.
+	 */
+	do_sample?: boolean;
+	/**
+	 * Controls the stopping condition for beam-based methods.
+	 */
+	early_stopping?: EarlyStoppingUnion;
+	/**
+	 * If set to float strictly between 0 and 1, only tokens with a conditional probability
+	 * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from
+	 * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language
+	 * Model Desmoothing](https://hf.co/papers/2210.15191) for more details.
+	 */
+	epsilon_cutoff?: number;
+	/**
+	 * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to
+	 * float strictly between 0 and 1, a token is only considered if it is greater than either
+	 * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter
+	 * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In
+	 * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
+	 * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191)
+	 * for more details.
+	 */
+	eta_cutoff?: number;
+	/**
+	 * The maximum length (in tokens) of the generated text, including the input.
+	 */
+	max_length?: number;
+	/**
+	 * The maximum number of tokens to generate. Takes precedence over max_length.
+	 */
+	max_new_tokens?: number;
+	/**
+	 * The minimum length (in tokens) of the generated text, including the input.
+	 */
+	min_length?: number;
+	/**
+	 * The minimum number of tokens to generate. Takes precedence over min_length.
+	 */
+	min_new_tokens?: number;
+	/**
+	 * Number of groups to divide num_beams into in order to ensure diversity among different
+	 * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details.
+	 */
+	num_beam_groups?: number;
+	/**
+	 * Number of beams to use for beam search.
+	 */
+	num_beams?: number;
+	/**
+	 * The value balances the model confidence and the degeneration penalty in contrastive
+	 * search decoding.
+	 */
+	penalty_alpha?: number;
+	/**
+	 * The value used to modulate the next token probabilities.
+	 */
+	temperature?: number;
+	/**
+	 * The number of highest probability vocabulary tokens to keep for top-k-filtering.
+	 */
+	top_k?: number;
+	/**
+	 * If set to float < 1, only the smallest set of most probable tokens with probabilities
+	 * that add up to top_p or higher are kept for generation.
+	 */
+	top_p?: number;
+	/**
+	 * Local typicality measures how similar the conditional probability of predicting a target
+	 * token next is to the expected conditional probability of predicting a random token next,
+	 * given the partial text already generated. If set to float < 1, the smallest set of the
+	 * most locally typical tokens with probabilities that add up to typical_p or higher are
+	 * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details.
+	 */
+	typical_p?: number;
+	/**
+	 * Whether the model should use the past last key/values attentions to speed up decoding
+	 */
+	use_cache?: boolean;
+	[property: string]: unknown;
+}
+
+/**
+ * Controls the stopping condition for beam-based methods.
+ */
+export type EarlyStoppingUnion = boolean | "never";
+
+/**
+ * Outputs for Text to Speech inference
+ *
+ * Outputs of inference for the Text To Audio task
+ */
+export interface TextToSpeechOutput {
+	/**
+	 * The generated audio waveform.
+	 */
+	audio: unknown;
+	samplingRate: unknown;
+	/**
+	 * The sampling rate of the generated audio waveform.
+	 */
+	sampling_rate?: number;
+	[property: string]: unknown;
+}

package/src/tasks/text-to-speech/spec/input.json

@@ -0,0 +1,31 @@
+{
+	"$id": "/inference/schemas/text-to-speech/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Text To Speech inference",
+	"title": "TextToSpeechInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/TextToSpeechParameters"
+		}
+	},
+	"$defs": {
+		"TextToSpeechParameters": {
+			"title": "TextToSpeechParameters",
+			"description": "Additional inference parameters for Text To Speech",
+			"type": "object",
+			"properties": {
+				"generation_parameters": {
+					"description": "Parametrization of the text generation process",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/text-to-speech/spec/output.json

@@ -0,0 +1,7 @@
+{
+	"$ref": "/inference/schemas/text-to-audio/output.json",
+	"$id": "/inference/schemas/text-to-speech/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"title": "TextToSpeechOutput",
+	"description": "Outputs for Text to Speech inference"
+}

package/src/tasks/text-to-video/about.md

@@ -0,0 +1,41 @@
+## Use Cases
+
+### Script-based Video Generation
+
+Text-to-video models can be used to create short-form video content from a provided text script. These models can be used to create engaging and informative marketing videos. For example, a company could use a text-to-video model to create a video that explains how their product works.
+
+### Content format conversion
+
+Text-to-video models can be used to generate videos from long-form text, including blog posts, articles, and text files. Text-to-video models can be used to create educational videos that are more engaging and interactive. An example of this is creating a video that explains a complex concept from an article.
+
+### Voice-overs and Speech
+
+Text-to-video models can be used to create an AI newscaster to deliver daily news, or for a film-maker to create a short film or a music video.
+
+## Task Variants
+Text-to-video models have different variants based on inputs and outputs.
+
+### Text-to-video Editing
+
+One text-to-video task is generating text-based video style and local attribute editing. Text-to-video editing models can make it easier to perform tasks like cropping, stabilization, color correction, resizing and audio editing consistently.
+
+### Text-to-video Search
+
+Text-to-video search is the task of retrieving videos that are relevant to a given text query. This can be challenging, as videos are a complex medium that can contain a lot of information. By using semantic analysis to extract the meaning of the text query, visual analysis to extract features from the videos, such as the objects and actions that are present in the video, and temporal analysis to categorize relationships between the objects and actions in the video, we can determine which videos are most likely to be relevant to the text query.
+
+### Text-driven Video Prediction
+
+Text-driven video prediction is the task of generating a video sequence from a text description. Text description can be anything from a simple sentence to a detailed story. The goal of this task is to generate a video that is both visually realistic and semantically consistent with the text description.
+
+### Video Translation
+
+Text-to-video translation models can translate videos from one language to another or allow to query the multilingual text-video model with non-English sentences. This can be useful for people who want to watch videos in a language that they don't understand, especially when multi-lingual captions are available for training.
+
+## Inference
+Contribute an inference snippet for text-to-video here!
+
+## Useful Resources
+
+In this area, you can insert useful resources about how to train or use a model for this task.
+
+- [Text-to-Video: The Task, Challenges and the Current State](https://huggingface.co/blog/text-to-video)

package/src/tasks/text-to-video/data.ts

@@ -0,0 +1,102 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Microsoft Research Video to Text is a large-scale dataset for open domain video captioning",
+			id: "iejMac/CLIP-MSR-VTT",
+		},
+		{
+			description: "UCF101 Human Actions dataset consists of 13,320 video clips from YouTube, with 101 classes.",
+			id: "quchenyuan/UCF101-ZIP",
+		},
+		{
+			description: "A high-quality dataset for human action recognition in YouTube videos.",
+			id: "nateraw/kinetics",
+		},
+		{
+			description: "A dataset of video clips of humans performing pre-defined basic actions with everyday objects.",
+			id: "HuggingFaceM4/something_something_v2",
+		},
+		{
+			description:
+				"This dataset consists of text-video pairs and contains noisy samples with irrelevant video descriptions",
+			id: "HuggingFaceM4/webvid",
+		},
+		{
+			description: "A dataset of short Flickr videos for the temporal localization of events with descriptions.",
+			id: "iejMac/CLIP-DiDeMo",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Input",
+				content: "Darth Vader is surfing on the waves.",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "text-to-video-output.gif",
+				type: "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"Inception Score uses an image classification model that predicts class labels and evaluates how distinct and diverse the images are. A higher score indicates better video generation.",
+			id: "is",
+		},
+		{
+			description:
+				"Frechet Inception Distance uses an image classification model to obtain image embeddings. The metric compares mean and standard deviation of the embeddings of real and generated images. A smaller score indicates better video generation.",
+			id: "fid",
+		},
+		{
+			description:
+				"Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.",
+			id: "fvd",
+		},
+		{
+			description:
+				"CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.",
+			id: "clipsim",
+		},
+	],
+	models: [
+		{
+			description: "A strong model for consistent video generation.",
+			id: "rain1011/pyramid-flow-sd3",
+		},
+		{
+			description: "A robust model for text-to-video generation.",
+			id: "VideoCrafter/VideoCrafter2",
+		},
+		{
+			description: "A cutting-edge text-to-video generation model.",
+			id: "TIGER-Lab/T2V-Turbo-V2",
+		},
+	],
+	spaces: [
+		{
+			description: "An application that generates video from text.",
+			id: "VideoCrafter/VideoCrafter",
+		},
+		{
+			description: "Consistent video generation application.",
+			id: "TIGER-Lab/T2V-Turbo-V2",
+		},
+		{
+			description: "A cutting edge video generation application.",
+			id: "Pyramid-Flow/pyramid-flow",
+		},
+	],
+	summary:
+		"Text-to-video models can be used in any application that requires generating consistent sequence of images from text. ",
+	widgetModels: [],
+	youtubeId: undefined,
+};
+
+export default taskData;

package/src/tasks/text2text-generation/inference.ts

@@ -0,0 +1,55 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Text2text Generation inference
+ */
+export interface Text2TextGenerationInput {
+	/**
+	 * The input text data
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: Text2TextGenerationParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Text2text Generation
+ */
+export interface Text2TextGenerationParameters {
+	/**
+	 * Whether to clean up the potential extra spaces in the text output.
+	 */
+	clean_up_tokenization_spaces?: boolean;
+	/**
+	 * Additional parametrization of the text generation algorithm
+	 */
+	generate_parameters?: { [key: string]: unknown };
+	/**
+	 * The truncation strategy to use
+	 */
+	truncation?: Text2TextGenerationTruncationStrategy;
+	[property: string]: unknown;
+}
+
+export type Text2TextGenerationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second";
+
+/**
+ * Outputs of inference for the Text2text Generation task
+ */
+export interface Text2TextGenerationOutput {
+	generatedText: unknown;
+	/**
+	 * The generated text.
+	 */
+	generated_text?: string;
+	[property: string]: unknown;
+}

package/src/tasks/text2text-generation/spec/input.json

@@ -0,0 +1,55 @@
+{
+	"$id": "/inference/schemas/text2text-generation/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Text2text Generation inference",
+	"title": "Text2TextGenerationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/Text2textGenerationParameters"
+		}
+	},
+	"$defs": {
+		"Text2textGenerationParameters": {
+			"title": "Text2textGenerationParameters",
+			"description": "Additional inference parameters for Text2text Generation",
+			"type": "object",
+			"properties": {
+				"clean_up_tokenization_spaces": {
+					"type": "boolean",
+					"description": "Whether to clean up the potential extra spaces in the text output."
+				},
+				"truncation": {
+					"title": "Text2textGenerationTruncationStrategy",
+					"type": "string",
+					"description": "The truncation strategy to use",
+					"oneOf": [
+						{
+							"const": "do_not_truncate"
+						},
+						{
+							"const": "longest_first"
+						},
+						{
+							"const": "only_first"
+						},
+						{
+							"const": "only_second"
+						}
+					]
+				},
+				"generate_parameters": {
+					"title": "generateParameters",
+					"type": "object",
+					"description": "Additional parametrization of the text generation algorithm"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/text2text-generation/spec/output.json

@@ -0,0 +1,14 @@
+{
+	"$id": "/inference/schemas/text2text-generation/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Text2text Generation task",
+	"title": "Text2TextGenerationOutput",
+	"type": "object",
+	"properties": {
+		"generated_text": {
+			"type": "string",
+			"description": "The generated text."
+		}
+	},
+	"required": ["generatedText"]
+}

package/src/tasks/token-classification/about.md

@@ -0,0 +1,76 @@
+## Use Cases
+
+### Information Extraction from Invoices
+
+You can extract entities of interest from invoices automatically using Named Entity Recognition (NER) models. Invoices can be read with Optical Character Recognition models and the output can be used to do inference with NER models. In this way, important information such as date, company name, and other named entities can be extracted.
+
+## Task Variants
+
+### Named Entity Recognition (NER)
+
+NER is the task of recognizing named entities in a text. These entities can be the names of people, locations, or organizations. The task is formulated as labeling each token with a class for each named entity and a class named "0" for tokens that do not contain any entities. The input for this task is text and the output is the annotated text with named entities.
+
+#### Inference
+
+You can use the 🤗 Transformers library `ner` pipeline to infer with NER models.
+
+```python
+from transformers import pipeline
+
+classifier = pipeline("ner")
+classifier("Hello I'm Omar and I live in Zürich.")
+```
+
+### Part-of-Speech (PoS) Tagging
+In PoS tagging, the model recognizes parts of speech, such as nouns, pronouns, adjectives, or verbs, in a given text. The task is formulated as labeling each word with a part of the speech.
+
+#### Inference
+
+You can use the 🤗 Transformers library `token-classification` pipeline with a POS tagging model of your choice. The model will return a json with PoS tags for each token.
+
+```python
+from transformers import pipeline
+
+classifier = pipeline("token-classification", model = "vblagoje/bert-english-uncased-finetuned-pos")
+classifier("Hello I'm Omar and I live in Zürich.")
+```
+
+This is not limited to transformers! You can also use other libraries such as Stanza, spaCy, and Flair to do inference! Here is an example using a canonical [spaCy](https://hf.co/blog/spacy) model.
+
+```python
+!pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
+
+import en_core_web_sm
+
+nlp = en_core_web_sm.load()
+doc = nlp("I'm Omar and I live in Zürich.")
+for token in doc:
+    print(token.text, token.pos_, token.dep_, token.ent_type_)
+
+## I PRON nsubj
+## 'm AUX ROOT
+## Omar PROPN attr PERSON
+### ...
+```
+
+## Useful Resources
+
+Would you like to learn more about token classification? Great! Here you can find some curated resources that you may find helpful!
+
+- [Course Chapter on Token Classification](https://huggingface.co/course/chapter7/2?fw=pt)
+- [Blog post: Welcome spaCy to the Hugging Face Hub](https://huggingface.co/blog/spacy)
+
+### Notebooks
+
+- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/token_classification.ipynb)
+- [TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/token_classification-tf.ipynb)
+
+### Scripts for training
+
+- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/token-classification)
+- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow)
+- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification)
+
+### Documentation
+
+- [Token classification task guide](https://huggingface.co/docs/transformers/tasks/token_classification)