@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/audio-to-audio/data.ts

@@ -0,0 +1,70 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "512-element X-vector embeddings of speakers from CMU ARCTIC dataset.",
+			id: "Matthijs/cmu-arctic-xvectors",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "input.wav",
+				type: "audio",
+			},
+		],
+		outputs: [
+			{
+				filename: "label-0.wav",
+				type: "audio",
+			},
+			{
+				filename: "label-1.wav",
+				type: "audio",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"The Signal-to-Noise ratio is the relationship between the target signal level and the background noise level. It is calculated as the logarithm of the target signal divided by the background noise, in decibels.",
+			id: "snri",
+		},
+		{
+			description:
+				"The Signal-to-Distortion ratio is the relationship between the target signal and the sum of noise, interference, and artifact errors",
+			id: "sdri",
+		},
+	],
+	models: [
+		{
+			description: "A solid model of audio source separation.",
+			id: "speechbrain/sepformer-wham",
+		},
+		{
+			description: "A speech enhancement model.",
+			id: "ResembleAI/resemble-enhance",
+		},
+		{
+			description: "A model that can change the voice in a speech recording.",
+			id: "microsoft/speecht5_vc",
+		},
+	],
+	spaces: [
+		{
+			description: "An application for speech separation.",
+			id: "younver/speechbrain-speech-separation",
+		},
+		{
+			description: "An application for audio style transfer.",
+			id: "nakas/audio-diffusion_style_transfer",
+		},
+	],
+	summary:
+		"Audio-to-Audio is a family of tasks in which the input is an audio and the output is one or multiple generated audios. Some example tasks are speech enhancement and source separation.",
+	widgetModels: ["speechbrain/sepformer-wham"],
+	youtubeId: "iohj7nCCYoM",
+};
+
+export default taskData;

package/src/tasks/automatic-speech-recognition/about.md

@@ -0,0 +1,90 @@
+## Use Cases
+
+### Virtual Speech Assistants
+
+Many edge devices have an embedded virtual assistant to interact with the end users better. These assistances rely on ASR models to recognize different voice commands to perform various tasks. For instance, you can ask your phone for dialing a phone number, ask a general question, or schedule a meeting.
+
+### Caption Generation
+
+A caption generation model takes audio as input from sources to generate automatic captions through transcription, for live-streamed or recorded videos. This can help with content accessibility. For example, an audience watching a video that includes a non-native language, can rely on captions to interpret the content. It can also help with information retention at online-classes environments improving knowledge assimilation while reading and taking notes faster.
+
+## Task Variants
+
+### Multilingual ASR
+
+Multilingual ASR models can convert audio inputs with multiple languages into transcripts. Some multilingual ASR models include [language identification](https://huggingface.co/tasks/audio-classification) blocks to improve the performance.
+
+The use of Multilingual ASR has become popular, the idea of maintaining just a single model for all language can simplify the production pipeline. Take a look at [Whisper](https://huggingface.co/openai/whisper-large-v2) to get an idea on how 100+ languages can be processed by a single model.
+
+## Inference
+
+The Hub contains over [17,000 ASR models](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&sort=downloads) that you can test right away in your browser using the model page widgets. You can also use any model as a service using the Serverless Inference API. We also support libraries such as [transformers](https://huggingface.co/models?library=transformers&pipeline_tag=automatic-speech-recognition&sort=downloads), [speechbrain](https://huggingface.co/models?library=speechbrain&pipeline_tag=automatic-speech-recognition&sort=downloads), [NeMo](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=nemo&sort=downloads) and [espnet](https://huggingface.co/models?library=espnet&pipeline_tag=automatic-speech-recognition&sort=downloads) via the Serverless Inference API. Here's a simple code snippet to run inference:
+
+```python
+import json
+import requests
+
+headers = {"Authorization": f"Bearer {API_TOKEN}"}
+API_URL = "https://api-inference.huggingface.co/models/openai/whisper-large-v3"
+
+def query(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+    response = requests.request("POST", API_URL, headers=headers, data=data)
+    return json.loads(response.content.decode("utf-8"))
+
+data = query("sample1.flac")
+```
+
+You can also use [huggingface.js](https://github.com/huggingface/huggingface.js), the JavaScript client, to transcribe audio with the Serverless Inference API.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.automaticSpeechRecognition({
+	data: await (await fetch("sample.flac")).blob(),
+	model: "openai/whisper-large-v3",
+});
+```
+
+For transformers-compatible models like Whisper, Wav2Vec2, and HuBERT, you can also run inference with the library as follows:
+
+```python
+# pip install --upgrade transformers
+
+from transformers import pipeline
+
+pipe = pipeline("automatic-speech-recognition", "openai/whisper-large-v3")
+
+pipe("sample.flac")
+# {'text': "GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES IN DRAUGHTY SCHOOL ROOMS DAY AFTER DAY FOR A FORTNIGHT HE'LL HAVE TO PUT IN AN APPEARANCE AT SOME PLACE OF WORSHIP ON SUNDAY MORNING AND HE CAN COME TO US IMMEDIATELY AFTERWARDS"}
+```
+
+## Solving ASR for your own data
+
+We have some great news! You can fine-tune (transfer learning) a foundational speech model on a specific language without tonnes of data. Pretrained models such as Whisper, Wav2Vec2-MMS and HuBERT exist. [OpenAI's Whisper model](https://huggingface.co/openai/whisper-large-v3) is a large multilingual model trained on 100+ languages and with 4 Million hours of speech.
+
+The following detailed [blog post](https://huggingface.co/blog/fine-tune-whisper) shows how to fine-tune a pre-trained Whisper checkpoint on labeled data for ASR. With the right data and strategy you can fine-tune a high-performant model on a free Google Colab instance too. We suggest to read the blog post for more info!
+
+## Hugging Face Whisper Event
+
+On December 2022, over 450 participants collaborated, fine-tuned and shared 600+ ASR Whisper models in 100+ different languages. You can compare these models on the event's speech recognition [leaderboard](https://huggingface.co/spaces/whisper-event/leaderboard?dataset=mozilla-foundation%2Fcommon_voice_11_0&config=ar&split=test).
+
+These events help democratize ASR for all languages, including low-resource languages. In addition to the trained models, the [event](https://github.com/huggingface/community-events/tree/main/whisper-fine-tuning-event) helps to build practical collaborative knowledge.
+
+## Useful Resources
+
+- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter5/introduction)
+- [Fine-tuning MetaAI's MMS Adapter Models for Multi-Lingual ASR](https://huggingface.co/blog/mms_adapters)
+- [Making automatic speech recognition work on large files with Wav2Vec2 in 🤗 Transformers](https://huggingface.co/blog/asr-chunking)
+- [Boosting Wav2Vec2 with n-grams in 🤗 Transformers](https://huggingface.co/blog/wav2vec2-with-ngram)
+- [ML for Audio Study Group - Intro to Audio and ASR Deep Dive](https://www.youtube.com/watch?v=D-MH6YjuIlE)
+- [Massively Multilingual ASR: 50 Languages, 1 Model, 1 Billion Parameters](https://arxiv.org/pdf/2007.03001.pdf)
+- An ASR toolkit made by [NVIDIA: NeMo](https://github.com/NVIDIA/NeMo) with code and pretrained models useful for new ASR models. Watch the [introductory video](https://www.youtube.com/embed/wBgpMf_KQVw) for an overview.
+- [An introduction to SpeechT5, a multi-purpose speech recognition and synthesis model](https://huggingface.co/blog/speecht5)
+- [Fine-tune Whisper For Multilingual ASR with 🤗Transformers](https://huggingface.co/blog/fine-tune-whisper)
+- [Automatic speech recognition task guide](https://huggingface.co/docs/transformers/tasks/asr)
+- [Speech Synthesis, Recognition, and More With SpeechT5](https://huggingface.co/blog/speecht5)
+- [Fine-Tune W2V2-Bert for low-resource ASR with 🤗 Transformers](https://huggingface.co/blog/fine-tune-w2v2-bert)
+- [Speculative Decoding for 2x Faster Whisper Inference](https://huggingface.co/blog/whisper-speculative-decoding)

package/src/tasks/automatic-speech-recognition/data.ts

@@ -0,0 +1,82 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "31,175 hours of multilingual audio-text dataset in 108 languages.",
+			id: "mozilla-foundation/common_voice_17_0",
+		},
+		{
+			description: "A dataset with 44.6k hours of English speaker data and 6k hours of other language speakers.",
+			id: "parler-tts/mls_eng",
+		},
+		{
+			description: "A multi-lingual audio dataset with 370K hours of audio.",
+			id: "espnet/yodas",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "input.flac",
+				type: "audio",
+			},
+		],
+		outputs: [
+			{
+				/// GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES I
+				label: "Transcript",
+				content: "Going along slushy country roads and speaking to damp audiences in...",
+				type: "text",
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "",
+			id: "wer",
+		},
+		{
+			description: "",
+			id: "cer",
+		},
+	],
+	models: [
+		{
+			description: "A powerful ASR model by OpenAI.",
+			id: "openai/whisper-large-v3",
+		},
+		{
+			description: "A good generic speech model by MetaAI for fine-tuning.",
+			id: "facebook/w2v-bert-2.0",
+		},
+		{
+			description: "An end-to-end model that performs ASR and Speech Translation by MetaAI.",
+			id: "facebook/seamless-m4t-v2-large",
+		},
+		{
+			description: "Powerful speaker diarization model.",
+			id: "pyannote/speaker-diarization-3.1",
+		},
+	],
+	spaces: [
+		{
+			description: "A powerful general-purpose speech recognition application.",
+			id: "hf-audio/whisper-large-v3",
+		},
+		{
+			description: "Fastest speech recognition application.",
+			id: "sanchit-gandhi/whisper-jax",
+		},
+		{
+			description: "A high quality speech and text translation model by Meta.",
+			id: "facebook/seamless_m4t",
+		},
+	],
+	summary:
+		"Automatic Speech Recognition (ASR), also known as Speech to Text (STT), is the task of transcribing a given audio to text. It has many applications, such as voice user interfaces.",
+	widgetModels: ["openai/whisper-large-v3"],
+	youtubeId: "TksaY_FDgnk",
+};
+
+export default taskData;

package/src/tasks/automatic-speech-recognition/inference.ts

@@ -0,0 +1,160 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Automatic Speech Recognition inference
+ */
+export interface AutomaticSpeechRecognitionInput {
+	/**
+	 * The input audio data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the audio data as a raw bytes payload.
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: AutomaticSpeechRecognitionParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Automatic Speech Recognition
+ */
+export interface AutomaticSpeechRecognitionParameters {
+	/**
+	 * Parametrization of the text generation process
+	 */
+	generation_parameters?: GenerationParameters;
+	/**
+	 * Whether to output corresponding timestamps with the generated text
+	 */
+	return_timestamps?: boolean;
+	[property: string]: unknown;
+}
+
+/**
+ * Parametrization of the text generation process
+ *
+ * Ad-hoc parametrization of the text generation process
+ */
+export interface GenerationParameters {
+	/**
+	 * Whether to use sampling instead of greedy decoding when generating new tokens.
+	 */
+	do_sample?: boolean;
+	/**
+	 * Controls the stopping condition for beam-based methods.
+	 */
+	early_stopping?: EarlyStoppingUnion;
+	/**
+	 * If set to float strictly between 0 and 1, only tokens with a conditional probability
+	 * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from
+	 * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language
+	 * Model Desmoothing](https://hf.co/papers/2210.15191) for more details.
+	 */
+	epsilon_cutoff?: number;
+	/**
+	 * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to
+	 * float strictly between 0 and 1, a token is only considered if it is greater than either
+	 * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter
+	 * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In
+	 * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
+	 * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191)
+	 * for more details.
+	 */
+	eta_cutoff?: number;
+	/**
+	 * The maximum length (in tokens) of the generated text, including the input.
+	 */
+	max_length?: number;
+	/**
+	 * The maximum number of tokens to generate. Takes precedence over max_length.
+	 */
+	max_new_tokens?: number;
+	/**
+	 * The minimum length (in tokens) of the generated text, including the input.
+	 */
+	min_length?: number;
+	/**
+	 * The minimum number of tokens to generate. Takes precedence over min_length.
+	 */
+	min_new_tokens?: number;
+	/**
+	 * Number of groups to divide num_beams into in order to ensure diversity among different
+	 * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details.
+	 */
+	num_beam_groups?: number;
+	/**
+	 * Number of beams to use for beam search.
+	 */
+	num_beams?: number;
+	/**
+	 * The value balances the model confidence and the degeneration penalty in contrastive
+	 * search decoding.
+	 */
+	penalty_alpha?: number;
+	/**
+	 * The value used to modulate the next token probabilities.
+	 */
+	temperature?: number;
+	/**
+	 * The number of highest probability vocabulary tokens to keep for top-k-filtering.
+	 */
+	top_k?: number;
+	/**
+	 * If set to float < 1, only the smallest set of most probable tokens with probabilities
+	 * that add up to top_p or higher are kept for generation.
+	 */
+	top_p?: number;
+	/**
+	 * Local typicality measures how similar the conditional probability of predicting a target
+	 * token next is to the expected conditional probability of predicting a random token next,
+	 * given the partial text already generated. If set to float < 1, the smallest set of the
+	 * most locally typical tokens with probabilities that add up to typical_p or higher are
+	 * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details.
+	 */
+	typical_p?: number;
+	/**
+	 * Whether the model should use the past last key/values attentions to speed up decoding
+	 */
+	use_cache?: boolean;
+	[property: string]: unknown;
+}
+
+/**
+ * Controls the stopping condition for beam-based methods.
+ */
+export type EarlyStoppingUnion = boolean | "never";
+
+/**
+ * Outputs of inference for the Automatic Speech Recognition task
+ */
+export interface AutomaticSpeechRecognitionOutput {
+	/**
+	 * When returnTimestamps is enabled, chunks contains a list of audio chunks identified by
+	 * the model.
+	 */
+	chunks?: AutomaticSpeechRecognitionOutputChunk[];
+	/**
+	 * The recognized text.
+	 */
+	text: string;
+	[property: string]: unknown;
+}
+
+export interface AutomaticSpeechRecognitionOutputChunk {
+	/**
+	 * A chunk of text identified by the model
+	 */
+	text: string;
+	/**
+	 * The start and end timestamps corresponding with the text
+	 */
+	timestamps: number[];
+	[property: string]: unknown;
+}

package/src/tasks/automatic-speech-recognition/spec/input.json

@@ -0,0 +1,35 @@
+{
+	"$id": "/inference/schemas/automatic-speech-recognition/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Automatic Speech Recognition inference",
+	"title": "AutomaticSpeechRecognitionInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input audio data as a base64-encoded string. If no `parameters` are provided, you can also provide the audio data as a raw bytes payload.",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/AutomaticSpeechRecognitionParameters"
+		}
+	},
+	"$defs": {
+		"AutomaticSpeechRecognitionParameters": {
+			"title": "AutomaticSpeechRecognitionParameters",
+			"description": "Additional inference parameters for Automatic Speech Recognition",
+			"type": "object",
+			"properties": {
+				"return_timestamps": {
+					"type": "boolean",
+					"description": "Whether to output corresponding timestamps with the generated text"
+				},
+				"generation_parameters": {
+					"description": "Parametrization of the text generation process",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/automatic-speech-recognition/spec/output.json

@@ -0,0 +1,38 @@
+{
+	"$id": "/inference/schemas/automatic-speech-recognition/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Automatic Speech Recognition task",
+	"title": "AutomaticSpeechRecognitionOutput",
+	"type": "object",
+	"properties": {
+		"text": {
+			"type": "string",
+			"description": "The recognized text."
+		},
+		"chunks": {
+			"type": "array",
+			"description": "When returnTimestamps is enabled, chunks contains a list of audio chunks identified by the model.",
+			"items": {
+				"type": "object",
+				"title": "AutomaticSpeechRecognitionOutputChunk",
+				"properties": {
+					"text": {
+						"type": "string",
+						"description": "A chunk of text identified by the model"
+					},
+					"timestamps": {
+						"type": "array",
+						"description": "The start and end timestamps corresponding with the text",
+						"items": {
+							"type": "number"
+						},
+						"minLength": 2,
+						"maxLength": 2
+					}
+				},
+				"required": ["text", "timestamps"]
+			}
+		}
+	},
+	"required": ["text"]
+}