@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/snippets/python.ts

@@ -0,0 +1,321 @@
+import type { PipelineType } from "../pipelines.js";
+import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
+import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
+import { getModelInputSnippet } from "./inputs.js";
+import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
+
+const snippetImportInferenceClient = (model: ModelDataMinimal, accessToken: string): string =>
+	`from huggingface_hub import InferenceClient
+client = InferenceClient("${model.id}", token="${accessToken || "{API_TOKEN}"}")
+`;
+
+export const snippetConversational = (
+	model: ModelDataMinimal,
+	accessToken: string,
+	opts?: {
+		streaming?: boolean;
+		messages?: ChatCompletionInputMessage[];
+		temperature?: GenerationParameters["temperature"];
+		max_tokens?: GenerationParameters["max_tokens"];
+		top_p?: GenerationParameters["top_p"];
+	}
+): InferenceSnippet[] => {
+	const streaming = opts?.streaming ?? true;
+	const exampleMessages = getModelInputSnippet(model) as ChatCompletionInputMessage[];
+	const messages = opts?.messages ?? exampleMessages;
+	const messagesStr = stringifyMessages(messages, { attributeKeyQuotes: true });
+
+	const config = {
+		...(opts?.temperature ? { temperature: opts.temperature } : undefined),
+		max_tokens: opts?.max_tokens ?? 500,
+		...(opts?.top_p ? { top_p: opts.top_p } : undefined),
+	};
+	const configStr = stringifyGenerationConfig(config, {
+		indent: "\n\t",
+		attributeValueConnector: "=",
+	});
+
+	if (streaming) {
+		return [
+			{
+				client: "huggingface_hub",
+				content: `from huggingface_hub import InferenceClient
+
+client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
+
+messages = ${messagesStr}
+
+stream = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr},
+	stream=True
+)
+
+for chunk in stream:
+    print(chunk.choices[0].delta.content, end="")`,
+			},
+			{
+				client: "openai",
+				content: `from openai import OpenAI
+
+client = OpenAI(
+	base_url="https://api-inference.huggingface.co/v1/",
+	api_key="${accessToken || "{API_TOKEN}"}"
+)
+
+messages = ${messagesStr}
+
+stream = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr},
+	stream=True
+)
+
+for chunk in stream:
+    print(chunk.choices[0].delta.content, end="")`,
+			},
+		];
+	} else {
+		return [
+			{
+				client: "huggingface_hub",
+				content: `from huggingface_hub import InferenceClient
+
+client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
+
+messages = ${messagesStr}
+
+completion = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr}
+)
+
+print(completion.choices[0].message)`,
+			},
+			{
+				client: "openai",
+				content: `from openai import OpenAI
+
+client = OpenAI(
+	base_url="https://api-inference.huggingface.co/v1/",
+	api_key="${accessToken || "{API_TOKEN}"}"
+)
+
+messages = ${messagesStr}
+
+completion = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr}
+)
+
+print(completion.choices[0].message)`,
+			},
+		];
+	}
+};
+
+export const snippetZeroShotClassification = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.json()
+
+output = query({
+    "inputs": ${getModelInputSnippet(model)},
+    "parameters": {"candidate_labels": ["refund", "legal", "faq"]},
+})`,
+});
+
+export const snippetZeroShotImageClassification = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(data):
+	with open(data["image_path"], "rb") as f:
+		img = f.read()
+	payload={
+		"parameters": data["parameters"],
+		"inputs": base64.b64encode(img).decode("utf-8")
+	}
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.json()
+
+output = query({
+    "image_path": ${getModelInputSnippet(model)},
+    "parameters": {"candidate_labels": ["cat", "dog", "llama"]},
+})`,
+});
+
+export const snippetBasic = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.json()
+	
+output = query({
+	"inputs": ${getModelInputSnippet(model)},
+})`,
+});
+
+export const snippetFile = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+    response = requests.post(API_URL, headers=headers, data=data)
+    return response.json()
+
+output = query(${getModelInputSnippet(model)})`,
+});
+
+export const snippetTextToImage = (model: ModelDataMinimal, accessToken: string): InferenceSnippet[] => [
+	{
+		client: "huggingface_hub",
+		content: `${snippetImportInferenceClient(model, accessToken)}
+# output is a PIL.Image object
+image = client.text_to_image(${getModelInputSnippet(model)})`,
+	},
+	{
+		client: "requests",
+		content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.content
+image_bytes = query({
+	"inputs": ${getModelInputSnippet(model)},
+})
+
+# You can access the image with PIL.Image for example
+import io
+from PIL import Image
+image = Image.open(io.BytesIO(image_bytes))`,
+	},
+];
+
+export const snippetTabular = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.content
+response = query({
+	"inputs": {"data": ${getModelInputSnippet(model)}},
+})`,
+});
+
+export const snippetTextToAudio = (model: ModelDataMinimal): InferenceSnippet => {
+	// Transformers TTS pipeline and api-inference-community (AIC) pipeline outputs are diverged
+	// with the latest update to inference-api (IA).
+	// Transformers IA returns a byte object (wav file), whereas AIC returns wav and sampling_rate.
+	if (model.library_name === "transformers") {
+		return {
+			content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.content
+
+audio_bytes = query({
+	"inputs": ${getModelInputSnippet(model)},
+})
+# You can access the audio with IPython.display for example
+from IPython.display import Audio
+Audio(audio_bytes)`,
+		};
+	} else {
+		return {
+			content: `def query(payload):
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.json()
+	
+audio, sampling_rate = query({
+	"inputs": ${getModelInputSnippet(model)},
+})
+# You can access the audio with IPython.display for example
+from IPython.display import Audio
+Audio(audio, rate=sampling_rate)`,
+		};
+	}
+};
+
+export const snippetDocumentQuestionAnswering = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
+ 	with open(payload["image"], "rb") as f:
+  		img = f.read()
+		payload["image"] = base64.b64encode(img).decode("utf-8")  
+	response = requests.post(API_URL, headers=headers, json=payload)
+	return response.json()
+
+output = query({
+    "inputs": ${getModelInputSnippet(model)},
+})`,
+});
+
+export const pythonSnippets: Partial<
+	Record<
+		PipelineType,
+		(
+			model: ModelDataMinimal,
+			accessToken: string,
+			opts?: Record<string, unknown>
+		) => InferenceSnippet | InferenceSnippet[]
+	>
+> = {
+	// Same order as in tasks/src/pipelines.ts
+	"text-classification": snippetBasic,
+	"token-classification": snippetBasic,
+	"table-question-answering": snippetBasic,
+	"question-answering": snippetBasic,
+	"zero-shot-classification": snippetZeroShotClassification,
+	translation: snippetBasic,
+	summarization: snippetBasic,
+	"feature-extraction": snippetBasic,
+	"text-generation": snippetBasic,
+	"text2text-generation": snippetBasic,
+	"image-text-to-text": snippetConversational,
+	"fill-mask": snippetBasic,
+	"sentence-similarity": snippetBasic,
+	"automatic-speech-recognition": snippetFile,
+	"text-to-image": snippetTextToImage,
+	"text-to-speech": snippetTextToAudio,
+	"text-to-audio": snippetTextToAudio,
+	"audio-to-audio": snippetFile,
+	"audio-classification": snippetFile,
+	"image-classification": snippetFile,
+	"tabular-regression": snippetTabular,
+	"tabular-classification": snippetTabular,
+	"object-detection": snippetFile,
+	"image-segmentation": snippetFile,
+	"document-question-answering": snippetDocumentQuestionAnswering,
+	"image-to-text": snippetFile,
+	"zero-shot-image-classification": snippetZeroShotImageClassification,
+};
+
+export function getPythonInferenceSnippet(
+	model: ModelDataMinimal,
+	accessToken: string,
+	opts?: Record<string, unknown>
+): InferenceSnippet | InferenceSnippet[] {
+	if (model.tags.includes("conversational")) {
+		// Conversational model detected, so we display a code snippet that features the Messages API
+		return snippetConversational(model, accessToken, opts);
+	} else {
+		let snippets =
+			model.pipeline_tag && model.pipeline_tag in pythonSnippets
+				? pythonSnippets[model.pipeline_tag]?.(model, accessToken) ?? { content: "" }
+				: { content: "" };
+
+		snippets = Array.isArray(snippets) ? snippets : [snippets];
+
+		return snippets.map((snippet) => {
+			return {
+				...snippet,
+				content: snippet.content.includes("requests")
+					? `import requests
+
+API_URL = "https://api-inference.huggingface.co/models/${model.id}"
+headers = {"Authorization": ${accessToken ? `"Bearer ${accessToken}"` : `f"Bearer {API_TOKEN}"`}}
+
+${snippet.content}`
+					: snippet.content,
+			};
+		});
+	}
+}
+
+export function hasPythonInferenceSnippet(model: ModelDataMinimal): boolean {
+	return !!model.pipeline_tag && model.pipeline_tag in pythonSnippets;
+}

package/src/snippets/types.ts

@@ -0,0 +1,16 @@
+import type { ModelData } from "../model-data.js";
+
+/**
+ * Minimal model data required for snippets.
+ *
+ * Add more fields as needed.
+ */
+export type ModelDataMinimal = Pick<
+	ModelData,
+	"id" | "pipeline_tag" | "mask_token" | "library_name" | "config" | "tags" | "inference"
+>;
+
+export interface InferenceSnippet {
+	content: string;
+	client?: string; // for instance: `client` could be `huggingface_hub` or `openai` client for Python snippets
+}

package/src/tasks/audio-classification/about.md

@@ -0,0 +1,86 @@
+## Use Cases
+
+### Command Recognition
+
+Command recognition or keyword spotting classifies utterances into a predefined set of commands. This is often done on-device for fast response time.
+
+As an example, using the Google Speech Commands dataset, given an input, a model can classify which of the following commands the user is typing:
+
+```
+'yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'unknown', 'silence'
+```
+
+Speechbrain models can easily perform this task with just a couple of lines of code!
+
+```python
+from speechbrain.pretrained import EncoderClassifier
+model = EncoderClassifier.from_hparams(
+  "speechbrain/google_speech_command_xvector"
+)
+model.classify_file("file.wav")
+```
+
+### Language Identification
+
+Datasets such as VoxLingua107 allow anyone to train language identification models for up to 107 languages! This can be extremely useful as a preprocessing step for other systems. Here's an example [model](https://huggingface.co/TalTechNLP/voxlingua107-epaca-tdnn)trained on VoxLingua107.
+
+### Emotion recognition
+
+Emotion recognition is self explanatory. In addition to trying the widgets, you can use Inference Endpoints to perform audio classification. Here is a simple example that uses a [HuBERT](https://huggingface.co/superb/hubert-large-superb-er) model fine-tuned for this task.
+
+```python
+import json
+import requests
+
+headers = {"Authorization": f"Bearer {API_TOKEN}"}
+API_URL = "https://api-inference.huggingface.co/models/superb/hubert-large-superb-er"
+
+def query(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+    response = requests.request("POST", API_URL, headers=headers, data=data)
+    return json.loads(response.content.decode("utf-8"))
+
+data = query("sample1.flac")
+# [{'label': 'neu', 'score': 0.60},
+# {'label': 'hap', 'score': 0.20},
+# {'label': 'ang', 'score': 0.13},
+# {'label': 'sad', 'score': 0.07}]
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer with audio classification models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.audioClassification({
+	data: await (await fetch("sample.flac")).blob(),
+	model: "facebook/mms-lid-126",
+});
+```
+
+### Speaker Identification
+
+Speaker Identification is classifying the audio of the person speaking. Speakers are usually predefined. You can try out this task with [this model](https://huggingface.co/superb/wav2vec2-base-superb-sid). A useful dataset for this task is VoxCeleb1.
+
+## Solving audio classification for your own data
+
+We have some great news! You can do fine-tuning (transfer learning) to train a well-performing model without requiring as much data. Pretrained models such as Wav2Vec2 and HuBERT exist. [Facebook's Wav2Vec2 XLS-R model](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2) is a large multilingual model trained on 128 languages and with 436K hours of speech. Similarly, you can also use [OpenAI's Whisper](https://huggingface.co/docs/transformers/model_doc/whisper) trained on up to 4 Million hours of multilingual speech data for this task too!
+
+## Useful Resources
+
+Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful!
+
+### Notebooks
+
+- [PyTorch](https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/audio_classification.ipynb)
+
+### Scripts for training
+
+- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/audio-classification)
+
+### Documentation
+
+- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter4/introduction)
+- [Audio classification task guide](https://huggingface.co/docs/transformers/tasks/audio_classification)

package/src/tasks/audio-classification/data.ts

@@ -0,0 +1,81 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A benchmark of 10 different audio tasks.",
+			id: "s3prl/superb",
+		},
+		{
+			description: "A dataset of YouTube clips and their sound categories.",
+			id: "agkphysics/AudioSet",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "audio.wav",
+				type: "audio",
+			},
+		],
+		outputs: [
+			{
+				data: [
+					{
+						label: "Up",
+						score: 0.2,
+					},
+					{
+						label: "Down",
+						score: 0.8,
+					},
+				],
+				type: "chart",
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "",
+			id: "accuracy",
+		},
+		{
+			description: "",
+			id: "recall",
+		},
+		{
+			description: "",
+			id: "precision",
+		},
+		{
+			description: "",
+			id: "f1",
+		},
+	],
+	models: [
+		{
+			description: "An easy-to-use model for command recognition.",
+			id: "speechbrain/google_speech_command_xvector",
+		},
+		{
+			description: "An emotion recognition model.",
+			id: "ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition",
+		},
+		{
+			description: "A language identification model.",
+			id: "facebook/mms-lid-126",
+		},
+	],
+	spaces: [
+		{
+			description: "An application that can classify music into different genre.",
+			id: "kurianbenoy/audioclassification",
+		},
+	],
+	summary:
+		"Audio classification is the task of assigning a label or class to a given audio. It can be used for recognizing which command a user is giving or the emotion of a statement, as well as identifying a speaker.",
+	widgetModels: ["MIT/ast-finetuned-audioset-10-10-0.4593"],
+	youtubeId: "KWwzcmG98Ds",
+};
+
+export default taskData;

package/src/tasks/audio-classification/inference.ts

@@ -0,0 +1,52 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Audio Classification inference
+ */
+export interface AudioClassificationInput {
+	/**
+	 * The input audio data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the audio data as a raw bytes payload.
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: AudioClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Audio Classification
+ */
+export interface AudioClassificationParameters {
+	function_to_apply?: ClassificationOutputTransform;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type AudioClassificationOutput = AudioClassificationOutputElement[];
+/**
+ * Outputs for Audio Classification inference
+ */
+export interface AudioClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/audio-classification/spec/input.json

@@ -0,0 +1,35 @@
+{
+	"$id": "/inference/schemas/audio-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Audio Classification inference",
+	"title": "AudioClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input audio data as a base64-encoded string. If no `parameters` are provided, you can also provide the audio data as a raw bytes payload.",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/AudioClassificationParameters"
+		}
+	},
+	"$defs": {
+		"AudioClassificationParameters": {
+			"title": "AudioClassificationParameters",
+			"description": "Additional inference parameters for Audio Classification",
+			"type": "object",
+			"properties": {
+				"function_to_apply": {
+					"title": "AudioClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/audio-classification/spec/output.json

@@ -0,0 +1,11 @@
+{
+	"$id": "/inference/schemas/audio-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"title": "AudioClassificationOutput",
+	"description": "Outputs for Audio Classification inference",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/audio-to-audio/about.md

@@ -0,0 +1,56 @@
+## Use Cases
+
+### Speech Enhancement (Noise removal)
+
+Speech Enhancement is a bit self explanatory. It improves (or enhances) the quality of an audio by removing noise. There are multiple libraries to solve this task, such as Speechbrain, Asteroid and ESPNet. Here is a simple example using Speechbrain
+
+```python
+from speechbrain.pretrained import SpectralMaskEnhancement
+model = SpectralMaskEnhancement.from_hparams(
+  "speechbrain/mtl-mimic-voicebank"
+)
+model.enhance_file("file.wav")
+```
+
+Alternatively, you can use [Inference Endpoints](https://huggingface.co/inference-endpoints) to solve this task
+
+```python
+import json
+import requests
+
+headers = {"Authorization": f"Bearer {API_TOKEN}"}
+API_URL = "https://api-inference.huggingface.co/models/speechbrain/mtl-mimic-voicebank"
+
+def query(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+    response = requests.request("POST", API_URL, headers=headers, data=data)
+    return json.loads(response.content.decode("utf-8"))
+
+data = query("sample1.flac")
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer with audio-to-audio models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.audioToAudio({
+	data: await (await fetch("sample.flac")).blob(),
+	model: "speechbrain/sepformer-wham",
+});
+```
+
+### Audio Source Separation
+
+Audio Source Separation allows you to isolate different sounds from individual sources. For example, if you have an audio file with multiple people speaking, you can get an audio file for each of them. You can then use an Automatic Speech Recognition system to extract the text from each of these sources as an initial step for your system!
+
+Audio-to-Audio can also be used to remove noise from audio files: you get one audio for the person speaking and another audio for the noise. This can also be useful when you have multi-person audio with some noise: yyou can get one audio for each person and then one audio for the noise.
+
+## Training a model for your own data
+
+If you want to learn how to train models for the Audio-to-Audio task, we recommend the following tutorials:
+
+- [Speech Enhancement](https://speechbrain.github.io/tutorial_enhancement.html)
+- [Source Separation](https://speechbrain.github.io/tutorial_separation.html)