@huggingface/tasks 0.2.1 → 0.3.0

package/src/model-libraries.ts

@@ -1,52 +1,383 @@
+import * as snippets from "./model-libraries-snippets";
+import type { ModelData } from "./model-data";
+import type { ElasticBoolQueryFilter } from "./model-libraries-downloads";
+
+/**
+ * Elements configurable by a model library.
+ */
+export interface LibraryUiElement {
+	/**
+	 * Pretty name of the library.
+	 * displayed in tags, and on the main
+	 * call-to-action button on the model page.
+	 */
+	prettyLabel: string;
+	/**
+	 * Repo name of the library's (usually on GitHub) code repo
+	 */
+	repoName: string;
+	/**
+	 * URL to library's (usually on GitHub) code repo
+	 */
+	repoUrl: string;
+	/**
+	 * URL to library's docs
+	 */
+	docsUrl?: string;
+	/**
+	 * Code snippet(s) displayed on model page
+	 */
+	snippets?: (model: ModelData) => string[];
+	/**
+	 * Elastic query used to count this library's model downloads
+	 *
+	 * By default, those files are counted:
+	 * "config.json", "config.yaml", "hyperparams.yaml", "meta.yaml"
+	 */
+	countDownloads?: ElasticBoolQueryFilter;
+	/**
+	 * should we display this library in hf.co/models filter
+	 * (only for popular libraries with > 100 models)
+	 */
+	filter?: boolean;
+}
+
 /**
  * Add your new library here.
  *
  * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc).
- * File formats live in an enum inside the internal codebase.
+ * (unlike libraries, file formats live in an enum inside the internal codebase.)
+ *
+ * Doc on how to add a library to the Hub:
+ *
+ * https://huggingface.co/docs/hub/models-adding-libraries
+ *
+ * /!\ IMPORTANT
+ *
+ * The key you choose is the tag your models have in their library_name on the Hub.
  */
-export enum ModelLibrary {
-	"adapter-transformers" = "Adapter Transformers",
-	"allennlp" = "allenNLP",
-	"asteroid" = "Asteroid",
-	"bertopic" = "BERTopic",
-	"diffusers" = "Diffusers",
-	"doctr" = "docTR",
-	"espnet" = "ESPnet",
-	"fairseq" = "Fairseq",
-	"flair" = "Flair",
-	"keras" = "Keras",
-	"k2" = "K2",
-	"mlx" = "MLX",
-	"nemo" = "NeMo",
-	"open_clip" = "OpenCLIP",
-	"paddlenlp" = "PaddleNLP",
-	"peft" = "PEFT",
-	"pyannote-audio" = "pyannote.audio",
-	"sample-factory" = "Sample Factory",
-	"sentence-transformers" = "Sentence Transformers",
-	"setfit" = "SetFit",
-	"sklearn" = "Scikit-learn",
-	"spacy" = "spaCy",
-	"span-marker" = "SpanMarker",
-	"speechbrain" = "speechbrain",
-	"tensorflowtts" = "TensorFlowTTS",
-	"timm" = "Timm",
-	"fastai" = "fastai",
-	"transformers" = "Transformers",
-	"transformers.js" = "Transformers.js",
-	"stanza" = "Stanza",
-	"fasttext" = "fastText",
-	"stable-baselines3" = "Stable-Baselines3",
-	"ml-agents" = "Unity ML-Agents",
-	"pythae" = "Pythae",
-	"mindspore" = "MindSpore",
-	"unity-sentis" = "Unity Sentis",
-}
 
-export type ModelLibraryKey = keyof typeof ModelLibrary;
+export const MODEL_LIBRARIES_UI_ELEMENTS = {
+	"adapter-transformers": {
+		prettyLabel: "Adapters",
+		repoName: "adapters",
+		repoUrl: "https://github.com/Adapter-Hub/adapters",
+		docsUrl: "https://huggingface.co/docs/hub/adapters",
+		snippets: snippets.adapters,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	allennlp: {
+		prettyLabel: "AllenNLP",
+		repoName: "AllenNLP",
+		repoUrl: "https://github.com/allenai/allennlp",
+		docsUrl: "https://huggingface.co/docs/hub/allennlp",
+		snippets: snippets.allennlp,
+		filter: true,
+	},
+	asteroid: {
+		prettyLabel: "Asteroid",
+		repoName: "Asteroid",
+		repoUrl: "https://github.com/asteroid-team/asteroid",
+		docsUrl: "https://huggingface.co/docs/hub/asteroid",
+		snippets: snippets.asteroid,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	bertopic: {
+		prettyLabel: "BERTopic",
+		repoName: "BERTopic",
+		repoUrl: "https://github.com/MaartenGr/BERTopic",
+		snippets: snippets.bertopic,
+		filter: true,
+	},
+	diffusers: {
+		prettyLabel: "Diffusers",
+		repoName: "🤗/diffusers",
+		repoUrl: "https://github.com/huggingface/diffusers",
+		docsUrl: "https://huggingface.co/docs/hub/diffusers",
+		snippets: snippets.diffusers,
+		filter: true,
+		/// diffusers has its own more complex "countDownloads" query
+	},
+	doctr: {
+		prettyLabel: "docTR",
+		repoName: "doctr",
+		repoUrl: "https://github.com/mindee/doctr",
+	},
+	espnet: {
+		prettyLabel: "ESPnet",
+		repoName: "ESPnet",
+		repoUrl: "https://github.com/espnet/espnet",
+		docsUrl: "https://huggingface.co/docs/hub/espnet",
+		snippets: snippets.espnet,
+		filter: true,
+	},
+	fairseq: {
+		prettyLabel: "Fairseq",
+		repoName: "fairseq",
+		repoUrl: "https://github.com/pytorch/fairseq",
+		snippets: snippets.fairseq,
+		filter: true,
+	},
+	fastai: {
+		prettyLabel: "fastai",
+		repoName: "fastai",
+		repoUrl: "https://github.com/fastai/fastai",
+		docsUrl: "https://huggingface.co/docs/hub/fastai",
+		snippets: snippets.fastai,
+		filter: true,
+	},
+	fasttext: {
+		prettyLabel: "fastText",
+		repoName: "fastText",
+		repoUrl: "https://fasttext.cc/",
+		snippets: snippets.fasttext,
+		filter: true,
+	},
+	flair: {
+		prettyLabel: "Flair",
+		repoName: "Flair",
+		repoUrl: "https://github.com/flairNLP/flair",
+		docsUrl: "https://huggingface.co/docs/hub/flair",
+		snippets: snippets.flair,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	keras: {
+		prettyLabel: "Keras",
+		repoName: "Keras",
+		repoUrl: "https://github.com/keras-team/keras",
+		docsUrl: "https://huggingface.co/docs/hub/keras",
+		snippets: snippets.keras,
+		filter: true,
+		countDownloads: { term: { path: "saved_model.pb" } },
+	},
+	k2: {
+		prettyLabel: "K2",
+		repoName: "k2",
+		repoUrl: "https://github.com/k2-fsa/k2",
+	},
+	mindspore: {
+		prettyLabel: "MindSpore",
+		repoName: "mindspore",
+		repoUrl: "https://github.com/mindspore-ai/mindspore",
+	},
+	"ml-agents": {
+		prettyLabel: "ml-agents",
+		repoName: "ml-agents",
+		repoUrl: "https://github.com/Unity-Technologies/ml-agents",
+		docsUrl: "https://huggingface.co/docs/hub/ml-agents",
+		snippets: snippets.mlAgents,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.onnx" } },
+	},
+	mlx: {
+		prettyLabel: "MLX",
+		repoName: "MLX",
+		repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main",
+		snippets: snippets.mlx,
+		filter: true,
+	},
+	nemo: {
+		prettyLabel: "NeMo",
+		repoName: "NeMo",
+		repoUrl: "https://github.com/NVIDIA/NeMo",
+		snippets: snippets.nemo,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.nemo" } },
+	},
+	open_clip: {
+		prettyLabel: "OpenCLIP",
+		repoName: "OpenCLIP",
+		repoUrl: "https://github.com/mlfoundations/open_clip",
+		snippets: snippets.open_clip,
+		filter: true,
+		countDownloads: { wildcard: { path: "*pytorch_model.bin" } },
+	},
+	paddlenlp: {
+		prettyLabel: "paddlenlp",
+		repoName: "PaddleNLP",
+		repoUrl: "https://github.com/PaddlePaddle/PaddleNLP",
+		docsUrl: "https://huggingface.co/docs/hub/paddlenlp",
+		snippets: snippets.paddlenlp,
+		filter: true,
+		countDownloads: {
+			term: { path: "model_config.json" },
+		},
+	},
+	peft: {
+		prettyLabel: "PEFT",
+		repoName: "PEFT",
+		repoUrl: "https://github.com/huggingface/peft",
+		snippets: snippets.peft,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	"pyannote-audio": {
+		prettyLabel: "pyannote.audio",
+		repoName: "pyannote-audio",
+		repoUrl: "https://github.com/pyannote/pyannote-audio",
+		snippets: snippets.pyannote_audio,
+		filter: true,
+	},
+	pythae: {
+		prettyLabel: "pythae",
+		repoName: "pythae",
+		repoUrl: "https://github.com/clementchadebec/benchmark_VAE",
+		snippets: snippets.pythae,
+		filter: true,
+	},
+	"sample-factory": {
+		prettyLabel: "sample-factory",
+		repoName: "sample-factory",
+		repoUrl: "https://github.com/alex-petrenko/sample-factory",
+		docsUrl: "https://huggingface.co/docs/hub/sample-factory",
+		snippets: snippets.sampleFactory,
+		filter: true,
+		countDownloads: { term: { path: "cfg.json" } },
+	},
+	"sentence-transformers": {
+		prettyLabel: "sentence-transformers",
+		repoName: "sentence-transformers",
+		repoUrl: "https://github.com/UKPLab/sentence-transformers",
+		docsUrl: "https://huggingface.co/docs/hub/sentence-transformers",
+		snippets: snippets.sentenceTransformers,
+		filter: true,
+	},
+	setfit: {
+		prettyLabel: "setfit",
+		repoName: "setfit",
+		repoUrl: "https://github.com/huggingface/setfit",
+		docsUrl: "https://huggingface.co/docs/hub/setfit",
+		snippets: snippets.setfit,
+		filter: true,
+	},
+	sklearn: {
+		prettyLabel: "Scikit-learn",
+		repoName: "Scikit-learn",
+		repoUrl: "https://github.com/scikit-learn/scikit-learn",
+		snippets: snippets.sklearn,
+		filter: true,
+		countDownloads: {
+			term: { path: "sklearn_model.joblib" },
+		},
+	},
+	spacy: {
+		prettyLabel: "spaCy",
+		repoName: "spaCy",
+		repoUrl: "https://github.com/explosion/spaCy",
+		docsUrl: "https://huggingface.co/docs/hub/spacy",
+		snippets: snippets.spacy,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.whl" },
+		},
+	},
+	"span-marker": {
+		prettyLabel: "SpanMarker",
+		repoName: "SpanMarkerNER",
+		repoUrl: "https://github.com/tomaarsen/SpanMarkerNER",
+		docsUrl: "https://huggingface.co/docs/hub/span_marker",
+		snippets: snippets.span_marker,
+		filter: true,
+	},
+	speechbrain: {
+		prettyLabel: "speechbrain",
+		repoName: "speechbrain",
+		repoUrl: "https://github.com/speechbrain/speechbrain",
+		docsUrl: "https://huggingface.co/docs/hub/speechbrain",
+		snippets: snippets.speechbrain,
+		filter: true,
+		countDownloads: {
+			term: { path: "hyperparams.yaml" },
+		},
+	},
+	"stable-baselines3": {
+		prettyLabel: "stable-baselines3",
+		repoName: "stable-baselines3",
+		repoUrl: "https://github.com/huggingface/huggingface_sb3",
+		docsUrl: "https://huggingface.co/docs/hub/stable-baselines3",
+		snippets: snippets.stableBaselines3,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.zip" },
+		},
+	},
+	stanza: {
+		prettyLabel: "Stanza",
+		repoName: "stanza",
+		repoUrl: "https://github.com/stanfordnlp/stanza",
+		docsUrl: "https://huggingface.co/docs/hub/stanza",
+		snippets: snippets.stanza,
+		filter: true,
+		countDownloads: {
+			term: { path: "models/default.zip" },
+		},
+	},
+	tensorflowtts: {
+		prettyLabel: "TensorFlowTTS",
+		repoName: "TensorFlowTTS",
+		repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS",
+		snippets: snippets.tensorflowtts,
+	},
+	timm: {
+		prettyLabel: "timm",
+		repoName: "pytorch-image-models",
+		repoUrl: "https://github.com/rwightman/pytorch-image-models",
+		docsUrl: "https://huggingface.co/docs/hub/timm",
+		snippets: snippets.timm,
+		filter: true,
+		countDownloads: {
+			terms: { path: ["pytorch_model.bin", "model.safetensors"] },
+		},
+	},
+	transformers: {
+		prettyLabel: "Transformers",
+		repoName: "🤗/transformers",
+		repoUrl: "https://github.com/huggingface/transformers",
+		docsUrl: "https://huggingface.co/docs/hub/transformers",
+		snippets: snippets.transformers,
+		filter: true,
+	},
+	"transformers.js": {
+		prettyLabel: "Transformers.js",
+		repoName: "transformers.js",
+		repoUrl: "https://github.com/xenova/transformers.js",
+		docsUrl: "https://huggingface.co/docs/hub/transformers-js",
+		snippets: snippets.transformersJS,
+		filter: true,
+	},
+	"unity-sentis": {
+		prettyLabel: "unity-sentis",
+		repoName: "unity-sentis",
+		repoUrl: "https://github.com/Unity-Technologies/sentis-samples",
+		snippets: snippets.sentis,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.sentis" },
+		},
+	},
+} satisfies Record<string, LibraryUiElement>;
+
+export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS;
 
-export const ALL_MODEL_LIBRARY_KEYS = Object.keys(ModelLibrary) as ModelLibraryKey[];
+export const ALL_MODEL_LIBRARY_KEYS = Object.keys(MODEL_LIBRARIES_UI_ELEMENTS) as ModelLibraryKey[];
 
-export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = ALL_MODEL_LIBRARY_KEYS.filter(
-	(k) => !["doctr", "k2", "mindspore", "tensorflowtts"].includes(k)
-);
+export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = (
+	Object.entries(MODEL_LIBRARIES_UI_ELEMENTS as Record<ModelLibraryKey, LibraryUiElement>) as [
+		ModelLibraryKey,
+		LibraryUiElement,
+	][]
+)
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	.filter(([_, v]) => v.filter)
+	.map(([k]) => k);

package/src/pipelines.ts

@@ -62,7 +62,7 @@ export interface PipelineData {
 /// This type is used in multiple places in the Hugging Face
 /// ecosystem:
 ///  - To determine which widget to show.
-///  - To determine which endpoint of Inference API to use.
+///  - To determine which endpoint of Inference Endpoints to use.
 ///  - As filters at the left of models and datasets page.
 ///
 /// Note that this is sensitive to order.

package/src/tasks/audio-classification/about.md

@@ -26,7 +26,7 @@ Datasets such as VoxLingua107 allow anyone to train language identification mode
 
 ### Emotion recognition
 
-Emotion recognition is self explanatory. In addition to trying the widgets, you can use the Inference API to perform audio classification. Here is a simple example that uses a [HuBERT](https://huggingface.co/superb/hubert-large-superb-er) model fine-tuned for this task.
+Emotion recognition is self explanatory. In addition to trying the widgets, you can use Inference Endpoints to perform audio classification. Here is a simple example that uses a [HuBERT](https://huggingface.co/superb/hubert-large-superb-er) model fine-tuned for this task.
 
 ```python
 import json

package/src/tasks/audio-classification/inference.ts

@@ -0,0 +1,51 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Audio Classification inference
+ */
+export interface AudioClassificationInput {
+	/**
+	 * The input audio data
+	 */
+	inputs: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: AudioClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Audio Classification
+ */
+export interface AudioClassificationParameters {
+	function_to_apply?: ClassificationOutputTransform;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type AudioClassificationOutput = AudioClassificationOutputElement[];
+/**
+ * Outputs for Audio Classification inference
+ */
+export interface AudioClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/audio-classification/spec/input.json

@@ -0,0 +1,34 @@
+{
+	"$id": "/inference/schemas/audio-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Audio Classification inference",
+	"title": "AudioClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input audio data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/AudioClassificationParameters"
+		}
+	},
+	"$defs": {
+		"AudioClassificationParameters": {
+			"title": "AudioClassificationParameters",
+			"description": "Additional inference parameters for Audio Classification",
+			"type": "object",
+			"properties": {
+				"function_to_apply": {
+					"title": "AudioClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/audio-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/audio-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"title": "AudioClassificationOutput",
+	"description": "Outputs for Audio Classification inference",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/audio-to-audio/about.md

@@ -12,7 +12,7 @@ model = SpectralMaskEnhancement.from_hparams(
 model.enhance_file("file.wav")
 ```
 
-Alternatively, you can use the [Inference API](https://huggingface.co/inference-api) to solve this task
+Alternatively, you can use [Inference Endpoints](https://huggingface.co/inference-endpoints) to solve this task
 
 ```python
 import json

package/src/tasks/automatic-speech-recognition/about.md

@@ -18,7 +18,7 @@ The use of Multilingual ASR has become popular, the idea of maintaining just a s
 
 ## Inference
 
-The Hub contains over [~9,000 ASR models](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&sort=downloads) that you can use right away by trying out the widgets directly in the browser or calling the models as a service using the Inference API. Here is a simple code snippet to do exactly this:
+The Hub contains over [~9,000 ASR models](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&sort=downloads) that you can use right away by trying out the widgets directly in the browser or calling the models as a service using Inference Endpoints. Here is a simple code snippet to do exactly this:
 
 ```python
 import json
@@ -83,6 +83,8 @@ These events help democratize ASR for all languages, including low-resource lang
 - [Massively Multilingual ASR: 50 Languages, 1 Model, 1 Billion Parameters](https://arxiv.org/pdf/2007.03001.pdf)
 - An ASR toolkit made by [NVIDIA: NeMo](https://github.com/NVIDIA/NeMo) with code and pretrained models useful for new ASR models. Watch the [introductory video](https://www.youtube.com/embed/wBgpMf_KQVw) for an overview.
 - [An introduction to SpeechT5, a multi-purpose speech recognition and synthesis model](https://huggingface.co/blog/speecht5)
-- [A guide on Fine-tuning Whisper For Multilingual ASR with 🤗Transformers](https://huggingface.co/blog/fine-tune-whisper)
+- [Fine-tune Whisper For Multilingual ASR with 🤗Transformers](https://huggingface.co/blog/fine-tune-whisper)
 - [Automatic speech recognition task guide](https://huggingface.co/docs/transformers/tasks/asr)
 - [Speech Synthesis, Recognition, and More With SpeechT5](https://huggingface.co/blog/speecht5)
+- [Fine-Tune W2V2-Bert for low-resource ASR with 🤗 Transformers](https://huggingface.co/blog/fine-tune-w2v2-bert)
+- [Speculative Decoding for 2x Faster Whisper Inference](https://huggingface.co/blog/whisper-speculative-decoding)