@huggingface/tasks 0.2.2 → 0.3.1

package/src/model-libraries.ts

@@ -1,52 +1,383 @@
+import * as snippets from "./model-libraries-snippets";
+import type { ModelData } from "./model-data";
+import type { ElasticBoolQueryFilter } from "./model-libraries-downloads";
+
+/**
+ * Elements configurable by a model library.
+ */
+export interface LibraryUiElement {
+	/**
+	 * Pretty name of the library.
+	 * displayed in tags, and on the main
+	 * call-to-action button on the model page.
+	 */
+	prettyLabel: string;
+	/**
+	 * Repo name of the library's (usually on GitHub) code repo
+	 */
+	repoName: string;
+	/**
+	 * URL to library's (usually on GitHub) code repo
+	 */
+	repoUrl: string;
+	/**
+	 * URL to library's docs
+	 */
+	docsUrl?: string;
+	/**
+	 * Code snippet(s) displayed on model page
+	 */
+	snippets?: (model: ModelData) => string[];
+	/**
+	 * Elastic query used to count this library's model downloads
+	 *
+	 * By default, those files are counted:
+	 * "config.json", "config.yaml", "hyperparams.yaml", "meta.yaml"
+	 */
+	countDownloads?: ElasticBoolQueryFilter;
+	/**
+	 * should we display this library in hf.co/models filter
+	 * (only for popular libraries with > 100 models)
+	 */
+	filter?: boolean;
+}
+
 /**
  * Add your new library here.
  *
  * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc).
- * File formats live in an enum inside the internal codebase.
+ * (unlike libraries, file formats live in an enum inside the internal codebase.)
+ *
+ * Doc on how to add a library to the Hub:
+ *
+ * https://huggingface.co/docs/hub/models-adding-libraries
+ *
+ * /!\ IMPORTANT
+ *
+ * The key you choose is the tag your models have in their library_name on the Hub.
  */
-export enum ModelLibrary {
-	"adapter-transformers" = "Adapters",
-	"allennlp" = "allenNLP",
-	"asteroid" = "Asteroid",
-	"bertopic" = "BERTopic",
-	"diffusers" = "Diffusers",
-	"doctr" = "docTR",
-	"espnet" = "ESPnet",
-	"fairseq" = "Fairseq",
-	"flair" = "Flair",
-	"keras" = "Keras",
-	"k2" = "K2",
-	"mlx" = "MLX",
-	"nemo" = "NeMo",
-	"open_clip" = "OpenCLIP",
-	"paddlenlp" = "PaddleNLP",
-	"peft" = "PEFT",
-	"pyannote-audio" = "pyannote.audio",
-	"sample-factory" = "Sample Factory",
-	"sentence-transformers" = "Sentence Transformers",
-	"setfit" = "SetFit",
-	"sklearn" = "Scikit-learn",
-	"spacy" = "spaCy",
-	"span-marker" = "SpanMarker",
-	"speechbrain" = "speechbrain",
-	"tensorflowtts" = "TensorFlowTTS",
-	"timm" = "Timm",
-	"fastai" = "fastai",
-	"transformers" = "Transformers",
-	"transformers.js" = "Transformers.js",
-	"stanza" = "Stanza",
-	"fasttext" = "fastText",
-	"stable-baselines3" = "Stable-Baselines3",
-	"ml-agents" = "Unity ML-Agents",
-	"pythae" = "Pythae",
-	"mindspore" = "MindSpore",
-	"unity-sentis" = "Unity Sentis",
-}
 
-export type ModelLibraryKey = keyof typeof ModelLibrary;
+export const MODEL_LIBRARIES_UI_ELEMENTS = {
+	"adapter-transformers": {
+		prettyLabel: "Adapters",
+		repoName: "adapters",
+		repoUrl: "https://github.com/Adapter-Hub/adapters",
+		docsUrl: "https://huggingface.co/docs/hub/adapters",
+		snippets: snippets.adapters,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	allennlp: {
+		prettyLabel: "AllenNLP",
+		repoName: "AllenNLP",
+		repoUrl: "https://github.com/allenai/allennlp",
+		docsUrl: "https://huggingface.co/docs/hub/allennlp",
+		snippets: snippets.allennlp,
+		filter: true,
+	},
+	asteroid: {
+		prettyLabel: "Asteroid",
+		repoName: "Asteroid",
+		repoUrl: "https://github.com/asteroid-team/asteroid",
+		docsUrl: "https://huggingface.co/docs/hub/asteroid",
+		snippets: snippets.asteroid,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	bertopic: {
+		prettyLabel: "BERTopic",
+		repoName: "BERTopic",
+		repoUrl: "https://github.com/MaartenGr/BERTopic",
+		snippets: snippets.bertopic,
+		filter: true,
+	},
+	diffusers: {
+		prettyLabel: "Diffusers",
+		repoName: "🤗/diffusers",
+		repoUrl: "https://github.com/huggingface/diffusers",
+		docsUrl: "https://huggingface.co/docs/hub/diffusers",
+		snippets: snippets.diffusers,
+		filter: true,
+		/// diffusers has its own more complex "countDownloads" query
+	},
+	doctr: {
+		prettyLabel: "docTR",
+		repoName: "doctr",
+		repoUrl: "https://github.com/mindee/doctr",
+	},
+	espnet: {
+		prettyLabel: "ESPnet",
+		repoName: "ESPnet",
+		repoUrl: "https://github.com/espnet/espnet",
+		docsUrl: "https://huggingface.co/docs/hub/espnet",
+		snippets: snippets.espnet,
+		filter: true,
+	},
+	fairseq: {
+		prettyLabel: "Fairseq",
+		repoName: "fairseq",
+		repoUrl: "https://github.com/pytorch/fairseq",
+		snippets: snippets.fairseq,
+		filter: true,
+	},
+	fastai: {
+		prettyLabel: "fastai",
+		repoName: "fastai",
+		repoUrl: "https://github.com/fastai/fastai",
+		docsUrl: "https://huggingface.co/docs/hub/fastai",
+		snippets: snippets.fastai,
+		filter: true,
+	},
+	fasttext: {
+		prettyLabel: "fastText",
+		repoName: "fastText",
+		repoUrl: "https://fasttext.cc/",
+		snippets: snippets.fasttext,
+		filter: true,
+	},
+	flair: {
+		prettyLabel: "Flair",
+		repoName: "Flair",
+		repoUrl: "https://github.com/flairNLP/flair",
+		docsUrl: "https://huggingface.co/docs/hub/flair",
+		snippets: snippets.flair,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	keras: {
+		prettyLabel: "Keras",
+		repoName: "Keras",
+		repoUrl: "https://github.com/keras-team/keras",
+		docsUrl: "https://huggingface.co/docs/hub/keras",
+		snippets: snippets.keras,
+		filter: true,
+		countDownloads: { term: { path: "saved_model.pb" } },
+	},
+	k2: {
+		prettyLabel: "K2",
+		repoName: "k2",
+		repoUrl: "https://github.com/k2-fsa/k2",
+	},
+	mindspore: {
+		prettyLabel: "MindSpore",
+		repoName: "mindspore",
+		repoUrl: "https://github.com/mindspore-ai/mindspore",
+	},
+	"ml-agents": {
+		prettyLabel: "ml-agents",
+		repoName: "ml-agents",
+		repoUrl: "https://github.com/Unity-Technologies/ml-agents",
+		docsUrl: "https://huggingface.co/docs/hub/ml-agents",
+		snippets: snippets.mlAgents,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.onnx" } },
+	},
+	mlx: {
+		prettyLabel: "MLX",
+		repoName: "MLX",
+		repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main",
+		snippets: snippets.mlx,
+		filter: true,
+	},
+	nemo: {
+		prettyLabel: "NeMo",
+		repoName: "NeMo",
+		repoUrl: "https://github.com/NVIDIA/NeMo",
+		snippets: snippets.nemo,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.nemo" } },
+	},
+	open_clip: {
+		prettyLabel: "OpenCLIP",
+		repoName: "OpenCLIP",
+		repoUrl: "https://github.com/mlfoundations/open_clip",
+		snippets: snippets.open_clip,
+		filter: true,
+		countDownloads: { wildcard: { path: "*pytorch_model.bin" } },
+	},
+	paddlenlp: {
+		prettyLabel: "paddlenlp",
+		repoName: "PaddleNLP",
+		repoUrl: "https://github.com/PaddlePaddle/PaddleNLP",
+		docsUrl: "https://huggingface.co/docs/hub/paddlenlp",
+		snippets: snippets.paddlenlp,
+		filter: true,
+		countDownloads: {
+			term: { path: "model_config.json" },
+		},
+	},
+	peft: {
+		prettyLabel: "PEFT",
+		repoName: "PEFT",
+		repoUrl: "https://github.com/huggingface/peft",
+		snippets: snippets.peft,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	"pyannote-audio": {
+		prettyLabel: "pyannote.audio",
+		repoName: "pyannote-audio",
+		repoUrl: "https://github.com/pyannote/pyannote-audio",
+		snippets: snippets.pyannote_audio,
+		filter: true,
+	},
+	pythae: {
+		prettyLabel: "pythae",
+		repoName: "pythae",
+		repoUrl: "https://github.com/clementchadebec/benchmark_VAE",
+		snippets: snippets.pythae,
+		filter: true,
+	},
+	"sample-factory": {
+		prettyLabel: "sample-factory",
+		repoName: "sample-factory",
+		repoUrl: "https://github.com/alex-petrenko/sample-factory",
+		docsUrl: "https://huggingface.co/docs/hub/sample-factory",
+		snippets: snippets.sampleFactory,
+		filter: true,
+		countDownloads: { term: { path: "cfg.json" } },
+	},
+	"sentence-transformers": {
+		prettyLabel: "sentence-transformers",
+		repoName: "sentence-transformers",
+		repoUrl: "https://github.com/UKPLab/sentence-transformers",
+		docsUrl: "https://huggingface.co/docs/hub/sentence-transformers",
+		snippets: snippets.sentenceTransformers,
+		filter: true,
+	},
+	setfit: {
+		prettyLabel: "setfit",
+		repoName: "setfit",
+		repoUrl: "https://github.com/huggingface/setfit",
+		docsUrl: "https://huggingface.co/docs/hub/setfit",
+		snippets: snippets.setfit,
+		filter: true,
+	},
+	sklearn: {
+		prettyLabel: "Scikit-learn",
+		repoName: "Scikit-learn",
+		repoUrl: "https://github.com/scikit-learn/scikit-learn",
+		snippets: snippets.sklearn,
+		filter: true,
+		countDownloads: {
+			term: { path: "sklearn_model.joblib" },
+		},
+	},
+	spacy: {
+		prettyLabel: "spaCy",
+		repoName: "spaCy",
+		repoUrl: "https://github.com/explosion/spaCy",
+		docsUrl: "https://huggingface.co/docs/hub/spacy",
+		snippets: snippets.spacy,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.whl" },
+		},
+	},
+	"span-marker": {
+		prettyLabel: "SpanMarker",
+		repoName: "SpanMarkerNER",
+		repoUrl: "https://github.com/tomaarsen/SpanMarkerNER",
+		docsUrl: "https://huggingface.co/docs/hub/span_marker",
+		snippets: snippets.span_marker,
+		filter: true,
+	},
+	speechbrain: {
+		prettyLabel: "speechbrain",
+		repoName: "speechbrain",
+		repoUrl: "https://github.com/speechbrain/speechbrain",
+		docsUrl: "https://huggingface.co/docs/hub/speechbrain",
+		snippets: snippets.speechbrain,
+		filter: true,
+		countDownloads: {
+			term: { path: "hyperparams.yaml" },
+		},
+	},
+	"stable-baselines3": {
+		prettyLabel: "stable-baselines3",
+		repoName: "stable-baselines3",
+		repoUrl: "https://github.com/huggingface/huggingface_sb3",
+		docsUrl: "https://huggingface.co/docs/hub/stable-baselines3",
+		snippets: snippets.stableBaselines3,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.zip" },
+		},
+	},
+	stanza: {
+		prettyLabel: "Stanza",
+		repoName: "stanza",
+		repoUrl: "https://github.com/stanfordnlp/stanza",
+		docsUrl: "https://huggingface.co/docs/hub/stanza",
+		snippets: snippets.stanza,
+		filter: true,
+		countDownloads: {
+			term: { path: "models/default.zip" },
+		},
+	},
+	tensorflowtts: {
+		prettyLabel: "TensorFlowTTS",
+		repoName: "TensorFlowTTS",
+		repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS",
+		snippets: snippets.tensorflowtts,
+	},
+	timm: {
+		prettyLabel: "timm",
+		repoName: "pytorch-image-models",
+		repoUrl: "https://github.com/rwightman/pytorch-image-models",
+		docsUrl: "https://huggingface.co/docs/hub/timm",
+		snippets: snippets.timm,
+		filter: true,
+		countDownloads: {
+			terms: { path: ["pytorch_model.bin", "model.safetensors"] },
+		},
+	},
+	transformers: {
+		prettyLabel: "Transformers",
+		repoName: "🤗/transformers",
+		repoUrl: "https://github.com/huggingface/transformers",
+		docsUrl: "https://huggingface.co/docs/hub/transformers",
+		snippets: snippets.transformers,
+		filter: true,
+	},
+	"transformers.js": {
+		prettyLabel: "Transformers.js",
+		repoName: "transformers.js",
+		repoUrl: "https://github.com/xenova/transformers.js",
+		docsUrl: "https://huggingface.co/docs/hub/transformers-js",
+		snippets: snippets.transformersJS,
+		filter: true,
+	},
+	"unity-sentis": {
+		prettyLabel: "unity-sentis",
+		repoName: "unity-sentis",
+		repoUrl: "https://github.com/Unity-Technologies/sentis-samples",
+		snippets: snippets.sentis,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.sentis" },
+		},
+	},
+} satisfies Record<string, LibraryUiElement>;
+
+export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS;
 
-export const ALL_MODEL_LIBRARY_KEYS = Object.keys(ModelLibrary) as ModelLibraryKey[];
+export const ALL_MODEL_LIBRARY_KEYS = Object.keys(MODEL_LIBRARIES_UI_ELEMENTS) as ModelLibraryKey[];
 
-export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = ALL_MODEL_LIBRARY_KEYS.filter(
-	(k) => !["doctr", "k2", "mindspore", "tensorflowtts"].includes(k)
-);
+export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = (
+	Object.entries(MODEL_LIBRARIES_UI_ELEMENTS as Record<ModelLibraryKey, LibraryUiElement>) as [
+		ModelLibraryKey,
+		LibraryUiElement,
+	][]
+)
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	.filter(([_, v]) => v.filter)
+	.map(([k]) => k);

package/src/pipelines.ts

@@ -238,7 +238,7 @@ export const PIPELINE_DATA = {
 	},
 	"feature-extraction": {
 		name: "Feature Extraction",
-		modality: "multimodal",
+		modality: "nlp",
 		color: "red",
 	},
 	"text-generation": {
@@ -419,7 +419,7 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-image": {
 		name: "Text-to-Image",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-text": {
@@ -430,7 +430,7 @@ export const PIPELINE_DATA = {
 				name: "Image Captioning",
 			},
 		],
-		modality: "multimodal",
+		modality: "cv",
 		color: "red",
 	},
 	"image-to-image": {
@@ -454,7 +454,7 @@ export const PIPELINE_DATA = {
 	},
 	"image-to-video": {
 		name: "Image-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "indigo",
 	},
 	"unconditional-image-generation": {
@@ -589,9 +589,15 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-video": {
 		name: "Text-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
+	"image-text-to-text": {
+		name: "Image + Text to Image (VLLMs)",
+		modality: "multimodal",
+		color: "red",
+		hideInDatasets: true,
+	},
 	"visual-question-answering": {
 		name: "Visual Question Answering",
 		subtasks: [
@@ -622,7 +628,7 @@ export const PIPELINE_DATA = {
 	},
 	"graph-ml": {
 		name: "Graph Machine Learning",
-		modality: "multimodal",
+		modality: "other",
 		color: "green",
 	},
 	"mask-generation": {
@@ -637,12 +643,12 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-3d": {
 		name: "Text-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-3d": {
 		name: "Image-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
 	other: {

package/src/tasks/audio-classification/inference.ts

@@ -10,7 +10,7 @@ export interface AudioClassificationInput {
 	/**
 	 * The input audio data
 	 */
-	data: unknown;
+	inputs: unknown;
 	/**
 	 * Additional inference parameters
 	 */
@@ -23,11 +23,11 @@ export interface AudioClassificationInput {
  * Additional inference parameters for Audio Classification
  */
 export interface AudioClassificationParameters {
-	functionToApply?: ClassificationOutputTransform;
+	function_to_apply?: ClassificationOutputTransform;
 	/**
 	 * When specified, limits the output to the top K most probable classes.
 	 */
-	topK?: number;
+	top_k?: number;
 	[property: string]: unknown;
 }
 /**
@@ -40,7 +40,7 @@ export type AudioClassificationOutput = AudioClassificationOutputElement[];
  */
 export interface AudioClassificationOutputElement {
 	/**
-	 * The predicted class label (model specific).
+	 * The predicted class label.
 	 */
 	label: string;
 	/**

package/src/tasks/audio-classification/spec/input.json

@@ -5,7 +5,7 @@
 	"title": "AudioClassificationInput",
 	"type": "object",
 	"properties": {
-		"data": {
+		"inputs": {
 			"description": "The input audio data"
 		},
 		"parameters": {
@@ -19,16 +19,16 @@
 			"description": "Additional inference parameters for Audio Classification",
 			"type": "object",
 			"properties": {
-				"functionToApply": {
+				"function_to_apply": {
 					"title": "AudioClassificationOutputTransform",
 					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
 				},
-				"topK": {
+				"top_k": {
 					"type": "integer",
 					"description": "When specified, limits the output to the top K most probable classes."
 				}
 			}
 		}
 	},
-	"required": ["data"]
+	"required": ["inputs"]
 }

package/src/tasks/audio-classification/spec/output.json

@@ -5,17 +5,6 @@
 	"description": "Outputs for Audio Classification inference",
 	"type": "array",
 	"items": {
-		"type": "object",
-		"properties": {
-			"label": {
-				"type": "string",
-				"description": "The predicted class label (model specific)."
-			},
-			"score": {
-				"type": "number",
-				"description": "The corresponding probability."
-			}
-		},
-		"required": ["label", "score"]
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
 	}
 }