@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/fill-mask/about.md

@@ -0,0 +1,51 @@
+## Use Cases
+
+### Domain Adaptation 👩‍⚕️
+
+Masked language models do not require labelled data! They are trained by masking a couple of words in sentences and the model is expected to guess the masked word. This makes it very practical!
+
+For example, masked language modeling is used to train large models for domain-specific problems. If you have to work on a domain-specific task, such as retrieving information from medical research papers, you can train a masked language model using those papers. 📄
+
+The resulting model has a statistical understanding of the language used in medical research papers, and can be further trained in a process called fine-tuning to solve different tasks, such as [Text Classification](/tasks/text-classification) or [Question Answering](/tasks/question-answering) to build a medical research papers information extraction system. 👩‍⚕️ Pre-training on domain-specific data tends to yield better results (see [this paper](https://arxiv.org/abs/2007.15779) for an example).
+
+If you don't have the data to train a masked language model, you can also use an existing [domain-specific masked language model](https://huggingface.co/microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext) from the Hub and fine-tune it with your smaller task dataset. That's the magic of Open Source and sharing your work! 🎉
+
+## Inference with Fill-Mask Pipeline
+
+You can use the 🤗 Transformers library `fill-mask` pipeline to do inference with masked language models. If a model name is not provided, the pipeline will be initialized with [distilroberta-base](/distilroberta-base). You can provide masked text and it will return a list of possible mask values ​​ranked according to the score.
+
+```python
+from transformers import pipeline
+
+classifier = pipeline("fill-mask")
+classifier("Paris is the <mask> of France.")
+
+# [{'score': 0.7, 'sequence': 'Paris is the capital of France.'},
+# {'score': 0.2, 'sequence': 'Paris is the birthplace of France.'},
+# {'score': 0.1, 'sequence': 'Paris is the heart of France.'}]
+```
+
+## Useful Resources
+
+Would you like to learn more about the topic? Awesome! Here you can find some curated resources that can be helpful to you!
+
+- [Course Chapter on Fine-tuning a Masked Language Model](https://huggingface.co/course/chapter7/3?fw=pt)
+- [Workshop on Pretraining Language Models and CodeParrot](https://www.youtube.com/watch?v=ExUR7w6xe94)
+- [BERT 101: State Of The Art NLP Model Explained](https://huggingface.co/blog/bert-101)
+- [Nyströmformer: Approximating self-attention in linear time and memory via the Nyström method](https://huggingface.co/blog/nystromformer)
+
+### Notebooks
+
+- [Pre-training an MLM for JAX/Flax](https://github.com/huggingface/notebooks/blob/master/examples/masked_language_modeling_flax.ipynb)
+- [Masked language modeling in TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling-tf.ipynb)
+- [Masked language modeling in PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling.ipynb)
+
+### Scripts for training
+
+- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling)
+- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling)
+- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling)
+
+### Documentation
+
+- [Masked language modeling task guide](https://huggingface.co/docs/transformers/tasks/masked_language_modeling)

package/src/tasks/fill-mask/data.ts

@@ -0,0 +1,79 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A common dataset that is used to train models for many languages.",
+			id: "wikipedia",
+		},
+		{
+			description: "A large English dataset with text crawled from the web.",
+			id: "c4",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Input",
+				content: "The <mask> barked at me",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "wolf",
+						score: 0.487,
+					},
+					{
+						label: "dog",
+						score: 0.061,
+					},
+					{
+						label: "cat",
+						score: 0.058,
+					},
+					{
+						label: "fox",
+						score: 0.047,
+					},
+					{
+						label: "squirrel",
+						score: 0.025,
+					},
+				],
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words",
+			id: "cross_entropy",
+		},
+		{
+			description:
+				"Perplexity is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance",
+			id: "perplexity",
+		},
+	],
+	models: [
+		{
+			description: "The famous BERT model.",
+			id: "google-bert/bert-base-uncased",
+		},
+		{
+			description: "A multilingual model trained on 100 languages.",
+			id: "FacebookAI/xlm-roberta-base",
+		},
+	],
+	spaces: [],
+	summary:
+		"Masked language modeling is the task of masking some of the words in a sentence and predicting which words should replace those masks. These models are useful when we want to get a statistical understanding of the language in which the model is trained in.",
+	widgetModels: ["distilroberta-base"],
+	youtubeId: "mqElG5QJWUg",
+};
+
+export default taskData;

package/src/tasks/fill-mask/inference.ts

@@ -0,0 +1,62 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Fill Mask inference
+ */
+export interface FillMaskInput {
+	/**
+	 * The text with masked tokens
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: FillMaskParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Fill Mask
+ */
+export interface FillMaskParameters {
+	/**
+	 * When passed, the model will limit the scores to the passed targets instead of looking up
+	 * in the whole vocabulary. If the provided targets are not in the model vocab, they will be
+	 * tokenized and the first resulting token will be used (with a warning, and that might be
+	 * slower).
+	 */
+	targets?: string[];
+	/**
+	 * When passed, overrides the number of predictions to return.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+export type FillMaskOutput = FillMaskOutputElement[];
+/**
+ * Outputs of inference for the Fill Mask task
+ */
+export interface FillMaskOutputElement {
+	/**
+	 * The corresponding probability
+	 */
+	score: number;
+	/**
+	 * The corresponding input with the mask token prediction.
+	 */
+	sequence: string;
+	/**
+	 * The predicted token id (to replace the masked one).
+	 */
+	token: number;
+	tokenStr: unknown;
+	/**
+	 * The predicted token (to replace the masked one).
+	 */
+	token_str?: string;
+	[property: string]: unknown;
+}

package/src/tasks/fill-mask/spec/input.json

@@ -0,0 +1,38 @@
+{
+	"$id": "/inference/schemas/fill-mask/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Fill Mask inference",
+	"title": "FillMaskInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The text with masked tokens",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/FillMaskParameters"
+		}
+	},
+	"$defs": {
+		"FillMaskParameters": {
+			"title": "FillMaskParameters",
+			"description": "Additional inference parameters for Fill Mask",
+			"type": "object",
+			"properties": {
+				"top_k": {
+					"type": "integer",
+					"description": "When passed, overrides the number of predictions to return."
+				},
+				"targets": {
+					"description": "When passed, the model will limit the scores to the passed targets instead of looking up in the whole vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first resulting token will be used (with a warning, and that might be slower).",
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/fill-mask/spec/output.json

@@ -0,0 +1,29 @@
+{
+	"$id": "/inference/schemas/fill-mask/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Fill Mask task",
+	"title": "FillMaskOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"sequence": {
+				"type": "string",
+				"description": "The corresponding input with the mask token prediction."
+			},
+			"score": {
+				"type": "number",
+				"description": "The corresponding probability"
+			},
+			"token": {
+				"type": "integer",
+				"description": "The predicted token id (to replace the masked one)."
+			},
+			"token_str": {
+				"type": "string",
+				"description": "The predicted token (to replace the masked one)."
+			}
+		},
+		"required": ["sequence", "score", "token", "tokenStr"]
+	}
+}

package/src/tasks/image-classification/about.md

@@ -0,0 +1,50 @@
+## Use Cases
+
+Image classification models can be used when we are not interested in specific instances of objects with location information or their shape.
+
+### Keyword Classification
+
+Image classification models are used widely in stock photography to assign each image a keyword.
+
+### Image Search
+
+Models trained in image classification can improve user experience by organizing and categorizing photo galleries on the phone or in the cloud, on multiple keywords or tags.
+
+## Inference
+
+With the `transformers` library, you can use the `image-classification` pipeline to infer with image classification models. You can initialize the pipeline with a model id from the Hub. If you do not provide a model id it will initialize with [google/vit-base-patch16-224](https://huggingface.co/google/vit-base-patch16-224) by default. When calling the pipeline you just need to specify a path, http link or an image loaded in PIL. You can also provide a `top_k` parameter which determines how many results it should return.
+
+```python
+from transformers import pipeline
+clf = pipeline("image-classification")
+clf("path_to_a_cat_image")
+
+[{'label': 'tabby cat', 'score': 0.731},
+...
+]
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to classify images using models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.imageClassification({
+	data: await (await fetch("https://picsum.photos/300/300")).blob(),
+	model: "microsoft/resnet-50",
+});
+```
+
+## Useful Resources
+
+- [Let's Play Pictionary with Machine Learning!](https://www.youtube.com/watch?v=LS9Y2wDVI0k)
+- [Fine-Tune ViT for Image Classification with 🤗Transformers](https://huggingface.co/blog/fine-tune-vit)
+- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8)
+- [Computer Vision Study Group: Swin Transformer](https://www.youtube.com/watch?v=Ngikt-K1Ecc)
+- [Computer Vision Study Group: Masked Autoencoders Paper Walkthrough](https://www.youtube.com/watch?v=Ngikt-K1Ecc)
+- [Image classification task guide](https://huggingface.co/docs/transformers/tasks/image_classification)
+
+### Creating your own image classifier in just a few minutes
+
+With [HuggingPics](https://github.com/nateraw/huggingpics), you can fine-tune Vision Transformers for anything using images found on the web. This project downloads images of classes defined by you, trains a model, and pushes it to the Hub. You even get to try out the model directly with a working widget in the browser, ready to be shared with all your friends!

package/src/tasks/image-classification/data.ts

@@ -0,0 +1,88 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			// TODO write proper description
+			description: "Benchmark dataset used for image classification with images that belong to 100 classes.",
+			id: "cifar100",
+		},
+		{
+			// TODO write proper description
+			description: "Dataset consisting of images of garments.",
+			id: "fashion_mnist",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "image-classification-input.jpeg",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "Egyptian cat",
+						score: 0.514,
+					},
+					{
+						label: "Tabby cat",
+						score: 0.193,
+					},
+					{
+						label: "Tiger cat",
+						score: 0.068,
+					},
+				],
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "",
+			id: "accuracy",
+		},
+		{
+			description: "",
+			id: "recall",
+		},
+		{
+			description: "",
+			id: "precision",
+		},
+		{
+			description: "",
+			id: "f1",
+		},
+	],
+	models: [
+		{
+			description: "A strong image classification model.",
+			id: "google/vit-base-patch16-224",
+		},
+		{
+			description: "A robust image classification model.",
+			id: "facebook/deit-base-distilled-patch16-224",
+		},
+		{
+			description: "A strong image classification model.",
+			id: "facebook/convnext-large-224",
+		},
+	],
+	spaces: [
+		{
+			// TO DO: write description
+			description: "An application that classifies what a given image is about.",
+			id: "nielsr/perceiver-image-classification",
+		},
+	],
+	summary:
+		"Image classification is the task of assigning a label or class to an entire image. Images are expected to have only one class for each image. Image classification models take an image as input and return a prediction about which class the image belongs to.",
+	widgetModels: ["google/vit-base-patch16-224"],
+	youtubeId: "tjAIM7BOYhw",
+};
+
+export default taskData;

package/src/tasks/image-classification/inference.ts

@@ -0,0 +1,52 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Classification inference
+ */
+export interface ImageClassificationInput {
+	/**
+	 * The input image data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the image data as a raw bytes payload.
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image Classification
+ */
+export interface ImageClassificationParameters {
+	function_to_apply?: ClassificationOutputTransform;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type ImageClassificationOutput = ImageClassificationOutputElement[];
+/**
+ * Outputs of inference for the Image Classification task
+ */
+export interface ImageClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/image-classification/spec/input.json

@@ -0,0 +1,35 @@
+{
+	"$id": "/inference/schemas/image-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Classification inference",
+	"title": "ImageClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"type": "string",
+			"description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload."
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ImageClassificationParameters": {
+			"title": "ImageClassificationParameters",
+			"description": "Additional inference parameters for Image Classification",
+			"type": "object",
+			"properties": {
+				"function_to_apply": {
+					"title": "ImageClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/image-classification/spec/output.json

@@ -0,0 +1,11 @@
+{
+	"$id": "/inference/schemas/image-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image Classification task",
+	"title": "ImageClassificationOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/image-feature-extraction/about.md

@@ -0,0 +1,23 @@
+## Use Cases
+
+### Transfer Learning
+
+Models trained on a specific dataset can learn features about the data. For instance, a model trained on a car classification dataset learns to recognize edges and curves on a very high level and car-specific features on a low level. This information can be transferred to a new model that is going to be trained on classifying trucks. This process of extracting features and transferring to another model is called transfer learning.
+
+### Similarity
+
+Features extracted from models contain semantically meaningful information about the world. These features can be used to detect the similarity between two images. Assume there are two images: a photo of a stray cat in a street setting and a photo of a cat at home. These images both contain cats, and the features will contain the information that there's a cat in the image. Thus, comparing the features of a stray cat photo to the features of a domestic cat photo will result in higher similarity compared to any other image that doesn't contain any cats.
+
+## Inference
+
+```python
+import torch
+from transformers import pipeline
+
+pipe = pipeline(task="image-feature-extraction", model_name="google/vit-base-patch16-384", framework="pt", pool=True)
+pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png")
+
+feature_extractor(text,return_tensors = "pt")[0].numpy().mean(axis=0)
+
+'[[[0.21236686408519745, 1.0919708013534546, 0.8512550592422485, ...]]]'
+```

package/src/tasks/image-feature-extraction/data.ts

@@ -0,0 +1,59 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description:
+				"ImageNet-1K is a image classification dataset in which images are used to train image-feature-extraction models.",
+			id: "imagenet-1k",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "mask-generation-input.png",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				table: [
+					["Dimension 1", "Dimension 2", "Dimension 3"],
+					["0.21236686408519745", "1.0919708013534546", "0.8512550592422485"],
+					["0.809657871723175", "-0.18544459342956543", "-0.7851548194885254"],
+					["1.3103108406066895", "-0.2479034662246704", "-0.9107287526130676"],
+					["1.8536205291748047", "-0.36419737339019775", "0.09717650711536407"],
+				],
+				type: "tabular",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "A powerful image feature extraction model.",
+			id: "timm/vit_large_patch14_dinov2.lvd142m",
+		},
+		{
+			description: "A strong image feature extraction model.",
+			id: "nvidia/MambaVision-T-1K",
+		},
+		{
+			description: "A robust image feature extraction model.",
+			id: "facebook/dino-vitb16",
+		},
+		{
+			description: "Strong image feature extraction model made for information retrieval from documents.",
+			id: "vidore/colpali",
+		},
+		{
+			description: "Strong image feature extraction model that can be used on images and documents.",
+			id: "OpenGVLab/InternViT-6B-448px-V1-2",
+		},
+	],
+	spaces: [],
+	summary: "Image feature extraction is the task of extracting features learnt in a computer vision model.",
+	widgetModels: [],
+};
+
+export default taskData;

package/src/tasks/image-segmentation/about.md

@@ -0,0 +1,63 @@
+## Use Cases
+
+### Autonomous Driving
+
+Segmentation models are used to identify road patterns such as lanes and obstacles for safer driving.
+
+### Background Removal
+
+Image Segmentation models are used in cameras to erase the background of certain objects and apply filters to them.
+
+### Medical Imaging
+
+Image Segmentation models are used to distinguish organs or tissues, improving medical imaging workflows. Models are used to segment dental instances, analyze X-Ray scans or even segment cells for pathological diagnosis. This [dataset](https://github.com/v7labs/covid-19-xray-dataset) contains images of lungs of healthy patients and patients with COVID-19 segmented with masks. Another [segmentation dataset](https://ivdm3seg.weebly.com/data.html) contains segmented MRI data of the lower spine to analyze the effect of spaceflight simulation.
+
+## Task Variants
+
+### Semantic Segmentation
+
+Semantic Segmentation is the task of segmenting parts of an image that belong to the same class. Semantic Segmentation models make predictions for each pixel and return the probabilities of the classes for each pixel. These models are evaluated on Mean Intersection Over Union (Mean IoU).
+
+### Instance Segmentation
+
+Instance Segmentation is the variant of Image Segmentation where every distinct object is segmented, instead of one segment per class.
+
+### Panoptic Segmentation
+
+Panoptic Segmentation is the Image Segmentation task that segments the image both by instance and by class, assigning each pixel a different instance of the class.
+
+## Inference
+
+You can infer with Image Segmentation models using the `image-segmentation` pipeline. You need to install [timm](https://github.com/rwightman/pytorch-image-models) first.
+
+```python
+!pip install timm
+model = pipeline("image-segmentation")
+model("cat.png")
+#[{'label': 'cat',
+#  'mask': mask_code,
+#  'score': 0.999}
+# ...]
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image segmentation models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_TOKEN);
+await inference.imageSegmentation({
+	data: await (await fetch("https://picsum.photos/300/300")).blob(),
+	model: "facebook/mask2former-swin-base-coco-panoptic",
+});
+```
+
+## Useful Resources
+
+Would you like to learn more about image segmentation? Great! Here you can find some curated resources that you may find helpful!
+
+- [Fine-Tune a Semantic Segmentation Model with a Custom Dataset](https://huggingface.co/blog/fine-tune-segformer)
+- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8)
+- [A Guide on Universal Image Segmentation with Mask2Former and OneFormer](https://huggingface.co/blog/mask2former)
+- [Zero-shot image segmentation with CLIPSeg](https://huggingface.co/blog/clipseg-zero-shot)
+- [Semantic segmentation task guide](https://huggingface.co/docs/transformers/tasks/semantic_segmentation)