@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/zero-shot-classification/about.md

@@ -0,0 +1,40 @@
+## About the Task
+
+Zero Shot Classification is the task of predicting a class that wasn't seen by the model during training. This method, which leverages a pre-trained language model, can be thought of as an instance of [transfer learning](https://www.youtube.com/watch?v=BqqfQnyjmgg) which generally refers to using a model trained for one task in a different application than what it was originally trained for. This is particularly useful for situations where the amount of labeled data is small.
+
+In zero shot classification, we provide the model with a prompt and a sequence of text that describes what we want our model to do, in natural language. Zero-shot classification excludes any examples of the desired task being completed. This differs from single or few-shot classification, as these tasks include a single or a few examples of the selected task.
+
+Zero, single and few-shot classification seem to be an emergent feature of large language models. This feature seems to come about around model sizes of +100M parameters. The effectiveness of a model at a zero, single or few-shot task seems to scale with model size, meaning that larger models (models with more trainable parameters or layers) generally do better at this task.
+
+Here is an example of a zero-shot prompt for classifying the sentiment of a sequence of text:
+
+```
+Classify the following input text into one of the following three categories: [positive, negative, neutral]
+
+Input Text: Hugging Face is awesome for making all of these
+state of the art models available!
+Sentiment: positive
+
+```
+
+One great example of this task with a nice off-the-shelf model is available at the widget of this page, where the user can input a sequence of text and candidate labels to the model. This is a _word level_ example of zero shot classification, more elaborate and lengthy generations are available with larger models. Testing these models out and getting a feel for prompt engineering is the best way to learn how to use them.
+
+## Inference
+
+You can use the 🤗 Transformers library zero-shot-classification pipeline to infer with zero shot text classification models.
+
+```python
+from transformers import pipeline
+
+pipe = pipeline(model="facebook/bart-large-mnli")
+pipe("I have a problem with my iphone that needs to be resolved asap!",
+    candidate_labels=["urgent", "not urgent", "phone", "tablet", "computer"],
+)
+# output
+>>> {'sequence': 'I have a problem with my iphone that needs to be resolved asap!!', 'labels': ['urgent', 'phone', 'computer', 'not urgent', 'tablet'], 'scores': [0.504, 0.479, 0.013, 0.003, 0.002]}
+```
+
+## Useful Resources
+
+- [Zero Shot Learning](https://joeddav.github.io/blog/2020/05/29/ZSL.html)
+- [Hugging Face on Transfer Learning](https://huggingface.co/course/en/chapter1/4?fw=pt#transfer-learning)

package/src/tasks/zero-shot-classification/data.ts

@@ -0,0 +1,70 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A widely used dataset used to benchmark multiple variants of text classification.",
+			id: "nyu-mll/glue",
+		},
+		{
+			description:
+				"The Multi-Genre Natural Language Inference (MultiNLI) corpus is a crowd-sourced collection of 433k sentence pairs annotated with textual entailment information.",
+			id: "nyu-mll/multi_nli",
+		},
+		{
+			description:
+				"FEVER is a publicly available dataset for fact extraction and verification against textual sources.",
+			id: "fever/fever",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Text Input",
+				content: "Dune is the best movie ever.",
+				type: "text",
+			},
+			{
+				label: "Candidate Labels",
+				content: "CINEMA, ART, MUSIC",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "CINEMA",
+						score: 0.9,
+					},
+					{
+						label: "ART",
+						score: 0.1,
+					},
+					{
+						label: "MUSIC",
+						score: 0.0,
+					},
+				],
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "Powerful zero-shot text classification model.",
+			id: "facebook/bart-large-mnli",
+		},
+		{
+			description: "Powerful zero-shot multilingual text classification model that can accomplish multiple tasks.",
+			id: "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
+		},
+	],
+	spaces: [],
+	summary:
+		"Zero-shot text classification is a task in natural language processing where a model is trained on a set of labeled examples but is then able to classify new examples from previously unseen classes.",
+	widgetModels: ["facebook/bart-large-mnli"],
+};
+
+export default taskData;

package/src/tasks/zero-shot-classification/inference.ts

@@ -0,0 +1,67 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Zero Shot Classification inference
+ */
+export interface ZeroShotClassificationInput {
+	/**
+	 * The input text data, with candidate labels
+	 */
+	inputs: ZeroShotClassificationInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ZeroShotClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * The input text data, with candidate labels
+ */
+export interface ZeroShotClassificationInputData {
+	/**
+	 * The set of possible class labels to classify the text into.
+	 */
+	candidateLabels: string[];
+	/**
+	 * The text to classify
+	 */
+	text: string;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Zero Shot Classification
+ */
+export interface ZeroShotClassificationParameters {
+	/**
+	 * The sentence used in conjunction with candidateLabels to attempt the text classification
+	 * by replacing the placeholder with the candidate labels.
+	 */
+	hypothesis_template?: string;
+	/**
+	 * Whether multiple candidate labels can be true. If false, the scores are normalized such
+	 * that the sum of the label likelihoods for each sequence is 1. If true, the labels are
+	 * considered independent and probabilities are normalized for each candidate.
+	 */
+	multi_label?: boolean;
+	[property: string]: unknown;
+}
+export type ZeroShotClassificationOutput = ZeroShotClassificationOutputElement[];
+/**
+ * Outputs of inference for the Zero Shot Classification task
+ */
+export interface ZeroShotClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/zero-shot-classification/spec/input.json

@@ -0,0 +1,50 @@
+{
+	"$id": "/inference/schemas/zero-shot-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Zero Shot Classification inference",
+	"title": "ZeroShotClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data, with candidate labels",
+			"type": "object",
+			"title": "ZeroShotClassificationInputData",
+			"properties": {
+				"text": {
+					"type": "string",
+					"description": "The text to classify"
+				},
+				"candidateLabels": {
+					"type": "array",
+					"description": "The set of possible class labels to classify the text into.",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": ["text", "candidateLabels"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ZeroShotClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ZeroShotClassificationParameters": {
+			"title": "ZeroShotClassificationParameters",
+			"description": "Additional inference parameters for Zero Shot Classification",
+			"type": "object",
+			"properties": {
+				"hypothesis_template": {
+					"type": "string",
+					"description": "The sentence used in conjunction with candidateLabels to attempt the text classification by replacing the placeholder with the candidate labels."
+				},
+				"multi_label": {
+					"type": "boolean",
+					"description": "Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of the label likelihoods for each sequence is 1. If true, the labels are considered independent and probabilities are normalized for each candidate."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/zero-shot-classification/spec/output.json

@@ -0,0 +1,11 @@
+{
+	"$id": "/inference/schemas/zero-shot-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Zero Shot Classification task",
+	"title": "ZeroShotClassificationOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/zero-shot-image-classification/about.md

@@ -0,0 +1,75 @@
+## About the Task
+
+Zero-shot image classification is a computer vision task to classify images into one of several classes, without any prior training or knowledge of the classes.
+
+Zero shot image classification works by transferring knowledge learnt during training of one model, to classify novel classes that was not present in the training data. So this is a variation of [transfer learning](https://www.youtube.com/watch?v=BqqfQnyjmgg). For instance, a model trained to differentiate cars from airplanes can be used to classify images of ships.
+
+The data in this learning paradigm consists of
+
+- Seen data - images and their corresponding labels
+- Unseen data - only labels and no images
+- Auxiliary information - additional information given to the model during training connecting the unseen and seen data. This can be in the form of textual description or word embeddings.
+
+## Use Cases
+
+### Image Retrieval
+
+Zero-shot learning resolves several challenges in image retrieval systems. For example, with the rapid growth of categories on the web, it is challenging to index images based on unseen categories. With zero-shot learning we can associate unseen categories to images by exploiting attributes to model the relationships among visual features and labels.
+
+### Action Recognition
+
+Action recognition is the task of identifying when a person in an image/video is performing a given action from a set of actions. If all the possible actions are not known beforehand, conventional deep learning models fail. With zero-shot learning, for a given domain of a set of actions, we can create a mapping connecting low-level features and a semantic description of auxiliary data to classify unknown classes of actions.
+
+## Task Variants
+
+You can contribute variants of this task [here](https://github.com/huggingface/hub-docs/blob/main/tasks/src/zero-shot-image-classification/about.md).
+
+## Inference
+
+The model can be loaded with the zero-shot-image-classification pipeline like so:
+
+```python
+from transformers import pipeline
+# More models in the model hub.
+model_name = "openai/clip-vit-large-patch14-336"
+classifier = pipeline("zero-shot-image-classification", model = model_name)
+```
+
+You can then use this pipeline to classify images into any of the class names you specify. You can specify more than two class labels too.
+
+```python
+image_to_classify = "path_to_cat_and_dog_image.jpeg"
+labels_for_classification =  ["cat and dog",
+                              "lion and cheetah",
+                              "rabbit and lion"]
+scores = classifier(image_to_classify,
+                    candidate_labels = labels_for_classification)
+```
+
+The classifier would return a list of dictionaries after the inference which is stored in the variable `scores` in the code snippet above. Variable `scores` would look as follows:
+
+```python
+[{'score': 0.9950482249259949, 'label': 'cat and dog'},
+{'score': 0.004863627254962921, 'label': 'rabbit and lion'},
+{'score': 8.816882473183796e-05, 'label': 'lion and cheetah'}]
+```
+
+The dictionary at the zeroth index of the list will contain the label with the highest score.
+
+```python
+print(f"The highest score is {scores[0]['score']:.3f} for the label {scores[0]['label']}")
+```
+
+The output from the print statement above would look as follows:
+
+```
+The highest probability is 0.995 for the label cat and dog
+```
+
+## Useful Resources
+
+- [Zero-shot image classification task guide](https://huggingface.co/docs/transformers/tasks/zero_shot_image_classification).
+- [Image-text Similarity Search](https://huggingface.co/learn/cookbook/faiss_with_hf_datasets_and_clip)
+
+This page was made possible thanks to the efforts of [Shamima Hossain](https://huggingface.co/Shamima), [Haider Zaidi
+](https://huggingface.co/chefhaider) and [Paarth Bhatnagar](https://huggingface.co/Paarth).

package/src/tasks/zero-shot-image-classification/data.ts

@@ -0,0 +1,84 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			// TODO write proper description
+			description: "",
+			id: "",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "image-classification-input.jpeg",
+				type: "img",
+			},
+			{
+				label: "Classes",
+				content: "cat, dog, bird",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "Cat",
+						score: 0.664,
+					},
+					{
+						label: "Dog",
+						score: 0.329,
+					},
+					{
+						label: "Bird",
+						score: 0.008,
+					},
+				],
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "Computes the number of times the correct label appears in top K labels predicted",
+			id: "top-K accuracy",
+		},
+	],
+	models: [
+		{
+			description: "Robust image classification model trained on publicly available image-caption data.",
+			id: "openai/clip-vit-base-patch16",
+		},
+		{
+			description: "Strong zero-shot image classification model.",
+			id: "google/siglip-so400m-patch14-224",
+		},
+		{
+			description: "Small yet powerful zero-shot image classification model that can run on edge devices.",
+			id: "apple/MobileCLIP-S1-OpenCLIP",
+		},
+		{
+			description: "Strong image classification model for biomedical domain.",
+			id: "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224",
+		},
+	],
+	spaces: [
+		{
+			description:
+				"An application that leverages zero-shot image classification to find best captions to generate an image. ",
+			id: "pharma/CLIP-Interrogator",
+		},
+		{
+			description: "An application to compare different zero-shot image classification models. ",
+			id: "merve/compare_clip_siglip",
+		},
+	],
+	summary:
+		"Zero-shot image classification is the task of classifying previously unseen classes during training of a model.",
+	widgetModels: ["google/siglip-so400m-patch14-224"],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/zero-shot-image-classification/inference.ts

@@ -0,0 +1,61 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Zero Shot Image Classification inference
+ */
+export interface ZeroShotImageClassificationInput {
+	/**
+	 * The input image data, with candidate labels
+	 */
+	inputs: ZeroShotImageClassificationInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ZeroShotImageClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * The input image data, with candidate labels
+ */
+export interface ZeroShotImageClassificationInputData {
+	/**
+	 * The candidate labels for this image
+	 */
+	candidateLabels: string[];
+	/**
+	 * The image data to classify
+	 */
+	image: unknown;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Zero Shot Image Classification
+ */
+export interface ZeroShotImageClassificationParameters {
+	/**
+	 * The sentence used in conjunction with candidateLabels to attempt the text classification
+	 * by replacing the placeholder with the candidate labels.
+	 */
+	hypothesis_template?: string;
+	[property: string]: unknown;
+}
+export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputElement[];
+/**
+ * Outputs of inference for the Zero Shot Image Classification task
+ */
+export interface ZeroShotImageClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/zero-shot-image-classification/spec/input.json

@@ -0,0 +1,45 @@
+{
+	"$id": "/inference/schemas/zero-shot-image-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Zero Shot Image Classification inference",
+	"title": "ZeroShotImageClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input image data, with candidate labels",
+			"type": "object",
+			"title": "ZeroShotImageClassificationInputData",
+			"properties": {
+				"image": {
+					"description": "The image data to classify"
+				},
+				"candidateLabels": {
+					"description": "The candidate labels for this image",
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": ["image", "candidateLabels"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ZeroShotImageClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ZeroShotImageClassificationParameters": {
+			"title": "ZeroShotImageClassificationParameters",
+			"description": "Additional inference parameters for Zero Shot Image Classification",
+			"type": "object",
+			"properties": {
+				"hypothesis_template": {
+					"type": "string",
+					"description": "The sentence used in conjunction with candidateLabels to attempt the text classification by replacing the placeholder with the candidate labels."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/zero-shot-image-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/zero-shot-image-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Zero Shot Image Classification task",
+	"title": "ZeroShotImageClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/zero-shot-object-detection/about.md

@@ -0,0 +1,45 @@
+## Use Cases
+
+Zero-shot object detection models can be used in any object detection application where the detection involves text queries for objects of interest.
+
+### Object Search
+
+Zero-shot object detection models can be used in image search. Smartphones, for example, use zero-shot object detection models to detect entities (such as specific places or objects) and allow the user to search for the entity on the internet.
+
+### Object Counting
+
+Zero-shot object detection models are used to count instances of objects in a given image. This can include counting the objects in warehouses or stores or the number of visitors in a store. They are also used to manage crowds at events to prevent disasters.
+
+### Object Tracking
+
+Zero-shot object detectors can track objects in videos.
+
+## Inference
+
+You can infer with zero-shot object detection models through the `zero-shot-object-detection` pipeline. When calling the pipeline, you just need to specify a path or HTTP link to an image and the candidate labels.
+
+```python
+from transformers import pipeline
+from PIL import Image
+
+image = Image.open("my-image.png").convert("RGB")
+
+detector = pipeline(model="google/owlvit-base-patch32", task="zero-shot-object-detection")
+
+predictions = detector(
+    image,
+    candidate_labels=["a photo of a cat", "a photo of a dog"],
+)
+
+# [{'score': 0.95,
+#   'label': 'a photo of a cat',
+#   'box': {'xmin': 180, 'ymin': 71, 'xmax': 271, 'ymax': 178}},
+#   ...
+# ]
+```
+
+# Useful Resources
+
+- [Zero-shot object detection task guide](https://huggingface.co/docs/transformers/tasks/zero_shot_object_detection)
+
+This page was made possible thanks to the efforts of [Victor Guichard](https://huggingface.co/VictorGuichard)

package/src/tasks/zero-shot-object-detection/data.ts

@@ -0,0 +1,67 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [],
+	demo: {
+		inputs: [
+			{
+				filename: "zero-shot-object-detection-input.jpg",
+				type: "img",
+			},
+			{
+				label: "Classes",
+				content: "cat, dog, bird",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "zero-shot-object-detection-output.jpg",
+				type: "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately",
+			id: "Average Precision",
+		},
+		{
+			description: "The Mean Average Precision (mAP) metric is the overall average of the AP values",
+			id: "Mean Average Precision",
+		},
+		{
+			description:
+				"The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75",
+			id: "APα",
+		},
+	],
+	models: [
+		{
+			description: "Solid zero-shot object detection model.",
+			id: "IDEA-Research/grounding-dino-base",
+		},
+		{
+			description: "Cutting-edge zero-shot object detection model.",
+			id: "google/owlv2-base-patch16-ensemble",
+		},
+	],
+	spaces: [
+		{
+			description: "A demo to try the state-of-the-art zero-shot object detection model, OWLv2.",
+			id: "merve/owlv2",
+		},
+		{
+			description:
+				"A demo that combines a zero-shot object detection and mask generation model for zero-shot segmentation.",
+			id: "merve/OWLSAM",
+		},
+	],
+	summary:
+		"Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.",
+	widgetModels: [],
+	youtubeId: "",
+};
+
+export default taskData;