@huggingface/tasks 0.2.1 → 0.3.0

package/src/tasks/token-classification/spec/output.json

@@ -0,0 +1,33 @@
+{
+	"$id": "/inference/schemas/token-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Token Classification task",
+	"title": "TokenClassificationOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"entity_group": {
+				"type": "string",
+				"description": "The predicted label for that group of tokens"
+			},
+			"score": {
+				"type": "number",
+				"description": "The associated score / probability"
+			},
+			"word": {
+				"type": "string",
+				"description": "The corresponding text"
+			},
+			"start": {
+				"type": "integer",
+				"description": "The character position in the input where this group begins."
+			},
+			"end": {
+				"type": "integer",
+				"description": "The character position in the input where this group ends."
+			}
+		},
+		"required": ["label", "score"]
+	}
+}

package/src/tasks/translation/data.ts

@@ -1,6 +1,7 @@
 import type { TaskDataCustom } from "..";
 
 const taskData: TaskDataCustom = {
+	canonicalId: "text2text-generation",
 	datasets: [
 		{
 			description: "A dataset of copyright-free books translated into 16 different languages.",

package/src/tasks/translation/inference.ts

@@ -0,0 +1,59 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Translation inference
+ *
+ * Inputs for Text2text Generation inference
+ */
+export interface TranslationInput {
+	/**
+	 * The input text data
+	 */
+	inputs: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: Text2TextGenerationParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Text2text Generation
+ */
+export interface Text2TextGenerationParameters {
+	/**
+	 * Whether to clean up the potential extra spaces in the text output.
+	 */
+	clean_up_tokenization_spaces?: boolean;
+	/**
+	 * Additional parametrization of the text generation algorithm
+	 */
+	generate_parameters?: { [key: string]: unknown };
+	/**
+	 * The truncation strategy to use
+	 */
+	truncation?: Text2TextGenerationTruncationStrategy;
+	[property: string]: unknown;
+}
+
+export type Text2TextGenerationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second";
+
+/**
+ * Outputs for Translation inference
+ *
+ * Outputs of inference for the Text2text Generation task
+ */
+export interface TranslationOutput {
+	generatedText: unknown;
+	/**
+	 * The generated text.
+	 */
+	generated_text?: string;
+	[property: string]: unknown;
+}

package/src/tasks/translation/spec/input.json

@@ -0,0 +1,7 @@
+{
+	"$ref": "/inference/schemas/text2text-generation/input.json",
+	"$id": "/inference/schemas/translation/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"title": "TranslationInput",
+	"description": "Inputs for Translation inference"
+}

package/src/tasks/translation/spec/output.json

@@ -0,0 +1,7 @@
+{
+	"$ref": "/inference/schemas/text2text-generation/output.json",
+	"$id": "/inference/schemas/translation/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"title": "TranslationOutput",
+	"description": "Outputs for Translation inference"
+}

package/src/tasks/video-classification/inference.ts

@@ -0,0 +1,59 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Video Classification inference
+ */
+export interface VideoClassificationInput {
+	/**
+	 * The input video data
+	 */
+	inputs: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: VideoClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Video Classification
+ */
+export interface VideoClassificationParameters {
+	/**
+	 * The sampling rate used to select frames from the video.
+	 */
+	frame_sampling_rate?: number;
+	function_to_apply?: ClassificationOutputTransform;
+	/**
+	 * The number of sampled frames to consider for classification.
+	 */
+	num_frames?: number;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type VideoClassificationOutput = VideoClassificationOutputElement[];
+/**
+ * Outputs of inference for the Video Classification task
+ */
+export interface VideoClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/video-classification/spec/input.json

@@ -0,0 +1,42 @@
+{
+	"$id": "/inference/schemas/video-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Video Classification inference",
+	"title": "VideoClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input video data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/VideoClassificationParameters"
+		}
+	},
+	"$defs": {
+		"VideoClassificationParameters": {
+			"title": "VideoClassificationParameters",
+			"description": "Additional inference parameters for Video Classification",
+			"type": "object",
+			"properties": {
+				"function_to_apply": {
+					"title": "TextClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"num_frames": {
+					"type": "integer",
+					"description": "The number of sampled frames to consider for classification."
+				},
+				"frame_sampling_rate": {
+					"type": "integer",
+					"description": "The sampling rate used to select frames from the video."
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/video-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/video-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Video Classification task",
+	"title": "VideoClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/visual-question-answering/inference.ts

@@ -0,0 +1,63 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Visual Question Answering inference
+ */
+export interface VisualQuestionAnsweringInput {
+	/**
+	 * One (image, question) pair to answer
+	 */
+	inputs: VisualQuestionAnsweringInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: VisualQuestionAnsweringParameters;
+	[property: string]: unknown;
+}
+/**
+ * One (image, question) pair to answer
+ */
+export interface VisualQuestionAnsweringInputData {
+	/**
+	 * The image.
+	 */
+	image: unknown;
+	/**
+	 * The question to answer based on the image.
+	 */
+	question: unknown;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Visual Question Answering
+ */
+export interface VisualQuestionAnsweringParameters {
+	/**
+	 * The number of answers to return (will be chosen by order of likelihood). Note that we
+	 * return less than topk answers if there are not enough options available within the
+	 * context.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+export type VisualQuestionAnsweringOutput = VisualQuestionAnsweringOutputElement[];
+/**
+ * Outputs of inference for the Visual Question Answering task
+ */
+export interface VisualQuestionAnsweringOutputElement {
+	/**
+	 * The answer to the question
+	 */
+	answer?: string;
+	label: unknown;
+	/**
+	 * The associated score / probability
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/visual-question-answering/spec/input.json

@@ -0,0 +1,41 @@
+{
+	"$id": "/inference/schemas/visual-question-answering/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Visual Question Answering inference",
+	"title": "VisualQuestionAnsweringInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "One (image, question) pair to answer",
+			"type": "object",
+			"title": "VisualQuestionAnsweringInputData",
+			"properties": {
+				"image": {
+					"description": "The image."
+				},
+				"question": {
+					"description": "The question to answer based on the image."
+				}
+			},
+			"required": ["question", "image"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/VisualQuestionAnsweringParameters"
+		}
+	},
+	"$defs": {
+		"VisualQuestionAnsweringParameters": {
+			"title": "VisualQuestionAnsweringParameters",
+			"description": "Additional inference parameters for Visual Question Answering",
+			"type": "object",
+			"properties": {
+				"top_k": {
+					"type": "integer",
+					"description": "The number of answers to return (will be chosen by order of likelihood). Note that we return less than topk answers if there are not enough options available within the context."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/visual-question-answering/spec/output.json

@@ -0,0 +1,21 @@
+{
+	"$id": "/inference/schemas/visual-question-answering/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Visual Question Answering task",
+	"title": "VisualQuestionAnsweringOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"answer": {
+				"type": "string",
+				"description": "The answer to the question"
+			},
+			"score": {
+				"type": "number",
+				"description": "The associated score / probability"
+			}
+		},
+		"required": ["label", "score"]
+	}
+}

package/src/tasks/zero-shot-classification/inference.ts

@@ -0,0 +1,67 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Zero Shot Classification inference
+ */
+export interface ZeroShotClassificationInput {
+	/**
+	 * The input text data, with candidate labels
+	 */
+	inputs: ZeroShotClassificationInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ZeroShotClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * The input text data, with candidate labels
+ */
+export interface ZeroShotClassificationInputData {
+	/**
+	 * The set of possible class labels to classify the text into.
+	 */
+	candidateLabels: string[];
+	/**
+	 * The text to classify
+	 */
+	text: string;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Zero Shot Classification
+ */
+export interface ZeroShotClassificationParameters {
+	/**
+	 * The sentence used in conjunction with candidateLabels to attempt the text classification
+	 * by replacing the placeholder with the candidate labels.
+	 */
+	hypothesis_template?: string;
+	/**
+	 * Whether multiple candidate labels can be true. If false, the scores are normalized such
+	 * that the sum of the label likelihoods for each sequence is 1. If true, the labels are
+	 * considered independent and probabilities are normalized for each candidate.
+	 */
+	multi_label?: boolean;
+	[property: string]: unknown;
+}
+export type ZeroShotClassificationOutput = ZeroShotClassificationOutputElement[];
+/**
+ * Outputs of inference for the Zero Shot Classification task
+ */
+export interface ZeroShotClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/zero-shot-classification/spec/input.json

@@ -0,0 +1,50 @@
+{
+	"$id": "/inference/schemas/zero-shot-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Zero Shot Classification inference",
+	"title": "ZeroShotClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input text data, with candidate labels",
+			"type": "object",
+			"title": "ZeroShotClassificationInputData",
+			"properties": {
+				"text": {
+					"type": "string",
+					"description": "The text to classify"
+				},
+				"candidateLabels": {
+					"type": "array",
+					"description": "The set of possible class labels to classify the text into.",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": ["text", "candidateLabels"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ZeroShotClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ZeroShotClassificationParameters": {
+			"title": "ZeroShotClassificationParameters",
+			"description": "Additional inference parameters for Zero Shot Classification",
+			"type": "object",
+			"properties": {
+				"hypothesis_template": {
+					"type": "string",
+					"description": "The sentence used in conjunction with candidateLabels to attempt the text classification by replacing the placeholder with the candidate labels."
+				},
+				"multi_label": {
+					"type": "boolean",
+					"description": "Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of the label likelihoods for each sequence is 1. If true, the labels are considered independent and probabilities are normalized for each candidate."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/zero-shot-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/zero-shot-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Zero Shot Classification task",
+	"title": "ZeroShotClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/zero-shot-image-classification/data.ts

@@ -52,9 +52,8 @@ const taskData: TaskDataCustom = {
 			id: "openai/clip-vit-base-patch16",
 		},
 		{
-			description:
-				"Robust image classification model trained on publicly available image-caption data trained on additional high pixel data for better performance.",
-			id: "openai/clip-vit-large-patch14-336",
+			description: "Strong zero-shot image classification model.",
+			id: "google/siglip-base-patch16-224",
 		},
 		{
 			description: "Strong image classification model for biomedical domain.",
@@ -64,12 +63,16 @@ const taskData: TaskDataCustom = {
 	spaces: [
 		{
 			description:
-				"An application that leverages zero shot image classification to find best captions to generate an image. ",
+				"An application that leverages zero-shot image classification to find best captions to generate an image. ",
 			id: "pharma/CLIP-Interrogator",
 		},
+		{
+			description: "An application to compare different zero-shot image classification models. ",
+			id: "merve/compare_clip_siglip",
+		},
 	],
 	summary:
-		"Zero shot image classification is the task of classifying previously unseen classes during training of a model.",
+		"Zero-shot image classification is the task of classifying previously unseen classes during training of a model.",
 	widgetModels: ["openai/clip-vit-large-patch14-336"],
 	youtubeId: "",
 };

package/src/tasks/zero-shot-image-classification/inference.ts

@@ -0,0 +1,61 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Zero Shot Image Classification inference
+ */
+export interface ZeroShotImageClassificationInput {
+	/**
+	 * The input image data, with candidate labels
+	 */
+	inputs: ZeroShotImageClassificationInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ZeroShotImageClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * The input image data, with candidate labels
+ */
+export interface ZeroShotImageClassificationInputData {
+	/**
+	 * The candidate labels for this image
+	 */
+	candidateLabels: string[];
+	/**
+	 * The image data to classify
+	 */
+	image: unknown;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Zero Shot Image Classification
+ */
+export interface ZeroShotImageClassificationParameters {
+	/**
+	 * The sentence used in conjunction with candidateLabels to attempt the text classification
+	 * by replacing the placeholder with the candidate labels.
+	 */
+	hypothesis_template?: string;
+	[property: string]: unknown;
+}
+export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputElement[];
+/**
+ * Outputs of inference for the Zero Shot Image Classification task
+ */
+export interface ZeroShotImageClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/zero-shot-image-classification/spec/input.json

@@ -0,0 +1,45 @@
+{
+	"$id": "/inference/schemas/zero-shot-image-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Zero Shot Image Classification inference",
+	"title": "ZeroShotImageClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input image data, with candidate labels",
+			"type": "object",
+			"title": "ZeroShotImageClassificationInputData",
+			"properties": {
+				"image": {
+					"description": "The image data to classify"
+				},
+				"candidateLabels": {
+					"description": "The candidate labels for this image",
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": ["image", "candidateLabels"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ZeroShotImageClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ZeroShotImageClassificationParameters": {
+			"title": "ZeroShotImageClassificationParameters",
+			"description": "Additional inference parameters for Zero Shot Image Classification",
+			"type": "object",
+			"properties": {
+				"hypothesis_template": {
+					"type": "string",
+					"description": "The sentence used in conjunction with candidateLabels to attempt the text classification by replacing the placeholder with the candidate labels."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/zero-shot-image-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/zero-shot-image-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Zero Shot Image Classification task",
+	"title": "ZeroShotImageClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/zero-shot-object-detection/about.md

@@ -1,5 +1,7 @@
 ## Use Cases
 
+Zero-shot object detection models can be used in any object detection application where the detection involves text queries for objects of interest.
+
 ### Object Search
 
 Zero-shot object detection models can be used in image search. Smartphones, for example, use zero-shot object detection models to detect entities (such as specific places or objects) and allow the user to search for the entity on the internet.
@@ -8,6 +10,10 @@ Zero-shot object detection models can be used in image search. Smartphones, for
 
 Zero-shot object detection models are used to count instances of objects in a given image. This can include counting the objects in warehouses or stores or the number of visitors in a store. They are also used to manage crowds at events to prevent disasters.
 
+### Object Tracking
+
+Zero-shot object detectors can track objects in videos.
+
 ## Inference
 
 You can infer with zero-shot object detection models through the `zero-shot-object-detection` pipeline. When calling the pipeline, you just need to specify a path or HTTP link to an image and the candidate labels.

package/src/tasks/zero-shot-object-detection/data.ts

@@ -47,7 +47,12 @@ const taskData: TaskDataCustom = {
 			id: "google/owlv2-base-patch16-ensemble",
 		},
 	],
-	spaces: [],
+	spaces: [
+		{
+			description: "A demo to try the state-of-the-art zero-shot object detection model, OWLv2.",
+			id: "merve/owlv2",
+		},
+	],
 	summary:
 		"Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.",
 	widgetModels: [],