@huggingface/tasks 0.2.1 → 0.3.0

package/src/tasks/image-to-text/inference.ts

@@ -0,0 +1,143 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Image To Text inference
+ */
+export interface ImageToTextInput {
+	/**
+	 * The input image data
+	 */
+	inputs: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageToTextParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image To Text
+ */
+export interface ImageToTextParameters {
+	/**
+	 * Parametrization of the text generation process
+	 */
+	generate?: GenerationParameters;
+	/**
+	 * The amount of maximum tokens to generate.
+	 */
+	max_new_tokens?: number;
+	[property: string]: unknown;
+}
+
+/**
+ * Parametrization of the text generation process
+ *
+ * Ad-hoc parametrization of the text generation process
+ */
+export interface GenerationParameters {
+	/**
+	 * Whether to use sampling instead of greedy decoding when generating new tokens.
+	 */
+	do_sample?: boolean;
+	/**
+	 * Controls the stopping condition for beam-based methods.
+	 */
+	early_stopping?: EarlyStoppingUnion;
+	/**
+	 * If set to float strictly between 0 and 1, only tokens with a conditional probability
+	 * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from
+	 * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language
+	 * Model Desmoothing](https://hf.co/papers/2210.15191) for more details.
+	 */
+	epsilon_cutoff?: number;
+	/**
+	 * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to
+	 * float strictly between 0 and 1, a token is only considered if it is greater than either
+	 * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter
+	 * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In
+	 * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
+	 * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191)
+	 * for more details.
+	 */
+	eta_cutoff?: number;
+	/**
+	 * The maximum length (in tokens) of the generated text, including the input.
+	 */
+	max_length?: number;
+	/**
+	 * The maximum number of tokens to generate. Takes precedence over maxLength.
+	 */
+	max_new_tokens?: number;
+	/**
+	 * The minimum length (in tokens) of the generated text, including the input.
+	 */
+	min_length?: number;
+	/**
+	 * The minimum number of tokens to generate. Takes precedence over maxLength.
+	 */
+	min_new_tokens?: number;
+	/**
+	 * Number of groups to divide num_beams into in order to ensure diversity among different
+	 * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details.
+	 */
+	num_beam_groups?: number;
+	/**
+	 * Number of beams to use for beam search.
+	 */
+	num_beams?: number;
+	/**
+	 * The value balances the model confidence and the degeneration penalty in contrastive
+	 * search decoding.
+	 */
+	penalty_alpha?: number;
+	/**
+	 * The value used to modulate the next token probabilities.
+	 */
+	temperature?: number;
+	/**
+	 * The number of highest probability vocabulary tokens to keep for top-k-filtering.
+	 */
+	top_k?: number;
+	/**
+	 * If set to float < 1, only the smallest set of most probable tokens with probabilities
+	 * that add up to top_p or higher are kept for generation.
+	 */
+	top_p?: number;
+	/**
+	 * Local typicality measures how similar the conditional probability of predicting a target
+	 * token next is to the expected conditional probability of predicting a random token next,
+	 * given the partial text already generated. If set to float < 1, the smallest set of the
+	 * most locally typical tokens with probabilities that add up to typical_p or higher are
+	 * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details.
+	 */
+	typical_p?: number;
+	/**
+	 * Whether the model should use the past last key/values attentions to speed up decoding
+	 */
+	use_cache?: boolean;
+	[property: string]: unknown;
+}
+
+/**
+ * Controls the stopping condition for beam-based methods.
+ */
+export type EarlyStoppingUnion = boolean | "never";
+
+/**
+ * Outputs of inference for the Image To Text task
+ */
+export interface ImageToTextOutput {
+	generatedText: unknown;
+	/**
+	 * The generated text.
+	 */
+	generated_text?: string;
+	[property: string]: unknown;
+}

package/src/tasks/image-to-text/spec/input.json

@@ -0,0 +1,34 @@
+{
+	"$id": "/inference/schemas/image-to-text/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image To Text inference",
+	"title": "ImageToTextInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageToTextParameters"
+		}
+	},
+	"$defs": {
+		"ImageToTextParameters": {
+			"title": "ImageToTextParameters",
+			"description": "Additional inference parameters for Image To Text",
+			"type": "object",
+			"properties": {
+				"max_new_tokens": {
+					"type": "integer",
+					"description": "The amount of maximum tokens to generate."
+				},
+				"generate": {
+					"description": "Parametrization of the text generation process",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/image-to-text/spec/output.json

@@ -0,0 +1,14 @@
+{
+	"$id": "/inference/schemas/image-to-text/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image To Text task",
+	"title": "ImageToTextOutput",
+	"type": "object",
+	"properties": {
+		"generated_text": {
+			"type": "string",
+			"description": "The generated text."
+		}
+	},
+	"required": ["generatedText"]
+}

package/src/tasks/index.ts

@@ -11,6 +11,7 @@ import imageClassification from "./image-classification/data";
 import imageToImage from "./image-to-image/data";
 import imageToText from "./image-to-text/data";
 import imageSegmentation from "./image-segmentation/data";
+import maskGeneration from "./mask-generation/data";
 import objectDetection from "./object-detection/data";
 import depthEstimation from "./depth-estimation/data";
 import placeholder from "./placeholder/data";
@@ -33,6 +34,7 @@ import videoClassification from "./video-classification/data";
 import visualQuestionAnswering from "./visual-question-answering/data";
 import zeroShotClassification from "./zero-shot-classification/data";
 import zeroShotImageClassification from "./zero-shot-image-classification/data";
+import zeroShotObjectDetection from "./zero-shot-object-detection/data";
 
 import type { ModelLibraryKey } from "../model-libraries";
 
@@ -131,7 +133,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": undefined,
-	"mask-generation": getData("mask-generation", placeholder),
+	"mask-generation": getData("mask-generation", maskGeneration),
 	"multiple-choice": undefined,
 	"object-detection": getData("object-detection", objectDetection),
 	"video-classification": getData("video-classification", videoClassification),
@@ -162,7 +164,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"voice-activity-detection": undefined,
 	"zero-shot-classification": getData("zero-shot-classification", zeroShotClassification),
 	"zero-shot-image-classification": getData("zero-shot-image-classification", zeroShotImageClassification),
-	"zero-shot-object-detection": getData("zero-shot-object-detection", placeholder),
+	"zero-shot-object-detection": getData("zero-shot-object-detection", zeroShotObjectDetection),
 	"text-to-3d": getData("text-to-3d", placeholder),
 	"image-to-3d": getData("image-to-3d", placeholder),
 } as const;
@@ -216,6 +218,7 @@ export interface TaskData {
 	datasets: ExampleRepo[];
 	demo: TaskDemo;
 	id: PipelineType;
+	canonicalId?: PipelineType;
 	isPlaceholder?: boolean;
 	label: string;
 	libraries: ModelLibraryKey[];

package/src/tasks/mask-generation/about.md

@@ -0,0 +1,65 @@
+## Use Cases
+
+### Filtering an Image
+
+When filtering for an image, the generated masks might serve as an initial filter to eliminate irrelevant information. For instance, when monitoring vegetation in satellite imaging, mask generation models identify green spots, highlighting the relevant region of the image.
+
+### Masked Image Modelling
+
+Generating masks can facilitate learning, especially in semi or unsupervised learning. For example, the [BEiT model](https://huggingface.co/docs/transformers/model_doc/beit) uses image-mask patches in the pre-training.
+
+### Human-in-the-loop Computer Vision Applications
+
+For applications where humans are in the loop, masks highlight certain regions of images for humans to validate.
+
+## Task Variants
+
+### Segmentation
+
+Image Segmentation divides an image into segments where each pixel is mapped to an object. This task has multiple variants, such as instance segmentation, panoptic segmentation, and semantic segmentation. You can learn more about segmentation on its [task page](https://huggingface.co/tasks/image-segmentation).
+
+## Inference
+
+Mask generation models often work in two modes: segment everything or prompt mode.
+The example below works in segment-everything-mode, where many masks will be returned.
+
+```python
+from transformers import pipeline
+
+generator = pipeline("mask-generation", model="Zigeng/SlimSAM-uniform-50", points_per_batch=64, device="cuda")
+image_url = "https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png"
+outputs = generator(image_url)
+outputs["masks"]
+# array of multiple binary masks returned for each generated mask
+```
+
+Prompt mode takes in three types of prompts:
+
+- **Point prompt:** The user can select a point on the image, and a meaningful segment around the point will be returned.
+- **Box prompt:** The user can draw a box on the image, and a meaningful segment within the box will be returned.
+- **Text prompt:** The user can input a text, and the objects of that type will be segmented. Note that this capability has not yet been released and has only been explored in research.
+
+Below you can see how to use an input-point prompt. It also demonstrates direct model inference without the `pipeline` abstraction. The input prompt here is a nested list where the outermost list is the batch size (`1`), then the number of points (also `1` in this example), and the innermost list contains the actual coordinates of the point (`[450, 600]`).
+
+```python
+from transformers import SamModel, SamProcessor
+from PIL import Image
+import requests
+
+model = SamModel.from_pretrained("Zigeng/SlimSAM-uniform-50").to("cuda")
+processor = SamProcessor.from_pretrained("Zigeng/SlimSAM-uniform-50")
+
+raw_image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+# pointing to the car window
+input_points = [[[450, 600]]]
+inputs = processor(raw_image, input_points=input_points, return_tensors="pt").to("cuda")
+outputs = model(**inputs)
+masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"].cpu(), inputs["reshaped_input_sizes"].cpu())
+scores = outputs.iou_scores
+```
+
+## Useful Resources
+
+Would you like to learn more about mask generation? Great! Here you can find some curated resources that you may find helpful!
+
+- [Segment anything model](https://huggingface.co/docs/transformers/main/model_doc/sam)

package/src/tasks/mask-generation/data.ts

@@ -3,14 +3,51 @@ import type { TaskDataCustom } from "..";
 const taskData: TaskDataCustom = {
 	datasets: [],
 	demo: {
-		inputs: [],
-		outputs: [],
+		inputs: [
+			{
+				filename: "mask-generation-input.png",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				filename: "mask-generation-output.png",
+				type: "img",
+			},
+		],
 	},
 	metrics: [],
-	models: [],
-	spaces: [],
+	models: [
+		{
+			description: "Small yet powerful mask generation model.",
+			id: "Zigeng/SlimSAM-uniform-50",
+		},
+		{
+			description: "Very strong mask generation model.",
+			id: "facebook/sam-vit-huge",
+		},
+	],
+	spaces: [
+		{
+			description:
+				"An application that combines a mask generation model with an image embedding model for open-vocabulary image segmentation.",
+			id: "SkalskiP/SAM_and_MetaCLIP",
+		},
+		{
+			description: "An application that compares the performance of a large and a small mask generation model.",
+			id: "merve/slimsam",
+		},
+		{
+			description: "An application based on an improved mask generation model.",
+			id: "linfanluntan/Grounded-SAM",
+		},
+		{
+			description: "An application to remove objects from videos using mask generation models.",
+			id: "SkalskiP/SAM_and_ProPainter",
+		},
+	],
 	summary:
-		"Mask generation is creating a binary image that identifies a specific object or region of interest in an input image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.",
+		"Mask generation is the task of generating masks that identify a specific object or region of interest in a given image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.",
 	widgetModels: [],
 	youtubeId: "",
 };

package/src/tasks/object-detection/inference.ts

@@ -0,0 +1,62 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Object Detection inference
+ */
+export interface ObjectDetectionInput {
+	/**
+	 * The input image data
+	 */
+	inputs: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ObjectDetectionParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Object Detection
+ */
+export interface ObjectDetectionParameters {
+	/**
+	 * The probability necessary to make a prediction.
+	 */
+	threshold?: number;
+	[property: string]: unknown;
+}
+/**
+ * The predicted bounding box. Coordinates are relative to the top left corner of the input
+ * image.
+ */
+export interface BoundingBox {
+	xmax: number;
+	xmin: number;
+	ymax: number;
+	ymin: number;
+	[property: string]: unknown;
+}
+export type ObjectDetectionOutput = ObjectDetectionOutputElement[];
+/**
+ * Outputs of inference for the Object Detection task
+ */
+export interface ObjectDetectionOutputElement {
+	/**
+	 * The predicted bounding box. Coordinates are relative to the top left corner of the input
+	 * image.
+	 */
+	box: BoundingBox;
+	/**
+	 * The predicted label for the bounding box
+	 */
+	label: string;
+	/**
+	 * The associated score / probability
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/object-detection/spec/input.json

@@ -0,0 +1,30 @@
+{
+	"$id": "/inference/schemas/object-detection/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Object Detection inference",
+	"title": "ObjectDetectionInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ObjectDetectionParameters"
+		}
+	},
+	"$defs": {
+		"ObjectDetectionParameters": {
+			"title": "ObjectDetectionParameters",
+			"description": "Additional inference parameters for Object Detection",
+			"type": "object",
+			"properties": {
+				"threshold": {
+					"type": "number",
+					"description": "The probability necessary to make a prediction."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/object-detection/spec/output.json

@@ -0,0 +1,46 @@
+{
+	"$id": "/inference/schemas/object-detection/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Object Detection task",
+	"title": "ObjectDetectionOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"label": {
+				"type": "string",
+				"description": "The predicted label for the bounding box"
+			},
+			"score": {
+				"type": "number",
+				"description": "The associated score / probability"
+			},
+			"box": {
+				"$ref": "#/$defs/BoundingBox",
+				"description": "The predicted bounding box. Coordinates are relative to the top left corner of the input image."
+			}
+		},
+		"required": ["box", "label", "score"]
+	},
+	"$defs": {
+		"BoundingBox": {
+			"type": "object",
+			"title": "BoundingBox",
+			"properties": {
+				"xmin": {
+					"type": "integer"
+				},
+				"xmax": {
+					"type": "integer"
+				},
+				"ymin": {
+					"type": "integer"
+				},
+				"ymax": {
+					"type": "integer"
+				}
+			},
+			"required": ["xmin", "xmax", "ymin", "ymax"]
+		}
+	}
+}

package/src/tasks/placeholder/data.ts

@@ -13,6 +13,9 @@ const taskData: TaskDataCustom = {
 	summary: "",
 	widgetModels: [],
 	youtubeId: undefined,
+	/// If this is a subtask, link to the most general task ID
+	/// (eg, text2text-generation is the canonical ID of translation)
+	canonicalId: undefined,
 };
 
 export default taskData;

package/src/tasks/placeholder/spec/input.json

@@ -0,0 +1,35 @@
+{
+	"$id": "/inference/schemas/<TASK_ID>/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for <TASK_ID> inference",
+	"title": "PlaceholderInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "TODO: describe the input here. This must be model & framework agnostic.",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/<TASK_ID>Parameters"
+		}
+	},
+	"$defs": {
+		"<TASK_ID>Parameters": {
+			"title": "<TASK_ID>Parameters",
+			"description": "TODO: describe additional parameters here.",
+			"type": "object",
+			"properties": {
+				"dummy_parameter_name": {
+					"type": "boolean",
+					"description": "TODO: describe the parameter here"
+				},
+				"dummy_parameter_name2": {
+					"type": "integer",
+					"description": "TODO: describe the parameter here"
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/placeholder/spec/output.json

@@ -0,0 +1,17 @@
+{
+	"$id": "/inference/schemas/<TASK_ID>/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs for <TASK_ID> inference",
+	"title": "PlaceholderOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"meaningful_output_name": {
+				"type": "string",
+				"description": "TODO: Describe what is outputed by the inference here"
+			}
+		},
+		"required": ["meaningfulOutputName"]
+	}
+}

package/src/tasks/question-answering/inference.ts

@@ -0,0 +1,99 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Question Answering inference
+ */
+export interface QuestionAnsweringInput {
+	/**
+	 * One (context, question) pair to answer
+	 */
+	inputs: QuestionAnsweringInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: QuestionAnsweringParameters;
+	[property: string]: unknown;
+}
+/**
+ * One (context, question) pair to answer
+ */
+export interface QuestionAnsweringInputData {
+	/**
+	 * The context to be used for answering the question
+	 */
+	context: string;
+	/**
+	 * The question to be answered
+	 */
+	question: string;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Question Answering
+ */
+export interface QuestionAnsweringParameters {
+	/**
+	 * Attempts to align the answer to real words. Improves quality on space separated
+	 * languages. Might hurt on non-space-separated languages (like Japanese or Chinese)
+	 */
+	align_to_words?: boolean;
+	/**
+	 * If the context is too long to fit with the question for the model, it will be split in
+	 * several chunks with some overlap. This argument controls the size of that overlap.
+	 */
+	doc_stride?: number;
+	/**
+	 * Whether to accept impossible as an answer.
+	 */
+	handle_impossible_answer?: boolean;
+	/**
+	 * The maximum length of predicted answers (e.g., only answers with a shorter length are
+	 * considered).
+	 */
+	max_answer_len?: number;
+	/**
+	 * The maximum length of the question after tokenization. It will be truncated if needed.
+	 */
+	max_question_len?: number;
+	/**
+	 * The maximum length of the total sentence (context + question) in tokens of each chunk
+	 * passed to the model. The context will be split in several chunks (using docStride as
+	 * overlap) if needed.
+	 */
+	max_seq_len?: number;
+	/**
+	 * The number of answers to return (will be chosen by order of likelihood). Note that we
+	 * return less than topk answers if there are not enough options available within the
+	 * context.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+export type QuestionAnsweringOutput = QuestionAnsweringOutputElement[];
+/**
+ * Outputs of inference for the Question Answering task
+ */
+export interface QuestionAnsweringOutputElement {
+	/**
+	 * The answer to the question.
+	 */
+	answer: string;
+	/**
+	 * The character position in the input where the answer ends.
+	 */
+	end: number;
+	/**
+	 * The probability associated to the answer.
+	 */
+	score: number;
+	/**
+	 * The character position in the input where the answer begins.
+	 */
+	start: number;
+	[property: string]: unknown;
+}