@huggingface/tasks 0.2.0 → 0.2.2

package/src/tasks/image-segmentation/spec/input.json

@@ -0,0 +1,54 @@
+{
+	"$id": "/inference/schemas/image-segmentation/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Segmentation inference",
+	"title": "ImageSegmentationInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageSegmentationParameters"
+		}
+	},
+	"$defs": {
+		"ImageSegmentationParameters": {
+			"title": "ImageSegmentationParameters",
+			"description": "Additional inference parameters for Image Segmentation",
+			"type": "object",
+			"properties": {
+				"maskThreshold": {
+					"type": "number",
+					"description": "Threshold to use when turning the predicted masks into binary values."
+				},
+				"overlapMaskAreaThreshold": {
+					"type": "number",
+					"description": "Mask overlap threshold to eliminate small, disconnected segments."
+				},
+				"subtask": {
+					"title": "ImageSegmentationSubtask",
+					"type": "string",
+					"description": "Segmentation task to be performed, depending on model capabilities.",
+					"oneOf": [
+						{
+							"const": "instance"
+						},
+						{
+							"const": "panoptic"
+						},
+						{
+							"const": "semantic"
+						}
+					]
+				},
+				"threshold": {
+					"type": "number",
+					"description": "Probability threshold to filter out predicted masks."
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/image-segmentation/spec/output.json

@@ -0,0 +1,25 @@
+{
+	"$id": "/inference/schemas/image-segmentation/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image Segmentation task",
+	"title": "ImageSegmentationOutput",
+	"type": "array",
+	"items": {
+		"description": "A predicted mask / segment",
+		"type": "object",
+		"properties": {
+			"label": {
+				"type": "string",
+				"description": "The label of the predicted segment"
+			},
+			"mask": {
+				"description": "The corresponding mask as a black-and-white image"
+			},
+			"score": {
+				"type": "number",
+				"description": "The score or confidence degreee the model has"
+			}
+		},
+		"required": ["label", "mask"]
+	}
+}

package/src/tasks/image-to-image/inference.ts

@@ -0,0 +1,67 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+/**
+ * Inputs for Image To Image inference
+ */
+export interface ImageToImageInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageToImageParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image To Image
+ */
+export interface ImageToImageParameters {
+	/**
+	 * For diffusion models. A higher guidance scale value encourages the model to generate
+	 * images closely linked to the text prompt at the expense of lower image quality.
+	 */
+	guidanceScale?: number;
+	/**
+	 * One or several prompt to guide what NOT to include in image generation.
+	 */
+	negativePrompt?: string[];
+	/**
+	 * For diffusion models. The number of denoising steps. More denoising steps usually lead to
+	 * a higher quality image at the expense of slower inference.
+	 */
+	numInferenceSteps?: number;
+	/**
+	 * The size in pixel of the output image
+	 */
+	targetSize?: TargetSize;
+	[property: string]: unknown;
+}
+
+/**
+ * The size in pixel of the output image
+ */
+export interface TargetSize {
+	height: number;
+	width: number;
+	[property: string]: unknown;
+}
+
+/**
+ * Outputs of inference for the Image To Image task
+ */
+export interface ImageToImageOutput {
+	/**
+	 * The output image
+	 */
+	image?: unknown;
+	[property: string]: unknown;
+}

package/src/tasks/image-to-image/spec/input.json

@@ -0,0 +1,52 @@
+{
+	"$id": "/inference/schemas/image-to-image/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image To Image inference",
+	"title": "ImageToImageInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageToImageParameters"
+		}
+	},
+	"$defs": {
+		"ImageToImageParameters": {
+			"title": "ImageToImageParameters",
+			"description": "Additional inference parameters for Image To Image",
+			"type": "object",
+			"properties": {
+				"guidanceScale": {
+					"type": "number",
+					"description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality."
+				},
+				"negativePrompt": {
+					"type": "array",
+					"items": { "type": "string" },
+					"description": "One or several prompt to guide what NOT to include in image generation."
+				},
+				"numInferenceSteps": {
+					"type": "integer",
+					"description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference."
+				},
+				"targetSize": {
+					"type": "object",
+					"description": "The size in pixel of the output image",
+					"properties": {
+						"width": {
+							"type": "integer"
+						},
+						"height": {
+							"type": "integer"
+						}
+					},
+					"required": ["width", "height"]
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/image-to-image/spec/output.json

@@ -0,0 +1,12 @@
+{
+	"$id": "/inference/schemas/image-to-image/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image To Image task",
+	"title": "ImageToImageOutput",
+	"type": "object",
+	"properties": {
+		"image": {
+			"description": "The output image"
+		}
+	}
+}

package/src/tasks/image-to-text/inference.ts

@@ -0,0 +1,138 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image To Text inference
+ */
+export interface ImageToTextInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageToTextParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image To Text
+ */
+export interface ImageToTextParameters {
+	/**
+	 * Parametrization of the text generation process
+	 */
+	generate?: GenerationParameters;
+	/**
+	 * The amount of maximum tokens to generate.
+	 */
+	maxNewTokens?: number;
+	[property: string]: unknown;
+}
+/**
+ * Parametrization of the text generation process
+ *
+ * Ad-hoc parametrization of the text generation process
+ */
+export interface GenerationParameters {
+	/**
+	 * Whether to use sampling instead of greedy decoding when generating new tokens.
+	 */
+	doSample?: boolean;
+	/**
+	 * Controls the stopping condition for beam-based methods.
+	 */
+	earlyStopping?: EarlyStoppingUnion;
+	/**
+	 * If set to float strictly between 0 and 1, only tokens with a conditional probability
+	 * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from
+	 * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language
+	 * Model Desmoothing](https://hf.co/papers/2210.15191) for more details.
+	 */
+	epsilonCutoff?: number;
+	/**
+	 * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to
+	 * float strictly between 0 and 1, a token is only considered if it is greater than either
+	 * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter
+	 * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In
+	 * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
+	 * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191)
+	 * for more details.
+	 */
+	etaCutoff?: number;
+	/**
+	 * The maximum length (in tokens) of the generated text, including the input.
+	 */
+	maxLength?: number;
+	/**
+	 * The maximum number of tokens to generate. Takes precedence over maxLength.
+	 */
+	maxNewTokens?: number;
+	/**
+	 * The minimum length (in tokens) of the generated text, including the input.
+	 */
+	minLength?: number;
+	/**
+	 * The minimum number of tokens to generate. Takes precedence over maxLength.
+	 */
+	minNewTokens?: number;
+	/**
+	 * Number of groups to divide num_beams into in order to ensure diversity among different
+	 * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details.
+	 */
+	numBeamGroups?: number;
+	/**
+	 * Number of beams to use for beam search.
+	 */
+	numBeams?: number;
+	/**
+	 * The value balances the model confidence and the degeneration penalty in contrastive
+	 * search decoding.
+	 */
+	penaltyAlpha?: number;
+	/**
+	 * The value used to modulate the next token probabilities.
+	 */
+	temperature?: number;
+	/**
+	 * The number of highest probability vocabulary tokens to keep for top-k-filtering.
+	 */
+	topK?: number;
+	/**
+	 * If set to float < 1, only the smallest set of most probable tokens with probabilities
+	 * that add up to top_p or higher are kept for generation.
+	 */
+	topP?: number;
+	/**
+	 * Local typicality measures how similar the conditional probability of predicting a target
+	 * token next is to the expected conditional probability of predicting a random token next,
+	 * given the partial text already generated. If set to float < 1, the smallest set of the
+	 * most locally typical tokens with probabilities that add up to typical_p or higher are
+	 * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details.
+	 */
+	typicalP?: number;
+	/**
+	 * Whether the model should use the past last key/values attentions to speed up decoding
+	 */
+	useCache?: boolean;
+	[property: string]: unknown;
+}
+/**
+ * Controls the stopping condition for beam-based methods.
+ */
+export type EarlyStoppingUnion = boolean | "never";
+export type ImageToTextOutput = ImageToTextOutputElement[];
+/**
+ * Outputs of inference for the Image To Text task
+ */
+export interface ImageToTextOutputElement {
+	/**
+	 * The generated text.
+	 */
+	generatedText: string;
+	[property: string]: unknown;
+}

package/src/tasks/image-to-text/spec/input.json

@@ -0,0 +1,34 @@
+{
+	"$id": "/inference/schemas/image-to-text/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image To Text inference",
+	"title": "ImageToTextInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageToTextParameters"
+		}
+	},
+	"$defs": {
+		"ImageToTextParameters": {
+			"title": "ImageToTextParameters",
+			"description": "Additional inference parameters for Image To Text",
+			"type": "object",
+			"properties": {
+				"maxNewTokens": {
+					"type": "integer",
+					"description": "The amount of maximum tokens to generate."
+				},
+				"generate": {
+					"description": "Parametrization of the text generation process",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters"
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/image-to-text/spec/output.json

@@ -0,0 +1,17 @@
+{
+	"$id": "/inference/schemas/image-to-text/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image To Text task",
+	"title": "ImageToTextOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"generatedText": {
+				"type": "string",
+				"description": "The generated text."
+			}
+		},
+		"required": ["generatedText"]
+	}
+}

package/src/tasks/index.ts

@@ -11,6 +11,7 @@ import imageClassification from "./image-classification/data";
 import imageToImage from "./image-to-image/data";
 import imageToText from "./image-to-text/data";
 import imageSegmentation from "./image-segmentation/data";
+import maskGeneration from "./mask-generation/data";
 import objectDetection from "./object-detection/data";
 import depthEstimation from "./depth-estimation/data";
 import placeholder from "./placeholder/data";
@@ -33,6 +34,7 @@ import videoClassification from "./video-classification/data";
 import visualQuestionAnswering from "./visual-question-answering/data";
 import zeroShotClassification from "./zero-shot-classification/data";
 import zeroShotImageClassification from "./zero-shot-image-classification/data";
+import zeroShotObjectDetection from "./zero-shot-object-detection/data";
 
 import type { ModelLibraryKey } from "../model-libraries";
 
@@ -131,7 +133,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": undefined,
-	"mask-generation": getData("mask-generation", placeholder),
+	"mask-generation": getData("mask-generation", maskGeneration),
 	"multiple-choice": undefined,
 	"object-detection": getData("object-detection", objectDetection),
 	"video-classification": getData("video-classification", videoClassification),
@@ -162,7 +164,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"voice-activity-detection": undefined,
 	"zero-shot-classification": getData("zero-shot-classification", zeroShotClassification),
 	"zero-shot-image-classification": getData("zero-shot-image-classification", zeroShotImageClassification),
-	"zero-shot-object-detection": getData("zero-shot-object-detection", placeholder),
+	"zero-shot-object-detection": getData("zero-shot-object-detection", zeroShotObjectDetection),
 	"text-to-3d": getData("text-to-3d", placeholder),
 	"image-to-3d": getData("image-to-3d", placeholder),
 } as const;
@@ -216,6 +218,7 @@ export interface TaskData {
 	datasets: ExampleRepo[];
 	demo: TaskDemo;
 	id: PipelineType;
+	canonicalId?: PipelineType;
 	isPlaceholder?: boolean;
 	label: string;
 	libraries: ModelLibraryKey[];

package/src/tasks/mask-generation/about.md

@@ -0,0 +1,65 @@
+## Use Cases
+
+### Filtering an Image
+
+When filtering for an image, the generated masks might serve as an initial filter to eliminate irrelevant information. For instance, when monitoring vegetation in satellite imaging, mask generation models identify green spots, highlighting the relevant region of the image.
+
+### Masked Image Modelling
+
+Generating masks can facilitate learning, especially in semi or unsupervised learning. For example, the [BEiT model](https://huggingface.co/docs/transformers/model_doc/beit) uses image-mask patches in the pre-training.
+
+### Human-in-the-loop Computer Vision Applications
+
+For applications where humans are in the loop, masks highlight certain regions of images for humans to validate.
+
+## Task Variants
+
+### Segmentation
+
+Image Segmentation divides an image into segments where each pixel is mapped to an object. This task has multiple variants, such as instance segmentation, panoptic segmentation, and semantic segmentation. You can learn more about segmentation on its [task page](https://huggingface.co/tasks/image-segmentation).
+
+## Inference
+
+Mask generation models often work in two modes: segment everything or prompt mode.
+The example below works in segment-everything-mode, where many masks will be returned.
+
+```python
+from transformers import pipeline
+
+generator = pipeline("mask-generation", model="Zigeng/SlimSAM-uniform-50", points_per_batch=64, device="cuda")
+image_url = "https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png"
+outputs = generator(image_url)
+outputs["masks"]
+# array of multiple binary masks returned for each generated mask
+```
+
+Prompt mode takes in three types of prompts:
+
+- **Point prompt:** The user can select a point on the image, and a meaningful segment around the point will be returned.
+- **Box prompt:** The user can draw a box on the image, and a meaningful segment within the box will be returned.
+- **Text prompt:** The user can input a text, and the objects of that type will be segmented. Note that this capability has not yet been released and has only been explored in research.
+
+Below you can see how to use an input-point prompt. It also demonstrates direct model inference without the `pipeline` abstraction. The input prompt here is a nested list where the outermost list is the batch size (`1`), then the number of points (also `1` in this example), and the innermost list contains the actual coordinates of the point (`[450, 600]`).
+
+```python
+from transformers import SamModel, SamProcessor
+from PIL import Image
+import requests
+
+model = SamModel.from_pretrained("Zigeng/SlimSAM-uniform-50").to("cuda")
+processor = SamProcessor.from_pretrained("Zigeng/SlimSAM-uniform-50")
+
+raw_image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+# pointing to the car window
+input_points = [[[450, 600]]]
+inputs = processor(raw_image, input_points=input_points, return_tensors="pt").to("cuda")
+outputs = model(**inputs)
+masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"].cpu(), inputs["reshaped_input_sizes"].cpu())
+scores = outputs.iou_scores
+```
+
+## Useful Resources
+
+Would you like to learn more about mask generation? Great! Here you can find some curated resources that you may find helpful!
+
+- [Segment anything model](https://huggingface.co/docs/transformers/main/model_doc/sam)

package/src/tasks/mask-generation/data.ts

@@ -0,0 +1,55 @@
+import type { TaskDataCustom } from "..";
+
+const taskData: TaskDataCustom = {
+	datasets: [],
+	demo: {
+		inputs: [
+			{
+				filename: "mask-generation-input.png",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				filename: "mask-generation-output.png",
+				type: "img",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "Small yet powerful mask generation model.",
+			id: "Zigeng/SlimSAM-uniform-50",
+		},
+		{
+			description: "Very strong mask generation model.",
+			id: "facebook/sam-vit-huge",
+		},
+	],
+	spaces: [
+		{
+			description:
+				"An application that combines a mask generation model with an image embedding model for open-vocabulary image segmentation.",
+			id: "SkalskiP/SAM_and_MetaCLIP",
+		},
+		{
+			description: "An application that compares the performance of a large and a small mask generation model.",
+			id: "merve/slimsam",
+		},
+		{
+			description: "An application based on an improved mask generation model.",
+			id: "linfanluntan/Grounded-SAM",
+		},
+		{
+			description: "An application to remove objects from videos using mask generation models.",
+			id: "SkalskiP/SAM_and_ProPainter",
+		},
+	],
+	summary:
+		"Mask generation is the task of generating masks that identify a specific object or region of interest in a given image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.",
+	widgetModels: [],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/object-detection/inference.ts

@@ -0,0 +1,62 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Object Detection inference
+ */
+export interface ObjectDetectionInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ObjectDetectionParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Object Detection
+ */
+export interface ObjectDetectionParameters {
+	/**
+	 * The probability necessary to make a prediction.
+	 */
+	threshold?: number;
+	[property: string]: unknown;
+}
+/**
+ * The predicted bounding box. Coordinates are relative to the top left corner of the input
+ * image.
+ */
+export interface BoundingBox {
+	xmax: number;
+	xmin: number;
+	ymax: number;
+	ymin: number;
+	[property: string]: unknown;
+}
+export type ObjectDetectionOutput = ObjectDetectionOutputElement[];
+/**
+ * Outputs of inference for the Object Detection task
+ */
+export interface ObjectDetectionOutputElement {
+	/**
+	 * The predicted bounding box. Coordinates are relative to the top left corner of the input
+	 * image.
+	 */
+	box: BoundingBox;
+	/**
+	 * The predicted label for the bounding box
+	 */
+	label: string;
+	/**
+	 * The associated score / probability
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/object-detection/spec/input.json

@@ -0,0 +1,30 @@
+{
+	"$id": "/inference/schemas/object-detection/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Object Detection inference",
+	"title": "ObjectDetectionInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ObjectDetectionParameters"
+		}
+	},
+	"$defs": {
+		"ObjectDetectionParameters": {
+			"title": "ObjectDetectionParameters",
+			"description": "Additional inference parameters for Object Detection",
+			"type": "object",
+			"properties": {
+				"threshold": {
+					"type": "number",
+					"description": "The probability necessary to make a prediction."
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}