@huggingface/tasks 0.2.0 → 0.2.2

package/src/tasks/depth-estimation/data.ts

@@ -28,8 +28,8 @@ const taskData: TaskDataCustom = {
 			id: "Intel/dpt-large",
 		},
 		{
-			description: "Strong Depth Estimation model trained on the KITTI dataset.",
-			id: "facebook/dpt-dinov2-large-kitti",
+			description: "Strong Depth Estimation model trained on a big compilation of datasets.",
+			id: "LiheYoung/depth-anything-large-hf",
 		},
 		{
 			description: "A strong monocular depth estimation model.",
@@ -42,8 +42,12 @@ const taskData: TaskDataCustom = {
 			id: "radames/dpt-depth-estimation-3d-voxels",
 		},
 		{
-			description: "An application that can estimate the depth in a given image.",
-			id: "keras-io/Monocular-Depth-Estimation",
+			description: "An application to compare the outputs of different depth estimation models.",
+			id: "LiheYoung/Depth-Anything",
+		},
+		{
+			description: "An application to try state-of-the-art depth estimation.",
+			id: "merve/compare_depth_models",
 		},
 	],
 	summary: "Depth estimation is the task of predicting depth of the objects present in an image.",

package/src/tasks/depth-estimation/inference.ts

@@ -0,0 +1,35 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+export type DepthEstimationOutput = unknown[];
+
+/**
+ * Inputs for Depth Estimation inference
+ */
+export interface DepthEstimationInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: DepthEstimationParameters;
+	[property: string]: unknown;
+}
+
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Depth Estimation
+ */
+export interface DepthEstimationParameters {
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	topK?: number;
+	[property: string]: unknown;
+}

package/src/tasks/depth-estimation/spec/input.json

@@ -0,0 +1,30 @@
+{
+	"$id": "/inference/schemas/depth-estimation/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Depth Estimation inference",
+	"title": "DepthEstimationInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/DepthEstimationParameters"
+		}
+	},
+	"$defs": {
+		"DepthEstimationParameters": {
+			"title": "DepthEstimationParameters",
+			"description": "Additional inference parameters for Depth Estimation",
+			"type": "object",
+			"properties": {
+				"topK": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/depth-estimation/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/depth-estimation/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Depth Estimation task",
+	"title": "DepthEstimationOutput",
+	"type": "array",
+	"items": {
+		"description": "The output depth labels"
+	}
+}

package/src/tasks/document-question-answering/inference.ts

@@ -0,0 +1,102 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Document Question Answering inference
+ */
+export interface DocumentQuestionAnsweringInput {
+	/**
+	 * One (document, question) pair to answer
+	 */
+	data: DocumentQuestionAnsweringInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: DocumentQuestionAnsweringParameters;
+	[property: string]: unknown;
+}
+/**
+ * One (document, question) pair to answer
+ */
+export interface DocumentQuestionAnsweringInputData {
+	/**
+	 * The image on which the question is asked
+	 */
+	image: unknown;
+	/**
+	 * A question to ask of the document
+	 */
+	question: string;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Document Question Answering
+ */
+export interface DocumentQuestionAnsweringParameters {
+	/**
+	 * If the words in the document are too long to fit with the question for the model, it will
+	 * be split in several chunks with some overlap. This argument controls the size of that
+	 * overlap.
+	 */
+	docStride?: number;
+	/**
+	 * Whether to accept impossible as an answer
+	 */
+	handleImpossibleAnswer?: boolean;
+	/**
+	 * Language to use while running OCR. Defaults to english.
+	 */
+	lang?: string;
+	/**
+	 * The maximum length of predicted answers (e.g., only answers with a shorter length are
+	 * considered).
+	 */
+	maxAnswerLen?: number;
+	/**
+	 * The maximum length of the question after tokenization. It will be truncated if needed.
+	 */
+	maxQuestionLen?: number;
+	/**
+	 * The maximum length of the total sentence (context + question) in tokens of each chunk
+	 * passed to the model. The context will be split in several chunks (using doc_stride as
+	 * overlap) if needed.
+	 */
+	maxSeqLen?: number;
+	/**
+	 * The number of answers to return (will be chosen by order of likelihood). Can return less
+	 * than top_k answers if there are not enough options available within the context.
+	 */
+	topK?: number;
+	/**
+	 * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will
+	 * skip the OCR step and use the provided bounding boxes instead.
+	 */
+	wordBoxes?: WordBox[];
+	[property: string]: unknown;
+}
+export type WordBox = number[] | string;
+export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[];
+/**
+ * Outputs of inference for the Document Question Answering task
+ */
+export interface DocumentQuestionAnsweringOutputElement {
+	/**
+	 * The answer to the question.
+	 */
+	answer: string;
+	end: number;
+	/**
+	 * The probability associated to the answer.
+	 */
+	score: number;
+	start: number;
+	/**
+	 * The index of each word/box pair that is in the answer
+	 */
+	words: number[];
+	[property: string]: unknown;
+}

package/src/tasks/document-question-answering/spec/input.json

@@ -0,0 +1,85 @@
+{
+	"$id": "/inference/schemas/document-question-answering/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Document Question Answering inference",
+	"title": "DocumentQuestionAnsweringInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "One (document, question) pair to answer",
+			"type": "object",
+			"title": "DocumentQuestionAnsweringInputData",
+			"properties": {
+				"image": {
+					"description": "The image on which the question is asked"
+				},
+				"question": {
+					"type": "string",
+					"description": "A question to ask of the document"
+				}
+			},
+			"required": ["image", "question"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/DocumentQuestionAnsweringParameters"
+		}
+	},
+	"$defs": {
+		"DocumentQuestionAnsweringParameters": {
+			"title": "DocumentQuestionAnsweringParameters",
+			"description": "Additional inference parameters for Document Question Answering",
+			"type": "object",
+			"properties": {
+				"docStride": {
+					"type": "integer",
+					"description": "If the words in the document are too long to fit with the question for the model, it will be split in several chunks with some overlap. This argument controls the size of that overlap."
+				},
+				"handleImpossibleAnswer": {
+					"type": "boolean",
+					"description": "Whether to accept impossible as an answer"
+				},
+				"lang": {
+					"type": "string",
+					"description": "Language to use while running OCR. Defaults to english."
+				},
+				"maxAnswerLen": {
+					"type": "integer",
+					"description": "The maximum length of predicted answers (e.g., only answers with a shorter length are considered)."
+				},
+				"maxSeqLen": {
+					"type": "integer",
+					"description": "The maximum length of the total sentence (context + question) in tokens of each chunk passed to the model. The context will be split in several chunks (using doc_stride as overlap) if needed."
+				},
+				"maxQuestionLen": {
+					"type": "integer",
+					"description": "The maximum length of the question after tokenization. It will be truncated if needed."
+				},
+				"topK": {
+					"type": "integer",
+					"description": "The number of answers to return (will be chosen by order of likelihood). Can return less than top_k answers if there are not enough options available within the context."
+				},
+				"wordBoxes": {
+					"type": "array",
+					"description": "A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR step and use the provided bounding boxes instead.",
+					"items": {
+						"anyOf": [
+							{
+								"type": "string"
+							},
+							{
+								"type": "array",
+								"items": {
+									"type": "number"
+								},
+								"maxLength": 4,
+								"minLength": 4
+							}
+						]
+					}
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/document-question-answering/spec/output.json

@@ -0,0 +1,36 @@
+{
+	"$id": "/inference/schemas/document-question-answering/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Document Question Answering task",
+	"title": "DocumentQuestionAnsweringOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"answer": {
+				"type": "string",
+				"description": "The answer to the question."
+			},
+			"score": {
+				"type": "number",
+				"description": "The probability associated to the answer."
+			},
+			"start": {
+				"type": "integer",
+				"descrtiption": "The start word index of the answer (in the OCR’d version of the input or provided word boxes)."
+			},
+			"end": {
+				"type": "integer",
+				"descrtiption": "The end word index of the answer (in the OCR’d version of the input or provided word boxes)."
+			},
+			"words": {
+				"type": "array",
+				"items": {
+					"type": "integer"
+				},
+				"description": "The index of each word/box pair that is in the answer"
+			}
+		},
+		"required": ["answer", "score", "start", "end", "words"]
+	}
+}

package/src/tasks/feature-extraction/inference.ts

@@ -0,0 +1,22 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+export type FeatureExtractionOutput = unknown[];
+
+/**
+ * Inputs for Text Embedding inference
+ */
+export interface FeatureExtractionInput {
+	/**
+	 * The text to get the embeddings of
+	 */
+	data: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: { [key: string]: unknown };
+	[property: string]: unknown;
+}

package/src/tasks/feature-extraction/spec/input.json

@@ -0,0 +1,26 @@
+{
+	"$id": "/inference/schemas/feature-extraction/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Text Embedding inference",
+	"title": "FeatureExtractionInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The text to get the embeddings of",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/FeatureExtractionParameters"
+		}
+	},
+	"$defs": {
+		"FeatureExtractionParameters": {
+			"title": "FeatureExtractionParameters",
+			"description": "Additional inference parameters for Feature Extraction",
+			"type": "object",
+			"properties": {}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/feature-extraction/spec/output.json

@@ -0,0 +1,7 @@
+{
+	"$id": "/inference/schemas/feature-extraction/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "The embedding for the input text, as a nested list (tensor) of floats",
+	"type": "array",
+	"title": "FeatureExtractionOutput"
+}

package/src/tasks/fill-mask/inference.ts

@@ -0,0 +1,61 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Fill Mask inference
+ */
+export interface FillMaskInput {
+	/**
+	 * The text with masked tokens
+	 */
+	data: string;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: FillMaskParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Fill Mask
+ */
+export interface FillMaskParameters {
+	/**
+	 * When passed, the model will limit the scores to the passed targets instead of looking up
+	 * in the whole vocabulary. If the provided targets are not in the model vocab, they will be
+	 * tokenized and the first resulting token will be used (with a warning, and that might be
+	 * slower).
+	 */
+	targets?: string[];
+	/**
+	 * When passed, overrides the number of predictions to return.
+	 */
+	topK?: number;
+	[property: string]: unknown;
+}
+export type FillMaskOutput = FillMaskOutputElement[];
+/**
+ * Outputs of inference for the Fill Mask task
+ */
+export interface FillMaskOutputElement {
+	/**
+	 * The corresponding probability
+	 */
+	score: number;
+	/**
+	 * The corresponding input with the mask token prediction.
+	 */
+	sequence: string;
+	/**
+	 * The predicted token id (to replace the masked one).
+	 */
+	token: number;
+	/**
+	 * The predicted token (to replace the masked one).
+	 */
+	tokenStr: string;
+	[property: string]: unknown;
+}

package/src/tasks/fill-mask/spec/input.json

@@ -0,0 +1,38 @@
+{
+	"$id": "/inference/schemas/fill-mask/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Fill Mask inference",
+	"title": "FillMaskInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The text with masked tokens",
+			"type": "string"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/FillMaskParameters"
+		}
+	},
+	"$defs": {
+		"FillMaskParameters": {
+			"title": "FillMaskParameters",
+			"description": "Additional inference parameters for Fill Mask",
+			"type": "object",
+			"properties": {
+				"topK": {
+					"type": "integer",
+					"description": "When passed, overrides the number of predictions to return."
+				},
+				"targets": {
+					"description": "When passed, the model will limit the scores to the passed targets instead of looking up in the whole vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first resulting token will be used (with a warning, and that might be slower).",
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/fill-mask/spec/output.json

@@ -0,0 +1,29 @@
+{
+	"$id": "/inference/schemas/fill-mask/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Fill Mask task",
+	"title": "FillMaskOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"sequence": {
+				"type": "string",
+				"description": "The corresponding input with the mask token prediction."
+			},
+			"score": {
+				"type": "number",
+				"description": "The corresponding probability"
+			},
+			"token": {
+				"type": "integer",
+				"description": "The predicted token id (to replace the masked one)."
+			},
+			"tokenStr": {
+				"type": "string",
+				"description": "The predicted token (to replace the masked one)."
+			}
+		},
+		"required": ["sequence", "score", "token", "tokenStr"]
+	}
+}

package/src/tasks/image-classification/inference.ts

@@ -0,0 +1,51 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Classification inference
+ */
+export interface ImageClassificationInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image Classification
+ */
+export interface ImageClassificationParameters {
+	functionToApply?: ClassificationOutputTransform;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	topK?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type ImageClassificationOutput = ImageClassificationOutputElement[];
+/**
+ * Outputs of inference for the Image Classification task
+ */
+export interface ImageClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/image-classification/spec/input.json

@@ -0,0 +1,34 @@
+{
+	"$id": "/inference/schemas/image-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Classification inference",
+	"title": "ImageClassificationInput",
+	"type": "object",
+	"properties": {
+		"data": {
+			"description": "The input image data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/ImageClassificationParameters"
+		}
+	},
+	"$defs": {
+		"ImageClassificationParameters": {
+			"title": "ImageClassificationParameters",
+			"description": "Additional inference parameters for Image Classification",
+			"type": "object",
+			"properties": {
+				"functionToApply": {
+					"title": "ImageClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"topK": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["data"]
+}

package/src/tasks/image-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/image-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Image Classification task",
+	"title": "ImageClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/image-segmentation/inference.ts

@@ -0,0 +1,65 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Segmentation inference
+ */
+export interface ImageSegmentationInput {
+	/**
+	 * The input image data
+	 */
+	data: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: ImageSegmentationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Image Segmentation
+ */
+export interface ImageSegmentationParameters {
+	/**
+	 * Threshold to use when turning the predicted masks into binary values.
+	 */
+	maskThreshold?: number;
+	/**
+	 * Mask overlap threshold to eliminate small, disconnected segments.
+	 */
+	overlapMaskAreaThreshold?: number;
+	/**
+	 * Segmentation task to be performed, depending on model capabilities.
+	 */
+	subtask?: ImageSegmentationSubtask;
+	/**
+	 * Probability threshold to filter out predicted masks.
+	 */
+	threshold?: number;
+	[property: string]: unknown;
+}
+export type ImageSegmentationSubtask = "instance" | "panoptic" | "semantic";
+export type ImageSegmentationOutput = ImageSegmentationOutputElement[];
+/**
+ * Outputs of inference for the Image Segmentation task
+ *
+ * A predicted mask / segment
+ */
+export interface ImageSegmentationOutputElement {
+	/**
+	 * The label of the predicted segment
+	 */
+	label: string;
+	/**
+	 * The corresponding mask as a black-and-white image
+	 */
+	mask: unknown;
+	/**
+	 * The score or confidence degreee the model has
+	 */
+	score?: number;
+	[property: string]: unknown;
+}