@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/depth-estimation/spec/output.json

@@ -0,0 +1,16 @@
+{
+	"$id": "/inference/schemas/depth-estimation/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Depth Estimation task",
+	"title": "DepthEstimationOutput",
+
+	"type": "object",
+	"properties": {
+		"predicted_depth": {
+			"description": "The predicted depth as a tensor"
+		},
+		"depth": {
+			"description": "The predicted depth as an image"
+		}
+	}
+}

package/src/tasks/document-question-answering/about.md

@@ -0,0 +1,53 @@
+## Use Cases
+
+Document Question Answering models can be used to answer natural language questions about documents. Typically, document QA models consider textual, layout and potentially visual information. This is useful when the question requires some understanding of the visual aspects of the document.
+Nevertheless, certain document QA models can work without document images. Hence the task is not limited to visually-rich documents and allows users to ask questions based on spreadsheets, text PDFs, etc!
+
+### Document Parsing
+
+One of the most popular use cases of document question answering models is the parsing of structured documents. For example, you can extract the name, address, and other information from a form. You can also use the model to extract information from a table, or even a resume.
+
+### Invoice Information Extraction
+
+Another very popular use case is invoice information extraction. For example, you can extract the invoice number, the invoice date, the total amount, the VAT number, and the invoice recipient.
+
+## Inference
+
+You can infer with Document QA models with the 🤗 Transformers library using the [`document-question-answering` pipeline](https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.DocumentQuestionAnsweringPipeline). If no model checkpoint is given, the pipeline will be initialized with [`impira/layoutlm-document-qa`](https://huggingface.co/impira/layoutlm-document-qa). This pipeline takes question(s) and document(s) as input, and returns the answer.  
+👉 Note that the question answering task solved here is extractive: the model extracts the answer from a context (the document).
+
+```python
+from transformers import pipeline
+from PIL import Image
+
+pipe = pipeline("document-question-answering", model="naver-clova-ix/donut-base-finetuned-docvqa")
+
+question = "What is the purchase amount?"
+image = Image.open("your-document.png")
+
+pipe(image=image, question=question)
+
+## [{'answer': '20,000$'}]
+```
+
+## Useful Resources
+
+Would you like to learn more about Document QA? Awesome! Here are some curated resources that you may find helpful!
+
+- [Document Visual Question Answering (DocVQA) challenge](https://rrc.cvc.uab.es/?ch=17)
+- [DocVQA: A Dataset for Document Visual Question Answering](https://arxiv.org/abs/2007.00398) (Dataset paper)
+- [ICDAR 2021 Competition on Document Visual Question Answering](https://lilianweng.github.io/lil-log/2020/10/29/open-domain-question-answering.html) (Conference paper)
+- [HuggingFace's Document Question Answering pipeline](https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.DocumentQuestionAnsweringPipeline)
+- [Github repo: DocQuery - Document Query Engine Powered by Large Language Models](https://github.com/impira/docquery)
+
+### Notebooks
+
+- [Fine-tuning Donut on DocVQA dataset](https://github.com/NielsRogge/Transformers-Tutorials/tree/0ea77f29d01217587d7e32a848f3691d9c15d6ab/Donut/DocVQA)
+- [Fine-tuning LayoutLMv2 on DocVQA dataset](https://github.com/NielsRogge/Transformers-Tutorials/tree/1b4bad710c41017d07a8f63b46a12523bfd2e835/LayoutLMv2/DocVQA)
+- [Accelerating Document AI](https://huggingface.co/blog/document-ai)
+
+### Documentation
+
+- [Document question answering task guide](https://huggingface.co/docs/transformers/tasks/document_question_answering)
+
+The contents of this page are contributed by [Eliott Zemour](https://huggingface.co/eliolio) and reviewed by [Kwadwo Agyapon-Ntra](https://huggingface.co/KayO) and [Ankur Goyal](https://huggingface.co/ankrgyl).

package/src/tasks/document-question-answering/data.ts

@@ -0,0 +1,85 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Largest document understanding dataset.",
+			id: "HuggingFaceM4/Docmatix",
+		},
+		{
+			description:
+				"Dataset from the 2020 DocVQA challenge. The documents are taken from the UCSF Industry Documents Library.",
+			id: "eliolio/docvqa",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Question",
+				content: "What is the idea behind the consumer relations efficiency team?",
+				type: "text",
+			},
+			{
+				filename: "document-question-answering-input.png",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				label: "Answer",
+				content: "Balance cost efficiency with quality customer service",
+				type: "text",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"The evaluation metric for the DocVQA challenge is the Average Normalized Levenshtein Similarity (ANLS). This metric is flexible to character regognition errors and compares the predicted answer with the ground truth answer.",
+			id: "anls",
+		},
+		{
+			description:
+				"Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0",
+			id: "exact-match",
+		},
+	],
+	models: [
+		{
+			description: "A robust document question answering model.",
+			id: "impira/layoutlm-document-qa",
+		},
+		{
+			description: "A document question answering model specialized in invoices.",
+			id: "impira/layoutlm-invoices",
+		},
+		{
+			description: "A special model for OCR-free document question answering.",
+			id: "microsoft/udop-large",
+		},
+		{
+			description: "A powerful model for document question answering.",
+			id: "google/pix2struct-docvqa-large",
+		},
+	],
+	spaces: [
+		{
+			description: "A robust document question answering application.",
+			id: "impira/docquery",
+		},
+		{
+			description: "An application that can answer questions from invoices.",
+			id: "impira/invoices",
+		},
+		{
+			description: "An application to compare different document question answering models.",
+			id: "merve/compare_docvqa_models",
+		},
+	],
+	summary:
+		"Document Question Answering (also known as Document Visual Question Answering) is the task of answering questions on document images. Document question answering models take a (document, question) pair as input and return an answer in natural language. Models usually rely on multi-modal features, combining text, position of words (bounding-boxes) and image.",
+	widgetModels: ["impira/layoutlm-invoices"],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/document-question-answering/inference.ts

@@ -0,0 +1,110 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Document Question Answering inference
+ */
+export interface DocumentQuestionAnsweringInput {
+	/**
+	 * One (document, question) pair to answer
+	 */
+	inputs: DocumentQuestionAnsweringInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: DocumentQuestionAnsweringParameters;
+	[property: string]: unknown;
+}
+/**
+ * One (document, question) pair to answer
+ */
+export interface DocumentQuestionAnsweringInputData {
+	/**
+	 * The image on which the question is asked
+	 */
+	image: unknown;
+	/**
+	 * A question to ask of the document
+	 */
+	question: string;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Document Question Answering
+ */
+export interface DocumentQuestionAnsweringParameters {
+	/**
+	 * If the words in the document are too long to fit with the question for the model, it will
+	 * be split in several chunks with some overlap. This argument controls the size of that
+	 * overlap.
+	 */
+	doc_stride?: number;
+	/**
+	 * Whether to accept impossible as an answer
+	 */
+	handle_impossible_answer?: boolean;
+	/**
+	 * Language to use while running OCR. Defaults to english.
+	 */
+	lang?: string;
+	/**
+	 * The maximum length of predicted answers (e.g., only answers with a shorter length are
+	 * considered).
+	 */
+	max_answer_len?: number;
+	/**
+	 * The maximum length of the question after tokenization. It will be truncated if needed.
+	 */
+	max_question_len?: number;
+	/**
+	 * The maximum length of the total sentence (context + question) in tokens of each chunk
+	 * passed to the model. The context will be split in several chunks (using doc_stride as
+	 * overlap) if needed.
+	 */
+	max_seq_len?: number;
+	/**
+	 * The number of answers to return (will be chosen by order of likelihood). Can return less
+	 * than top_k answers if there are not enough options available within the context.
+	 */
+	top_k?: number;
+	/**
+	 * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will
+	 * skip the OCR step and use the provided bounding boxes instead.
+	 */
+	word_boxes?: WordBox[];
+	[property: string]: unknown;
+}
+export type WordBox = number[] | string;
+export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[];
+/**
+ * Outputs of inference for the Document Question Answering task
+ */
+export interface DocumentQuestionAnsweringOutputElement {
+	/**
+	 * The answer to the question.
+	 */
+	answer: string;
+	/**
+	 * The end word index of the answer (in the OCR’d version of the input or provided word
+	 * boxes).
+	 */
+	end: number;
+	/**
+	 * The probability associated to the answer.
+	 */
+	score: number;
+	/**
+	 * The start word index of the answer (in the OCR’d version of the input or provided word
+	 * boxes).
+	 */
+	start: number;
+	/**
+	 * The index of each word/box pair that is in the answer
+	 */
+	words: number[];
+	[property: string]: unknown;
+}

package/src/tasks/document-question-answering/spec/input.json

@@ -0,0 +1,85 @@
+{
+	"$id": "/inference/schemas/document-question-answering/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Document Question Answering inference",
+	"title": "DocumentQuestionAnsweringInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "One (document, question) pair to answer",
+			"type": "object",
+			"title": "DocumentQuestionAnsweringInputData",
+			"properties": {
+				"image": {
+					"description": "The image on which the question is asked"
+				},
+				"question": {
+					"type": "string",
+					"description": "A question to ask of the document"
+				}
+			},
+			"required": ["image", "question"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/DocumentQuestionAnsweringParameters"
+		}
+	},
+	"$defs": {
+		"DocumentQuestionAnsweringParameters": {
+			"title": "DocumentQuestionAnsweringParameters",
+			"description": "Additional inference parameters for Document Question Answering",
+			"type": "object",
+			"properties": {
+				"doc_stride": {
+					"type": "integer",
+					"description": "If the words in the document are too long to fit with the question for the model, it will be split in several chunks with some overlap. This argument controls the size of that overlap."
+				},
+				"handle_impossible_answer": {
+					"type": "boolean",
+					"description": "Whether to accept impossible as an answer"
+				},
+				"lang": {
+					"type": "string",
+					"description": "Language to use while running OCR. Defaults to english."
+				},
+				"max_answer_len": {
+					"type": "integer",
+					"description": "The maximum length of predicted answers (e.g., only answers with a shorter length are considered)."
+				},
+				"max_seq_len": {
+					"type": "integer",
+					"description": "The maximum length of the total sentence (context + question) in tokens of each chunk passed to the model. The context will be split in several chunks (using doc_stride as overlap) if needed."
+				},
+				"max_question_len": {
+					"type": "integer",
+					"description": "The maximum length of the question after tokenization. It will be truncated if needed."
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "The number of answers to return (will be chosen by order of likelihood). Can return less than top_k answers if there are not enough options available within the context."
+				},
+				"word_boxes": {
+					"type": "array",
+					"description": "A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR step and use the provided bounding boxes instead.",
+					"items": {
+						"anyOf": [
+							{
+								"type": "string"
+							},
+							{
+								"type": "array",
+								"items": {
+									"type": "number"
+								},
+								"maxLength": 4,
+								"minLength": 4
+							}
+						]
+					}
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/document-question-answering/spec/output.json

@@ -0,0 +1,36 @@
+{
+	"$id": "/inference/schemas/document-question-answering/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Document Question Answering task",
+	"title": "DocumentQuestionAnsweringOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"answer": {
+				"type": "string",
+				"description": "The answer to the question."
+			},
+			"score": {
+				"type": "number",
+				"description": "The probability associated to the answer."
+			},
+			"start": {
+				"type": "integer",
+				"description": "The start word index of the answer (in the OCR\u2019d version of the input or provided word boxes)."
+			},
+			"end": {
+				"type": "integer",
+				"description": "The end word index of the answer (in the OCR\u2019d version of the input or provided word boxes)."
+			},
+			"words": {
+				"type": "array",
+				"items": {
+					"type": "integer"
+				},
+				"description": "The index of each word/box pair that is in the answer"
+			}
+		},
+		"required": ["answer", "score", "start", "end", "words"]
+	}
+}

package/src/tasks/feature-extraction/about.md

@@ -0,0 +1,72 @@
+## Use Cases
+
+### Transfer Learning
+
+Models trained on a specific dataset can learn features about the data. For instance, a model trained on an English poetry dataset learns English grammar at a very high level. This information can be transferred to a new model that is going to be trained on tweets. This process of extracting features and transferring to another model is called transfer learning. One can pass their dataset through a feature extraction pipeline and feed the result to a classifier.
+
+### Retrieval and Reranking
+
+Retrieval is the process of obtaining relevant documents or information based on a user's search query. In the context of NLP, retrieval systems aim to find relevant text passages or documents from a large corpus of data that match the user's query. The goal is to return a set of results that are likely to be useful to the user. On the other hand, reranking is a technique used to improve the quality of retrieval results by reordering them based on their relevance to the query.
+
+### Retrieval Augmented Generation
+
+Retrieval-augmented generation (RAG) is a technique in which user inputs to generative models are first queried through a knowledge base, and the most relevant information from the knowledge base is used to augment the prompt to reduce hallucinations during generation. Feature extraction models (primarily retrieval and reranking models) can be used in RAG to reduce model hallucinations and ground the model.
+
+## Inference
+
+You can infer feature extraction models using `pipeline` of transformers library.
+
+```python
+from transformers import pipeline
+checkpoint = "facebook/bart-base"
+feature_extractor = pipeline("feature-extraction", framework="pt", model=checkpoint)
+text = "Transformers is an awesome library!"
+
+#Reducing along the first dimension to get a 768 dimensional array
+feature_extractor(text,return_tensors = "pt")[0].numpy().mean(axis=0)
+
+'''tensor([[[ 2.5834,  2.7571,  0.9024,  ...,  1.5036, -0.0435, -0.8603],
+         [-1.2850, -1.0094, -2.0826,  ...,  1.5993, -0.9017,  0.6426],
+         [ 0.9082,  0.3896, -0.6843,  ...,  0.7061,  0.6517,  1.0550],
+         ...,
+         [ 0.6919, -1.1946,  0.2438,  ...,  1.3646, -1.8661, -0.1642],
+         [-0.1701, -2.0019, -0.4223,  ...,  0.3680, -1.9704, -0.0068],
+         [ 0.2520, -0.6869, -1.0582,  ...,  0.5198, -2.2106,  0.4547]]])'''
+```
+
+A very popular library for training similarity and search models is called `sentence-transformers`.  To get started, install the library.
+
+```bash
+pip install -U sentence-transformers
+```
+
+You can infer with `sentence-transformers` models as follows.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+sentences = [
+    "The weather is lovely today.",
+    "It's so sunny outside!",
+    "He drove to the stadium.",
+]
+
+embeddings = model.encode(sentences)
+similarities = model.similarity(embeddings, embeddings)
+print(similarities)
+# tensor([[1.0000, 0.6660, 0.1046],
+#         [0.6660, 1.0000, 0.1411],
+#         [0.1046, 0.1411, 1.0000]])
+```
+
+### Text Embedding Inference
+
+[Text Embeddings Inference (TEI)](https://github.com/huggingface/text-embeddings-inference) is a toolkit to easily serve feature extraction models using few lines of code.
+
+## Useful resources
+
+- [Documentation for feature extraction task in 🤗Transformers](https://huggingface.co/docs/transformers/main_classes/feature_extractor)
+- [Introduction to MTEB Benchmark](https://huggingface.co/blog/mteb)
+- [Cookbook: Simple RAG for GitHub issues using Hugging Face Zephyr and LangChain](https://huggingface.co/learn/cookbook/rag_zephyr_langchain)
+- [sentence-transformers organization on Hugging Face Hub](https://huggingface.co/sentence-transformers)

package/src/tasks/feature-extraction/data.ts

@@ -0,0 +1,57 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description:
+				"Wikipedia dataset containing cleaned articles of all languages. Can be used to train `feature-extraction` models.",
+			id: "wikipedia",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				label: "Input",
+				content: "India, officially the Republic of India, is a country in South Asia.",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				table: [
+					["Dimension 1", "Dimension 2", "Dimension 3"],
+					["2.583383083343506", "2.757075071334839", "0.9023529887199402"],
+					["8.29393482208252", "1.1071064472198486", "2.03399395942688"],
+					["-0.7754912972450256", "-1.647324562072754", "-0.6113331913948059"],
+					["0.07087723910808563", "1.5942802429199219", "1.4610432386398315"],
+				],
+				type: "tabular",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "A powerful feature extraction model for natural language processing tasks.",
+			id: "thenlper/gte-large",
+		},
+		{
+			description: "A strong feature extraction model for retrieval.",
+			id: "Alibaba-NLP/gte-Qwen1.5-7B-instruct",
+		},
+	],
+	spaces: [
+		{
+			description: "A leaderboard to rank text feature extraction models based on a benchmark.",
+			id: "mteb/leaderboard",
+		},
+		{
+			description: "A leaderboard to rank best feature extraction models based on human feedback.",
+			id: "mteb/arena",
+		},
+	],
+	summary: "Feature extraction is the task of extracting features learnt in a model.",
+	widgetModels: ["facebook/bart-base"],
+};
+
+export default taskData;

package/src/tasks/feature-extraction/inference.ts

@@ -0,0 +1,40 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+
+export type FeatureExtractionOutput = Array<number[]>;
+
+/**
+ * Feature Extraction Input.
+ *
+ * Auto-generated from TEI specs.
+ * For more details, check out
+ * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.
+ */
+export interface FeatureExtractionInput {
+	/**
+	 * The text to embed.
+	 */
+	inputs: string;
+	normalize?: boolean;
+	/**
+	 * The name of the prompt that should be used by for encoding. If not set, no prompt
+	 * will be applied.
+	 *
+	 * Must be a key in the `Sentence Transformers` configuration `prompts` dictionary.
+	 *
+	 * For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",
+	 * ...},
+	 * then the sentence "What is the capital of France?" will be encoded as
+	 * "query: What is the capital of France?" because the prompt text will be prepended before
+	 * any text to encode.
+	 */
+	prompt_name?: string;
+	truncate?: boolean;
+	truncation_direction?: FeatureExtractionInputTruncationDirection;
+	[property: string]: unknown;
+}
+
+export type FeatureExtractionInputTruncationDirection = "Left" | "Right";

package/src/tasks/feature-extraction/spec/input.json

@@ -0,0 +1,47 @@
+{
+	"$id": "/inference/schemas/feature-extraction/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Feature Extraction Input.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.",
+	"title": "FeatureExtractionInput",
+	"type": "object",
+	"required": ["inputs"],
+	"properties": {
+		"inputs": {
+			"type": "string",
+			"description": "The text to embed."
+		},
+		"normalize": {
+			"type": "boolean",
+			"default": "true",
+			"example": "true"
+		},
+		"prompt_name": {
+			"type": "string",
+			"description": "The name of the prompt that should be used by for encoding. If not set, no prompt\nwill be applied.\n\nMust be a key in the `Sentence Transformers` configuration `prompts` dictionary.\n\nFor example if ``prompt_name`` is \"query\" and the ``prompts`` is {\"query\": \"query: \", ...},\nthen the sentence \"What is the capital of France?\" will be encoded as\n\"query: What is the capital of France?\" because the prompt text will be prepended before\nany text to encode.",
+			"default": "null",
+			"example": "null",
+			"nullable": true
+		},
+		"truncate": {
+			"type": "boolean",
+			"default": "false",
+			"example": "false",
+			"nullable": true
+		},
+		"truncation_direction": {
+			"allOf": [
+				{
+					"$ref": "#/$defs/FeatureExtractionInputTruncationDirection"
+				}
+			],
+			"default": "right"
+		}
+	},
+	"$defs": {
+		"FeatureExtractionInputTruncationDirection": {
+			"type": "string",
+			"enum": ["Left", "Right"],
+			"title": "FeatureExtractionInputTruncationDirection"
+		}
+	}
+}

package/src/tasks/feature-extraction/spec/output.json

@@ -0,0 +1,15 @@
+{
+	"$id": "/inference/schemas/feature-extraction/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Feature Extraction Output.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.",
+	"title": "FeatureExtractionOutput",
+	"type": "array",
+	"$defs": {},
+	"items": {
+		"type": "array",
+		"items": {
+			"type": "number",
+			"format": "float"
+		}
+	}
+}