@huggingface/tasks 0.13.0-test5 → 0.13.1-test

package/src/tasks/document-question-answering/inference.ts

@@ -1,110 +0,0 @@
-/**
- * Inference code generated from the JSON schema spec in ./spec
- *
- * Using src/scripts/inference-codegen
- */
-/**
- * Inputs for Document Question Answering inference
- */
-export interface DocumentQuestionAnsweringInput {
-	/**
-	 * One (document, question) pair to answer
-	 */
-	inputs: DocumentQuestionAnsweringInputData;
-	/**
-	 * Additional inference parameters
-	 */
-	parameters?: DocumentQuestionAnsweringParameters;
-	[property: string]: unknown;
-}
-/**
- * One (document, question) pair to answer
- */
-export interface DocumentQuestionAnsweringInputData {
-	/**
-	 * The image on which the question is asked
-	 */
-	image: unknown;
-	/**
-	 * A question to ask of the document
-	 */
-	question: string;
-	[property: string]: unknown;
-}
-/**
- * Additional inference parameters
- *
- * Additional inference parameters for Document Question Answering
- */
-export interface DocumentQuestionAnsweringParameters {
-	/**
-	 * If the words in the document are too long to fit with the question for the model, it will
-	 * be split in several chunks with some overlap. This argument controls the size of that
-	 * overlap.
-	 */
-	doc_stride?: number;
-	/**
-	 * Whether to accept impossible as an answer
-	 */
-	handle_impossible_answer?: boolean;
-	/**
-	 * Language to use while running OCR. Defaults to english.
-	 */
-	lang?: string;
-	/**
-	 * The maximum length of predicted answers (e.g., only answers with a shorter length are
-	 * considered).
-	 */
-	max_answer_len?: number;
-	/**
-	 * The maximum length of the question after tokenization. It will be truncated if needed.
-	 */
-	max_question_len?: number;
-	/**
-	 * The maximum length of the total sentence (context + question) in tokens of each chunk
-	 * passed to the model. The context will be split in several chunks (using doc_stride as
-	 * overlap) if needed.
-	 */
-	max_seq_len?: number;
-	/**
-	 * The number of answers to return (will be chosen by order of likelihood). Can return less
-	 * than top_k answers if there are not enough options available within the context.
-	 */
-	top_k?: number;
-	/**
-	 * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will
-	 * skip the OCR step and use the provided bounding boxes instead.
-	 */
-	word_boxes?: WordBox[];
-	[property: string]: unknown;
-}
-export type WordBox = number[] | string;
-export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[];
-/**
- * Outputs of inference for the Document Question Answering task
- */
-export interface DocumentQuestionAnsweringOutputElement {
-	/**
-	 * The answer to the question.
-	 */
-	answer: string;
-	/**
-	 * The end word index of the answer (in the OCR’d version of the input or provided word
-	 * boxes).
-	 */
-	end: number;
-	/**
-	 * The probability associated to the answer.
-	 */
-	score: number;
-	/**
-	 * The start word index of the answer (in the OCR’d version of the input or provided word
-	 * boxes).
-	 */
-	start: number;
-	/**
-	 * The index of each word/box pair that is in the answer
-	 */
-	words: number[];
-	[property: string]: unknown;
-}

package/src/tasks/document-question-answering/spec/input.json

@@ -1,85 +0,0 @@
-{
-	"$id": "/inference/schemas/document-question-answering/input.json",
-	"$schema": "http://json-schema.org/draft-06/schema#",
-	"description": "Inputs for Document Question Answering inference",
-	"title": "DocumentQuestionAnsweringInput",
-	"type": "object",
-	"properties": {
-		"inputs": {
-			"description": "One (document, question) pair to answer",
-			"type": "object",
-			"title": "DocumentQuestionAnsweringInputData",
-			"properties": {
-				"image": {
-					"description": "The image on which the question is asked"
-				},
-				"question": {
-					"type": "string",
-					"description": "A question to ask of the document"
-				}
-			},
-			"required": ["image", "question"]
-		},
-		"parameters": {
-			"description": "Additional inference parameters",
-			"$ref": "#/$defs/DocumentQuestionAnsweringParameters"
-		}
-	},
-	"$defs": {
-		"DocumentQuestionAnsweringParameters": {
-			"title": "DocumentQuestionAnsweringParameters",
-			"description": "Additional inference parameters for Document Question Answering",
-			"type": "object",
-			"properties": {
-				"doc_stride": {
-					"type": "integer",
-					"description": "If the words in the document are too long to fit with the question for the model, it will be split in several chunks with some overlap. This argument controls the size of that overlap."
-				},
-				"handle_impossible_answer": {
-					"type": "boolean",
-					"description": "Whether to accept impossible as an answer"
-				},
-				"lang": {
-					"type": "string",
-					"description": "Language to use while running OCR. Defaults to english."
-				},
-				"max_answer_len": {
-					"type": "integer",
-					"description": "The maximum length of predicted answers (e.g., only answers with a shorter length are considered)."
-				},
-				"max_seq_len": {
-					"type": "integer",
-					"description": "The maximum length of the total sentence (context + question) in tokens of each chunk passed to the model. The context will be split in several chunks (using doc_stride as overlap) if needed."
-				},
-				"max_question_len": {
-					"type": "integer",
-					"description": "The maximum length of the question after tokenization. It will be truncated if needed."
-				},
-				"top_k": {
-					"type": "integer",
-					"description": "The number of answers to return (will be chosen by order of likelihood). Can return less than top_k answers if there are not enough options available within the context."
-				},
-				"word_boxes": {
-					"type": "array",
-					"description": "A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR step and use the provided bounding boxes instead.",
-					"items": {
-						"anyOf": [
-							{
-								"type": "string"
-							},
-							{
-								"type": "array",
-								"items": {
-									"type": "number"
-								},
-								"maxLength": 4,
-								"minLength": 4
-							}
-						]
-					}
-				}
-			}
-		}
-	},
-	"required": ["inputs"]
-}

package/src/tasks/document-question-answering/spec/output.json

@@ -1,36 +0,0 @@
-{
-	"$id": "/inference/schemas/document-question-answering/output.json",
-	"$schema": "http://json-schema.org/draft-06/schema#",
-	"description": "Outputs of inference for the Document Question Answering task",
-	"title": "DocumentQuestionAnsweringOutput",
-	"type": "array",
-	"items": {
-		"type": "object",
-		"properties": {
-			"answer": {
-				"type": "string",
-				"description": "The answer to the question."
-			},
-			"score": {
-				"type": "number",
-				"description": "The probability associated to the answer."
-			},
-			"start": {
-				"type": "integer",
-				"description": "The start word index of the answer (in the OCR\u2019d version of the input or provided word boxes)."
-			},
-			"end": {
-				"type": "integer",
-				"description": "The end word index of the answer (in the OCR\u2019d version of the input or provided word boxes)."
-			},
-			"words": {
-				"type": "array",
-				"items": {
-					"type": "integer"
-				},
-				"description": "The index of each word/box pair that is in the answer"
-			}
-		},
-		"required": ["answer", "score", "start", "end", "words"]
-	}
-}

package/src/tasks/feature-extraction/about.md

@@ -1,72 +0,0 @@
-## Use Cases
-
-### Transfer Learning
-
-Models trained on a specific dataset can learn features about the data. For instance, a model trained on an English poetry dataset learns English grammar at a very high level. This information can be transferred to a new model that is going to be trained on tweets. This process of extracting features and transferring to another model is called transfer learning. One can pass their dataset through a feature extraction pipeline and feed the result to a classifier.
-
-### Retrieval and Reranking
-
-Retrieval is the process of obtaining relevant documents or information based on a user's search query. In the context of NLP, retrieval systems aim to find relevant text passages or documents from a large corpus of data that match the user's query. The goal is to return a set of results that are likely to be useful to the user. On the other hand, reranking is a technique used to improve the quality of retrieval results by reordering them based on their relevance to the query.
-
-### Retrieval Augmented Generation
-
-Retrieval-augmented generation (RAG) is a technique in which user inputs to generative models are first queried through a knowledge base, and the most relevant information from the knowledge base is used to augment the prompt to reduce hallucinations during generation. Feature extraction models (primarily retrieval and reranking models) can be used in RAG to reduce model hallucinations and ground the model.
-
-## Inference
-
-You can infer feature extraction models using `pipeline` of transformers library.
-
-```python
-from transformers import pipeline
-checkpoint = "facebook/bart-base"
-feature_extractor = pipeline("feature-extraction", framework="pt", model=checkpoint)
-text = "Transformers is an awesome library!"
-
-#Reducing along the first dimension to get a 768 dimensional array
-feature_extractor(text,return_tensors = "pt")[0].numpy().mean(axis=0)
-
-'''tensor([[[ 2.5834,  2.7571,  0.9024,  ...,  1.5036, -0.0435, -0.8603],
-         [-1.2850, -1.0094, -2.0826,  ...,  1.5993, -0.9017,  0.6426],
-         [ 0.9082,  0.3896, -0.6843,  ...,  0.7061,  0.6517,  1.0550],
-         ...,
-         [ 0.6919, -1.1946,  0.2438,  ...,  1.3646, -1.8661, -0.1642],
-         [-0.1701, -2.0019, -0.4223,  ...,  0.3680, -1.9704, -0.0068],
-         [ 0.2520, -0.6869, -1.0582,  ...,  0.5198, -2.2106,  0.4547]]])'''
-```
-
-A very popular library for training similarity and search models is called `sentence-transformers`.  To get started, install the library.
-
-```bash
-pip install -U sentence-transformers
-```
-
-You can infer with `sentence-transformers` models as follows.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-sentences = [
-    "The weather is lovely today.",
-    "It's so sunny outside!",
-    "He drove to the stadium.",
-]
-
-embeddings = model.encode(sentences)
-similarities = model.similarity(embeddings, embeddings)
-print(similarities)
-# tensor([[1.0000, 0.6660, 0.1046],
-#         [0.6660, 1.0000, 0.1411],
-#         [0.1046, 0.1411, 1.0000]])
-```
-
-### Text Embedding Inference
-
-[Text Embeddings Inference (TEI)](https://github.com/huggingface/text-embeddings-inference) is a toolkit to easily serve feature extraction models using few lines of code.
-
-## Useful resources
-
-- [Documentation for feature extraction task in 🤗Transformers](https://huggingface.co/docs/transformers/main_classes/feature_extractor)
-- [Introduction to MTEB Benchmark](https://huggingface.co/blog/mteb)
-- [Cookbook: Simple RAG for GitHub issues using Hugging Face Zephyr and LangChain](https://huggingface.co/learn/cookbook/rag_zephyr_langchain)
-- [sentence-transformers organization on Hugging Face Hub](https://huggingface.co/sentence-transformers)

package/src/tasks/feature-extraction/data.ts

@@ -1,57 +0,0 @@
-import type { TaskDataCustom } from "../index.js";
-
-const taskData: TaskDataCustom = {
-	datasets: [
-		{
-			description:
-				"Wikipedia dataset containing cleaned articles of all languages. Can be used to train `feature-extraction` models.",
-			id: "wikipedia",
-		},
-	],
-	demo: {
-		inputs: [
-			{
-				label: "Input",
-				content: "India, officially the Republic of India, is a country in South Asia.",
-				type: "text",
-			},
-		],
-		outputs: [
-			{
-				table: [
-					["Dimension 1", "Dimension 2", "Dimension 3"],
-					["2.583383083343506", "2.757075071334839", "0.9023529887199402"],
-					["8.29393482208252", "1.1071064472198486", "2.03399395942688"],
-					["-0.7754912972450256", "-1.647324562072754", "-0.6113331913948059"],
-					["0.07087723910808563", "1.5942802429199219", "1.4610432386398315"],
-				],
-				type: "tabular",
-			},
-		],
-	},
-	metrics: [],
-	models: [
-		{
-			description: "A powerful feature extraction model for natural language processing tasks.",
-			id: "thenlper/gte-large",
-		},
-		{
-			description: "A strong feature extraction model for retrieval.",
-			id: "Alibaba-NLP/gte-Qwen1.5-7B-instruct",
-		},
-	],
-	spaces: [
-		{
-			description: "A leaderboard to rank text feature extraction models based on a benchmark.",
-			id: "mteb/leaderboard",
-		},
-		{
-			description: "A leaderboard to rank best feature extraction models based on human feedback.",
-			id: "mteb/arena",
-		},
-	],
-	summary: "Feature extraction is the task of extracting features learnt in a model.",
-	widgetModels: ["facebook/bart-base"],
-};
-
-export default taskData;

package/src/tasks/feature-extraction/inference.ts

@@ -1,40 +0,0 @@
-/**
- * Inference code generated from the JSON schema spec in ./spec
- *
- * Using src/scripts/inference-codegen
- */
-
-export type FeatureExtractionOutput = Array<number[]>;
-
-/**
- * Feature Extraction Input.
- *
- * Auto-generated from TEI specs.
- * For more details, check out
- * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.
- */
-export interface FeatureExtractionInput {
-	/**
-	 * The text to embed.
-	 */
-	inputs: string;
-	normalize?: boolean;
-	/**
-	 * The name of the prompt that should be used by for encoding. If not set, no prompt
-	 * will be applied.
-	 *
-	 * Must be a key in the `Sentence Transformers` configuration `prompts` dictionary.
-	 *
-	 * For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",
-	 * ...},
-	 * then the sentence "What is the capital of France?" will be encoded as
-	 * "query: What is the capital of France?" because the prompt text will be prepended before
-	 * any text to encode.
-	 */
-	prompt_name?: string;
-	truncate?: boolean;
-	truncation_direction?: FeatureExtractionInputTruncationDirection;
-	[property: string]: unknown;
-}
-
-export type FeatureExtractionInputTruncationDirection = "Left" | "Right";

package/src/tasks/feature-extraction/spec/input.json

@@ -1,47 +0,0 @@
-{
-	"$id": "/inference/schemas/feature-extraction/input.json",
-	"$schema": "http://json-schema.org/draft-06/schema#",
-	"description": "Feature Extraction Input.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.",
-	"title": "FeatureExtractionInput",
-	"type": "object",
-	"required": ["inputs"],
-	"properties": {
-		"inputs": {
-			"type": "string",
-			"description": "The text to embed."
-		},
-		"normalize": {
-			"type": "boolean",
-			"default": "true",
-			"example": "true"
-		},
-		"prompt_name": {
-			"type": "string",
-			"description": "The name of the prompt that should be used by for encoding. If not set, no prompt\nwill be applied.\n\nMust be a key in the `Sentence Transformers` configuration `prompts` dictionary.\n\nFor example if ``prompt_name`` is \"query\" and the ``prompts`` is {\"query\": \"query: \", ...},\nthen the sentence \"What is the capital of France?\" will be encoded as\n\"query: What is the capital of France?\" because the prompt text will be prepended before\nany text to encode.",
-			"default": "null",
-			"example": "null",
-			"nullable": true
-		},
-		"truncate": {
-			"type": "boolean",
-			"default": "false",
-			"example": "false",
-			"nullable": true
-		},
-		"truncation_direction": {
-			"allOf": [
-				{
-					"$ref": "#/$defs/FeatureExtractionInputTruncationDirection"
-				}
-			],
-			"default": "right"
-		}
-	},
-	"$defs": {
-		"FeatureExtractionInputTruncationDirection": {
-			"type": "string",
-			"enum": ["Left", "Right"],
-			"title": "FeatureExtractionInputTruncationDirection"
-		}
-	}
-}

package/src/tasks/feature-extraction/spec/output.json

@@ -1,15 +0,0 @@
-{
-	"$id": "/inference/schemas/feature-extraction/output.json",
-	"$schema": "http://json-schema.org/draft-06/schema#",
-	"description": "Feature Extraction Output.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.",
-	"title": "FeatureExtractionOutput",
-	"type": "array",
-	"$defs": {},
-	"items": {
-		"type": "array",
-		"items": {
-			"type": "number",
-			"format": "float"
-		}
-	}
-}

package/src/tasks/fill-mask/about.md

@@ -1,51 +0,0 @@
-## Use Cases
-
-### Domain Adaptation 👩‍⚕️
-
-Masked language models do not require labelled data! They are trained by masking a couple of words in sentences and the model is expected to guess the masked word. This makes it very practical!
-
-For example, masked language modeling is used to train large models for domain-specific problems. If you have to work on a domain-specific task, such as retrieving information from medical research papers, you can train a masked language model using those papers. 📄
-
-The resulting model has a statistical understanding of the language used in medical research papers, and can be further trained in a process called fine-tuning to solve different tasks, such as [Text Classification](/tasks/text-classification) or [Question Answering](/tasks/question-answering) to build a medical research papers information extraction system. 👩‍⚕️ Pre-training on domain-specific data tends to yield better results (see [this paper](https://arxiv.org/abs/2007.15779) for an example).
-
-If you don't have the data to train a masked language model, you can also use an existing [domain-specific masked language model](https://huggingface.co/microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext) from the Hub and fine-tune it with your smaller task dataset. That's the magic of Open Source and sharing your work! 🎉
-
-## Inference with Fill-Mask Pipeline
-
-You can use the 🤗 Transformers library `fill-mask` pipeline to do inference with masked language models. If a model name is not provided, the pipeline will be initialized with [distilroberta-base](/distilroberta-base). You can provide masked text and it will return a list of possible mask values ​​ranked according to the score.
-
-```python
-from transformers import pipeline
-
-classifier = pipeline("fill-mask")
-classifier("Paris is the <mask> of France.")
-
-# [{'score': 0.7, 'sequence': 'Paris is the capital of France.'},
-# {'score': 0.2, 'sequence': 'Paris is the birthplace of France.'},
-# {'score': 0.1, 'sequence': 'Paris is the heart of France.'}]
-```
-
-## Useful Resources
-
-Would you like to learn more about the topic? Awesome! Here you can find some curated resources that can be helpful to you!
-
-- [Course Chapter on Fine-tuning a Masked Language Model](https://huggingface.co/course/chapter7/3?fw=pt)
-- [Workshop on Pretraining Language Models and CodeParrot](https://www.youtube.com/watch?v=ExUR7w6xe94)
-- [BERT 101: State Of The Art NLP Model Explained](https://huggingface.co/blog/bert-101)
-- [Nyströmformer: Approximating self-attention in linear time and memory via the Nyström method](https://huggingface.co/blog/nystromformer)
-
-### Notebooks
-
-- [Pre-training an MLM for JAX/Flax](https://github.com/huggingface/notebooks/blob/master/examples/masked_language_modeling_flax.ipynb)
-- [Masked language modeling in TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling-tf.ipynb)
-- [Masked language modeling in PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling.ipynb)
-
-### Scripts for training
-
-- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling)
-- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling)
-- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling)
-
-### Documentation
-
-- [Masked language modeling task guide](https://huggingface.co/docs/transformers/tasks/masked_language_modeling)

package/src/tasks/fill-mask/data.ts

@@ -1,79 +0,0 @@
-import type { TaskDataCustom } from "../index.js";
-
-const taskData: TaskDataCustom = {
-	datasets: [
-		{
-			description: "A common dataset that is used to train models for many languages.",
-			id: "wikipedia",
-		},
-		{
-			description: "A large English dataset with text crawled from the web.",
-			id: "c4",
-		},
-	],
-	demo: {
-		inputs: [
-			{
-				label: "Input",
-				content: "The <mask> barked at me",
-				type: "text",
-			},
-		],
-		outputs: [
-			{
-				type: "chart",
-				data: [
-					{
-						label: "wolf",
-						score: 0.487,
-					},
-					{
-						label: "dog",
-						score: 0.061,
-					},
-					{
-						label: "cat",
-						score: 0.058,
-					},
-					{
-						label: "fox",
-						score: 0.047,
-					},
-					{
-						label: "squirrel",
-						score: 0.025,
-					},
-				],
-			},
-		],
-	},
-	metrics: [
-		{
-			description:
-				"Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words",
-			id: "cross_entropy",
-		},
-		{
-			description:
-				"Perplexity is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance",
-			id: "perplexity",
-		},
-	],
-	models: [
-		{
-			description: "The famous BERT model.",
-			id: "google-bert/bert-base-uncased",
-		},
-		{
-			description: "A multilingual model trained on 100 languages.",
-			id: "FacebookAI/xlm-roberta-base",
-		},
-	],
-	spaces: [],
-	summary:
-		"Masked language modeling is the task of masking some of the words in a sentence and predicting which words should replace those masks. These models are useful when we want to get a statistical understanding of the language in which the model is trained in.",
-	widgetModels: ["distilroberta-base"],
-	youtubeId: "mqElG5QJWUg",
-};
-
-export default taskData;

package/src/tasks/fill-mask/inference.ts

@@ -1,62 +0,0 @@
-/**
- * Inference code generated from the JSON schema spec in ./spec
- *
- * Using src/scripts/inference-codegen
- */
-/**
- * Inputs for Fill Mask inference
- */
-export interface FillMaskInput {
-	/**
-	 * The text with masked tokens
-	 */
-	inputs: string;
-	/**
-	 * Additional inference parameters
-	 */
-	parameters?: FillMaskParameters;
-	[property: string]: unknown;
-}
-/**
- * Additional inference parameters
- *
- * Additional inference parameters for Fill Mask
- */
-export interface FillMaskParameters {
-	/**
-	 * When passed, the model will limit the scores to the passed targets instead of looking up
-	 * in the whole vocabulary. If the provided targets are not in the model vocab, they will be
-	 * tokenized and the first resulting token will be used (with a warning, and that might be
-	 * slower).
-	 */
-	targets?: string[];
-	/**
-	 * When passed, overrides the number of predictions to return.
-	 */
-	top_k?: number;
-	[property: string]: unknown;
-}
-export type FillMaskOutput = FillMaskOutputElement[];
-/**
- * Outputs of inference for the Fill Mask task
- */
-export interface FillMaskOutputElement {
-	/**
-	 * The corresponding probability
-	 */
-	score: number;
-	/**
-	 * The corresponding input with the mask token prediction.
-	 */
-	sequence: string;
-	/**
-	 * The predicted token id (to replace the masked one).
-	 */
-	token: number;
-	tokenStr: unknown;
-	/**
-	 * The predicted token (to replace the masked one).
-	 */
-	token_str?: string;
-	[property: string]: unknown;
-}