@huggingface/tasks 0.10.15 → 0.10.16

package/src/model-libraries-snippets.ts

@@ -56,6 +56,12 @@ export const bertopic = (model: ModelData): string[] => [
 model = BERTopic.load("${model.id}")`,
 ];
 
+export const bm25s = (model: ModelData): string[] => [
+	`from bm25s.hf import BM25HF
+
+retriever = BM25HF.load_from_hub("${model.id}")`,
+];
+
 const diffusers_default = (model: ModelData) => [
 	`from diffusers import DiffusionPipeline
 

package/src/model-libraries.ts

@@ -111,6 +111,16 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 			wildcard: { path: "*.npz" },
 		},
 	},
+	bm25s: {
+		prettyLabel: "BM25S",
+		repoName: "bm25s",
+		repoUrl: "https://github.com/xhluca/bm25s",
+		snippets: snippets.bm25s,
+		filter: false,
+		countDownloads: {
+			term: { path: "params.index.json" },
+		},
+	},
 	chat_tts: {
 		prettyLabel: "ChatTTS",
 		repoName: "ChatTTS",
@@ -223,6 +233,14 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 			terms: { path: ["ckpt/tensor00000_000", "ckpt-0/tensor00000_000"] },
 		},
 	},
+	"hunyuan-dit": {
+		prettyLabel: "HunyuanDiT",
+		repoName: "HunyuanDiT",
+		repoUrl: "https://github.com/Tencent/HunyuanDiT",
+		countDownloads: {
+			terms: { path: ["pytorch_model_ema.pt", "pytorch_model_distill.pt"] },
+		},
+	},
 	keras: {
 		prettyLabel: "Keras",
 		repoName: "Keras",

package/src/tasks/image-text-to-text/about.md

@@ -0,0 +1,74 @@
+## Different Types of Vision Language Models
+
+Vision language models come in three types:
+
+- **Base:** Pre-trained models that can be fine-tuned. A good example of base models is the [PaliGemma models family](https://huggingface.co/models?sort=trending&search=google%2Fpaligemma-3b-pt) by Google.
+- **Instruction:** Base models fine-tuned on instruction datasets. A good example of instruction fine-tuned models is [idefics2-8b](https://huggingface.co/HuggingFaceM4/idefics2-8b).
+- **Chatty/Conversational:** Base models fine-tuned on conversation datasets. A good example of chatty models is [deepseek-vl-7b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-7b-chat).
+
+![VLM uses](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/vlm/visual.jpg)
+
+## Use Cases
+
+### Multimodal Dialogue
+
+Vision language models can be used as multimodal assistants, keeping context about the conversation and keeping the image to have multiple-turn dialogues.
+
+### Zero-shot Object Detection, Image Segmentation and Localization
+
+Some vision language models can detect or segment a set of objects or describe the positions or relative positions of the objects. For example, one could prompt such a model to ask if one object is behind another. Such a model can also output bounding box coordination or segmentation masks directly in the text output, unlike the traditional models explicitly trained on only object detection or image segmentation.
+
+### Visual Question Answering
+
+Vision language models trained on image-text pairs can be used for visual question answering and generating captions for images.
+
+### Document Question Answering and Retrieval
+
+Documents often consist of different layouts, charts, tables, images, and more. Vision language models trained on formatted documents can extract information from them. This is an OCR-free approach; the inputs skip OCR, and documents are directly fed to vision language models.
+
+### Image Recognition with Instructions
+
+Vision language models can recognize images through descriptions. When given detailed descriptions of specific entities, it can classify the entities in an image.
+
+## Inference
+
+You can use the Transformers library to interact with vision-language models. You can load the model like below.
+
+```python
+from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
+import torch
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
+model = LlavaNextForConditionalGeneration.from_pretrained(
+    "llava-hf/llava-v1.6-mistral-7b-hf",
+    torch_dtype=torch.float16
+)
+model.to(device)
+```
+
+We can infer by passing image and text dialogues.
+
+```python
+from PIL import Image
+import requests
+
+# image of a radar chart
+url = "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true"
+image = Image.open(requests.get(url, stream=True).raw)
+prompt = "[INST] <image>\nWhat is shown in this image? [/INST]"
+
+inputs = processor(prompt, image, return_tensors="pt").to(device)
+output = model.generate(**inputs, max_new_tokens=100)
+
+print(processor.decode(output[0], skip_special_tokens=True))
+# The image appears to be a radar chart, which is a type of multivariate chart that displays values for multiple variables represented on axes
+# starting from the same point. This particular radar chart is showing the performance of different models or systems across various metrics.
+# The axes represent different metrics or benchmarks, such as MM-Vet, MM-Vet, MM-Vet, MM-Vet, MM-Vet, MM-V
+```
+
+## Useful Resources
+
+- [Vision Language Models Explained](https://huggingface.co/blog/vlms)
+- [Open-source Multimodality and How to Achieve it using Hugging Face](https://www.youtube.com/watch?v=IoGaGfU1CIg&t=601s)
+- [Introducing Idefics2: A Powerful 8B Vision-Language Model for the community](https://huggingface.co/blog/idefics2)

package/src/tasks/image-text-to-text/data.ts

@@ -0,0 +1,90 @@
+import type { TaskDataCustom } from "..";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Instructions composed of image and text.",
+			id: "liuhaotian/LLaVA-Instruct-150K",
+		},
+		{
+			description: "Conversation turns where questions involve image and text.",
+			id: "liuhaotian/LLaVA-Pretrain",
+		},
+		{
+			description: "A collection of datasets made for model fine-tuning.",
+			id: "HuggingFaceM4/the_cauldron",
+		},
+		{
+			description: "Screenshots of websites with their HTML/CSS codes.",
+			id: "HuggingFaceM4/WebSight",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "mask-generation-input.png",
+				type: "img",
+			},
+			{
+				label: "Text Prompt",
+				content: "Describe the position of the bee in detail.",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				label: "Answer",
+				content:
+					"The bee is sitting on a pink flower, surrounded by other flowers. The bee is positioned in the center of the flower, with its head and front legs sticking out.",
+				type: "text",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "Cutting-edge vision language model that can also localize texts in images.",
+			id: "liuhaotian/llava-v1.6-34b",
+		},
+		{
+			description: "Cutting-edge conversational vision language model that can take multiple image inputs.",
+			id: "HuggingFaceM4/idefics2-8b-chatty",
+		},
+		{
+			description: "Small yet powerful model.",
+			id: "vikhyatk/moondream2",
+		},
+		{
+			description: "Strong image-text-to-text model made to understand documents.",
+			id: "mPLUG/DocOwl1.5",
+		},
+	],
+	spaces: [
+		{
+			description: "Leaderboard to evaluate vision language models.",
+			id: "opencompass/open_vlm_leaderboard",
+		},
+		{
+			description: "Vision language models arena, where models are ranked by votes of users.",
+			id: "WildVision/vision-arena",
+		},
+		{
+			description: "Powerful vision-language model assistant.",
+			id: "liuhaotian/LLaVA-1.6",
+		},
+		{
+			description: "An application to compare outputs of different vision language models.",
+			id: "merve/compare_VLMs",
+		},
+		{
+			description: "An application for document vision language tasks.",
+			id: "mPLUG/DocOwl",
+		},
+	],
+	summary:
+		"Image-text-to-text models take in an image and text prompt and output text. These models are also called vision-language models, or VLMs. The difference from image-to-text models is that these models take an additional text input, not restricting the model to certain use cases like image captioning, and may also be trained to accept a conversation as input.",
+	widgetModels: ["microsoft/kosmos-2-patch14-224"],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/image-to-text/about.md

@@ -9,10 +9,6 @@ This can help the visually impaired people to understand what's happening in the
 
 OCR models convert the text present in an image, e.g. a scanned document, to text.
 
-## Pix2Struct
-
-Pix2Struct is a state-of-the-art model built and released by Google AI. The model itself has to be trained on a downstream task to be used. These tasks include, captioning UI components, images including text, visual questioning infographics, charts, scientific diagrams and more. You can find these models on recommended models of this page.
-
 ## Inference
 
 ### Image Captioning
@@ -22,24 +18,11 @@ You can use the 🤗 Transformers library's `image-to-text` pipeline to generate
 ```python
 from transformers import pipeline
 
-captioner = pipeline("image-to-text",model="Salesforce/blip-image-captioning-base")
+captioner = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")
 captioner("https://huggingface.co/datasets/Narsil/image_dummy/resolve/main/parrots.png")
 ## [{'generated_text': 'two birds are standing next to each other '}]
 ```
 
-### Conversation about the Image
-
-Some text generation models also take image inputs. These are called vision language models. You can use `image-to-text` pipeline to use these models like below.
-
-```python
-from transformers import pipeline
-
-mm_pipeline = pipeline("image-to-text",model="llava-hf/llava-1.5-7b-hf")
-mm_pipeline("https://huggingface.co/spaces/llava-hf/llava-4bit/resolve/main/examples/baklava.png", "How to make this pastry?")
-
-## [{'generated_text': 'To create these pastries, you will need a few key ingredients and tools. Firstly, gather the dough by combining flour with water in your mixing bowl until it forms into an elastic ball that can be easily rolled out on top of another surface or table without breaking apart (like pizza).'}]
-```
-
 ### OCR
 
 This code snippet uses Microsoft’s TrOCR, an encoder-decoder model consisting of an image Transformer encoder and a text Transformer decoder for state-of-the-art optical character recognition (OCR) on single-text line images.
@@ -71,7 +54,7 @@ await inference.imageToText({
 ## Useful Resources
 
 - [Image Captioning](https://huggingface.co/docs/transformers/main/en/tasks/image_captioning)
-- [Image captioning use case](https://blog.google/outreach-initiatives/accessibility/get-image-descriptions/)
+- [Image Captioning Use Case](https://blog.google/outreach-initiatives/accessibility/get-image-descriptions/)
 - [Train Image Captioning model on your dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/GIT/Fine_tune_GIT_on_an_image_captioning_dataset.ipynb)
 - [Train OCR model on your dataset ](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/TrOCR)
 

package/src/tasks/index.ts

@@ -11,6 +11,7 @@ import imageClassification from "./image-classification/data";
 import imageFeatureExtraction from "./image-feature-extraction/data";
 import imageToImage from "./image-to-image/data";
 import imageToText from "./image-to-text/data";
+import imageTextToText from "./image-text-to-text/data";
 import imageSegmentation from "./image-segmentation/data";
 import maskGeneration from "./mask-generation/data";
 import objectDetection from "./object-detection/data";
@@ -202,8 +203,8 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"image-classification": getData("image-classification", imageClassification),
 	"image-feature-extraction": getData("image-feature-extraction", imageFeatureExtraction),
 	"image-segmentation": getData("image-segmentation", imageSegmentation),
-	"image-text-to-text": undefined,
 	"image-to-image": getData("image-to-image", imageToImage),
+	"image-text-to-text": getData("image-text-to-text", imageTextToText),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": undefined,
 	"mask-generation": getData("mask-generation", maskGeneration),

package/src/tasks/text-generation/about.md

@@ -1,10 +1,10 @@
-This task covers guides on both [text-generation](https://huggingface.co/models?pipeline_tag=text-generation&sort=downloads) and [text-to-text generation](https://huggingface.co/models?pipeline_tag=text2text-generation&sort=downloads) models. Popular large language models that are used for chats or following instructions are also covered in this task. You can find the list of selected open-source large language models [here](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard), ranked by their performance scores.
+This task covers guides on both [text-generation](https://huggingface.co/models?pipeline_tag=text-generation&sort=downloads) and [text-to-text generation](https://huggingface.co/models?pipeline_tag=text2text-generation&sort=downloads) models. Popular large language models that are used for chats or following instructions are also covered in this task. You can find the list of selected open-source large language models [here](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard), ranked by their performance scores.
 
 ## Use Cases
 
 ### Instruction Models
 
-A model trained for text generation can be later adapted to follow instructions. One of the most used open-source models for instruction is OpenAssistant, which you can try [at Hugging Chat](https://huggingface.co/chat).
+A model trained for text generation can be later adapted to follow instructions. You can try some of the most powerful instruction-tuned open-access models like Mixtral 8x7B, Cohere Command R+, and Meta Llama3 70B [at Hugging Chat](https://huggingface.co/chat).
 
 ### Code Generation
 
@@ -36,15 +36,15 @@ These models are trained to learn the mapping between a pair of texts (e.g. tran
 
 When it comes to text generation, the underlying language model can come in several types:
 
-- **Base models:** refers to plain language models like [Mistral 7B](mistralai/Mistral-7B-v0.1) and [Llama-2-70b](https://huggingface.co/meta-llama/Llama-2-70b-hf). These models are good for fine-tuning and few-shot prompting.
+- **Base models:** refers to plain language models like [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.3) and [Meta Llama-3-70b](https://huggingface.co/meta-llama/Meta-Llama-3-70B). These models are good for fine-tuning and few-shot prompting.
 
-- **Instruction-trained models:** these models are trained in a multi-task manner to follow a broad range of instructions like "Write me a recipe for chocolate cake". Models like [Flan-T5](https://huggingface.co/google/flan-t5-xl), [Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1), and [falcon-40b-instruct](https://huggingface.co/tiiuae/falcon-40b-instruct) are examples of instruction-trained models. In general, instruction-trained models will produce better responses to instructions than base models.
+- **Instruction-trained models:** these models are trained in a multi-task manner to follow a broad range of instructions like "Write me a recipe for chocolate cake". Models like [Qwen 2 7B](https://huggingface.co/Qwen/Qwen2-7B-Instruct), [Yi 1.5 34B Chat](https://huggingface.co/01-ai/Yi-1.5-34B-Chat), and [Meta Llama 70B Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) are examples of instruction-trained models. In general, instruction-trained models will produce better responses to instructions than base models.
 
-- **Human feedback models:** these models extend base and instruction-trained models by incorporating human feedback that rates the quality of the generated text according to criteria like [helpfulness, honesty, and harmlessness](https://arxiv.org/abs/2112.00861). The human feedback is then combined with an optimization technique like reinforcement learning to align the original model to be closer with human preferences. The overall methodology is often called [Reinforcement Learning from Human Feedback](https://huggingface.co/blog/rlhf), or RLHF for short. [Llama2-Chat](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) is an open-source model aligned through human feedback.
+- **Human feedback models:** these models extend base and instruction-trained models by incorporating human feedback that rates the quality of the generated text according to criteria like [helpfulness, honesty, and harmlessness](https://arxiv.org/abs/2112.00861). The human feedback is then combined with an optimization technique like reinforcement learning to align the original model to be closer with human preferences. The overall methodology is often called [Reinforcement Learning from Human Feedback](https://huggingface.co/blog/rlhf), or RLHF for short. [Zephyr ORPO 141B A35B](https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1) is an open-source model aligned through human feedback.
 
 ## Text Generation from Image and Text
 
-There are language models that can input both text and image and output text, called vision language models. [LLaVA](https://huggingface.co/llava-hf/llava-1.5-7b-hf) and [BLIP-2](https://huggingface.co/Salesforce/blip2-opt-2.7b) are good examples. Although they work just like other language models by means of input parameters for generation, since they also take input images, you can use them with `image-to-text` pipeline. You can find information about the pipeline in [image-to-text](https://huggingface.co/tasks/image-to-text) task page.
+There are language models that can input both text and image and output text, called vision language models. [IDEFICS 2](https://huggingface.co/HuggingFaceM4/idefics2-8b) and [MiniCPM Llama3 V](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5) are good examples. They accept the same generation parameters as other language models. However, since they also take images as input, you have to use them with the `image-to-text` pipeline. You can find more information about this in the [image-to-text task page](https://huggingface.co/tasks/image-to-text).
 
 ## Inference
 
@@ -102,7 +102,7 @@ Would you like to learn more about the topic? Awesome! Here you can find some cu
 - You can use [PEFT](https://github.com/huggingface/peft) to adapt large language models in efficient way.
 - [ChatUI](https://github.com/huggingface/chat-ui) is the open-source interface to conversate with Large Language Models.
 - [text-generation-inferface](https://github.com/huggingface/text-generation-inference)
-- [HuggingChat](https://huggingface.co/chat/) is a chat interface powered by Hugging Face to chat with powerful models like Llama 2 70B.
+- [HuggingChat](https://huggingface.co/chat/) is a chat interface powered by Hugging Face to chat with powerful models like Meta Llama 3 70B, Mixtral 8x7B, etc.
 
 ### Documentation