@huggingface/tasks 0.13.1-test → 0.13.1-test2

package/src/tasks/video-classification/about.md

@@ -0,0 +1,37 @@
+## Use Cases
+
+Video classification models can be used to categorize what a video is all about.
+
+### Activity Recognition
+
+Video classification models are used to perform activity recognition which is useful for fitness applications. Activity recognition is also helpful for vision-impaired individuals especially when they're commuting.
+
+### Video Search
+
+Models trained in video classification can improve user experience by organizing and categorizing video galleries on the phone or in the cloud, on multiple keywords or tags.
+
+## Inference
+
+Below you can find code for inferring with a pre-trained video classification model.
+
+```python
+from transformers import pipeline
+
+pipe = pipeline(task = "video-classification", model="nateraw/videomae-base-finetuned-ucf101-subset")
+pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/basketball.avi?download=true")
+
+#[{'score': 0.90, 'label': 'BasketballDunk'},
+# {'score': 0.02, 'label': 'BalanceBeam'},
+# ... ]
+```
+
+## Useful Resources
+
+- [Developing a simple video classification model](https://keras.io/examples/vision/video_classification)
+- [Video classification with Transformers](https://keras.io/examples/vision/video_transformers)
+- [Building a video archive](https://www.youtube.com/watch?v=_IeS1m8r6SY)
+- [Video classification task guide](https://huggingface.co/docs/transformers/tasks/video_classification)
+
+### Creating your own video classifier in minutes
+
+- [Fine-tuning tutorial notebook (PyTorch)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb)

package/src/tasks/video-classification/data.ts

@@ -0,0 +1,84 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			// TODO write proper description
+			description: "Benchmark dataset used for video classification with videos that belong to 400 classes.",
+			id: "kinetics400",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "video-classification-input.gif",
+				type: "img",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "Playing Guitar",
+						score: 0.514,
+					},
+					{
+						label: "Playing Tennis",
+						score: 0.193,
+					},
+					{
+						label: "Cooking",
+						score: 0.068,
+					},
+				],
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "",
+			id: "accuracy",
+		},
+		{
+			description: "",
+			id: "recall",
+		},
+		{
+			description: "",
+			id: "precision",
+		},
+		{
+			description: "",
+			id: "f1",
+		},
+	],
+	models: [
+		{
+			// TO DO: write description
+			description: "Strong Video Classification model trained on the Kinetics 400 dataset.",
+			id: "google/vivit-b-16x2-kinetics400",
+		},
+		{
+			// TO DO: write description
+			description: "Strong Video Classification model trained on the Kinetics 400 dataset.",
+			id: "microsoft/xclip-base-patch32",
+		},
+	],
+	spaces: [
+		{
+			description: "An application that classifies video at different timestamps.",
+			id: "nateraw/lavila",
+		},
+		{
+			description: "An application that classifies video.",
+			id: "fcakyon/video-classification",
+		},
+	],
+	summary:
+		"Video classification is the task of assigning a label or class to an entire video. Videos are expected to have only one class for each video. Video classification models take a video as input and return a prediction about which class the video belongs to.",
+	widgetModels: [],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/video-classification/inference.ts

@@ -0,0 +1,59 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Video Classification inference
+ */
+export interface VideoClassificationInput {
+	/**
+	 * The input video data
+	 */
+	inputs: unknown;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: VideoClassificationParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Video Classification
+ */
+export interface VideoClassificationParameters {
+	/**
+	 * The sampling rate used to select frames from the video.
+	 */
+	frame_sampling_rate?: number;
+	function_to_apply?: ClassificationOutputTransform;
+	/**
+	 * The number of sampled frames to consider for classification.
+	 */
+	num_frames?: number;
+	/**
+	 * When specified, limits the output to the top K most probable classes.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+/**
+ * The function to apply to the model outputs in order to retrieve the scores.
+ */
+export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none";
+export type VideoClassificationOutput = VideoClassificationOutputElement[];
+/**
+ * Outputs of inference for the Video Classification task
+ */
+export interface VideoClassificationOutputElement {
+	/**
+	 * The predicted class label.
+	 */
+	label: string;
+	/**
+	 * The corresponding probability.
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/video-classification/spec/input.json

@@ -0,0 +1,42 @@
+{
+	"$id": "/inference/schemas/video-classification/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Video Classification inference",
+	"title": "VideoClassificationInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "The input video data"
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/VideoClassificationParameters"
+		}
+	},
+	"$defs": {
+		"VideoClassificationParameters": {
+			"title": "VideoClassificationParameters",
+			"description": "Additional inference parameters for Video Classification",
+			"type": "object",
+			"properties": {
+				"function_to_apply": {
+					"title": "TextClassificationOutputTransform",
+					"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform"
+				},
+				"num_frames": {
+					"type": "integer",
+					"description": "The number of sampled frames to consider for classification."
+				},
+				"frame_sampling_rate": {
+					"type": "integer",
+					"description": "The sampling rate used to select frames from the video."
+				},
+				"top_k": {
+					"type": "integer",
+					"description": "When specified, limits the output to the top K most probable classes."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/video-classification/spec/output.json

@@ -0,0 +1,10 @@
+{
+	"$id": "/inference/schemas/video-classification/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Video Classification task",
+	"title": "VideoClassificationOutput",
+	"type": "array",
+	"items": {
+		"$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput"
+	}
+}

package/src/tasks/video-text-to-text/about.md

@@ -0,0 +1,98 @@
+Most of the video language models can take in videos, multiple videos, images and multiple images. Some of these models can also take interleaved inputs, which can have images and videos inside the text, where you can refer to the input images and input videos within the text prompt.
+
+## Different Types of Video Language Models
+
+Video language models come in three types:
+
+- **Base:** Pre-trained models that can be fine-tuned.
+- **Instruction:** Base models fine-tuned on video-instruction pairs and answers.
+- **Chatty/Conversational:** Base models fine-tuned on video conversation datasets.
+
+## Use Cases
+
+### Video Question Answering
+
+Video language models trained on video-question-answer pairs can be used for video question answering and generating captions for videos.
+
+### Video Chat
+
+Video language models can be used to have a dialogue about a video.
+
+### Video Recognition with Instructions
+
+Video language models can recognize images through descriptions. When given detailed descriptions of specific entities, they can classify the entities in a video.
+
+## Inference
+
+You can use the Transformers library to interact with video-language models.
+Below we load [a video language model](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-hf), write a simple utility to sample videos, use chat template to format the text prompt, process the video and the text prompt and infer. To run the snippet below, please install [OpenCV](https://pypi.org/project/opencv-python/) by running `pip install opencv-python`.
+
+```python
+import uuid
+import requests
+import cv2
+import torch
+from transformers import LlavaNextVideoProcessor, LlavaNextVideoForConditionalGeneration
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_id = "llava-hf/LLaVA-NeXT-Video-7B-hf"
+
+model = LlavaNextVideoForConditionalGeneration.from_pretrained(
+    model_id,
+    torch_dtype=torch.float16,
+    low_cpu_mem_usage=True,
+).to(device)
+
+processor = LlavaNextVideoProcessor.from_pretrained(model_id)
+
+def sample_frames(url, num_frames):
+    response = requests.get(url)
+    path_id = str(uuid.uuid4())
+
+    path = f"./{path_id}.mp4"
+
+    with open(path, "wb") as f:
+         f.write(response.content)
+
+    video = cv2.VideoCapture(path)
+    total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+    interval = total_frames // num_frames
+    frames = []
+    for i in range(total_frames):
+        ret, frame = video.read()
+        if not ret:
+            continue
+        if i % interval == 0:
+            pil_img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+            frames.append(pil_img)
+    video.release()
+    return frames
+
+conversation = [
+    {
+
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "Why is this video funny?"},
+            {"type": "video"},
+            ],
+    },
+]
+
+prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
+
+video_url = "https://huggingface.co/spaces/merve/llava-interleave/resolve/main/cats_1.mp4"
+video = sample_frames(video, 8)
+
+inputs = processor(text=prompt, videos=video, padding=True, return_tensors="pt").to(model.device)
+
+output = model.generate(**inputs, max_new_tokens=100, do_sample=False)
+print(processor.decode(output[0][2:], skip_special_tokens=True))
+
+# Why is this video funny? ASSISTANT: The humor in this video comes from the cat's facial expression and body language. The cat appears to be making a funny face, with its eyes squinted and mouth open, which can be interpreted as a playful or mischievous expression. Cats often make such faces when they are in a good mood or are playful, and this can be amusing to people who are familiar with their behavior. The combination of the cat's expression and the close-
+
+```
+
+## Useful Resources
+
+- [Transformers task guide on video-text-to-text](https://huggingface.co/docs/transformers/tasks/video_text_to_text)

package/src/tasks/video-text-to-text/data.ts

@@ -0,0 +1,66 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Multiple-choice questions and answers about videos.",
+			id: "lmms-lab/Video-MME",
+		},
+		{
+			description: "A dataset of instructions and question-answer pairs about videos.",
+			id: "lmms-lab/VideoChatGPT",
+		},
+		{
+			description: "Large video understanding dataset.",
+			id: "HuggingFaceFV/finevideo",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "video-text-to-text-input.gif",
+				type: "img",
+			},
+			{
+				label: "Text Prompt",
+				content: "What is happening in this video?",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				label: "Answer",
+				content:
+					"The video shows a series of images showing a fountain with water jets and a variety of colorful flowers and butterflies in the background.",
+				type: "text",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "A robust video-text-to-text model that can take in image and video inputs.",
+			id: "llava-hf/llava-onevision-qwen2-72b-ov-hf",
+		},
+		{
+			description: "Large and powerful video-text-to-text model that can take in image and video inputs.",
+			id: "llava-hf/LLaVA-NeXT-Video-34B-hf",
+		},
+	],
+	spaces: [
+		{
+			description: "An application to chat with a video-text-to-text model.",
+			id: "llava-hf/video-llava",
+		},
+		{
+			description: "A leaderboard for various video-text-to-text models.",
+			id: "opencompass/openvlm_video_leaderboard",
+		},
+	],
+	summary:
+		"Video-text-to-text models take in a video and a text prompt and output text. These models are also called video-language models.",
+	widgetModels: [""],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/visual-question-answering/about.md

@@ -0,0 +1,48 @@
+## Use Cases
+
+### Aid the Visually Impaired Persons
+
+VQA models can be used to reduce visual barriers for visually impaired individuals by allowing them to get information about images from the web and the real world.
+
+### Education
+
+VQA models can be used to improve experiences at museums by allowing observers to directly ask questions they interested in.
+
+### Improved Image Retrieval
+
+Visual question answering models can be used to retrieve images with specific characteristics. For example, the user can ask "Is there a dog?" to find all images with dogs from a set of images.
+
+### Video Search
+
+Specific snippets/timestamps of a video can be retrieved based on search queries. For example, the user can ask "At which part of the video does the guitar appear?" and get a specific timestamp range from the whole video.
+
+## Task Variants
+
+### Video Question Answering
+
+Video Question Answering aims to answer questions asked about the content of a video.
+
+## Inference
+
+You can infer with Visual Question Answering models using the `vqa` (or `visual-question-answering`) pipeline. This pipeline requires [the Python Image Library (PIL)](https://pillow.readthedocs.io/en/stable/) to process images. You can install it with (`pip install pillow`).
+
+```python
+from PIL import Image
+from transformers import pipeline
+
+vqa_pipeline = pipeline("visual-question-answering")
+
+image =  Image.open("elephant.jpeg")
+question = "Is there an elephant?"
+
+vqa_pipeline(image, question, top_k=1)
+#[{'score': 0.9998154044151306, 'answer': 'yes'}]
+```
+
+## Useful Resources
+
+- [An introduction to Visual Question Answering - AllenAI](https://blog.allenai.org/vanilla-vqa-adcaaaa94336)
+- [Multi Modal Framework (MMF) - Meta Research](https://mmf.sh/docs/getting_started/video_overview/)
+
+The contents of this page are contributed by [
+Bharat Raghunathan](https://huggingface.co/bharat-raghunathan) and [Jose Londono Botero](https://huggingface.co/jlondonobo).

package/src/tasks/visual-question-answering/data.ts

@@ -0,0 +1,97 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A widely used dataset containing questions (with answers) about images.",
+			id: "Graphcore/vqa",
+		},
+		{
+			description: "A dataset to benchmark visual reasoning based on text in images.",
+			id: "facebook/textvqa",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "elephant.jpeg",
+				type: "img",
+			},
+			{
+				label: "Question",
+				content: "What is in this image?",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				type: "chart",
+				data: [
+					{
+						label: "elephant",
+						score: 0.97,
+					},
+					{
+						label: "elephants",
+						score: 0.06,
+					},
+					{
+						label: "animal",
+						score: 0.003,
+					},
+				],
+			},
+		],
+	},
+	isPlaceholder: false,
+	metrics: [
+		{
+			description: "",
+			id: "accuracy",
+		},
+		{
+			description:
+				"Measures how much a predicted answer differs from the ground truth based on the difference in their semantic meaning.",
+			id: "wu-palmer similarity",
+		},
+	],
+	models: [
+		{
+			description: "A visual question answering model trained to convert charts and plots to text.",
+			id: "google/deplot",
+		},
+		{
+			description:
+				"A visual question answering model trained for mathematical reasoning and chart derendering from images.",
+			id: "google/matcha-base",
+		},
+		{
+			description: "A strong visual question answering that answers questions from book covers.",
+			id: "google/pix2struct-ocrvqa-large",
+		},
+	],
+	spaces: [
+		{
+			description: "An application that compares visual question answering models across different tasks.",
+			id: "merve/pix2struct",
+		},
+		{
+			description: "An application that can answer questions based on images.",
+			id: "nielsr/vilt-vqa",
+		},
+		{
+			description: "An application that can caption images and answer questions about a given image. ",
+			id: "Salesforce/BLIP",
+		},
+		{
+			description: "An application that can caption images and answer questions about a given image. ",
+			id: "vumichien/Img2Prompt",
+		},
+	],
+	summary:
+		"Visual Question Answering is the task of answering open-ended questions based on an image. They output natural language responses to natural language questions.",
+	widgetModels: ["dandelin/vilt-b32-finetuned-vqa"],
+	youtubeId: "",
+};
+
+export default taskData;

package/src/tasks/visual-question-answering/inference.ts

@@ -0,0 +1,62 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Visual Question Answering inference
+ */
+export interface VisualQuestionAnsweringInput {
+	/**
+	 * One (image, question) pair to answer
+	 */
+	inputs: VisualQuestionAnsweringInputData;
+	/**
+	 * Additional inference parameters
+	 */
+	parameters?: VisualQuestionAnsweringParameters;
+	[property: string]: unknown;
+}
+/**
+ * One (image, question) pair to answer
+ */
+export interface VisualQuestionAnsweringInputData {
+	/**
+	 * The image.
+	 */
+	image: unknown;
+	/**
+	 * The question to answer based on the image.
+	 */
+	question: unknown;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters
+ *
+ * Additional inference parameters for Visual Question Answering
+ */
+export interface VisualQuestionAnsweringParameters {
+	/**
+	 * The number of answers to return (will be chosen by order of likelihood). Note that we
+	 * return less than topk answers if there are not enough options available within the
+	 * context.
+	 */
+	top_k?: number;
+	[property: string]: unknown;
+}
+export type VisualQuestionAnsweringOutput = VisualQuestionAnsweringOutputElement[];
+/**
+ * Outputs of inference for the Visual Question Answering task
+ */
+export interface VisualQuestionAnsweringOutputElement {
+	/**
+	 * The answer to the question
+	 */
+	answer?: string;
+	/**
+	 * The associated score / probability
+	 */
+	score: number;
+	[property: string]: unknown;
+}

package/src/tasks/visual-question-answering/spec/input.json

@@ -0,0 +1,41 @@
+{
+	"$id": "/inference/schemas/visual-question-answering/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Visual Question Answering inference",
+	"title": "VisualQuestionAnsweringInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"description": "One (image, question) pair to answer",
+			"type": "object",
+			"title": "VisualQuestionAnsweringInputData",
+			"properties": {
+				"image": {
+					"description": "The image."
+				},
+				"question": {
+					"description": "The question to answer based on the image."
+				}
+			},
+			"required": ["question", "image"]
+		},
+		"parameters": {
+			"description": "Additional inference parameters",
+			"$ref": "#/$defs/VisualQuestionAnsweringParameters"
+		}
+	},
+	"$defs": {
+		"VisualQuestionAnsweringParameters": {
+			"title": "VisualQuestionAnsweringParameters",
+			"description": "Additional inference parameters for Visual Question Answering",
+			"type": "object",
+			"properties": {
+				"top_k": {
+					"type": "integer",
+					"description": "The number of answers to return (will be chosen by order of likelihood). Note that we return less than topk answers if there are not enough options available within the context."
+				}
+			}
+		}
+	},
+	"required": ["inputs"]
+}

package/src/tasks/visual-question-answering/spec/output.json

@@ -0,0 +1,21 @@
+{
+	"$id": "/inference/schemas/visual-question-answering/output.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Outputs of inference for the Visual Question Answering task",
+	"title": "VisualQuestionAnsweringOutput",
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+			"answer": {
+				"type": "string",
+				"description": "The answer to the question"
+			},
+			"score": {
+				"type": "number",
+				"description": "The associated score / probability"
+			}
+		},
+		"required": ["score"]
+	}
+}