@huggingface/tasks 0.0.1

package/src/image-segmentation/about.md

@@ -0,0 +1,63 @@
+## Use Cases
+
+### Autonomous Driving
+
+Segmentation models are used to identify road patterns such as lanes and obstacles for safer driving.
+
+### Background Removal
+
+Image Segmentation models are used in cameras to erase the background of certain objects and apply filters to them.
+
+### Medical Imaging
+
+Image Segmentation models are used to distinguish organs or tissues, improving medical imaging workflows. Models are used to segment dental instances, analyze X-Ray scans or even segment cells for pathological diagnosis. This [dataset](https://github.com/v7labs/covid-19-xray-dataset) contains images of lungs of healthy patients and patients with COVID-19 segmented with masks. Another [segmentation dataset](https://ivdm3seg.weebly.com/data.html) contains segmented MRI data of the lower spine to analyze the effect of spaceflight simulation.
+
+## Task Variants
+
+### Semantic Segmentation
+
+Semantic Segmentation is the task of segmenting parts of an image that belong to the same class. Semantic Segmentation models make predictions for each pixel and return the probabilities of the classes for each pixel. These models are evaluated on Mean Intersection Over Union (Mean IoU).
+
+### Instance Segmentation
+
+Instance Segmentation is the variant of Image Segmentation where every distinct object is segmented, instead of one segment per class.
+
+### Panoptic Segmentation
+
+Panoptic Segmentation is the Image Segmentation task that segments the image both by instance and by class, assigning each pixel a different instance of the class.
+
+## Inference
+
+You can infer with Image Segmentation models using the `image-segmentation` pipeline. You need to install [timm](https://github.com/rwightman/pytorch-image-models) first.
+
+```python
+!pip install timm
+model = pipeline("image-segmentation")
+model("cat.png")
+#[{'label': 'cat',
+#  'mask': mask_code,
+#  'score': 0.999}
+# ...]
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image segmentation models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_ACCESS_TOKEN);
+await inference.imageSegmentation({
+  data: await (await fetch('https://picsum.photos/300/300')).blob(),
+  model: 'facebook/detr-resnet-50-panoptic',  
+})
+```
+
+## Useful Resources
+
+Would you like to learn more about image segmentation? Great! Here you can find some curated resources that you may find helpful!
+
+- [Fine-Tune a Semantic Segmentation Model with a Custom Dataset](https://huggingface.co/blog/fine-tune-segformer)
+- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8)
+- [A Guide on Universal Image Segmentation with Mask2Former and OneFormer](https://huggingface.co/blog/mask2former)
+- [Zero-shot image segmentation with CLIPSeg](https://huggingface.co/blog/clipseg-zero-shot)
+- [Semantic segmentation task guide](https://huggingface.co/docs/transformers/tasks/semantic_segmentation)

package/src/image-segmentation/data.ts

@@ -0,0 +1,96 @@
+import type { TaskDataCustom } from "../Types";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Scene segmentation dataset.",
+			id:          "scene_parse_150",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "image-segmentation-input.jpeg",
+				type:     "img",
+			},
+		],
+		outputs: [
+			{
+				filename: "image-segmentation-output.png",
+				type:     "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "Average Precision (AP) is the Area Under the PR Curve (AUC-PR). It is calculated for each semantic class separately",
+			id:          "Average Precision",
+		},
+		{
+			description: "Mean Average Precision (mAP) is the overall average of the AP values",
+			id:          "Mean Average Precision",
+		},
+		{
+			description: "Intersection over Union (IoU) is the overlap of segmentation masks. Mean IoU is the average of the IoU of all semantic classes",
+			id:          "Mean Intersection over Union",
+		},
+		{
+			description: "APα is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75",
+			id:          "APα",
+		},
+	],
+	models: [
+		{
+			// TO DO: write description
+			description: "Solid panoptic segmentation model trained on the COCO 2017 benchmark dataset.",
+			id:          "facebook/detr-resnet-50-panoptic",
+		},
+		{
+			description: "Semantic segmentation model trained on ADE20k benchmark dataset.",
+			id:          "microsoft/beit-large-finetuned-ade-640-640",
+		},
+		{
+			description: "Semantic segmentation model trained on ADE20k benchmark dataset with 512x512 resolution.",
+			id:          "nvidia/segformer-b0-finetuned-ade-512-512",
+		},
+		{
+			description: "Semantic segmentation model trained Cityscapes dataset.",
+			id:          "facebook/mask2former-swin-large-cityscapes-semantic",
+		},
+		{
+			description: "Panoptic segmentation model trained COCO (common objects) dataset.",
+			id:          "facebook/mask2former-swin-large-coco-panoptic",
+		},
+	],
+	spaces:       [
+		{
+			description: "A semantic segmentation application that can predict unseen instances out of the box.",
+			id:          "facebook/ov-seg",
+		},
+		{
+			description: "One of the strongest segmentation applications.",
+			id:          "jbrinkma/segment-anything",
+		},
+		{
+			description: "A semantic segmentation application that predicts human silhouettes.",
+			id:          "keras-io/Human-Part-Segmentation",
+		},
+		{
+			description: "An instance segmentation application to predict neuronal cell types from microscopy images.",
+			id:          "rashmi/sartorius-cell-instance-segmentation",
+		},
+		{
+			description: "An application that segments videos.",
+			id:          "ArtGAN/Segment-Anything-Video",
+		},
+		{
+			description: "An panoptic segmentation application built for outdoor environments.",
+			id:          "segments/panoptic-segment-anything",
+		},
+	],
+	summary:      "Image Segmentation divides an image into segments where each pixel in the image is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation.",
+	widgetModels: ["facebook/detr-resnet-50-panoptic"],
+	youtubeId:    "dKE8SIt9C-w",
+};
+
+export default taskData;

package/src/image-to-image/about.md

@@ -0,0 +1,81 @@
+## Use Cases
+
+### Style transfer
+
+One of the most popular use cases of image to image is the style transfer. Style transfer models can convert a regular photography into a painting in the style of a famous painter.
+
+## Task Variants
+
+### Image inpainting
+
+Image inpainting is widely used during photography editing to remove unwanted objects, such as poles, wires or sensor
+dust.
+
+### Image colorization
+
+Old, black and white images can be brought up to life using an image colorization model.
+
+### Super Resolution
+
+Super resolution models increase the resolution of an image, allowing for higher quality viewing and printing.
+
+## Inference
+
+You can use pipelines for image-to-image in 🧨diffusers library to easily use image-to-image models. See an example for `StableDiffusionImg2ImgPipeline` below.
+
+```python
+from PIL import Image
+from diffusers import StableDiffusionImg2ImgPipeline
+
+model_id_or_path = "runwayml/stable-diffusion-v1-5"
+pipe = StableDiffusionImg2ImgPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16)
+pipe = pipe.to(cuda)
+
+init_image = Image.open("mountains_image.jpeg").convert("RGB").resize((768, 512))
+prompt = "A fantasy landscape, trending on artstation"
+
+images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images
+images[0].save("fantasy_landscape.png")
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image-to-image models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_ACCESS_TOKEN);
+await inference.imageToImage({
+  data: await (await fetch('image')).blob(),
+  model: "timbrooks/instruct-pix2pix", 
+  parameters: {
+    prompt: "Deblur this image"
+  }
+})
+```
+
+## ControlNet
+
+Controlling outputs of diffusion models only with a text prompt is a challenging problem. ControlNet is a neural network type that provides an image based control to diffusion models. These controls can be edges or landmarks in an image.
+
+Many ControlNet models were trained in our community event, JAX Diffusers sprint. You can see the full list of the ControlNet models available [here](https://huggingface.co/spaces/jax-diffusers-event/leaderboard). 
+
+## Most Used Model for the Task
+
+Pix2Pix is a popular model used for image to image translation tasks. It is based on a conditional-GAN (generative adversarial network) where instead of a noise vector a 2D image is given as input. More information about Pix2Pix can be retrieved from this [link](https://phillipi.github.io/pix2pix/) where the associated paper and the GitHub repository can be found.
+
+
+Below images show some of the examples shared in the paper that can be obtained using Pix2Pix. There are various cases this model can be applied on. It is capable of relatively simpler things, e.g. converting a grayscale image to its colored version. But more importantly, it can generate realistic pictures from rough sketches (can be seen in the purse example) or from painting-like images (can be seen in the street and facade examples below).
+
+<img src="/tasks/assets/image-to-image/pix2pix_examples.jpg" alt="Alt text" title="Optional title">
+
+
+## Useful Resources
+
+- [Train your ControlNet with diffusers 🧨](https://huggingface.co/blog/train-your-controlnet)
+- [Ultra fast ControlNet with 🧨 Diffusers](https://huggingface.co/blog/controlnet)
+
+## References 
+
+[1] P. Isola, J. -Y. Zhu, T. Zhou and A. A. Efros, "Image-to-Image Translation with Conditional Adversarial Networks," 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2017, pp. 5967-5976, doi: 10.1109/CVPR.2017.632.
+
+This page was made possible thanks to the efforts of [Paul Gafton](https://github.com/Paul92) and [Osman Alenbey](https://huggingface.co/osman93).

package/src/image-to-image/data.ts

@@ -0,0 +1,97 @@
+import type { TaskDataCustom } from "../Types";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "Synthetic dataset, for image relighting",
+			id:          "VIDIT",
+		},
+		{
+			description: "Multiple images of celebrities, used for facial expression translation",
+			id:          "huggan/CelebA-faces",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "image-to-image-input.jpeg",
+				type:     "img",
+			},
+		],
+		outputs: [
+			{
+				filename: "image-to-image-output.png",
+				type:     "img",
+			},
+		],
+	},
+	isPlaceholder: false,
+	metrics:       [
+		{
+			description: "Peak Signal to Noise Ratio (PSNR) is an approximation of the human perception, considering the ratio of the absolute intensity with respect to the variations. Measured in dB, a high value indicates a high fidelity.",
+			id:          "PSNR",
+		},
+		{
+			description: "Structural Similarity Index (SSIM) is a perceptual metric which compares the luminance, contrast and structure of two images. The values of SSIM range between -1 and 1, and higher values indicate closer resemblance to the original image.",
+			id:          "SSIM",
+		},
+		{
+			description: "Inception Score (IS) is an analysis of the labels predicted by an image classification model when presented with a sample of the generated images.",
+			id:          "IS",
+		},
+	],
+	models: [
+		{
+			description: "A model that enhances images captured in low light conditions.",
+			id:          "keras-io/low-light-image-enhancement",
+		},
+		{
+			description: "A model that increases the resolution of an image.",
+			id:          "keras-io/super-resolution",
+		},
+		{
+			description: "A model that creates a set of variations of the input image in the style of DALL-E using Stable Diffusion.",
+			id:          "lambdalabs/sd-image-variations-diffusers",
+		},
+		{
+			description: "A model that generates images based on segments in the input image and the text prompt.",
+			id:          "mfidabel/controlnet-segment-anything",
+		},
+		{
+			description: "A model that takes an image and an instruction to edit the image.",
+			id:          "timbrooks/instruct-pix2pix",
+		},
+	],
+	spaces:       [
+		{
+			description: "Image enhancer application for low light.",
+			id:          "keras-io/low-light-image-enhancement",
+		},
+		{
+			description: "Style transfer application.",
+			id:          "keras-io/neural-style-transfer",
+		},
+		{
+			description: "An application that generates images based on segment control.",
+			id:          "mfidabel/controlnet-segment-anything",
+		},
+		{
+			description: "Image generation application that takes image control and text prompt.",
+			id:          "hysts/ControlNet",
+		},
+		{
+			description: "Colorize any image using this app.",
+			id:          "ioclab/brightness-controlnet",
+		},
+		{
+			description: "Edit images with instructions.",
+			id:          "timbrooks/instruct-pix2pix",
+		},
+
+	],
+	summary:      "Image-to-image is the task of transforming a source image to match the characteristics of a target image or a target image domain. Any image manipulation and enhancement is possible with image to image models.",
+	widgetModels: ["lllyasviel/sd-controlnet-canny"],
+	youtubeId:    "",
+};
+
+export default taskData;

package/src/image-to-text/about.md

@@ -0,0 +1,58 @@
+## Use Cases
+### Image Captioning
+Image Captioning is the process of generating textual description of an image. 
+This can help the visually impaired people to understand what's happening in their surroundings.
+
+### Optical Character Recognition (OCR)
+OCR models convert the text present in an image, e.g. a scanned document, to text.
+
+
+
+## Pix2Struct
+
+Pix2Struct is a state-of-the-art model built and released by Google AI. The model itself has to be trained on a downstream task to be used. These tasks include, captioning UI components, images including text, visual questioning infographics, charts, scientific diagrams and more. You can find these models on recommended models of this page.
+
+## Inference
+### Image Captioning
+You can use the 🤗 Transformers library's `image-to-text` pipeline to generate caption for the Image input.
+```python
+from transformers import pipeline
+
+captioner = pipeline("image-to-text",model="Salesforce/blip-image-captioning-base")
+captioner("https://huggingface.co/datasets/Narsil/image_dummy/resolve/main/parrots.png")
+## [{'generated_text': 'two birds are standing next to each other '}]
+```
+
+### OCR
+This code snippet uses Microsoft’s TrOCR, an encoder-decoder model consisting of an image Transformer encoder and a text Transformer decoder for state-of-the-art optical character recognition (OCR) on single-text line images.
+```python
+from transformers import TrOCRProcessor, VisionEncoderDecoderModel
+
+processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten')
+model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten')
+pixel_values = processor(images="image.jpeg", return_tensors="pt").pixel_values
+
+generated_ids = model.generate(pixel_values)
+generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+```
+
+You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image-to-text models on Hugging Face Hub.
+
+```javascript
+import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference(HF_ACCESS_TOKEN);
+await inference.imageToText({
+  data: await (await fetch('https://picsum.photos/300/300')).blob(),
+  model: 'Salesforce/blip-image-captioning-base',  
+})
+```
+
+## Useful Resources
+- [Image Captioning](https://huggingface.co/docs/transformers/main/en/tasks/image_captioning)
+- [Image captioning use case](https://blog.google/outreach-initiatives/accessibility/get-image-descriptions/)
+- [Train Image Captioning model on your dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/GIT/Fine_tune_GIT_on_an_image_captioning_dataset.ipynb)
+- [Train OCR model on your dataset ](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/TrOCR)
+
+This page was made possible thanks to efforts of [Sukesh Perla](https://huggingface.co/hitchhiker3010) and [Johannes Kolbe](https://huggingface.co/johko).

package/src/image-to-text/data.ts

@@ -0,0 +1,87 @@
+import type { TaskDataCustom } from "../Types";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			// TODO write proper description
+			description: "Dataset from 12M image-text of Reddit",
+			id:          "red_caps",
+		},
+		{
+			// TODO write proper description
+			description: "Dataset from 3.3M images of Google",
+			id:          "datasets/conceptual_captions",
+		},
+
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "savanna.jpg",
+				type:     "img",
+			},
+		],
+		outputs: [
+			{
+				label:   "Detailed description",
+				content: "a herd of giraffes and zebras grazing in a field",
+				type:    "text",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "A robust image captioning model.",
+			id:          "Salesforce/blip-image-captioning-large",
+		},
+		{
+			description: "A strong image captioning model.",
+			id:          "nlpconnect/vit-gpt2-image-captioning",
+		},
+		{
+			description: "A strong optical character recognition model.",
+			id:          "microsoft/trocr-base-printed",
+		},
+		{
+			description: "A strong visual question answering model for scientific diagrams.",
+			id:          "google/pix2struct-ai2d-base",
+		},
+		{
+			description: "A strong captioning model for UI components.",
+			id:          "google/pix2struct-widget-captioning-base",
+		},
+		{
+			description: "A captioning model for images that contain text.",
+			id:          "google/pix2struct-textcaps-base",
+		},
+	],
+	spaces:       [
+		{
+			description: "A robust image captioning application.",
+			id:          "flax-community/image-captioning",
+		},
+		{
+			description: "An application that transcribes handwritings into text.",
+			id:          "nielsr/TrOCR-handwritten",
+		},
+		{
+			description: "An application that can caption images and answer questions about a given image.",
+			id:          "Salesforce/BLIP",
+		},
+		{
+			description: "An application that can caption images and answer questions with a conversational agent.",
+			id:          "Salesforce/BLIP2",
+		},
+		{
+			description: "An image captioning application that demonstrates the effect of noise on captions.",
+			id:          "johko/capdec-image-captioning",
+		},
+	],
+	summary:      "Image to text models output a text from a given image. Image captioning or optical character recognition can be considered as the most common applications of image to text.",
+	widgetModels: ["Salesforce/blip-image-captioning-base"],
+	youtubeId:    "",
+};
+
+
+export default taskData;

package/src/index.ts

@@ -0,0 +1,2 @@
+export type { TaskData, TaskDemo, TaskDemoEntry } from "./Types";
+export { TASKS_DATA } from "./tasksData";

package/src/object-detection/about.md

@@ -0,0 +1,36 @@
+## Use Cases
+
+### Autonomous Driving
+
+Object Detection is widely used in computer vision for autonomous driving. Self-driving cars use Object Detection models to detect pedestrians, bicycles, traffic lights and road signs to decide which step to take.
+
+### Object Tracking in Matches
+
+Object Detection models are widely used in sports where the ball or a player is tracked for monitoring and refereeing during matches.
+
+### Image Search
+
+Object Detection models are widely used in image search. Smartphones use Object Detection models to detect entities (such as specific places or objects) and allow the user to search for the entity on the Internet.
+
+### Object Counting
+
+Object Detection models are used to count instances of objects in a given image, this can include counting the objects in warehouses or stores, or counting the number of visitors in a store. They are also used to manage crowds at events to prevent disasters.
+
+## Inference
+
+You can infer with Object Detection models through the `object-detection` pipeline. When calling the pipeline you just need to specify a path or http link to an image.
+
+```python
+model = pipeline("object-detection")
+
+model("path_to_cat_image")
+
+# [{'label': 'blanket',
+#  'mask': mask_string,
+#  'score': 0.917},
+#...]
+```
+
+# Useful Resources
+- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8)
+- [Object detection task guide](https://huggingface.co/docs/transformers/tasks/object_detection)

package/src/object-detection/data.ts

@@ -0,0 +1,73 @@
+import type { TaskDataCustom } from "../Types";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			// TODO write proper description
+			description: "Widely used benchmark dataset for multiple Vision tasks.",
+			id:          "merve/coco2017",
+		},
+	],
+	demo: {
+		inputs: [
+			{
+				filename: "object-detection-input.jpg",
+				type:     "img",
+			},
+		],
+		outputs: [
+			{
+				filename: "object-detection-output.jpg",
+				type:     "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description: "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately",
+			id:          "Average Precision",
+		},
+		{
+			description: "The Mean Average Precision (mAP) metric is the overall average of the AP values",
+			id:          "Mean Average Precision",
+		},
+		{
+			description: "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75",
+			id:          "APα",
+		},
+	],
+	models: [
+		{
+			// TO DO: write description
+			description: "Solid object detection model trained on the benchmark dataset COCO 2017.",
+			id:          "facebook/detr-resnet-50",
+		},
+		{
+			description: "Strong object detection model trained on ImageNet-21k dataset.",
+			id:          "microsoft/beit-base-patch16-224-pt22k-ft22k",
+		},
+	],
+	spaces:       [
+		{
+			description: "An object detection application that can detect unseen objects out of the box.",
+			id:          "adirik/OWL-ViT",
+		},
+		{
+			description: "An application that contains various object detection models to try from.",
+			id:          "Gradio-Blocks/Object-Detection-With-DETR-and-YOLOS",
+		},
+		{
+			description: "An application that shows multiple cutting edge techniques for object detection and tracking.",
+			id:          "kadirnar/torchyolo",
+		},
+		{
+			description: "An object tracking, segmentation and inpainting application.",
+			id:          "VIPLab/Track-Anything",
+		},
+	],
+	summary:      "Object Detection models allow users to identify objects of certain defined classes. Object detection models receive an image as input and output the images with bounding boxes and labels on detected objects.",
+	widgetModels: ["facebook/detr-resnet-50"],
+	youtubeId:    "WdAeKSOpxhw",
+};
+
+export default taskData;

package/src/placeholder/about.md

@@ -0,0 +1,15 @@
+## Use Cases
+
+You can contribute this area with common use cases of the task! 
+
+## Task Variants 
+
+This place can be filled with variants of this task if there's any. 
+
+## Inference
+
+This section should have useful information about how to pull a model from Hugging Face Hub that is a part of a library specialized in a task and use it.
+
+## Useful Resources
+
+In this area, you can insert useful resources about how to train or use a model for this task.

package/src/placeholder/data.ts

@@ -0,0 +1,18 @@
+import type { TaskDataCustom } from "../Types";
+
+const taskData: TaskDataCustom = {
+	datasets: [],
+	demo:     {
+		inputs:  [],
+		outputs: [],
+	},
+	isPlaceholder: true,
+	metrics:       [],
+	models:        [],
+	spaces:        [],
+	summary:       "",
+	widgetModels:  [],
+	youtubeId:     undefined,
+};
+
+export default taskData;