@huggingface/tasks 0.12.22 → 0.12.24

package/src/snippets/python.ts

@@ -1,22 +1,128 @@
 import type { PipelineType } from "../pipelines.js";
+import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
+import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
 import { getModelInputSnippet } from "./inputs.js";
-import type { ModelDataMinimal } from "./types.js";
+import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
 
-export const snippetConversational = (model: ModelDataMinimal, accessToken: string): string =>
-	`from huggingface_hub import InferenceClient
+export const snippetConversational = (
+	model: ModelDataMinimal,
+	accessToken: string,
+	opts?: {
+		streaming?: boolean;
+		messages?: ChatCompletionInputMessage[];
+		temperature?: GenerationParameters["temperature"];
+		max_tokens?: GenerationParameters["max_tokens"];
+		top_p?: GenerationParameters["top_p"];
+	}
+): InferenceSnippet[] => {
+	const streaming = opts?.streaming ?? true;
+	const messages: ChatCompletionInputMessage[] = opts?.messages ?? [
+		{ role: "user", content: "What is the capital of France?" },
+	];
+	const messagesStr = stringifyMessages(messages, {
+		sep: ",\n\t",
+		start: `[\n\t`,
+		end: `\n]`,
+		attributeKeyQuotes: true,
+	});
+
+	const config = {
+		...(opts?.temperature ? { temperature: opts.temperature } : undefined),
+		max_tokens: opts?.max_tokens ?? 500,
+		...(opts?.top_p ? { top_p: opts.top_p } : undefined),
+	};
+	const configStr = stringifyGenerationConfig(config, {
+		sep: ",\n\t",
+		start: "",
+		end: "",
+		attributeValueConnector: "=",
+	});
+
+	if (streaming) {
+		return [
+			{
+				client: "huggingface_hub",
+				content: `from huggingface_hub import InferenceClient
 
 client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
 
-for message in client.chat_completion(
-	model="${model.id}",
-	messages=[{"role": "user", "content": "What is the capital of France?"}],
-	max_tokens=500,
-	stream=True,
-):
-    print(message.choices[0].delta.content, end="")`;
+messages = ${messagesStr}
 
-export const snippetConversationalWithImage = (model: ModelDataMinimal, accessToken: string): string =>
-	`from huggingface_hub import InferenceClient
+stream = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr},
+	stream=True
+)
+
+for chunk in stream:
+    print(chunk.choices[0].delta.content)`,
+			},
+			{
+				client: "openai",
+				content: `from openai import OpenAI
+
+client = OpenAI(
+	base_url="https://api-inference.huggingface.co/v1/",
+	api_key="${accessToken || "{API_TOKEN}"}"
+)
+
+messages = ${messagesStr}
+
+stream = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr},
+	stream=True
+)
+
+for chunk in stream:
+    print(chunk.choices[0].delta.content)`,
+			},
+		];
+	} else {
+		return [
+			{
+				client: "huggingface_hub",
+				content: `from huggingface_hub import InferenceClient
+
+client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
+
+messages = ${messagesStr}
+
+completion = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr}
+)
+
+print(completion.choices[0].message)`,
+			},
+			{
+				client: "openai",
+				content: `from openai import OpenAI
+
+client = OpenAI(
+	base_url="https://api-inference.huggingface.co/v1/",
+	api_key="${accessToken || "{API_TOKEN}"}"
+)
+
+messages = ${messagesStr}
+
+completion = client.chat.completions.create(
+    model="${model.id}", 
+	messages=messages, 
+	${configStr}
+)
+
+print(completion.choices[0].message)`,
+			},
+		];
+	}
+};
+
+export const snippetConversationalWithImage = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => ({
+	content: `from huggingface_hub import InferenceClient
 
 client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
 
@@ -36,20 +142,22 @@ for message in client.chat_completion(
 	max_tokens=500,
 	stream=True,
 ):
-	print(message.choices[0].delta.content, end="")`;
+	print(message.choices[0].delta.content, end="")`,
+});
 
-export const snippetZeroShotClassification = (model: ModelDataMinimal): string =>
-	`def query(payload):
+export const snippetZeroShotClassification = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.json()
 
 output = query({
     "inputs": ${getModelInputSnippet(model)},
     "parameters": {"candidate_labels": ["refund", "legal", "faq"]},
-})`;
+})`,
+});
 
-export const snippetZeroShotImageClassification = (model: ModelDataMinimal): string =>
-	`def query(data):
+export const snippetZeroShotImageClassification = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(data):
 	with open(data["image_path"], "rb") as f:
 		img = f.read()
 	payload={
@@ -62,28 +170,31 @@ export const snippetZeroShotImageClassification = (model: ModelDataMinimal): str
 output = query({
     "image_path": ${getModelInputSnippet(model)},
     "parameters": {"candidate_labels": ["cat", "dog", "llama"]},
-})`;
+})`,
+});
 
-export const snippetBasic = (model: ModelDataMinimal): string =>
-	`def query(payload):
+export const snippetBasic = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.json()
 	
 output = query({
 	"inputs": ${getModelInputSnippet(model)},
-})`;
+})`,
+});
 
-export const snippetFile = (model: ModelDataMinimal): string =>
-	`def query(filename):
+export const snippetFile = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(filename):
     with open(filename, "rb") as f:
         data = f.read()
     response = requests.post(API_URL, headers=headers, data=data)
     return response.json()
 
-output = query(${getModelInputSnippet(model)})`;
+output = query(${getModelInputSnippet(model)})`,
+});
 
-export const snippetTextToImage = (model: ModelDataMinimal): string =>
-	`def query(payload):
+export const snippetTextToImage = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.content
 image_bytes = query({
@@ -92,22 +203,25 @@ image_bytes = query({
 # You can access the image with PIL.Image for example
 import io
 from PIL import Image
-image = Image.open(io.BytesIO(image_bytes))`;
+image = Image.open(io.BytesIO(image_bytes))`,
+});
 
-export const snippetTabular = (model: ModelDataMinimal): string =>
-	`def query(payload):
+export const snippetTabular = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.content
 response = query({
 	"inputs": {"data": ${getModelInputSnippet(model)}},
-})`;
+})`,
+});
 
-export const snippetTextToAudio = (model: ModelDataMinimal): string => {
+export const snippetTextToAudio = (model: ModelDataMinimal): InferenceSnippet => {
 	// Transformers TTS pipeline and api-inference-community (AIC) pipeline outputs are diverged
 	// with the latest update to inference-api (IA).
 	// Transformers IA returns a byte object (wav file), whereas AIC returns wav and sampling_rate.
 	if (model.library_name === "transformers") {
-		return `def query(payload):
+		return {
+			content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.content
 
@@ -116,9 +230,11 @@ audio_bytes = query({
 })
 # You can access the audio with IPython.display for example
 from IPython.display import Audio
-Audio(audio_bytes)`;
+Audio(audio_bytes)`,
+		};
 	} else {
-		return `def query(payload):
+		return {
+			content: `def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
 	return response.json()
 	
@@ -127,12 +243,13 @@ audio, sampling_rate = query({
 })
 # You can access the audio with IPython.display for example
 from IPython.display import Audio
-Audio(audio, rate=sampling_rate)`;
+Audio(audio, rate=sampling_rate)`,
+		};
 	}
 };
 
-export const snippetDocumentQuestionAnswering = (model: ModelDataMinimal): string =>
-	`def query(payload):
+export const snippetDocumentQuestionAnswering = (model: ModelDataMinimal): InferenceSnippet => ({
+	content: `def query(payload):
  	with open(payload["image"], "rb") as f:
   		img = f.read()
 		payload["image"] = base64.b64encode(img).decode("utf-8")  
@@ -141,9 +258,19 @@ export const snippetDocumentQuestionAnswering = (model: ModelDataMinimal): strin
 
 output = query({
     "inputs": ${getModelInputSnippet(model)},
-})`;
+})`,
+});
 
-export const pythonSnippets: Partial<Record<PipelineType, (model: ModelDataMinimal, accessToken: string) => string>> = {
+export const pythonSnippets: Partial<
+	Record<
+		PipelineType,
+		(
+			model: ModelDataMinimal,
+			accessToken: string,
+			opts?: Record<string, unknown>
+		) => InferenceSnippet | InferenceSnippet[]
+	>
+> = {
 	// Same order as in tasks/src/pipelines.ts
 	"text-classification": snippetBasic,
 	"token-classification": snippetBasic,
@@ -174,25 +301,36 @@ export const pythonSnippets: Partial<Record<PipelineType, (model: ModelDataMinim
 	"zero-shot-image-classification": snippetZeroShotImageClassification,
 };
 
-export function getPythonInferenceSnippet(model: ModelDataMinimal, accessToken: string): string {
+export function getPythonInferenceSnippet(
+	model: ModelDataMinimal,
+	accessToken: string,
+	opts?: Record<string, unknown>
+): InferenceSnippet | InferenceSnippet[] {
 	if (model.pipeline_tag === "text-generation" && model.tags.includes("conversational")) {
 		// Conversational model detected, so we display a code snippet that features the Messages API
-		return snippetConversational(model, accessToken);
+		return snippetConversational(model, accessToken, opts);
 	} else if (model.pipeline_tag === "image-text-to-text" && model.tags.includes("conversational")) {
 		// Example sending an image to the Message API
 		return snippetConversationalWithImage(model, accessToken);
 	} else {
-		const body =
+		let snippets =
 			model.pipeline_tag && model.pipeline_tag in pythonSnippets
-				? pythonSnippets[model.pipeline_tag]?.(model, accessToken) ?? ""
-				: "";
+				? pythonSnippets[model.pipeline_tag]?.(model, accessToken) ?? { content: "" }
+				: { content: "" };
 
-		return `import requests
+		snippets = Array.isArray(snippets) ? snippets : [snippets];
 
+		return snippets.map((snippet) => {
+			return {
+				...snippet,
+				content: `import requests
+	
 API_URL = "https://api-inference.huggingface.co/models/${model.id}"
 headers = {"Authorization": ${accessToken ? `"Bearer ${accessToken}"` : `f"Bearer {API_TOKEN}"`}}
-
-${body}`;
+	
+${snippet.content}`,
+			};
+		});
 	}
 }
 

package/src/snippets/types.ts

@@ -9,3 +9,8 @@ export type ModelDataMinimal = Pick<
 	ModelData,
 	"id" | "pipeline_tag" | "mask_token" | "library_name" | "config" | "tags" | "inference"
 >;
+
+export interface InferenceSnippet {
+	content: string;
+	client?: string; // for instance: `client` could be `huggingface_hub` or `openai` client for Python snippets
+}

package/src/tasks/depth-estimation/data.ts

@@ -33,11 +33,15 @@ const taskData: TaskDataCustom = {
 		},
 		{
 			description: "A strong monocular depth estimation model.",
-			id: "Bingxin/Marigold",
+			id: "jingheya/lotus-depth-g-v1-0",
 		},
 		{
-			description: "A metric depth estimation model trained on NYU dataset.",
-			id: "Intel/zoedepth-nyu",
+			description: "A depth estimation model that predicts depth in videos.",
+			id: "tencent/DepthCrafter",
+		},
+		{
+			description: "A robust depth estimation model.",
+			id: "apple/DepthPro",
 		},
 	],
 	spaces: [
@@ -46,12 +50,16 @@ const taskData: TaskDataCustom = {
 			id: "radames/dpt-depth-estimation-3d-voxels",
 		},
 		{
-			description: "An application on cutting-edge depth estimation.",
-			id: "depth-anything/Depth-Anything-V2",
+			description: "An application for bleeding-edge depth estimation.",
+			id: "akhaliq/depth-pro",
+		},
+		{
+			description: "An application on cutting-edge depth estimation in videos.",
+			id: "tencent/DepthCrafter",
 		},
 		{
-			description: "An application to try state-of-the-art depth estimation.",
-			id: "merve/compare_depth_models",
+			description: "A human-centric depth estimation application.",
+			id: "facebook/sapiens-depth",
 		},
 	],
 	summary: "Depth estimation is the task of predicting depth of the objects present in an image.",

package/src/tasks/image-segmentation/about.md

@@ -48,7 +48,7 @@ import { HfInference } from "@huggingface/inference";
 const inference = new HfInference(HF_TOKEN);
 await inference.imageSegmentation({
 	data: await (await fetch("https://picsum.photos/300/300")).blob(),
-	model: "facebook/detr-resnet-50-panoptic",
+	model: "facebook/mask2former-swin-base-coco-panoptic",
 });
 ```
 

package/src/tasks/image-segmentation/data.ts

@@ -44,23 +44,24 @@ const taskData: TaskDataCustom = {
 	models: [
 		{
 			// TO DO: write description
-			description: "Solid panoptic segmentation model trained on the COCO 2017 benchmark dataset.",
-			id: "facebook/detr-resnet-50-panoptic",
+			description:
+				"Solid semantic segmentation model trained on ADE20k.",
+			id: "openmmlab/upernet-convnext-small",
 		},
 		{
 			description: "Background removal model.",
 			id: "briaai/RMBG-1.4",
 		},
-		{
-			description: "Semantic segmentation model trained on ADE20k benchmark dataset with 512x512 resolution.",
-			id: "nvidia/segformer-b0-finetuned-ade-512-512",
-		},
 		{
 			description: "A multipurpose image segmentation model for high resolution images.",
 			id: "ZhengPeng7/BiRefNet",
 		},
 		{
-			description: "Panoptic segmentation model trained COCO (common objects) dataset.",
+			description: "Powerful human-centric image segmentation model.",
+			id: "facebook/sapiens-seg-1b",
+		},
+		{
+			description: "Panoptic segmentation model trained on the COCO (common objects) dataset.",
 			id: "facebook/mask2former-swin-large-coco-panoptic",
 		},
 	],
@@ -74,8 +75,8 @@ const taskData: TaskDataCustom = {
 			id: "jbrinkma/segment-anything",
 		},
 		{
-			description: "A semantic segmentation application that predicts human silhouettes.",
-			id: "keras-io/Human-Part-Segmentation",
+			description: "A human-centric segmentation model.",
+			id: "facebook/sapiens-pose",
 		},
 		{
 			description: "An instance segmentation application to predict neuronal cell types from microscopy images.",

package/src/tasks/image-text-to-text/data.ts

@@ -47,20 +47,24 @@ const taskData: TaskDataCustom = {
 			id: "meta-llama/Llama-3.2-11B-Vision-Instruct",
 		},
 		{
-			description: "Cutting-edge conversational vision language model that can take multiple image inputs.",
-			id: "HuggingFaceM4/idefics2-8b-chatty",
+			description: "Cutting-edge vision language models.",
+			id: "allenai/Molmo-7B-D-0924",
 		},
 		{
 			description: "Small yet powerful model.",
 			id: "vikhyatk/moondream2",
 		},
 		{
-			description: "Strong image-text-to-text model made to understand documents.",
-			id: "mPLUG/DocOwl1.5",
+			description: "Strong image-text-to-text model.",
+			id: "Qwen/Qwen2-VL-7B-Instruct",
 		},
 		{
 			description: "Strong image-text-to-text model.",
-			id: "microsoft/Phi-3.5-vision-instruct",
+			id: "mistralai/Pixtral-12B-2409",
+		},
+		{
+			description: "Strong image-text-to-text model focused on documents.",
+			id: "stepfun-ai/GOT-OCR2_0",
 		},
 	],
 	spaces: [
@@ -74,20 +78,24 @@ const taskData: TaskDataCustom = {
 		},
 		{
 			description: "Powerful vision-language model assistant.",
-			id: "liuhaotian/LLaVA-1.6",
+			id: "akhaliq/Molmo-7B-D-0924",
+		},
+		{
+			description: "An image-text-to-text application focused on documents.",
+			id: "stepfun-ai/GOT_official_online_demo",
 		},
 		{
 			description: "An application to compare outputs of different vision language models.",
 			id: "merve/compare_VLMs",
 		},
 		{
-			description: "An application for document vision language tasks.",
-			id: "mPLUG/DocOwl",
+			description: "An application for chatting with an image-text-to-text model.",
+			id: "GanymedeNil/Qwen2-VL-7B",
 		},
 	],
 	summary:
 		"Image-text-to-text models take in an image and text prompt and output text. These models are also called vision-language models, or VLMs. The difference from image-to-text models is that these models take an additional text input, not restricting the model to certain use cases like image captioning, and may also be trained to accept a conversation as input.",
-	widgetModels: ["microsoft/kosmos-2-patch14-224"],
+	widgetModels: ["meta-llama/Llama-3.2-11B-Vision-Instruct"],
 	youtubeId: "IoGaGfU1CIg",
 };
 

package/src/tasks/keypoint-detection/data.ts

@@ -29,7 +29,7 @@ const taskData: TaskDataCustom = {
 		},
 		{
 			description: "Strong keypoint detection model used to detect human pose.",
-			id: "qualcomm/MediaPipe-Pose-Estimation",
+			id: "facebook/sapiens-pose-1b",
 		},
 	],
 	spaces: [

package/src/tasks/object-detection/data.ts

@@ -43,15 +43,16 @@ const taskData: TaskDataCustom = {
 	],
 	models: [
 		{
-			description: "Solid object detection model trained on the benchmark dataset COCO 2017.",
+			description: "Solid object detection model pre-trained on the COCO 2017 dataset.",
 			id: "facebook/detr-resnet-50",
 		},
 		{
-			description: "Strong object detection model trained on ImageNet-21k dataset.",
-			id: "microsoft/beit-base-patch16-224-pt22k-ft22k",
+			description: "Real-time and accurate object detection model.",
+			id: "jameslahm/yolov10x",
 		},
 		{
-			description: "Fast and accurate object detection model trained on COCO dataset.",
+			description:
+				"Fast and accurate object detection model trained on COCO and Object365 datasets.",
 			id: "PekingU/rtdetr_r18vd_coco_o365",
 		},
 	],

package/src/tasks/text-generation/data.ts

@@ -58,10 +58,6 @@ const taskData: TaskDataCustom = {
 			description: "A text-generation model trained to follow instructions.",
 			id: "google/gemma-2-2b-it",
 		},
-		{
-			description: "A code generation model that can generate code in 80+ languages.",
-			id: "bigcode/starcoder",
-		},
 		{
 			description: "Very powerful text generation model trained to follow instructions.",
 			id: "meta-llama/Meta-Llama-3.1-8B-Instruct",
@@ -75,12 +71,12 @@ const taskData: TaskDataCustom = {
 			id: "AI-MO/NuminaMath-7B-TIR",
 		},
 		{
-			description: "Strong coding assistant model.",
-			id: "HuggingFaceH4/starchat2-15b-v0.1",
+			description: "Strong text generation model to follow instructions.",
+			id: "Qwen/Qwen2.5-7B-Instruct",
 		},
 		{
 			description: "Very strong open-source large language model.",
-			id: "mistralai/Mistral-Nemo-Instruct-2407",
+			id: "nvidia/Llama-3.1-Nemotron-70B-Instruct",
 		},
 	],
 	spaces: [
@@ -88,6 +84,10 @@ const taskData: TaskDataCustom = {
 			description: "A leaderboard to compare different open-source text generation models based on various benchmarks.",
 			id: "open-llm-leaderboard/open_llm_leaderboard",
 		},
+		{
+			description: "A leaderboard for comparing chain-of-thought performance of models.",
+			id: "logikon/open_cot_leaderboard",
+		},
 		{
 			description: "An text generation based application based on a very powerful LLaMA2 model.",
 			id: "ysharma/Explore_llamav2_with_TGI",

package/src/tasks/text-to-image/data.ts

@@ -71,8 +71,8 @@ const taskData: TaskDataCustom = {
 			id: "jbilcke-hf/ai-comic-factory",
 		},
 		{
-			description: "A text-to-image application that can generate coherent text inside the image.",
-			id: "DeepFloyd/IF",
+			description: "An application to match multiple custom image generation models.",
+			id: "multimodalart/flux-lora-lab",
 		},
 		{
 			description: "A powerful yet very fast image generation application.",

package/src/tasks/text-to-speech/data.ts

@@ -57,9 +57,13 @@ const taskData: TaskDataCustom = {
 			id: "suno/bark",
 		},
 		{
-			description: "XTTS is a Voice generation model that lets you clone voices into different languages.",
+			description: "An application on XTTS, a voice generation model that lets you clone voices into different languages.",
 			id: "coqui/xtts",
 		},
+		{
+			description: "An application that generates speech in different styles in English and Chinese.",
+			id: "mrfakename/E2-F5-TTS",
+		},
 		{
 			description: "An application that synthesizes speech for diverse speaker prompts.",
 			id: "parler-tts/parler_tts_mini",

package/src/tasks/text-to-video/data.ts

@@ -67,30 +67,30 @@ const taskData: TaskDataCustom = {
 	],
 	models: [
 		{
-			description: "A strong model for video generation.",
-			id: "Vchitect/LaVie",
+			description: "A strong model for consistent video generation.",
+			id: "rain1011/pyramid-flow-sd3",
 		},
 		{
 			description: "A robust model for text-to-video generation.",
-			id: "damo-vilab/text-to-video-ms-1.7b",
+			id: "VideoCrafter/VideoCrafter2",
 		},
 		{
-			description: "A text-to-video generation model with high quality and smooth outputs.",
-			id: "hotshotco/Hotshot-XL",
+			description: "A cutting-edge text-to-video generation model.",
+			id: "TIGER-Lab/T2V-Turbo-V2",
 		},
 	],
 	spaces: [
 		{
 			description: "An application that generates video from text.",
-			id: "fffiloni/zeroscope",
+			id: "VideoCrafter/VideoCrafter",
 		},
 		{
-			description: "An application that generates video from image and text.",
-			id: "Vchitect/LaVie",
+			description: "Consistent video generation application.",
+			id: "TIGER-Lab/T2V-Turbo-V2",
 		},
 		{
-			description: "An application that generates videos from text and provides multi-model support.",
-			id: "ArtGAN/Video-Diffusion-WebUI",
+			description: "A cutting edge video generation application.",
+			id: "Pyramid-Flow/pyramid-flow",
 		},
 	],
 	summary:

package/src/tasks/token-classification/inference.ts

@@ -60,12 +60,15 @@ export interface TokenClassificationOutputElement {
 	/**
 	 * The character position in the input where this group ends.
 	 */
-	end?: number;
+	end: number;
 	/**
-	 * The predicted label for that group of tokens
+	 * The predicted label for a single token
+	 */
+	entity?: string;
+	/**
+	 * The predicted label for a group of one or more tokens
 	 */
 	entity_group?: string;
-	label: unknown;
 	/**
 	 * The associated score / probability
 	 */
@@ -73,10 +76,10 @@ export interface TokenClassificationOutputElement {
 	/**
 	 * The character position in the input where this group begins.
 	 */
-	start?: number;
+	start: number;
 	/**
 	 * The corresponding text
 	 */
-	word?: string;
+	word: string;
 	[property: string]: unknown;
 }