@huggingface/tasks 0.0.6 → 0.0.8

package/src/model-data.ts

@@ -0,0 +1,239 @@
+import type { PipelineType } from "./pipelines";
+
+type TableData = Record<string, (string | number)[]>;
+
+//#region outputs
+export type WidgetExampleOutputLabels = Array<{ label: string; score: number }>;
+export interface WidgetExampleOutputAnswerScore {
+	answer: string;
+	score: number;
+}
+export interface WidgetExampleOutputText {
+	text: string;
+}
+export interface WidgetExampleOutputUrl {
+	url: string;
+}
+
+export type WidgetExampleOutput =
+	| WidgetExampleOutputLabels
+	| WidgetExampleOutputAnswerScore
+	| WidgetExampleOutputText
+	| WidgetExampleOutputUrl;
+//#endregion
+
+export interface WidgetExampleBase<TOutput> {
+	example_title?: string;
+	group?: string;
+	/**
+	 * Potential overrides to API parameters for this specific example
+	 * (takes precedences over the model card metadata's inference.parameters)
+	 */
+	parameters?: {
+		/// token-classification
+		aggregation_strategy?: string;
+		/// text-generation
+		top_k?: number;
+		top_p?: number;
+		temperature?: number;
+		max_new_tokens?: number;
+		do_sample?: boolean;
+		/// text-to-image
+		negative_prompt?: string;
+		guidance_scale?: number;
+		num_inference_steps?: number;
+	};
+	/**
+	 * Optional output
+	 */
+	output?: TOutput;
+}
+
+export interface WidgetExampleTextInput<TOutput = WidgetExampleOutput> extends WidgetExampleBase<TOutput> {
+	text: string;
+}
+
+export interface WidgetExampleTextAndContextInput<TOutput = WidgetExampleOutput>
+	extends WidgetExampleTextInput<TOutput> {
+	context: string;
+}
+
+export interface WidgetExampleTextAndTableInput<TOutput = WidgetExampleOutput> extends WidgetExampleTextInput<TOutput> {
+	table: TableData;
+}
+
+export interface WidgetExampleAssetInput<TOutput = WidgetExampleOutput> extends WidgetExampleBase<TOutput> {
+	src: string;
+}
+export interface WidgetExampleAssetAndPromptInput<TOutput = WidgetExampleOutput>
+	extends WidgetExampleAssetInput<TOutput> {
+	prompt: string;
+}
+
+export type WidgetExampleAssetAndTextInput<TOutput = WidgetExampleOutput> = WidgetExampleAssetInput<TOutput> &
+	WidgetExampleTextInput<TOutput>;
+
+export type WidgetExampleAssetAndZeroShotInput<TOutput = WidgetExampleOutput> = WidgetExampleAssetInput<TOutput> &
+	WidgetExampleZeroShotTextInput<TOutput>;
+
+export interface WidgetExampleStructuredDataInput<TOutput = WidgetExampleOutput> extends WidgetExampleBase<TOutput> {
+	structured_data: TableData;
+}
+
+export interface WidgetExampleTableDataInput<TOutput = WidgetExampleOutput> extends WidgetExampleBase<TOutput> {
+	table: TableData;
+}
+
+export interface WidgetExampleZeroShotTextInput<TOutput = WidgetExampleOutput> extends WidgetExampleTextInput<TOutput> {
+	text: string;
+	candidate_labels: string;
+	multi_class: boolean;
+}
+
+export interface WidgetExampleSentenceSimilarityInput<TOutput = WidgetExampleOutput>
+	extends WidgetExampleBase<TOutput> {
+	source_sentence: string;
+	sentences: string[];
+}
+
+//#endregion
+
+export type WidgetExample<TOutput = WidgetExampleOutput> =
+	| WidgetExampleTextInput<TOutput>
+	| WidgetExampleTextAndContextInput<TOutput>
+	| WidgetExampleTextAndTableInput<TOutput>
+	| WidgetExampleAssetInput<TOutput>
+	| WidgetExampleAssetAndPromptInput<TOutput>
+	| WidgetExampleAssetAndTextInput<TOutput>
+	| WidgetExampleAssetAndZeroShotInput<TOutput>
+	| WidgetExampleStructuredDataInput<TOutput>
+	| WidgetExampleTableDataInput<TOutput>
+	| WidgetExampleZeroShotTextInput<TOutput>
+	| WidgetExampleSentenceSimilarityInput<TOutput>;
+
+type KeysOfUnion<T> = T extends unknown ? keyof T : never;
+
+export type WidgetExampleAttribute = KeysOfUnion<WidgetExample>;
+
+export enum InferenceDisplayability {
+	/**
+	 * Yes
+	 */
+	Yes = "Yes",
+	/**
+	 * And then, all the possible reasons why it's no:
+	 */
+	ExplicitOptOut = "ExplicitOptOut",
+	CustomCode = "CustomCode",
+	LibraryNotDetected = "LibraryNotDetected",
+	PipelineNotDetected = "PipelineNotDetected",
+	PipelineLibraryPairNotSupported = "PipelineLibraryPairNotSupported",
+}
+
+/**
+ * Public interface for model metadata
+ */
+export interface ModelData {
+	/**
+	 * id of model (e.g. 'user/repo_name')
+	 */
+	id: string;
+	/**
+	 * Kept for backward compatibility
+	 */
+	modelId?: string;
+	/**
+	 * Whether or not to enable inference widget for this model
+	 */
+	inference: InferenceDisplayability;
+	/**
+	 * is this model private?
+	 */
+	private?: boolean;
+	/**
+	 * this dictionary has useful information about the model configuration
+	 */
+	config?: Record<string, unknown> & {
+		adapter_transformers?: { model_class?: string; model_name?: string };
+		architectures?: string[];
+		sklearn?: {
+			filename?: string;
+			model_format?: string;
+		};
+		speechbrain?: {
+			interface?: string;
+		};
+		peft?: {
+			base_model_name?: string;
+			task_type?: string;
+		};
+	};
+	/**
+	 * all the model tags
+	 */
+	tags?: string[];
+	/**
+	 * transformers-specific info to display in the code sample.
+	 */
+	transformersInfo?: TransformersInfo;
+	/**
+	 * Pipeline type
+	 */
+	pipeline_tag?: PipelineType | undefined;
+	/**
+	 * for relevant models, get mask token
+	 */
+	mask_token?: string | undefined;
+	/**
+	 * Example data that will be fed into the widget.
+	 *
+	 * can be set in the model card metadata (under `widget`),
+	 * or by default in `DefaultWidget.ts`
+	 */
+	widgetData?: WidgetExample[] | undefined;
+	/**
+	 * Parameters that will be used by the widget when calling Inference API
+	 * https://huggingface.co/docs/api-inference/detailed_parameters
+	 *
+	 * can be set in the model card metadata (under `inference/parameters`)
+	 * Example:
+	 * inference:
+	 *     parameters:
+	 *         key: val
+	 */
+	cardData?: {
+		inference?:
+			| boolean
+			| {
+					parameters?: Record<string, unknown>;
+			  };
+		base_model?: string;
+	};
+	/**
+	 * Library name
+	 * Example: transformers, SpeechBrain, Stanza, etc.
+	 */
+	library_name?: string;
+}
+
+/**
+ * transformers-specific info to display in the code sample.
+ */
+export interface TransformersInfo {
+	/**
+	 * e.g. AutoModelForSequenceClassification
+	 */
+	auto_model: string;
+	/**
+	 * if set in config.json's auto_map
+	 */
+	custom_class?: string;
+	/**
+	 * e.g. text-classification
+	 */
+	pipeline_tag?: PipelineType;
+	/**
+	 * e.g. "AutoTokenizer" | "AutoFeatureExtractor" | "AutoProcessor"
+	 */
+	processor?: string;
+}

package/src/pipelines.ts

@@ -435,9 +435,28 @@ export const PIPELINE_DATA = {
 	},
 	"image-to-image": {
 		name: "Image-to-Image",
+		subtasks: [
+			{
+				type: "image-inpainting",
+				name: "Image Inpainting",
+			},
+			{
+				type: "image-colorization",
+				name: "Image Colorization",
+			},
+			{
+				type: "super-resolution",
+				name: "Super Resolution",
+			},
+		],
 		modality: "cv",
 		color: "indigo",
 	},
+	"image-to-video": {
+		name: "Image-to-Video",
+		modality: "multimodal",
+		color: "indigo",
+	},
 	"unconditional-image-generation": {
 		name: "Unconditional Image Generation",
 		modality: "cv",
@@ -606,6 +625,26 @@ export const PIPELINE_DATA = {
 		modality: "multimodal",
 		color: "green",
 	},
+	"mask-generation": {
+		name: "Mask Generation",
+		modality: "cv",
+		color: "indigo",
+	},
+	"zero-shot-object-detection": {
+		name: "Zero-Shot Object Detection",
+		modality: "cv",
+		color: "yellow",
+	},
+	"text-to-3d": {
+		name: "Text-to-3D",
+		modality: "multimodal",
+		color: "yellow",
+	},
+	"image-to-3d": {
+		name: "Image-to-3D",
+		modality: "multimodal",
+		color: "green",
+	},
 	other: {
 		name: "Other",
 		modality: "other",

package/src/snippets/curl.ts

@@ -0,0 +1,63 @@
+import type { ModelData } from "../model-data.js";
+import type { PipelineType } from "../pipelines.js";
+import { getModelInputSnippet } from "./inputs.js";
+
+export const snippetBasic = (model: ModelData, accessToken: string): string =>
+	`curl https://api-inference.huggingface.co/models/${model.id} \\
+	-X POST \\
+	-d '{"inputs": ${getModelInputSnippet(model, true)}}' \\
+	-H 'Content-Type: application/json' \\
+	-H "Authorization: Bearer ${accessToken || `{API_TOKEN}`}"
+`;
+
+export const snippetZeroShotClassification = (model: ModelData, accessToken: string): string =>
+	`curl https://api-inference.huggingface.co/models/${model.id} \\
+	-X POST \\
+	-d '{"inputs": ${getModelInputSnippet(model, true)}, "parameters": {"candidate_labels": ["refund", "legal", "faq"]}}' \\
+	-H 'Content-Type: application/json' \\
+	-H "Authorization: Bearer ${accessToken || `{API_TOKEN}`}"
+`;
+
+export const snippetFile = (model: ModelData, accessToken: string): string =>
+	`curl https://api-inference.huggingface.co/models/${model.id} \\
+	-X POST \\
+	--data-binary '@${getModelInputSnippet(model, true, true)}' \\
+	-H "Authorization: Bearer ${accessToken || `{API_TOKEN}`}"
+`;
+
+export const curlSnippets: Partial<Record<PipelineType, (model: ModelData, accessToken: string) => string>> = {
+	// Same order as in js/src/lib/interfaces/Types.ts
+	"text-classification": snippetBasic,
+	"token-classification": snippetBasic,
+	"table-question-answering": snippetBasic,
+	"question-answering": snippetBasic,
+	"zero-shot-classification": snippetZeroShotClassification,
+	translation: snippetBasic,
+	summarization: snippetBasic,
+	conversational: snippetBasic,
+	"feature-extraction": snippetBasic,
+	"text-generation": snippetBasic,
+	"text2text-generation": snippetBasic,
+	"fill-mask": snippetBasic,
+	"sentence-similarity": snippetBasic,
+	"automatic-speech-recognition": snippetFile,
+	"text-to-image": snippetBasic,
+	"text-to-speech": snippetBasic,
+	"text-to-audio": snippetBasic,
+	"audio-to-audio": snippetFile,
+	"audio-classification": snippetFile,
+	"image-classification": snippetFile,
+	"image-to-text": snippetFile,
+	"object-detection": snippetFile,
+	"image-segmentation": snippetFile,
+};
+
+export function getCurlInferenceSnippet(model: ModelData, accessToken: string): string {
+	return model.pipeline_tag && model.pipeline_tag in curlSnippets
+		? curlSnippets[model.pipeline_tag]?.(model, accessToken) ?? ""
+		: "";
+}
+
+export function hasCurlInferenceSnippet(model: ModelData): boolean {
+	return !!model.pipeline_tag && model.pipeline_tag in curlSnippets;
+}

package/src/snippets/index.ts

@@ -0,0 +1,6 @@
+import * as inputs from "./inputs";
+import * as curl from "./curl";
+import * as python from "./python";
+import * as js from "./js";
+
+export { inputs, curl, python, js };

package/src/snippets/inputs.ts

@@ -0,0 +1,144 @@
+import type { ModelData } from "../model-data";
+import type { PipelineType } from "../pipelines";
+
+const inputsZeroShotClassification = () =>
+	`"Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"`;
+
+const inputsTranslation = () => `"Меня зовут Вольфганг и я живу в Берлине"`;
+
+const inputsSummarization = () =>
+	`"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`;
+
+const inputsConversational = () =>
+	`{
+		"past_user_inputs": ["Which movie is the best ?"],
+		"generated_responses": ["It is Die Hard for sure."],
+		"text": "Can you explain why ?"
+	}`;
+
+const inputsTableQuestionAnswering = () =>
+	`{
+		"query": "How many stars does the transformers repository have?",
+		"table": {
+			"Repository": ["Transformers", "Datasets", "Tokenizers"],
+			"Stars": ["36542", "4512", "3934"],
+			"Contributors": ["651", "77", "34"],
+			"Programming language": [
+				"Python",
+				"Python",
+				"Rust, Python and NodeJS"
+			]
+		}
+	}`;
+
+const inputsVisualQuestionAnswering = () =>
+	`{
+		"image": "cat.png",
+		"question": "What is in this image?"
+	}`;
+
+const inputsQuestionAnswering = () =>
+	`{
+		"question": "What is my name?",
+		"context": "My name is Clara and I live in Berkeley."
+	}`;
+
+const inputsTextClassification = () => `"I like you. I love you"`;
+
+const inputsTokenClassification = () => `"My name is Sarah Jessica Parker but you can call me Jessica"`;
+
+const inputsTextGeneration = () => `"Can you please let us know more details about your "`;
+
+const inputsText2TextGeneration = () => `"The answer to the universe is"`;
+
+const inputsFillMask = (model: ModelData) => `"The answer to the universe is ${model.mask_token}."`;
+
+const inputsSentenceSimilarity = () =>
+	`{
+		"source_sentence": "That is a happy person",
+		"sentences": [
+			"That is a happy dog",
+			"That is a very happy person",
+			"Today is a sunny day"
+		]
+	}`;
+
+const inputsFeatureExtraction = () => `"Today is a sunny day and I will get some ice cream."`;
+
+const inputsImageClassification = () => `"cats.jpg"`;
+
+const inputsImageToText = () => `"cats.jpg"`;
+
+const inputsImageSegmentation = () => `"cats.jpg"`;
+
+const inputsObjectDetection = () => `"cats.jpg"`;
+
+const inputsAudioToAudio = () => `"sample1.flac"`;
+
+const inputsAudioClassification = () => `"sample1.flac"`;
+
+const inputsTextToImage = () => `"Astronaut riding a horse"`;
+
+const inputsTextToSpeech = () => `"The answer to the universe is 42"`;
+
+const inputsTextToAudio = () => `"liquid drum and bass, atmospheric synths, airy sounds"`;
+
+const inputsAutomaticSpeechRecognition = () => `"sample1.flac"`;
+
+const inputsTabularPrediction = () =>
+	`'{"Height":[11.52,12.48],"Length1":[23.2,24.0],"Length2":[25.4,26.3],"Species": ["Bream","Bream"]}'`;
+
+const inputsZeroShotImageClassification = () => `"cats.jpg"`;
+
+const modelInputSnippets: {
+	[key in PipelineType]?: (model: ModelData) => string;
+} = {
+	"audio-to-audio": inputsAudioToAudio,
+	"audio-classification": inputsAudioClassification,
+	"automatic-speech-recognition": inputsAutomaticSpeechRecognition,
+	conversational: inputsConversational,
+	"document-question-answering": inputsVisualQuestionAnswering,
+	"feature-extraction": inputsFeatureExtraction,
+	"fill-mask": inputsFillMask,
+	"image-classification": inputsImageClassification,
+	"image-to-text": inputsImageToText,
+	"image-segmentation": inputsImageSegmentation,
+	"object-detection": inputsObjectDetection,
+	"question-answering": inputsQuestionAnswering,
+	"sentence-similarity": inputsSentenceSimilarity,
+	summarization: inputsSummarization,
+	"table-question-answering": inputsTableQuestionAnswering,
+	"tabular-regression": inputsTabularPrediction,
+	"tabular-classification": inputsTabularPrediction,
+	"text-classification": inputsTextClassification,
+	"text-generation": inputsTextGeneration,
+	"text-to-image": inputsTextToImage,
+	"text-to-speech": inputsTextToSpeech,
+	"text-to-audio": inputsTextToAudio,
+	"text2text-generation": inputsText2TextGeneration,
+	"token-classification": inputsTokenClassification,
+	translation: inputsTranslation,
+	"zero-shot-classification": inputsZeroShotClassification,
+	"zero-shot-image-classification": inputsZeroShotImageClassification,
+};
+
+// Use noWrap to put the whole snippet on a single line (removing new lines and tabulations)
+// Use noQuotes to strip quotes from start & end (example: "abc" -> abc)
+export function getModelInputSnippet(model: ModelData, noWrap = false, noQuotes = false): string {
+	if (model.pipeline_tag) {
+		const inputs = modelInputSnippets[model.pipeline_tag];
+		if (inputs) {
+			let result = inputs(model);
+			if (noWrap) {
+				result = result.replace(/(?:(?:\r?\n|\r)\t*)|\t+/g, " ");
+			}
+			if (noQuotes) {
+				const REGEX_QUOTES = /^"(.+)"$/s;
+				const match = result.match(REGEX_QUOTES);
+				result = match ? match[1] : result;
+			}
+			return result;
+		}
+	}
+	return "No input example has been defined for this model task.";
+}

package/src/snippets/js.ts

@@ -0,0 +1,150 @@
+import type { ModelData } from "../model-data.js";
+import type { PipelineType } from "../pipelines.js";
+import { getModelInputSnippet } from "./inputs.js";
+
+export const snippetBasic = (model: ModelData, accessToken: string): string =>
+	`async function query(data) {
+	const response = await fetch(
+		"https://api-inference.huggingface.co/models/${model.id}",
+		{
+			headers: { Authorization: "Bearer ${accessToken || `{API_TOKEN}`}" },
+			method: "POST",
+			body: JSON.stringify(data),
+		}
+	);
+	const result = await response.json();
+	return result;
+}
+
+query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
+	console.log(JSON.stringify(response));
+});`;
+
+export const snippetZeroShotClassification = (model: ModelData, accessToken: string): string =>
+	`async function query(data) {
+	const response = await fetch(
+		"https://api-inference.huggingface.co/models/${model.id}",
+		{
+			headers: { Authorization: "Bearer ${accessToken || `{API_TOKEN}`}" },
+			method: "POST",
+			body: JSON.stringify(data),
+		}
+	);
+	const result = await response.json();
+	return result;
+}
+
+query({"inputs": ${getModelInputSnippet(
+		model
+	)}, "parameters": {"candidate_labels": ["refund", "legal", "faq"]}}).then((response) => {
+	console.log(JSON.stringify(response));
+});`;
+
+export const snippetTextToImage = (model: ModelData, accessToken: string): string =>
+	`async function query(data) {
+	const response = await fetch(
+		"https://api-inference.huggingface.co/models/${model.id}",
+		{
+			headers: { Authorization: "Bearer ${accessToken || `{API_TOKEN}`}" },
+			method: "POST",
+			body: JSON.stringify(data),
+		}
+	);
+	const result = await response.blob();
+	return result;
+}
+query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
+	// Use image
+});`;
+
+export const snippetTextToAudio = (model: ModelData, accessToken: string): string => {
+	const commonSnippet = `async function query(data) {
+		const response = await fetch(
+			"https://api-inference.huggingface.co/models/${model.id}",
+			{
+				headers: { Authorization: "Bearer ${accessToken || `{API_TOKEN}`}" },
+				method: "POST",
+				body: JSON.stringify(data),
+			}
+		);`;
+	if (model.library_name === "transformers") {
+		return (
+			commonSnippet +
+			`
+			const result = await response.blob();
+			return result;
+		}
+		query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
+			// Returns a byte object of the Audio wavform. Use it directly!
+		});`
+		);
+	} else {
+		return (
+			commonSnippet +
+			`
+			const result = await response.json();
+			return result;
+		}
+		
+		query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
+			console.log(JSON.stringify(response));
+		});`
+		);
+	}
+};
+
+export const snippetFile = (model: ModelData, accessToken: string): string =>
+	`async function query(filename) {
+	const data = fs.readFileSync(filename);
+	const response = await fetch(
+		"https://api-inference.huggingface.co/models/${model.id}",
+		{
+			headers: { Authorization: "Bearer ${accessToken || `{API_TOKEN}`}" },
+			method: "POST",
+			body: data,
+		}
+	);
+	const result = await response.json();
+	return result;
+}
+
+query(${getModelInputSnippet(model)}).then((response) => {
+	console.log(JSON.stringify(response));
+});`;
+
+export const jsSnippets: Partial<Record<PipelineType, (model: ModelData, accessToken: string) => string>> = {
+	// Same order as in js/src/lib/interfaces/Types.ts
+	"text-classification": snippetBasic,
+	"token-classification": snippetBasic,
+	"table-question-answering": snippetBasic,
+	"question-answering": snippetBasic,
+	"zero-shot-classification": snippetZeroShotClassification,
+	translation: snippetBasic,
+	summarization: snippetBasic,
+	conversational: snippetBasic,
+	"feature-extraction": snippetBasic,
+	"text-generation": snippetBasic,
+	"text2text-generation": snippetBasic,
+	"fill-mask": snippetBasic,
+	"sentence-similarity": snippetBasic,
+	"automatic-speech-recognition": snippetFile,
+	"text-to-image": snippetTextToImage,
+	"text-to-speech": snippetTextToAudio,
+	"text-to-audio": snippetTextToAudio,
+	"audio-to-audio": snippetFile,
+	"audio-classification": snippetFile,
+	"image-classification": snippetFile,
+	"image-to-text": snippetFile,
+	"object-detection": snippetFile,
+	"image-segmentation": snippetFile,
+};
+
+export function getJsInferenceSnippet(model: ModelData, accessToken: string): string {
+	return model.pipeline_tag && model.pipeline_tag in jsSnippets
+		? jsSnippets[model.pipeline_tag]?.(model, accessToken) ?? ""
+		: "";
+}
+
+export function hasJsInferenceSnippet(model: ModelData): boolean {
+	return !!model.pipeline_tag && model.pipeline_tag in jsSnippets;
+}