@huggingface/inference 2.3.3 → 2.5.0

package/README.md

@@ -149,6 +149,16 @@ await hf.audioClassification({
   data: readFileSync('test/sample1.flac')
 })
 
+await hf.textToSpeech({
+  model: 'espnet/kan-bayashi_ljspeech_vits',
+  inputs: 'Hello world!'
+})
+
+await hf.audioToAudio({
+  model: 'speechbrain/sepformer-wham',
+  data: readFileSync('test/sample1.flac')
+})
+
 // Computer Vision
 
 await hf.imageClassification({
@@ -187,6 +197,16 @@ await hf.imageToImage({
   model: "lllyasviel/sd-controlnet-depth",
 });
 
+await hf.zeroShotImageClassification({
+  model: 'openai/clip-vit-large-patch14-336',
+  inputs: {
+    image: await (await fetch('https://placekitten.com/300/300')).blob()
+  },  
+  parameters: {
+    candidate_labels: ['cat', 'dog']
+  }
+})
+
 // Multimodal
 
 await hf.visualQuestionAnswering({
@@ -211,12 +231,31 @@ await hf.tabularRegression({
   model: "scikit-learn/Fish-Weight",
   inputs: {
     data: {
-      "Height":["11.52", "12.48", "12.3778"],
-      "Length1":["23.2", "24", "23.9"],
-      "Length2":["25.4", "26.3", "26.5"],
-      "Length3":["30", "31.2", "31.1"],
-      "Species":["Bream", "Bream", "Bream"],
-      "Width":["4.02", "4.3056", "4.6961"]
+      "Height": ["11.52", "12.48", "12.3778"],
+      "Length1": ["23.2", "24", "23.9"],
+      "Length2": ["25.4", "26.3", "26.5"],
+      "Length3": ["30", "31.2", "31.1"],
+      "Species": ["Bream", "Bream", "Bream"],
+      "Width": ["4.02", "4.3056", "4.6961"]
+    },
+  },
+})
+
+await hf.tabularClassification({
+  model: "vvmnnnkv/wine-quality",
+  inputs: {
+    data: {
+      "fixed_acidity": ["7.4", "7.8", "10.3"],
+      "volatile_acidity": ["0.7", "0.88", "0.32"],
+      "citric_acid": ["0", "0", "0.45"],
+      "residual_sugar": ["1.9", "2.6", "6.4"],
+      "chlorides": ["0.076", "0.098", "0.073"],
+      "free_sulfur_dioxide": ["11", "25", "5"],
+      "total_sulfur_dioxide": ["34", "67", "13"],
+      "density": ["0.9978", "0.9968", "0.9976"],
+      "pH": ["3.51", "3.2", "3.23"],
+      "sulphates": ["0.56", "0.68", "0.82"],
+      "alcohol": ["9.4", "9.8", "12.6"]
     },
   },
 })
@@ -269,6 +308,8 @@ const { generated_text } = await gpt2.textGeneration({inputs: 'The answer to the
 
 - [x] Automatic speech recognition
 - [x] Audio classification
+- [x] Text to speech
+- [x] Audio to audio
 
 ### Computer Vision
 
@@ -278,6 +319,7 @@ const { generated_text } = await gpt2.textGeneration({inputs: 'The answer to the
 - [x] Text to image
 - [x] Image to text - [demo](https://huggingface.co/spaces/huggingfacejs/image-to-text)
 - [x] Image to Image
+- [x] Zero-shot image classification
 
 ### Multimodal
 
@@ -287,6 +329,7 @@ const { generated_text } = await gpt2.textGeneration({inputs: 'The answer to the
 ### Tabular
 
 - [x] Tabular regression
+- [x] Tabular classification
 
 ## Tree-shaking
 

package/dist/index.d.ts

@@ -26,6 +26,8 @@ export interface Options {
 	fetch?: typeof fetch;
 }
 
+export type InferenceTask = "text-classification" | "feature-extraction" | "sentence-similarity";
+
 export interface BaseArgs {
 	/**
 	 * The access token to use. Without it, you'll get rate-limited quickly.
@@ -72,6 +74,34 @@ export function audioClassification(
 	args: AudioClassificationArgs,
 	options?: Options
 ): Promise<AudioClassificationReturn>;
+export type AudioToAudioArgs = BaseArgs & {
+	/**
+	 * Binary audio data
+	 */
+	data: Blob | ArrayBuffer;
+};
+export type AudioToAudioReturn = AudioToAudioOutputValue[];
+export interface AudioToAudioOutputValue {
+	/**
+	 * The label for the audio output (model specific)
+	 */
+	label: string;
+
+	/**
+	 * Base64 encoded audio output.
+	 */
+	blob: string;
+
+	/**
+	 * Content-type for blob, e.g. audio/flac
+	 */
+	"content-type": string;
+}
+/**
+ * This task reads some audio input and outputs one or multiple audio files.
+ * Example model: speechbrain/sepformer-wham does audio source separation.
+ */
+export function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioReturn>;
 export type AutomaticSpeechRecognitionArgs = BaseArgs & {
 	/**
 	 * Binary audio data
@@ -112,6 +142,8 @@ export function request<T>(
 	options?: Options & {
 		/** For internal HF use, which is why it's not exposed in {@link Options} */
 		includeCredentials?: boolean;
+		/** When a model can be used for multiple tasks, and we want to run a non-default task */
+		task?: string | InferenceTask;
 	}
 ): Promise<T>;
 /**
@@ -122,6 +154,8 @@ export function streamingRequest<T>(
 	options?: Options & {
 		/** For internal HF use, which is why it's not exposed in {@link Options} */
 		includeCredentials?: boolean;
+		/** When a model can be used for multiple tasks, and we want to run a non-default task */
+		task?: string | InferenceTask;
 	}
 ): AsyncGenerator<T>;
 export type ImageClassificationArgs = BaseArgs & {
@@ -315,6 +349,33 @@ export type TextToImageOutput = Blob;
  * Recommended model: stabilityai/stable-diffusion-2
  */
 export function textToImage(args: TextToImageArgs, options?: Options): Promise<TextToImageOutput>;
+export type ZeroShotImageClassificationArgs = BaseArgs & {
+	inputs: {
+		/**
+		 * Binary image data
+		 */
+		image: Blob | ArrayBuffer;
+	};
+	parameters: {
+		/**
+		 * A list of strings that are potential classes for inputs. (max 10)
+		 */
+		candidate_labels: string[];
+	};
+};
+export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputValue[];
+export interface ZeroShotImageClassificationOutputValue {
+	label: string;
+	score: number;
+}
+/**
+ * Classify an image to specified classes.
+ * Recommended model: openai/clip-vit-large-patch14-336
+ */
+export function zeroShotImageClassification(
+	args: ZeroShotImageClassificationArgs,
+	options?: Options
+): Promise<ZeroShotImageClassificationOutput>;
 export type DocumentQuestionAnsweringArgs = BaseArgs & {
 	inputs: {
 		/**
@@ -448,9 +509,9 @@ export type FeatureExtractionArgs = BaseArgs & {
 	inputs: string | string[];
 };
 /**
- * Returned values are a list of floats, or a list of list of floats (depending on if you sent a string or a list of string, and if the automatic reduction, usually mean_pooling for instance was applied for you or not. This should be explained on the model's README.
+ * Returned values are a list of floats, or a list of list of floats, or a list of list of list of floats (depending on if you sent a string or a list of string, and if the automatic reduction, usually mean_pooling for instance was applied for you or not. This should be explained on the model's README.
  */
-export type FeatureExtractionOutput = (number | number[])[];
+export type FeatureExtractionOutput = (number | number[] | number[][])[];
 /**
  * This task reads some text and outputs raw float values, that are usually consumed as part of a semantic database/semantic search.
  */
@@ -875,6 +936,27 @@ export function zeroShotClassification(
 	args: ZeroShotClassificationArgs,
 	options?: Options
 ): Promise<ZeroShotClassificationOutput>;
+export type TabularClassificationArgs = BaseArgs & {
+	inputs: {
+		/**
+		 * A table of data represented as a dict of list where entries are headers and the lists are all the values, all lists must have the same size.
+		 */
+		data: Record<string, string[]>;
+	};
+};
+/**
+ * A list of predicted labels for each row
+ */
+export type TabularClassificationOutput = number[];
+/**
+ * Predicts target label for a given set of features in tabular form.
+ * Typically, you will want to train a classification model on your training data and use it with your new data of the same format.
+ * Example model: vvmnnnkv/wine-quality
+ */
+export function tabularClassification(
+	args: TabularClassificationArgs,
+	options?: Options
+): Promise<TabularClassificationOutput>;
 export type TabularRegressionArgs = BaseArgs & {
 	inputs: {
 		/**
@@ -910,6 +992,11 @@ export class HfInference {
 		args: Omit<AudioClassificationArgs, 'accessToken'>,
 		options?: Options
 	): Promise<AudioClassificationReturn>;
+	/**
+	 * This task reads some audio input and outputs one or multiple audio files.
+	 * Example model: speechbrain/sepformer-wham does audio source separation.
+	 */
+	audioToAudio(args: Omit<AudioToAudioArgs, 'accessToken'>, options?: Options): Promise<AudioToAudioReturn>;
 	/**
 	 * This task reads some audio input and outputs the said words within the audio files.
 	 * Recommended model (english language): facebook/wav2vec2-large-960h-lv60-self
@@ -931,6 +1018,8 @@ export class HfInference {
 		options?: Options & {
 			/** For internal HF use, which is why it's not exposed in {@link Options} */
 			includeCredentials?: boolean;
+			/** When a model can be used for multiple tasks, and we want to run a non-default task */
+			task?: string | InferenceTask;
 		}
 	): Promise<T>;
 	/**
@@ -941,6 +1030,8 @@ export class HfInference {
 		options?: Options & {
 			/** For internal HF use, which is why it's not exposed in {@link Options} */
 			includeCredentials?: boolean;
+			/** When a model can be used for multiple tasks, and we want to run a non-default task */
+			task?: string | InferenceTask;
 		}
 	): AsyncGenerator<T>;
 	/**
@@ -978,6 +1069,14 @@ export class HfInference {
 	 * Recommended model: stabilityai/stable-diffusion-2
 	 */
 	textToImage(args: Omit<TextToImageArgs, 'accessToken'>, options?: Options): Promise<TextToImageOutput>;
+	/**
+	 * Classify an image to specified classes.
+	 * Recommended model: openai/clip-vit-large-patch14-336
+	 */
+	zeroShotImageClassification(
+		args: Omit<ZeroShotImageClassificationArgs, 'accessToken'>,
+		options?: Options
+	): Promise<ZeroShotImageClassificationOutput>;
 	/**
 	 * Answers a question on a document image. Recommended model: impira/layoutlm-document-qa.
 	 */
@@ -1069,6 +1168,15 @@ export class HfInference {
 		args: Omit<ZeroShotClassificationArgs, 'accessToken'>,
 		options?: Options
 	): Promise<ZeroShotClassificationOutput>;
+	/**
+	 * Predicts target label for a given set of features in tabular form.
+	 * Typically, you will want to train a classification model on your training data and use it with your new data of the same format.
+	 * Example model: vvmnnnkv/wine-quality
+	 */
+	tabularClassification(
+		args: Omit<TabularClassificationArgs, 'accessToken'>,
+		options?: Options
+	): Promise<TabularClassificationOutput>;
 	/**
 	 * Predicts target value for a given set of features in tabular form.
 	 * Typically, you will want to train a regression model on your training data and use it with your new data of the same format.
@@ -1089,6 +1197,11 @@ export class HfInferenceEndpoint {
 		args: Omit<AudioClassificationArgs, 'accessToken' | 'model'>,
 		options?: Options
 	): Promise<AudioClassificationReturn>;
+	/**
+	 * This task reads some audio input and outputs one or multiple audio files.
+	 * Example model: speechbrain/sepformer-wham does audio source separation.
+	 */
+	audioToAudio(args: Omit<AudioToAudioArgs, 'accessToken' | 'model'>, options?: Options): Promise<AudioToAudioReturn>;
 	/**
 	 * This task reads some audio input and outputs the said words within the audio files.
 	 * Recommended model (english language): facebook/wav2vec2-large-960h-lv60-self
@@ -1110,6 +1223,8 @@ export class HfInferenceEndpoint {
 		options?: Options & {
 			/** For internal HF use, which is why it's not exposed in {@link Options} */
 			includeCredentials?: boolean;
+			/** When a model can be used for multiple tasks, and we want to run a non-default task */
+			task?: string | InferenceTask;
 		}
 	): Promise<T>;
 	/**
@@ -1120,6 +1235,8 @@ export class HfInferenceEndpoint {
 		options?: Options & {
 			/** For internal HF use, which is why it's not exposed in {@link Options} */
 			includeCredentials?: boolean;
+			/** When a model can be used for multiple tasks, and we want to run a non-default task */
+			task?: string | InferenceTask;
 		}
 	): AsyncGenerator<T>;
 	/**
@@ -1157,6 +1274,14 @@ export class HfInferenceEndpoint {
 	 * Recommended model: stabilityai/stable-diffusion-2
 	 */
 	textToImage(args: Omit<TextToImageArgs, 'accessToken' | 'model'>, options?: Options): Promise<TextToImageOutput>;
+	/**
+	 * Classify an image to specified classes.
+	 * Recommended model: openai/clip-vit-large-patch14-336
+	 */
+	zeroShotImageClassification(
+		args: Omit<ZeroShotImageClassificationArgs, 'accessToken' | 'model'>,
+		options?: Options
+	): Promise<ZeroShotImageClassificationOutput>;
 	/**
 	 * Answers a question on a document image. Recommended model: impira/layoutlm-document-qa.
 	 */
@@ -1248,6 +1373,15 @@ export class HfInferenceEndpoint {
 		args: Omit<ZeroShotClassificationArgs, 'accessToken' | 'model'>,
 		options?: Options
 	): Promise<ZeroShotClassificationOutput>;
+	/**
+	 * Predicts target label for a given set of features in tabular form.
+	 * Typically, you will want to train a classification model on your training data and use it with your new data of the same format.
+	 * Example model: vvmnnnkv/wine-quality
+	 */
+	tabularClassification(
+		args: Omit<TabularClassificationArgs, 'accessToken' | 'model'>,
+		options?: Options
+	): Promise<TabularClassificationOutput>;
 	/**
 	 * Predicts target value for a given set of features in tabular form.
 	 * Typically, you will want to train a regression model on your training data and use it with your new data of the same format.

package/dist/index.js

@@ -25,6 +25,7 @@ __export(src_exports, {
   HfInferenceEndpoint: () => HfInferenceEndpoint,
   InferenceOutputError: () => InferenceOutputError,
   audioClassification: () => audioClassification,
+  audioToAudio: () => audioToAudio,
   automaticSpeechRecognition: () => automaticSpeechRecognition,
   conversational: () => conversational,
   documentQuestionAnswering: () => documentQuestionAnswering,
@@ -41,6 +42,7 @@ __export(src_exports, {
   streamingRequest: () => streamingRequest,
   summarization: () => summarization,
   tableQuestionAnswering: () => tableQuestionAnswering,
+  tabularClassification: () => tabularClassification,
   tabularRegression: () => tabularRegression,
   textClassification: () => textClassification,
   textGeneration: () => textGeneration,
@@ -50,7 +52,8 @@ __export(src_exports, {
   tokenClassification: () => tokenClassification,
   translation: () => translation,
   visualQuestionAnswering: () => visualQuestionAnswering,
-  zeroShotClassification: () => zeroShotClassification
+  zeroShotClassification: () => zeroShotClassification,
+  zeroShotImageClassification: () => zeroShotImageClassification
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -58,6 +61,7 @@ module.exports = __toCommonJS(src_exports);
 var tasks_exports = {};
 __export(tasks_exports, {
   audioClassification: () => audioClassification,
+  audioToAudio: () => audioToAudio,
   automaticSpeechRecognition: () => automaticSpeechRecognition,
   conversational: () => conversational,
   documentQuestionAnswering: () => documentQuestionAnswering,
@@ -74,6 +78,7 @@ __export(tasks_exports, {
   streamingRequest: () => streamingRequest,
   summarization: () => summarization,
   tableQuestionAnswering: () => tableQuestionAnswering,
+  tabularClassification: () => tabularClassification,
   tabularRegression: () => tabularRegression,
   textClassification: () => textClassification,
   textGeneration: () => textGeneration,
@@ -83,13 +88,20 @@ __export(tasks_exports, {
   tokenClassification: () => tokenClassification,
   translation: () => translation,
   visualQuestionAnswering: () => visualQuestionAnswering,
-  zeroShotClassification: () => zeroShotClassification
+  zeroShotClassification: () => zeroShotClassification,
+  zeroShotImageClassification: () => zeroShotImageClassification
 });
 
+// src/lib/isUrl.ts
+function isUrl(modelOrUrl) {
+  return /^http(s?):/.test(modelOrUrl) || modelOrUrl.startsWith("/");
+}
+
 // src/lib/makeRequestOptions.ts
-var HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co/models/";
+var HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co";
 function makeRequestOptions(args, options) {
   const { model, accessToken, ...otherArgs } = args;
+  const { task, includeCredentials, ...otherOptions } = options ?? {};
   const headers = {};
   if (accessToken) {
     headers["Authorization"] = `Bearer ${accessToken}`;
@@ -108,15 +120,23 @@ function makeRequestOptions(args, options) {
       headers["X-Load-Model"] = "0";
     }
   }
-  const url = /^http(s?):/.test(model) || model.startsWith("/") ? model : `${HF_INFERENCE_API_BASE_URL}${model}`;
+  const url = (() => {
+    if (isUrl(model)) {
+      return model;
+    }
+    if (task) {
+      return `${HF_INFERENCE_API_BASE_URL}/pipeline/${task}/${model}`;
+    }
+    return `${HF_INFERENCE_API_BASE_URL}/models/${model}`;
+  })();
   const info = {
     headers,
     method: "POST",
     body: binary ? args.data : JSON.stringify({
       ...otherArgs,
-      options
+      options: options && otherOptions
     }),
-    credentials: options?.includeCredentials ? "include" : "same-origin"
+    credentials: includeCredentials ? "include" : "same-origin"
   };
   return { url, info };
 }
@@ -348,6 +368,18 @@ async function textToSpeech(args, options) {
   return res;
 }
 
+// src/tasks/audio/audioToAudio.ts
+async function audioToAudio(args, options) {
+  const res = await request(args, options);
+  const isValidOutput = Array.isArray(res) && res.every(
+    (x) => typeof x.label === "string" && typeof x.blob === "string" && typeof x["content-type"] === "string"
+  );
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected Array<{label: string, blob: string, content-type: string}>");
+  }
+  return res;
+}
+
 // src/tasks/cv/imageClassification.ts
 async function imageClassification(args, options) {
   const res = await request(args, options);
@@ -443,6 +475,26 @@ async function imageToImage(args, options) {
   return res;
 }
 
+// src/tasks/cv/zeroShotImageClassification.ts
+async function zeroShotImageClassification(args, options) {
+  const reqArgs = {
+    ...args,
+    inputs: {
+      image: base64FromBytes(
+        new Uint8Array(
+          args.inputs.image instanceof ArrayBuffer ? args.inputs.image : await args.inputs.image.arrayBuffer()
+        )
+      )
+    }
+  };
+  const res = await request(reqArgs, options);
+  const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected Array<{label: string, score: number}>");
+  }
+  return res;
+}
+
 // src/tasks/nlp/conversational.ts
 async function conversational(args, options) {
   const res = await request(args, options);
@@ -455,27 +507,60 @@ async function conversational(args, options) {
   return res;
 }
 
+// src/lib/getDefaultTask.ts
+var taskCache = /* @__PURE__ */ new Map();
+var CACHE_DURATION = 10 * 60 * 1e3;
+var MAX_CACHE_ITEMS = 1e3;
+var HF_HUB_URL = "https://huggingface.co";
+async function getDefaultTask(model, accessToken) {
+  if (isUrl(model)) {
+    return null;
+  }
+  const key = `${model}:${accessToken}`;
+  let cachedTask = taskCache.get(key);
+  if (cachedTask && cachedTask.date < new Date(Date.now() - CACHE_DURATION)) {
+    taskCache.delete(key);
+    cachedTask = void 0;
+  }
+  if (cachedTask === void 0) {
+    const modelTask = await fetch(`${HF_HUB_URL}/api/models/${model}?expand[]=pipeline_tag`, {
+      headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}
+    }).then((resp) => resp.json()).then((json) => json.pipeline_tag).catch(() => null);
+    if (!modelTask) {
+      return null;
+    }
+    cachedTask = { task: modelTask, date: /* @__PURE__ */ new Date() };
+    taskCache.set(key, { task: modelTask, date: /* @__PURE__ */ new Date() });
+    if (taskCache.size > MAX_CACHE_ITEMS) {
+      taskCache.delete(taskCache.keys().next().value);
+    }
+  }
+  return cachedTask.task;
+}
+
 // src/tasks/nlp/featureExtraction.ts
 async function featureExtraction(args, options) {
-  const res = await request(args, options);
+  const defaultTask = await getDefaultTask(args.model, args.accessToken);
+  const res = await request(
+    args,
+    defaultTask === "sentence-similarity" ? {
+      ...options,
+      task: "feature-extraction"
+    } : options
+  );
   let isValidOutput = true;
-  if (Array.isArray(res)) {
-    for (const e of res) {
-      if (Array.isArray(e)) {
-        isValidOutput = e.every((x) => typeof x === "number");
-        if (!isValidOutput) {
-          break;
-        }
-      } else if (typeof e !== "number") {
-        isValidOutput = false;
-        break;
-      }
+  const isNumArrayRec = (arr, maxDepth, curDepth = 0) => {
+    if (curDepth > maxDepth)
+      return false;
+    if (arr.every((x) => Array.isArray(x))) {
+      return arr.every((x) => isNumArrayRec(x, maxDepth, curDepth + 1));
+    } else {
+      return arr.every((x) => typeof x === "number");
     }
-  } else {
-    isValidOutput = false;
-  }
+  };
+  isValidOutput = Array.isArray(res) && isNumArrayRec(res, 2, 0);
   if (!isValidOutput) {
-    throw new InferenceOutputError("Expected Array<number[] | number>");
+    throw new InferenceOutputError("Expected Array<number[][] | number[] | number>");
   }
   return res;
 }
@@ -506,7 +591,14 @@ async function questionAnswering(args, options) {
 
 // src/tasks/nlp/sentenceSimilarity.ts
 async function sentenceSimilarity(args, options) {
-  const res = await request(args, options);
+  const defaultTask = await getDefaultTask(args.model, args.accessToken);
+  const res = await request(
+    args,
+    defaultTask === "feature-extraction" ? {
+      ...options,
+      task: "sentence-similarity"
+    } : options
+  );
   const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
   if (!isValidOutput) {
     throw new InferenceOutputError("Expected number[]");
@@ -663,6 +755,16 @@ async function tabularRegression(args, options) {
   return res;
 }
 
+// src/tasks/tabular/tabularClassification.ts
+async function tabularClassification(args, options) {
+  const res = await request(args, options);
+  const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected number[]");
+  }
+  return res;
+}
+
 // src/HfInference.ts
 var HfInference = class {
   accessToken;
@@ -708,6 +810,7 @@ var HfInferenceEndpoint = class {
   HfInferenceEndpoint,
   InferenceOutputError,
   audioClassification,
+  audioToAudio,
   automaticSpeechRecognition,
   conversational,
   documentQuestionAnswering,
@@ -724,6 +827,7 @@ var HfInferenceEndpoint = class {
   streamingRequest,
   summarization,
   tableQuestionAnswering,
+  tabularClassification,
   tabularRegression,
   textClassification,
   textGeneration,
@@ -733,5 +837,6 @@ var HfInferenceEndpoint = class {
   tokenClassification,
   translation,
   visualQuestionAnswering,
-  zeroShotClassification
+  zeroShotClassification,
+  zeroShotImageClassification
 });

package/dist/index.mjs

@@ -9,6 +9,7 @@ var __export = (target, all) => {
 var tasks_exports = {};
 __export(tasks_exports, {
   audioClassification: () => audioClassification,
+  audioToAudio: () => audioToAudio,
   automaticSpeechRecognition: () => automaticSpeechRecognition,
   conversational: () => conversational,
   documentQuestionAnswering: () => documentQuestionAnswering,
@@ -25,6 +26,7 @@ __export(tasks_exports, {
   streamingRequest: () => streamingRequest,
   summarization: () => summarization,
   tableQuestionAnswering: () => tableQuestionAnswering,
+  tabularClassification: () => tabularClassification,
   tabularRegression: () => tabularRegression,
   textClassification: () => textClassification,
   textGeneration: () => textGeneration,
@@ -34,13 +36,20 @@ __export(tasks_exports, {
   tokenClassification: () => tokenClassification,
   translation: () => translation,
   visualQuestionAnswering: () => visualQuestionAnswering,
-  zeroShotClassification: () => zeroShotClassification
+  zeroShotClassification: () => zeroShotClassification,
+  zeroShotImageClassification: () => zeroShotImageClassification
 });
 
+// src/lib/isUrl.ts
+function isUrl(modelOrUrl) {
+  return /^http(s?):/.test(modelOrUrl) || modelOrUrl.startsWith("/");
+}
+
 // src/lib/makeRequestOptions.ts
-var HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co/models/";
+var HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co";
 function makeRequestOptions(args, options) {
   const { model, accessToken, ...otherArgs } = args;
+  const { task, includeCredentials, ...otherOptions } = options ?? {};
   const headers = {};
   if (accessToken) {
     headers["Authorization"] = `Bearer ${accessToken}`;
@@ -59,15 +68,23 @@ function makeRequestOptions(args, options) {
       headers["X-Load-Model"] = "0";
     }
   }
-  const url = /^http(s?):/.test(model) || model.startsWith("/") ? model : `${HF_INFERENCE_API_BASE_URL}${model}`;
+  const url = (() => {
+    if (isUrl(model)) {
+      return model;
+    }
+    if (task) {
+      return `${HF_INFERENCE_API_BASE_URL}/pipeline/${task}/${model}`;
+    }
+    return `${HF_INFERENCE_API_BASE_URL}/models/${model}`;
+  })();
   const info = {
     headers,
     method: "POST",
     body: binary ? args.data : JSON.stringify({
       ...otherArgs,
-      options
+      options: options && otherOptions
     }),
-    credentials: options?.includeCredentials ? "include" : "same-origin"
+    credentials: includeCredentials ? "include" : "same-origin"
   };
   return { url, info };
 }
@@ -299,6 +316,18 @@ async function textToSpeech(args, options) {
   return res;
 }
 
+// src/tasks/audio/audioToAudio.ts
+async function audioToAudio(args, options) {
+  const res = await request(args, options);
+  const isValidOutput = Array.isArray(res) && res.every(
+    (x) => typeof x.label === "string" && typeof x.blob === "string" && typeof x["content-type"] === "string"
+  );
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected Array<{label: string, blob: string, content-type: string}>");
+  }
+  return res;
+}
+
 // src/tasks/cv/imageClassification.ts
 async function imageClassification(args, options) {
   const res = await request(args, options);
@@ -394,6 +423,26 @@ async function imageToImage(args, options) {
   return res;
 }
 
+// src/tasks/cv/zeroShotImageClassification.ts
+async function zeroShotImageClassification(args, options) {
+  const reqArgs = {
+    ...args,
+    inputs: {
+      image: base64FromBytes(
+        new Uint8Array(
+          args.inputs.image instanceof ArrayBuffer ? args.inputs.image : await args.inputs.image.arrayBuffer()
+        )
+      )
+    }
+  };
+  const res = await request(reqArgs, options);
+  const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected Array<{label: string, score: number}>");
+  }
+  return res;
+}
+
 // src/tasks/nlp/conversational.ts
 async function conversational(args, options) {
   const res = await request(args, options);
@@ -406,27 +455,60 @@ async function conversational(args, options) {
   return res;
 }
 
+// src/lib/getDefaultTask.ts
+var taskCache = /* @__PURE__ */ new Map();
+var CACHE_DURATION = 10 * 60 * 1e3;
+var MAX_CACHE_ITEMS = 1e3;
+var HF_HUB_URL = "https://huggingface.co";
+async function getDefaultTask(model, accessToken) {
+  if (isUrl(model)) {
+    return null;
+  }
+  const key = `${model}:${accessToken}`;
+  let cachedTask = taskCache.get(key);
+  if (cachedTask && cachedTask.date < new Date(Date.now() - CACHE_DURATION)) {
+    taskCache.delete(key);
+    cachedTask = void 0;
+  }
+  if (cachedTask === void 0) {
+    const modelTask = await fetch(`${HF_HUB_URL}/api/models/${model}?expand[]=pipeline_tag`, {
+      headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}
+    }).then((resp) => resp.json()).then((json) => json.pipeline_tag).catch(() => null);
+    if (!modelTask) {
+      return null;
+    }
+    cachedTask = { task: modelTask, date: /* @__PURE__ */ new Date() };
+    taskCache.set(key, { task: modelTask, date: /* @__PURE__ */ new Date() });
+    if (taskCache.size > MAX_CACHE_ITEMS) {
+      taskCache.delete(taskCache.keys().next().value);
+    }
+  }
+  return cachedTask.task;
+}
+
 // src/tasks/nlp/featureExtraction.ts
 async function featureExtraction(args, options) {
-  const res = await request(args, options);
+  const defaultTask = await getDefaultTask(args.model, args.accessToken);
+  const res = await request(
+    args,
+    defaultTask === "sentence-similarity" ? {
+      ...options,
+      task: "feature-extraction"
+    } : options
+  );
   let isValidOutput = true;
-  if (Array.isArray(res)) {
-    for (const e of res) {
-      if (Array.isArray(e)) {
-        isValidOutput = e.every((x) => typeof x === "number");
-        if (!isValidOutput) {
-          break;
-        }
-      } else if (typeof e !== "number") {
-        isValidOutput = false;
-        break;
-      }
+  const isNumArrayRec = (arr, maxDepth, curDepth = 0) => {
+    if (curDepth > maxDepth)
+      return false;
+    if (arr.every((x) => Array.isArray(x))) {
+      return arr.every((x) => isNumArrayRec(x, maxDepth, curDepth + 1));
+    } else {
+      return arr.every((x) => typeof x === "number");
     }
-  } else {
-    isValidOutput = false;
-  }
+  };
+  isValidOutput = Array.isArray(res) && isNumArrayRec(res, 2, 0);
   if (!isValidOutput) {
-    throw new InferenceOutputError("Expected Array<number[] | number>");
+    throw new InferenceOutputError("Expected Array<number[][] | number[] | number>");
   }
   return res;
 }
@@ -457,7 +539,14 @@ async function questionAnswering(args, options) {
 
 // src/tasks/nlp/sentenceSimilarity.ts
 async function sentenceSimilarity(args, options) {
-  const res = await request(args, options);
+  const defaultTask = await getDefaultTask(args.model, args.accessToken);
+  const res = await request(
+    args,
+    defaultTask === "feature-extraction" ? {
+      ...options,
+      task: "sentence-similarity"
+    } : options
+  );
   const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
   if (!isValidOutput) {
     throw new InferenceOutputError("Expected number[]");
@@ -614,6 +703,16 @@ async function tabularRegression(args, options) {
   return res;
 }
 
+// src/tasks/tabular/tabularClassification.ts
+async function tabularClassification(args, options) {
+  const res = await request(args, options);
+  const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
+  if (!isValidOutput) {
+    throw new InferenceOutputError("Expected number[]");
+  }
+  return res;
+}
+
 // src/HfInference.ts
 var HfInference = class {
   accessToken;
@@ -658,6 +757,7 @@ export {
   HfInferenceEndpoint,
   InferenceOutputError,
   audioClassification,
+  audioToAudio,
   automaticSpeechRecognition,
   conversational,
   documentQuestionAnswering,
@@ -674,6 +774,7 @@ export {
   streamingRequest,
   summarization,
   tableQuestionAnswering,
+  tabularClassification,
   tabularRegression,
   textClassification,
   textGeneration,
@@ -683,5 +784,6 @@ export {
   tokenClassification,
   translation,
   visualQuestionAnswering,
-  zeroShotClassification
+  zeroShotClassification,
+  zeroShotImageClassification
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/inference",
-  "version": "2.3.3",
+  "version": "2.5.0",
   "packageManager": "pnpm@8.3.1",
   "license": "MIT",
   "author": "Tim Mikeladze <tim.mikeladze@gmail.com>",

package/src/lib/getDefaultTask.ts

@@ -0,0 +1,53 @@
+import { isUrl } from "./isUrl";
+
+/**
+ * We want to make calls to the huggingface hub the least possible, eg if
+ * someone is calling the inference API 1000 times per second, we don't want
+ * to make 1000 calls to the hub to get the task name.
+ */
+const taskCache = new Map<string, { task: string; date: Date }>();
+const CACHE_DURATION = 10 * 60 * 1000;
+const MAX_CACHE_ITEMS = 1000;
+const HF_HUB_URL = "https://huggingface.co";
+
+/**
+ * Get the default task. Use a LRU cache of 1000 items with 10 minutes expiration
+ * to avoid making too many calls to the HF hub.
+ *
+ * @returns The default task for the model, or `null` if it was impossible to get it
+ */
+export async function getDefaultTask(model: string, accessToken: string | undefined): Promise<string | null> {
+	if (isUrl(model)) {
+		return null;
+	}
+
+	const key = `${model}:${accessToken}`;
+	let cachedTask = taskCache.get(key);
+
+	if (cachedTask && cachedTask.date < new Date(Date.now() - CACHE_DURATION)) {
+		taskCache.delete(key);
+		cachedTask = undefined;
+	}
+
+	if (cachedTask === undefined) {
+		const modelTask = await fetch(`${HF_HUB_URL}/api/models/${model}?expand[]=pipeline_tag`, {
+			headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {},
+		})
+			.then((resp) => resp.json())
+			.then((json) => json.pipeline_tag)
+			.catch(() => null);
+
+		if (!modelTask) {
+			return null;
+		}
+
+		cachedTask = { task: modelTask, date: new Date() };
+		taskCache.set(key, { task: modelTask, date: new Date() });
+
+		if (taskCache.size > MAX_CACHE_ITEMS) {
+			taskCache.delete(taskCache.keys().next().value);
+		}
+	}
+
+	return cachedTask.task;
+}

package/src/lib/isUrl.ts

@@ -0,0 +1,3 @@
+export function isUrl(modelOrUrl: string): boolean {
+	return /^http(s?):/.test(modelOrUrl) || modelOrUrl.startsWith("/");
+}

package/src/lib/makeRequestOptions.ts

@@ -1,6 +1,7 @@
-import type { Options, RequestArgs } from "../types";
+import type { InferenceTask, Options, RequestArgs } from "../types";
+import { isUrl } from "./isUrl";
 
-const HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co/models/";
+const HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co";
 
 /**
  * Helper that prepares request arguments
@@ -13,9 +14,12 @@ export function makeRequestOptions(
 	options?: Options & {
 		/** For internal HF use, which is why it's not exposed in {@link Options} */
 		includeCredentials?: boolean;
+		/** When a model can be used for multiple tasks, and we want to run a non-default task */
+		task?: string | InferenceTask;
 	}
 ): { url: string; info: RequestInit } {
 	const { model, accessToken, ...otherArgs } = args;
+	const { task, includeCredentials, ...otherOptions } = options ?? {};
 
 	const headers: Record<string, string> = {};
 	if (accessToken) {
@@ -38,7 +42,18 @@ export function makeRequestOptions(
 		}
 	}
 
-	const url = /^http(s?):/.test(model) || model.startsWith("/") ? model : `${HF_INFERENCE_API_BASE_URL}${model}`;
+	const url = (() => {
+		if (isUrl(model)) {
+			return model;
+		}
+
+		if (task) {
+			return `${HF_INFERENCE_API_BASE_URL}/pipeline/${task}/${model}`;
+		}
+
+		return `${HF_INFERENCE_API_BASE_URL}/models/${model}`;
+	})();
+
 	const info: RequestInit = {
 		headers,
 		method: "POST",
@@ -46,9 +61,9 @@ export function makeRequestOptions(
 			? args.data
 			: JSON.stringify({
 					...otherArgs,
-					options,
+					options: options && otherOptions,
 			  }),
-		credentials: options?.includeCredentials ? "include" : "same-origin",
+		credentials: includeCredentials ? "include" : "same-origin",
 	};
 
 	return { url, info };

package/src/tasks/audio/audioToAudio.ts

@@ -0,0 +1,46 @@
+import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import type { BaseArgs, Options } from "../../types";
+import { request } from "../custom/request";
+
+export type AudioToAudioArgs = BaseArgs & {
+	/**
+	 * Binary audio data
+	 */
+	data: Blob | ArrayBuffer;
+};
+
+export interface AudioToAudioOutputValue {
+	/**
+	 * The label for the audio output (model specific)
+	 */
+	label: string;
+
+	/**
+	 * Base64 encoded audio output.
+	 */
+	blob: string;
+
+	/**
+	 * Content-type for blob, e.g. audio/flac
+	 */
+	"content-type": string;
+}
+
+export type AudioToAudioReturn = AudioToAudioOutputValue[];
+
+/**
+ * This task reads some audio input and outputs one or multiple audio files.
+ * Example model: speechbrain/sepformer-wham does audio source separation.
+ */
+export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioReturn> {
+	const res = await request<AudioToAudioReturn>(args, options);
+	const isValidOutput =
+		Array.isArray(res) &&
+		res.every(
+			(x) => typeof x.label === "string" && typeof x.blob === "string" && typeof x["content-type"] === "string"
+		);
+	if (!isValidOutput) {
+		throw new InferenceOutputError("Expected Array<{label: string, blob: string, content-type: string}>");
+	}
+	return res;
+}

package/src/tasks/custom/request.ts

@@ -1,4 +1,4 @@
-import type { Options, RequestArgs } from "../../types";
+import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { makeRequestOptions } from "../../lib/makeRequestOptions";
 
 /**
@@ -9,6 +9,8 @@ export async function request<T>(
 	options?: Options & {
 		/** For internal HF use, which is why it's not exposed in {@link Options} */
 		includeCredentials?: boolean;
+		/** When a model can be used for multiple tasks, and we want to run a non-default task */
+		task?: string | InferenceTask;
 	}
 ): Promise<T> {
 	const { url, info } = makeRequestOptions(args, options);

package/src/tasks/custom/streamingRequest.ts

@@ -1,4 +1,4 @@
-import type { Options, RequestArgs } from "../../types";
+import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { makeRequestOptions } from "../../lib/makeRequestOptions";
 import type { EventSourceMessage } from "../../vendor/fetch-event-source/parse";
 import { getLines, getMessages } from "../../vendor/fetch-event-source/parse";
@@ -11,6 +11,8 @@ export async function* streamingRequest<T>(
 	options?: Options & {
 		/** For internal HF use, which is why it's not exposed in {@link Options} */
 		includeCredentials?: boolean;
+		/** When a model can be used for multiple tasks, and we want to run a non-default task */
+		task?: string | InferenceTask;
 	}
 ): AsyncGenerator<T> {
 	const { url, info } = makeRequestOptions({ ...args, stream: true }, options);

package/src/tasks/cv/zeroShotImageClassification.ts

@@ -0,0 +1,55 @@
+import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import type { BaseArgs, Options } from "../../types";
+import { request } from "../custom/request";
+import type { RequestArgs } from "../../types";
+import { base64FromBytes } from "../../../../shared";
+
+export type ZeroShotImageClassificationArgs = BaseArgs & {
+	inputs: {
+		/**
+		 * Binary image data
+		 */
+		image: Blob | ArrayBuffer;
+	};
+	parameters: {
+		/**
+		 * A list of strings that are potential classes for inputs. (max 10)
+		 */
+		candidate_labels: string[];
+	};
+};
+
+export interface ZeroShotImageClassificationOutputValue {
+	label: string;
+	score: number;
+}
+
+export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputValue[];
+
+/**
+ * Classify an image to specified classes.
+ * Recommended model: openai/clip-vit-large-patch14-336
+ */
+export async function zeroShotImageClassification(
+	args: ZeroShotImageClassificationArgs,
+	options?: Options
+): Promise<ZeroShotImageClassificationOutput> {
+	const reqArgs: RequestArgs = {
+		...args,
+		inputs: {
+			image: base64FromBytes(
+				new Uint8Array(
+					args.inputs.image instanceof ArrayBuffer ? args.inputs.image : await args.inputs.image.arrayBuffer()
+				)
+			),
+		},
+	} as RequestArgs;
+
+	const res = await request<ZeroShotImageClassificationOutput>(reqArgs, options);
+	const isValidOutput =
+		Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+	if (!isValidOutput) {
+		throw new InferenceOutputError("Expected Array<{label: string, score: number}>");
+	}
+	return res;
+}

package/src/tasks/index.ts

@@ -6,6 +6,7 @@ export * from "./custom/streamingRequest";
 export * from "./audio/audioClassification";
 export * from "./audio/automaticSpeechRecognition";
 export * from "./audio/textToSpeech";
+export * from "./audio/audioToAudio";
 
 // Computer Vision tasks
 export * from "./cv/imageClassification";
@@ -14,6 +15,7 @@ export * from "./cv/imageToText";
 export * from "./cv/objectDetection";
 export * from "./cv/textToImage";
 export * from "./cv/imageToImage";
+export * from "./cv/zeroShotImageClassification";
 
 // Natural Language Processing tasks
 export * from "./nlp/conversational";
@@ -36,3 +38,4 @@ export * from "./multimodal/visualQuestionAnswering";
 
 // Tabular tasks
 export * from "./tabular/tabularRegression";
+export * from "./tabular/tabularClassification";

package/src/tasks/nlp/featureExtraction.ts

@@ -1,4 +1,5 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getDefaultTask } from "../../lib/getDefaultTask";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
 
@@ -13,9 +14,9 @@ export type FeatureExtractionArgs = BaseArgs & {
 };
 
 /**
- * Returned values are a list of floats, or a list of list of floats (depending on if you sent a string or a list of string, and if the automatic reduction, usually mean_pooling for instance was applied for you or not. This should be explained on the model's README.
+ * Returned values are a list of floats, or a list of list of floats, or a list of list of list of floats (depending on if you sent a string or a list of string, and if the automatic reduction, usually mean_pooling for instance was applied for you or not. This should be explained on the model's README.
  */
-export type FeatureExtractionOutput = (number | number[])[];
+export type FeatureExtractionOutput = (number | number[] | number[][])[];
 
 /**
  * This task reads some text and outputs raw float values, that are usually consumed as part of a semantic database/semantic search.
@@ -24,28 +25,31 @@ export async function featureExtraction(
 	args: FeatureExtractionArgs,
 	options?: Options
 ): Promise<FeatureExtractionOutput> {
-	const res = await request<FeatureExtractionOutput>(args, options);
+	const defaultTask = await getDefaultTask(args.model, args.accessToken);
+	const res = await request<FeatureExtractionOutput>(
+		args,
+		defaultTask === "sentence-similarity"
+			? {
+					...options,
+					task: "feature-extraction",
+			  }
+			: options
+	);
 	let isValidOutput = true;
-	// Check if output is an array
-	if (Array.isArray(res)) {
-		for (const e of res) {
-			// Check if output is an array of arrays or numbers
-			if (Array.isArray(e)) {
-				// if all elements are numbers, continue
-				isValidOutput = e.every((x) => typeof x === "number");
-				if (!isValidOutput) {
-					break;
-				}
-			} else if (typeof e !== "number") {
-				isValidOutput = false;
-				break;
-			}
+
+	const isNumArrayRec = (arr: unknown[], maxDepth: number, curDepth = 0): boolean => {
+		if (curDepth > maxDepth) return false;
+		if (arr.every((x) => Array.isArray(x))) {
+			return arr.every((x) => isNumArrayRec(x as unknown[], maxDepth, curDepth + 1));
+		} else {
+			return arr.every((x) => typeof x === "number");
 		}
-	} else {
-		isValidOutput = false;
-	}
+	};
+
+	isValidOutput = Array.isArray(res) && isNumArrayRec(res, 2, 0);
+
 	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Array<number[] | number>");
+		throw new InferenceOutputError("Expected Array<number[][] | number[] | number>");
 	}
 	return res;
 }

package/src/tasks/nlp/sentenceSimilarity.ts

@@ -1,4 +1,5 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getDefaultTask } from "../../lib/getDefaultTask";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
 
@@ -24,7 +25,16 @@ export async function sentenceSimilarity(
 	args: SentenceSimilarityArgs,
 	options?: Options
 ): Promise<SentenceSimilarityOutput> {
-	const res = await request<SentenceSimilarityOutput>(args, options);
+	const defaultTask = await getDefaultTask(args.model, args.accessToken);
+	const res = await request<SentenceSimilarityOutput>(
+		args,
+		defaultTask === "feature-extraction"
+			? {
+					...options,
+					task: "sentence-similarity",
+			  }
+			: options
+	);
 
 	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
 	if (!isValidOutput) {

package/src/tasks/tabular/tabularClassification.ts

@@ -0,0 +1,34 @@
+import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import type { BaseArgs, Options } from "../../types";
+import { request } from "../custom/request";
+
+export type TabularClassificationArgs = BaseArgs & {
+	inputs: {
+		/**
+		 * A table of data represented as a dict of list where entries are headers and the lists are all the values, all lists must have the same size.
+		 */
+		data: Record<string, string[]>;
+	};
+};
+
+/**
+ * A list of predicted labels for each row
+ */
+export type TabularClassificationOutput = number[];
+
+/**
+ * Predicts target label for a given set of features in tabular form.
+ * Typically, you will want to train a classification model on your training data and use it with your new data of the same format.
+ * Example model: vvmnnnkv/wine-quality
+ */
+export async function tabularClassification(
+	args: TabularClassificationArgs,
+	options?: Options
+): Promise<TabularClassificationOutput> {
+	const res = await request<TabularClassificationOutput>(args, options);
+	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
+	if (!isValidOutput) {
+		throw new InferenceOutputError("Expected number[]");
+	}
+	return res;
+}

package/src/types.ts

@@ -26,6 +26,8 @@ export interface Options {
 	fetch?: typeof fetch;
 }
 
+export type InferenceTask = "text-classification" | "feature-extraction" | "sentence-similarity";
+
 export interface BaseArgs {
 	/**
 	 * The access token to use. Without it, you'll get rate-limited quickly.