@huggingface/inference 3.0.1 → 3.1.0

package/dist/src/tasks/nlp/tokenClassification.d.ts

@@ -1,49 +1,6 @@
+import type { TokenClassificationInput, TokenClassificationOutput } from "@huggingface/tasks";
 import type { BaseArgs, Options } from "../../types";
-export type TokenClassificationArgs = BaseArgs & {
-    /**
-     * A string to be classified
-     */
-    inputs: string;
-    parameters?: {
-        /**
-         * (Default: simple). There are several aggregation strategies:
-         *
-         * none: Every token gets classified without further aggregation.
-         *
-         * simple: Entities are grouped according to the default schema (B-, I- tags get merged when the tag is similar).
-         *
-         * first: Same as the simple strategy except words cannot end up with different tags. Words will use the tag of the first token when there is ambiguity.
-         *
-         * average: Same as the simple strategy except words cannot end up with different tags. Scores are averaged across tokens and then the maximum label is applied.
-         *
-         * max: Same as the simple strategy except words cannot end up with different tags. Word entity will be the token with the maximum score.
-         */
-        aggregation_strategy?: "none" | "simple" | "first" | "average" | "max";
-    };
-};
-export interface TokenClassificationOutputValue {
-    /**
-     * The offset stringwise where the answer is located. Useful to disambiguate if word occurs multiple times.
-     */
-    end: number;
-    /**
-     * The type for the entity being recognized (model specific).
-     */
-    entity_group: string;
-    /**
-     * How likely the entity was recognized.
-     */
-    score: number;
-    /**
-     * The offset stringwise where the answer is located. Useful to disambiguate if word occurs multiple times.
-     */
-    start: number;
-    /**
-     * The string that was captured
-     */
-    word: string;
-}
-export type TokenClassificationOutput = TokenClassificationOutputValue[];
+export type TokenClassificationArgs = BaseArgs & TokenClassificationInput;
 /**
  * Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. Recommended model: dbmdz/bert-large-cased-finetuned-conll03-english
  */

package/dist/src/tasks/nlp/tokenClassification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenClassification.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/tokenClassification.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAIrD,MAAM,MAAM,uBAAuB,GAAG,QAAQ,GAAG;IAChD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACZ;;;;;;;;;;;;WAYG;QACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC;KACvE,CAAC;CACF,CAAC;AAEF,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,yBAAyB,GAAG,8BAA8B,EAAE,CAAC;AAEzE;;GAEG;AACH,wBAAsB,mBAAmB,CACxC,IAAI,EAAE,uBAAuB,EAC7B,OAAO,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,yBAAyB,CAAC,CAuBpC"}
+{"version":3,"file":"tokenClassification.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/tokenClassification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAIrD,MAAM,MAAM,uBAAuB,GAAG,QAAQ,GAAG,wBAAwB,CAAC;AAE1E;;GAEG;AACH,wBAAsB,mBAAmB,CACxC,IAAI,EAAE,uBAAuB,EAC7B,OAAO,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,yBAAyB,CAAC,CAuBpC"}

package/dist/src/tasks/nlp/translation.d.ts

@@ -1,17 +1,6 @@
+import type { TranslationInput, TranslationOutput } from "@huggingface/tasks";
 import type { BaseArgs, Options } from "../../types";
-export type TranslationArgs = BaseArgs & {
-    /**
-     * A string to be translated
-     */
-    inputs: string | string[];
-};
-export interface TranslationOutputValue {
-    /**
-     * The string after translation
-     */
-    translation_text: string;
-}
-export type TranslationOutput = TranslationOutputValue | TranslationOutputValue[];
+export type TranslationArgs = BaseArgs & TranslationInput;
 /**
  * This task is well known to translate text from one language to another. Recommended model: Helsinki-NLP/opus-mt-ru-en.
  */

package/dist/src/tasks/nlp/translation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"translation.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/translation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACxC;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC1B,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,iBAAiB,GAAG,sBAAsB,GAAG,sBAAsB,EAAE,CAAC;AAElF;;GAEG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAUtG"}
+{"version":3,"file":"translation.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/translation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,gBAAgB,CAAC;AAC1D;;GAEG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAUtG"}

package/dist/src/tasks/nlp/zeroShotClassification.d.ts

@@ -1,26 +1,6 @@
+import type { ZeroShotClassificationInput, ZeroShotClassificationOutput } from "@huggingface/tasks";
 import type { BaseArgs, Options } from "../../types";
-export type ZeroShotClassificationArgs = BaseArgs & {
-    /**
-     * a string or list of strings
-     */
-    inputs: string | string[];
-    parameters: {
-        /**
-         * a list of strings that are potential classes for inputs. (max 10 candidate_labels, for more, simply run multiple requests, results are going to be misleading if using too many candidate_labels anyway. If you want to keep the exact same, you can simply run multi_label=True and do the scaling on your end.
-         */
-        candidate_labels: string[];
-        /**
-         * (Default: false) Boolean that is set to True if classes can overlap
-         */
-        multi_label?: boolean;
-    };
-};
-export interface ZeroShotClassificationOutputValue {
-    labels: string[];
-    scores: number[];
-    sequence: string;
-}
-export type ZeroShotClassificationOutput = ZeroShotClassificationOutputValue[];
+export type ZeroShotClassificationArgs = BaseArgs & ZeroShotClassificationInput;
 /**
  * This task is super useful to try out classification with zero code, you simply pass a sentence/paragraph and the possible labels for that sentence, and you get a result. Recommended model: facebook/bart-large-mnli.
  */

package/dist/src/tasks/nlp/zeroShotClassification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"zeroShotClassification.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/zeroShotClassification.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAIrD,MAAM,MAAM,0BAA0B,GAAG,QAAQ,GAAG;IACnD;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,UAAU,EAAE;QACX;;WAEG;QACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;CACF,CAAC;AAEF,MAAM,WAAW,iCAAiC;IACjD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,4BAA4B,GAAG,iCAAiC,EAAE,CAAC;AAE/E;;GAEG;AACH,wBAAsB,sBAAsB,CAC3C,IAAI,EAAE,0BAA0B,EAChC,OAAO,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,4BAA4B,CAAC,CAqBvC"}
+{"version":3,"file":"zeroShotClassification.d.ts","sourceRoot":"","sources":["../../../../src/tasks/nlp/zeroShotClassification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAEpG,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAIrD,MAAM,MAAM,0BAA0B,GAAG,QAAQ,GAAG,2BAA2B,CAAC;AAEhF;;GAEG;AACH,wBAAsB,sBAAsB,CAC3C,IAAI,EAAE,0BAA0B,EAChC,OAAO,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,4BAA4B,CAAC,CAqBvC"}

package/dist/src/types.d.ts

@@ -76,6 +76,10 @@ export type RequestArgs = BaseArgs & ({
     data: Blob | ArrayBuffer;
 } | {
     inputs: unknown;
+} | {
+    prompt: string;
+} | {
+    audio_url: string;
 } | ChatCompletionInput) & {
     parameters?: Record<string, unknown>;
     accessToken?: string;

package/dist/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B,MAAM,WAAW,OAAO;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;AAE3D,eAAO,MAAM,mBAAmB,2EAA4E,CAAC;AAC7G,MAAM,MAAM,iBAAiB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAErE,MAAM,WAAW,QAAQ;IACxB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,WAAW,GAAG,QAAQ,GACjC,CAAC;IAAE,IAAI,EAAE,IAAI,GAAG,WAAW,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GAAG,mBAAmB,CAAC,GAAG;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B,MAAM,WAAW,OAAO;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;AAE3D,eAAO,MAAM,mBAAmB,2EAA4E,CAAC;AAC7G,MAAM,MAAM,iBAAiB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAErE,MAAM,WAAW,QAAQ;IACxB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,WAAW,GAAG,QAAQ,GACjC,CACG;IAAE,IAAI,EAAE,IAAI,GAAG,WAAW,CAAA;CAAE,GAC5B;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GACnB;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAClB;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB,mBAAmB,CACrB,GAAG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/inference",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "packageManager": "pnpm@8.10.5",
   "license": "MIT",
   "author": "Tim Mikeladze <tim.mikeladze@gmail.com>",
@@ -39,7 +39,7 @@
   },
   "type": "module",
   "dependencies": {
-    "@huggingface/tasks": "^0.13.17"
+    "@huggingface/tasks": "^0.14.0"
   },
   "devDependencies": {
     "@types/node": "18.13.0"

package/src/providers/fal-ai.ts

@@ -20,4 +20,8 @@ export const FAL_AI_SUPPORTED_MODEL_IDS: ProviderMapping<FalAiId> = {
 	"automatic-speech-recognition": {
 		"openai/whisper-large-v3": "fal-ai/whisper",
 	},
+	"text-to-video": {
+		"genmo/mochi-1-preview": "fal-ai/mochi-v1",
+		"tencent/HunyuanVideo": "fal-ai/hunyuan-video",
+	},
 };

package/src/providers/replicate.ts

@@ -13,4 +13,7 @@ export const REPLICATE_SUPPORTED_MODEL_IDS: ProviderMapping<ReplicateId> = {
 	"text-to-speech": {
 		"OuteAI/OuteTTS-0.3-500M": "jbilcke/oute-tts:39a59319327b27327fa3095149c5a746e7f2aee18c75055c3368237a6503cd26",
 	},
+	"text-to-video": {
+		"genmo/mochi-1-preview": "genmoai/mochi-1:1944af04d098ef69bed7f9d335d102e652203f268ec4aaa2d836f6217217e460",
+	},
 };

package/src/tasks/audio/audioClassification.ts

@@ -1,27 +1,11 @@
+import type { AudioClassificationInput, AudioClassificationOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
+import type { LegacyAudioInput } from "./utils";
+import { preparePayload } from "./utils";
 
-export type AudioClassificationArgs = BaseArgs & {
-	/**
-	 * Binary audio data
-	 */
-	data: Blob | ArrayBuffer;
-};
-
-export interface AudioClassificationOutputValue {
-	/**
-	 * The label for the class (model specific)
-	 */
-	label: string;
-
-	/**
-	 * A float that represents how likely it is that the audio file belongs to this class.
-	 */
-	score: number;
-}
-
-export type AudioClassificationReturn = AudioClassificationOutputValue[];
+export type AudioClassificationArgs = BaseArgs & (AudioClassificationInput | LegacyAudioInput);
 
 /**
  * This task reads some audio input and outputs the likelihood of classes.
@@ -30,8 +14,9 @@ export type AudioClassificationReturn = AudioClassificationOutputValue[];
 export async function audioClassification(
 	args: AudioClassificationArgs,
 	options?: Options
-): Promise<AudioClassificationReturn> {
-	const res = await request<AudioClassificationReturn>(args, {
+): Promise<AudioClassificationOutput> {
+	const payload = preparePayload(args);
+	const res = await request<AudioClassificationOutput>(payload, {
 		...options,
 		taskHint: "audio-classification",
 	});

package/src/tasks/audio/audioToAudio.ts

@@ -1,15 +1,19 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
+import type { LegacyAudioInput } from "./utils";
+import { preparePayload } from "./utils";
 
-export type AudioToAudioArgs = BaseArgs & {
-	/**
-	 * Binary audio data
-	 */
-	data: Blob | ArrayBuffer;
-};
+export type AudioToAudioArgs =
+	| (BaseArgs & {
+			/**
+			 * Binary audio data
+			 */
+			inputs: Blob;
+	  })
+	| LegacyAudioInput;
 
-export interface AudioToAudioOutputValue {
+export interface AudioToAudioOutputElem {
 	/**
 	 * The label for the audio output (model specific)
 	 */
@@ -18,32 +22,48 @@ export interface AudioToAudioOutputValue {
 	/**
 	 * Base64 encoded audio output.
 	 */
-	blob: string;
+	audio: Blob;
+}
 
-	/**
-	 * Content-type for blob, e.g. audio/flac
-	 */
+export interface AudioToAudioOutput {
+	blob: string;
 	"content-type": string;
+	label: string;
 }
 
-export type AudioToAudioReturn = AudioToAudioOutputValue[];
-
 /**
  * This task reads some audio input and outputs one or multiple audio files.
  * Example model: speechbrain/sepformer-wham does audio source separation.
  */
-export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioReturn> {
-	const res = await request<AudioToAudioReturn>(args, {
+export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioOutput[]> {
+	const payload = preparePayload(args);
+	const res = await request<AudioToAudioOutput>(payload, {
 		...options,
 		taskHint: "audio-to-audio",
 	});
-	const isValidOutput =
-		Array.isArray(res) &&
-		res.every(
-			(x) => typeof x.label === "string" && typeof x.blob === "string" && typeof x["content-type"] === "string"
-		);
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Array<{label: string, blob: string, content-type: string}>");
+
+	return validateOutput(res);
+}
+
+function validateOutput(output: unknown): AudioToAudioOutput[] {
+	if (!Array.isArray(output)) {
+		throw new InferenceOutputError("Expected Array");
+	}
+	if (
+		!output.every((elem): elem is AudioToAudioOutput => {
+			return (
+				typeof elem === "object" &&
+				elem &&
+				"label" in elem &&
+				typeof elem.label === "string" &&
+				"content-type" in elem &&
+				typeof elem["content-type"] === "string" &&
+				"blob" in elem &&
+				typeof elem.blob === "string"
+			);
+		})
+	) {
+		throw new InferenceOutputError("Expected Array<{label: string, audio: Blob}>");
 	}
-	return res;
+	return output;
 }

package/src/tasks/audio/automaticSpeechRecognition.ts

@@ -1,22 +1,13 @@
+import type { AutomaticSpeechRecognitionInput, AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options, RequestArgs } from "../../types";
 import { base64FromBytes } from "../../utils/base64FromBytes";
 import { request } from "../custom/request";
+import type { LegacyAudioInput } from "./utils";
+import { preparePayload } from "./utils";
+import { omit } from "../../utils/omit";
 
-export type AutomaticSpeechRecognitionArgs = BaseArgs & {
-	/**
-	 * Binary audio data
-	 */
-	data: Blob | ArrayBuffer;
-};
-
-export interface AutomaticSpeechRecognitionOutput {
-	/**
-	 * The text that was recognized from the audio
-	 */
-	text: string;
-}
-
+export type AutomaticSpeechRecognitionArgs = BaseArgs & (AutomaticSpeechRecognitionInput | LegacyAudioInput);
 /**
  * This task reads some audio input and outputs the said words within the audio files.
  * Recommended model (english language): facebook/wav2vec2-large-960h-lv60-self
@@ -25,15 +16,8 @@ export async function automaticSpeechRecognition(
 	args: AutomaticSpeechRecognitionArgs,
 	options?: Options
 ): Promise<AutomaticSpeechRecognitionOutput> {
-	if (args.provider === "fal-ai") {
-		const contentType = args.data instanceof Blob ? args.data.type : "audio/mpeg";
-		const base64audio = base64FromBytes(
-			new Uint8Array(args.data instanceof ArrayBuffer ? args.data : await args.data.arrayBuffer())
-		);
-		(args as RequestArgs & { audio_url: string }).audio_url = `data:${contentType};base64,${base64audio}`;
-		delete (args as RequestArgs & { data: unknown }).data;
-	}
-	const res = await request<AutomaticSpeechRecognitionOutput>(args, {
+	const payload = await buildPayload(args);
+	const res = await request<AutomaticSpeechRecognitionOutput>(payload, {
 		...options,
 		taskHint: "automatic-speech-recognition",
 	});
@@ -43,3 +27,31 @@ export async function automaticSpeechRecognition(
 	}
 	return res;
 }
+
+const FAL_AI_SUPPORTED_BLOB_TYPES = ["audio/mpeg", "audio/mp4", "audio/wav", "audio/x-wav"];
+
+async function buildPayload(args: AutomaticSpeechRecognitionArgs): Promise<RequestArgs> {
+	if (args.provider === "fal-ai") {
+		const blob = "data" in args && args.data instanceof Blob ? args.data : "inputs" in args ? args.inputs : undefined;
+		const contentType = blob?.type;
+		if (!contentType) {
+			throw new Error(
+				`Unable to determine the input's content-type. Make sure your are passing a Blob when using provider fal-ai.`
+			);
+		}
+		if (!FAL_AI_SUPPORTED_BLOB_TYPES.includes(contentType)) {
+			throw new Error(
+				`Provider fal-ai does not support blob type ${contentType} - supported content types are: ${FAL_AI_SUPPORTED_BLOB_TYPES.join(
+					", "
+				)}`
+			);
+		}
+		const base64audio = base64FromBytes(new Uint8Array(await blob.arrayBuffer()));
+		return {
+			...("data" in args ? omit(args, "data") : omit(args, "inputs")),
+			audio_url: `data:${contentType};base64,${base64audio}`,
+		};
+	} else {
+		return preparePayload(args);
+	}
+}

package/src/tasks/audio/textToSpeech.ts

@@ -1,15 +1,10 @@
+import type { TextToSpeechInput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
 
-export type TextToSpeechArgs = BaseArgs & {
-	/**
-	 * The text to generate an audio from
-	 */
-	inputs: string;
-};
+type TextToSpeechArgs = BaseArgs & TextToSpeechInput;
 
-export type TextToSpeechOutput = Blob;
 interface OutputUrlTextToSpeechGeneration {
 	output: string | string[];
 }
@@ -17,11 +12,14 @@ interface OutputUrlTextToSpeechGeneration {
  * This task synthesize an audio of a voice pronouncing a given text.
  * Recommended model: espnet/kan-bayashi_ljspeech_vits
  */
-export async function textToSpeech(args: TextToSpeechArgs, options?: Options): Promise<TextToSpeechOutput> {
-	const res = await request<TextToSpeechOutput | OutputUrlTextToSpeechGeneration>(args, {
+export async function textToSpeech(args: TextToSpeechArgs, options?: Options): Promise<Blob> {
+	const res = await request<Blob | OutputUrlTextToSpeechGeneration>(args, {
 		...options,
 		taskHint: "text-to-speech",
 	});
+	if (res instanceof Blob) {
+		return res;
+	}
 	if (res && typeof res === "object") {
 		if ("output" in res) {
 			if (typeof res.output === "string") {
@@ -35,9 +33,5 @@ export async function textToSpeech(args: TextToSpeechArgs, options?: Options): P
 			}
 		}
 	}
-	const isValidOutput = res && res instanceof Blob;
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Blob");
-	}
-	return res;
+	throw new InferenceOutputError("Expected Blob or object with output");
 }

package/src/tasks/audio/utils.ts

@@ -0,0 +1,18 @@
+import type { BaseArgs, RequestArgs } from "../../types";
+import { omit } from "../../utils/omit";
+
+/**
+ * @deprecated
+ */
+export interface LegacyAudioInput {
+	data: Blob | ArrayBuffer;
+}
+
+export function preparePayload(args: BaseArgs & ({ inputs: Blob } | LegacyAudioInput)): RequestArgs {
+	return "data" in args
+		? args
+		: {
+				...omit(args, "inputs"),
+				data: args.inputs,
+		  };
+}

package/src/tasks/cv/imageClassification.ts

@@ -1,26 +1,10 @@
+import type { ImageClassificationInput, ImageClassificationOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
+import { preparePayload, type LegacyImageInput } from "./utils";
 
-export type ImageClassificationArgs = BaseArgs & {
-	/**
-	 * Binary image data
-	 */
-	data: Blob | ArrayBuffer;
-};
-
-export interface ImageClassificationOutputValue {
-	/**
-	 * The label for the class (model specific)
-	 */
-	label: string;
-	/**
-	 * A float that represents how likely it is that the image file belongs to this class.
-	 */
-	score: number;
-}
-
-export type ImageClassificationOutput = ImageClassificationOutputValue[];
+export type ImageClassificationArgs = BaseArgs & (ImageClassificationInput | LegacyImageInput);
 
 /**
  * This task reads some image input and outputs the likelihood of classes.
@@ -30,7 +14,8 @@ export async function imageClassification(
 	args: ImageClassificationArgs,
 	options?: Options
 ): Promise<ImageClassificationOutput> {
-	const res = await request<ImageClassificationOutput>(args, {
+	const payload = preparePayload(args);
+	const res = await request<ImageClassificationOutput>(payload, {
 		...options,
 		taskHint: "image-classification",
 	});

package/src/tasks/cv/imageSegmentation.ts

@@ -1,30 +1,10 @@
+import type { ImageSegmentationInput, ImageSegmentationOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
+import { preparePayload, type LegacyImageInput } from "./utils";
 
-export type ImageSegmentationArgs = BaseArgs & {
-	/**
-	 * Binary image data
-	 */
-	data: Blob | ArrayBuffer;
-};
-
-export interface ImageSegmentationOutputValue {
-	/**
-	 * The label for the class (model specific) of a segment.
-	 */
-	label: string;
-	/**
-	 * A str (base64 str of a single channel black-and-white img) representing the mask of a segment.
-	 */
-	mask: string;
-	/**
-	 * A float that represents how likely it is that the detected object belongs to the given class.
-	 */
-	score: number;
-}
-
-export type ImageSegmentationOutput = ImageSegmentationOutputValue[];
+export type ImageSegmentationArgs = BaseArgs & (ImageSegmentationInput | LegacyImageInput);
 
 /**
  * This task reads some image input and outputs the likelihood of classes & bounding boxes of detected objects.
@@ -34,7 +14,8 @@ export async function imageSegmentation(
 	args: ImageSegmentationArgs,
 	options?: Options
 ): Promise<ImageSegmentationOutput> {
-	const res = await request<ImageSegmentationOutput>(args, {
+	const payload = preparePayload(args);
+	const res = await request<ImageSegmentationOutput>(payload, {
 		...options,
 		taskHint: "image-segmentation",
 	});

package/src/tasks/cv/imageToImage.ts

@@ -1,64 +1,16 @@
+import type { ImageToImageInput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options, RequestArgs } from "../../types";
 import { base64FromBytes } from "../../utils/base64FromBytes";
 import { request } from "../custom/request";
 
-export type ImageToImageArgs = BaseArgs & {
-	/**
-	 * The initial image condition
-	 *
-	 **/
-	inputs: Blob | ArrayBuffer;
-
-	parameters?: {
-		/**
-		 * The text prompt to guide the image generation.
-		 */
-		prompt?: string;
-		/**
-		 * strengh param only works for SD img2img and alt diffusion img2img models
-		 * Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image`
-		 * will be used as a starting point, adding more noise to it the larger the `strength`. The number of
-		 * denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will
-		 * be maximum and the denoising process will run for the full number of iterations specified in
-		 * `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.
-		 **/
-		strength?: number;
-		/**
-		 * An optional negative prompt for the image generation
-		 */
-		negative_prompt?: string;
-		/**
-		 * The height in pixels of the generated image
-		 */
-		height?: number;
-		/**
-		 * The width in pixels of the generated image
-		 */
-		width?: number;
-		/**
-		 * The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference.
-		 */
-		num_inference_steps?: number;
-		/**
-		 * Guidance scale: Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality.
-		 */
-		guidance_scale?: number;
-		/**
-		 * guess_mode only works for ControlNet models, defaults to False In this mode, the ControlNet encoder will try best to recognize the content of the input image even if
-		 * you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended.
-		 */
-		guess_mode?: boolean;
-	};
-};
-
-export type ImageToImageOutput = Blob;
+export type ImageToImageArgs = BaseArgs & ImageToImageInput;
 
 /**
  * This task reads some text input and outputs an image.
  * Recommended model: lllyasviel/sd-controlnet-depth
  */
-export async function imageToImage(args: ImageToImageArgs, options?: Options): Promise<ImageToImageOutput> {
+export async function imageToImage(args: ImageToImageArgs, options?: Options): Promise<Blob> {
 	let reqArgs: RequestArgs;
 	if (!args.parameters) {
 		reqArgs = {
@@ -74,7 +26,7 @@ export async function imageToImage(args: ImageToImageArgs, options?: Options): P
 			),
 		};
 	}
-	const res = await request<ImageToImageOutput>(reqArgs, {
+	const res = await request<Blob>(reqArgs, {
 		...options,
 		taskHint: "image-to-image",
 	});

package/src/tasks/cv/imageToText.ts

@@ -1,27 +1,18 @@
+import type { ImageToTextInput, ImageToTextOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
+import type { LegacyImageInput } from "./utils";
+import { preparePayload } from "./utils";
 
-export type ImageToTextArgs = BaseArgs & {
-	/**
-	 * Binary image data
-	 */
-	data: Blob | ArrayBuffer;
-};
-
-export interface ImageToTextOutput {
-	/**
-	 * The generated caption
-	 */
-	generated_text: string;
-}
-
+export type ImageToTextArgs = BaseArgs & (ImageToTextInput | LegacyImageInput);
 /**
  * This task reads some image input and outputs the text caption.
  */
 export async function imageToText(args: ImageToTextArgs, options?: Options): Promise<ImageToTextOutput> {
+	const payload = preparePayload(args);
 	const res = (
-		await request<[ImageToTextOutput]>(args, {
+		await request<[ImageToTextOutput]>(payload, {
 			...options,
 			taskHint: "image-to-text",
 		})