@huggingface/inference 3.9.2 → 3.11.0

package/src/InferenceClient.ts

@@ -1,73 +1,62 @@
 import * as tasks from "./tasks";
-import type { Options, RequestArgs } from "./types";
-import type { DistributiveOmit } from "./utils/distributive-omit";
+import type { Options } from "./types";
+import { omit } from "./utils/omit";
+import { typedEntries } from "./utils/typedEntries";
 
 /* eslint-disable @typescript-eslint/no-empty-interface */
 /* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */
 
 type Task = typeof tasks;
 
-type TaskWithNoAccessToken = {
-	[key in keyof Task]: (
-		args: DistributiveOmit<Parameters<Task[key]>[0], "accessToken">,
-		options?: Parameters<Task[key]>[1]
-	) => ReturnType<Task[key]>;
-};
-
-type TaskWithNoAccessTokenNoEndpointUrl = {
-	[key in keyof Task]: (
-		args: DistributiveOmit<Parameters<Task[key]>[0], "accessToken" | "endpointUrl">,
-		options?: Parameters<Task[key]>[1]
-	) => ReturnType<Task[key]>;
-};
-
 export class InferenceClient {
 	private readonly accessToken: string;
 	private readonly defaultOptions: Options;
 
-	constructor(accessToken = "", defaultOptions: Options = {}) {
+	constructor(
+		accessToken = "",
+		defaultOptions: Options & {
+			endpointUrl?: string;
+		} = {}
+	) {
 		this.accessToken = accessToken;
 		this.defaultOptions = defaultOptions;
 
-		for (const [name, fn] of Object.entries(tasks)) {
+		for (const [name, fn] of typedEntries(tasks)) {
 			Object.defineProperty(this, name, {
 				enumerable: false,
-				value: (params: RequestArgs, options: Options) =>
+				value: (params: Parameters<typeof fn>[0], options: Parameters<typeof fn>[1]) =>
 					// eslint-disable-next-line @typescript-eslint/no-explicit-any
-					fn({ ...params, accessToken } as any, { ...defaultOptions, ...options }),
+					(fn as any)(
+						/// ^ The cast of fn to any is necessary, otherwise TS can't compile because the generated union type is too complex
+						{ endpointUrl: defaultOptions.endpointUrl, accessToken, ...params },
+						{
+							...omit(defaultOptions, ["endpointUrl"]),
+							...options,
+						}
+					),
 			});
 		}
 	}
 
 	/**
-	 * Returns copy of InferenceClient tied to a specified endpoint.
+	 * Returns a new instance of InferenceClient tied to a specified endpoint.
+	 *
+	 * For backward compatibility mostly.
 	 */
-	public endpoint(endpointUrl: string): InferenceClientEndpoint {
-		return new InferenceClientEndpoint(endpointUrl, this.accessToken, this.defaultOptions);
-	}
-}
-
-export class InferenceClientEndpoint {
-	constructor(endpointUrl: string, accessToken = "", defaultOptions: Options = {}) {
-		accessToken;
-		defaultOptions;
-
-		for (const [name, fn] of Object.entries(tasks)) {
-			Object.defineProperty(this, name, {
-				enumerable: false,
-				value: (params: RequestArgs, options: Options) =>
-					// eslint-disable-next-line @typescript-eslint/no-explicit-any
-					fn({ ...params, accessToken, endpointUrl } as any, { ...defaultOptions, ...options }),
-			});
-		}
+	public endpoint(endpointUrl: string): InferenceClient {
+		return new InferenceClient(this.accessToken, { ...this.defaultOptions, endpointUrl });
 	}
 }
 
-export interface InferenceClient extends TaskWithNoAccessToken {}
-
-export interface InferenceClientEndpoint extends TaskWithNoAccessTokenNoEndpointUrl {}
+export interface InferenceClient extends Task {}
 
 /**
- * For backward compatibility only.
+ * For backward compatibility only, will remove soon.
+ * @deprecated replace with InferenceClient
  */
 export class HfInference extends InferenceClient {}
+/**
+ * For backward compatibility only, will remove soon.
+ * @deprecated replace with InferenceClient
+ */
+export class InferenceClientEndpoint extends InferenceClient {}

package/src/lib/getInferenceProviderMapping.ts

@@ -1,8 +1,8 @@
 import type { WidgetType } from "@huggingface/tasks";
-import type { InferenceProvider, ModelId } from "../types";
 import { HF_HUB_URL } from "../config";
 import { HARDCODED_MODEL_INFERENCE_MAPPING } from "../providers/consts";
 import { EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS } from "../providers/hf-inference";
+import type { InferenceProvider, InferenceProviderOrPolicy, ModelId } from "../types";
 import { typedInclude } from "../utils/typedInclude";
 
 export const inferenceProviderMappingCache = new Map<ModelId, InferenceProviderMapping>();
@@ -20,44 +20,62 @@ export interface InferenceProviderModelMapping {
 	task: WidgetType;
 }
 
-export async function getInferenceProviderMapping(
-	params: {
-		accessToken?: string;
-		modelId: ModelId;
-		provider: InferenceProvider;
-		task: WidgetType;
-	},
-	options: {
+export async function fetchInferenceProviderMappingForModel(
+	modelId: ModelId,
+	accessToken?: string,
+	options?: {
 		fetch?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
 	}
-): Promise<InferenceProviderModelMapping | null> {
-	if (HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId]) {
-		return HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId];
-	}
+): Promise<InferenceProviderMapping> {
 	let inferenceProviderMapping: InferenceProviderMapping | null;
-	if (inferenceProviderMappingCache.has(params.modelId)) {
+	if (inferenceProviderMappingCache.has(modelId)) {
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		inferenceProviderMapping = inferenceProviderMappingCache.get(params.modelId)!;
+		inferenceProviderMapping = inferenceProviderMappingCache.get(modelId)!;
 	} else {
 		const resp = await (options?.fetch ?? fetch)(
-			`${HF_HUB_URL}/api/models/${params.modelId}?expand[]=inferenceProviderMapping`,
+			`${HF_HUB_URL}/api/models/${modelId}?expand[]=inferenceProviderMapping`,
 			{
-				headers: params.accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${params.accessToken}` } : {},
+				headers: accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${accessToken}` } : {},
 			}
 		);
 		if (resp.status === 404) {
-			throw new Error(`Model ${params.modelId} does not exist`);
+			throw new Error(`Model ${modelId} does not exist`);
 		}
 		inferenceProviderMapping = await resp
 			.json()
 			.then((json) => json.inferenceProviderMapping)
 			.catch(() => null);
+
+		if (inferenceProviderMapping) {
+			inferenceProviderMappingCache.set(modelId, inferenceProviderMapping);
+		}
 	}
 
 	if (!inferenceProviderMapping) {
-		throw new Error(`We have not been able to find inference provider information for model ${params.modelId}.`);
+		throw new Error(`We have not been able to find inference provider information for model ${modelId}.`);
 	}
+	return inferenceProviderMapping;
+}
 
+export async function getInferenceProviderMapping(
+	params: {
+		accessToken?: string;
+		modelId: ModelId;
+		provider: InferenceProvider;
+		task: WidgetType;
+	},
+	options: {
+		fetch?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
+	}
+): Promise<InferenceProviderModelMapping | null> {
+	if (HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId]) {
+		return HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId];
+	}
+	const inferenceProviderMapping = await fetchInferenceProviderMappingForModel(
+		params.modelId,
+		params.accessToken,
+		options
+	);
 	const providerMapping = inferenceProviderMapping[params.provider];
 	if (providerMapping) {
 		const equivalentTasks =
@@ -78,3 +96,34 @@ export async function getInferenceProviderMapping(
 	}
 	return null;
 }
+
+export async function resolveProvider(
+	provider?: InferenceProviderOrPolicy,
+	modelId?: string,
+	endpointUrl?: string
+): Promise<InferenceProvider> {
+	if (endpointUrl) {
+		if (provider) {
+			throw new Error("Specifying both endpointUrl and provider is not supported.");
+		}
+		/// Defaulting to hf-inference helpers / API
+		return "hf-inference";
+	}
+	if (!provider) {
+		console.log(
+			"Defaulting to 'auto' which will select the first provider available for the model, sorted by the user's order in https://hf.co/settings/inference-providers."
+		);
+		provider = "auto";
+	}
+	if (provider === "auto") {
+		if (!modelId) {
+			throw new Error("Specifying a model is required when provider is 'auto'");
+		}
+		const inferenceProviderMapping = await fetchInferenceProviderMappingForModel(modelId);
+		provider = Object.keys(inferenceProviderMapping)[0] as InferenceProvider | undefined;
+	}
+	if (!provider) {
+		throw new Error(`No Inference Provider available for model ${modelId}.`);
+	}
+	return provider;
+}

package/src/lib/getProviderHelper.ts

@@ -11,6 +11,7 @@ import * as Nebius from "../providers/nebius";
 import * as Novita from "../providers/novita";
 import * as Nscale from "../providers/nscale";
 import * as OpenAI from "../providers/openai";
+import * as OvhCloud from "../providers/ovhcloud";
 import type {
 	AudioClassificationTaskHelper,
 	AudioToAudioTaskHelper,
@@ -126,6 +127,10 @@ export const PROVIDERS: Record<InferenceProvider, Partial<Record<InferenceTask,
 	openai: {
 		conversational: new OpenAI.OpenAIConversationalTask(),
 	},
+	ovhcloud: {
+		conversational: new OvhCloud.OvhCloudConversationalTask(),
+		"text-generation": new OvhCloud.OvhCloudTextGenerationTask(),
+	},
 	replicate: {
 		"text-to-image": new Replicate.ReplicateTextToImageTask(),
 		"text-to-speech": new Replicate.ReplicateTextToSpeechTask(),

package/src/lib/makeRequestOptions.ts

@@ -27,8 +27,8 @@ export async function makeRequestOptions(
 		task?: InferenceTask;
 	}
 ): Promise<{ url: string; info: RequestInit }> {
-	const { provider: maybeProvider, model: maybeModel } = args;
-	const provider = maybeProvider ?? "hf-inference";
+	const { model: maybeModel } = args;
+	const provider = providerHelper.provider;
 	const { task } = options ?? {};
 
 	// Validate inputs
@@ -113,8 +113,9 @@ export function makeRequestOptionsFromResolvedModel(
 ): { url: string; info: RequestInit } {
 	const { accessToken, endpointUrl, provider: maybeProvider, model, ...remainingArgs } = args;
 	void model;
+	void maybeProvider;
 
-	const provider = maybeProvider ?? "hf-inference";
+	const provider = providerHelper.provider;
 
 	const { includeCredentials, task, signal, billTo } = options ?? {};
 	const authMethod = (() => {

package/src/providers/consts.ts

@@ -32,6 +32,7 @@ export const HARDCODED_MODEL_INFERENCE_MAPPING: Record<
 	novita: {},
 	nscale: {},
 	openai: {},
+	ovhcloud: {},
 	replicate: {},
 	sambanova: {},
 	together: {},

package/src/providers/ovhcloud.ts

@@ -0,0 +1,75 @@
+/**
+ * See the registered mapping of HF model ID => OVHcloud model ID here:
+ *
+ * https://huggingface.co/api/partners/ovhcloud/models
+ *
+ * This is a publicly available mapping.
+ *
+ * If you want to try to run inference for a new model locally before it's registered on huggingface.co,
+ * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes.
+ *
+ * - If you work at OVHcloud and want to update this mapping, please use the model mapping API we provide on huggingface.co
+ * - If you're a community member and want to add a new supported HF model to OVHcloud, please open an issue on the present repo
+ * and we will tag OVHcloud team members.
+ *
+ * Thanks!
+ */
+
+import { BaseConversationalTask, BaseTextGenerationTask } from "./providerHelper";
+import type { ChatCompletionOutput, TextGenerationOutput, TextGenerationOutputFinishReason } from "@huggingface/tasks";
+import { InferenceOutputError } from "../lib/InferenceOutputError";
+import type { BodyParams } from "../types";
+import { omit } from "../utils/omit";
+import type { TextGenerationInput } from "@huggingface/tasks";
+
+const OVHCLOUD_API_BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net";
+
+interface OvhCloudTextCompletionOutput extends Omit<ChatCompletionOutput, "choices"> {
+	choices: Array<{
+		text: string;
+		finish_reason: TextGenerationOutputFinishReason;
+		logprobs: unknown;
+		index: number;
+	}>;
+}
+
+export class OvhCloudConversationalTask extends BaseConversationalTask {
+	constructor() {
+		super("ovhcloud", OVHCLOUD_API_BASE_URL);
+	}
+}
+
+export class OvhCloudTextGenerationTask extends BaseTextGenerationTask {
+	constructor() {
+		super("ovhcloud", OVHCLOUD_API_BASE_URL);
+	}
+
+	override preparePayload(params: BodyParams<TextGenerationInput>): Record<string, unknown> {
+		return {
+			model: params.model,
+			...omit(params.args, ["inputs", "parameters"]),
+			...(params.args.parameters
+				? {
+						max_tokens: (params.args.parameters as Record<string, unknown>).max_new_tokens,
+						...omit(params.args.parameters as Record<string, unknown>, "max_new_tokens"),
+				  }
+				: undefined),
+			prompt: params.args.inputs,
+		};
+	}
+
+	override async getResponse(response: OvhCloudTextCompletionOutput): Promise<TextGenerationOutput> {
+		if (
+			typeof response === "object" &&
+			"choices" in response &&
+			Array.isArray(response?.choices) &&
+			typeof response?.model === "string"
+		) {
+			const completion = response.choices[0];
+			return {
+				generated_text: completion.text,
+			};
+		}
+		throw new InferenceOutputError("Expected OVHcloud text generation response format");
+	}
+}

package/src/providers/providerHelper.ts

@@ -56,7 +56,7 @@ import { toArray } from "../utils/toArray";
  */
 export abstract class TaskProviderHelper {
 	constructor(
-		private provider: InferenceProvider,
+		readonly provider: InferenceProvider,
 		private baseUrl: string,
 		readonly clientSideRoutingOnly: boolean = false
 	) {}

package/src/snippets/getInferenceSnippets.ts

@@ -8,11 +8,11 @@ import {
 } from "@huggingface/tasks";
 import type { PipelineType, WidgetType } from "@huggingface/tasks/src/pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "@huggingface/tasks/src/tasks/index.js";
+import type { InferenceProviderModelMapping } from "../lib/getInferenceProviderMapping";
+import { getProviderHelper } from "../lib/getProviderHelper";
 import { makeRequestOptionsFromResolvedModel } from "../lib/makeRequestOptions";
 import type { InferenceProvider, InferenceTask, RequestArgs } from "../types";
 import { templates } from "./templates.exported";
-import type { InferenceProviderModelMapping } from "../lib/getInferenceProviderMapping";
-import { getProviderHelper } from "../lib/getProviderHelper";
 
 export type InferenceSnippetOptions = { streaming?: boolean; billTo?: string } & Record<string, unknown>;
 
@@ -112,6 +112,7 @@ const HF_JS_METHODS: Partial<Record<WidgetType, string>> = {
 	"text-generation": "textGeneration",
 	"text2text-generation": "textGeneration",
 	"token-classification": "tokenClassification",
+	"text-to-speech": "textToSpeech",
 	translation: "translation",
 };
 
@@ -271,7 +272,7 @@ const prepareConversationalInput = (
 	return {
 		messages: opts?.messages ?? getModelInputSnippet(model),
 		...(opts?.temperature ? { temperature: opts?.temperature } : undefined),
-		max_tokens: opts?.max_tokens ?? 512,
+		...(opts?.max_tokens ? { max_tokens: opts?.max_tokens } : undefined),
 		...(opts?.top_p ? { top_p: opts?.top_p } : undefined),
 	};
 };
@@ -310,7 +311,7 @@ const snippets: Partial<
 	"text-generation": snippetGenerator("basic"),
 	"text-to-audio": snippetGenerator("textToAudio"),
 	"text-to-image": snippetGenerator("textToImage"),
-	"text-to-speech": snippetGenerator("textToAudio"),
+	"text-to-speech": snippetGenerator("textToSpeech"),
 	"text-to-video": snippetGenerator("textToVideo"),
 	"text2text-generation": snippetGenerator("basic"),
 	"token-classification": snippetGenerator("basic"),

package/src/snippets/templates.exported.ts

@@ -7,6 +7,7 @@ export const templates: Record<string, Record<string, Record<string, string>>> =
       "basicImage": "async function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"image/jpeg\",\n{% if billTo %}\n\t\t\t\t\"X-HF-Bill-To\": \"{{ billTo }}\",\n{% endif %}\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n\tconst result = await response.json();\n\treturn result;\n}\n\nquery({ inputs: {{ providerInputs.asObj.inputs }} }).then((response) => {\n    console.log(JSON.stringify(response));\n});",
       "textToAudio": "{% if model.library_name == \"transformers\" %}\nasync function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n{% if billTo %}\n\t\t\t\t\"X-HF-Bill-To\": \"{{ billTo }}\",\n{% endif %}\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n\tconst result = await response.blob();\n    return result;\n}\n\nquery({ inputs: {{ providerInputs.asObj.inputs }} }).then((response) => {\n    // Returns a byte object of the Audio wavform. Use it directly!\n});\n{% else %}\nasync function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n    const result = await response.json();\n    return result;\n}\n\nquery({ inputs: {{ providerInputs.asObj.inputs }} }).then((response) => {\n    console.log(JSON.stringify(response));\n});\n{% endif %} ",
       "textToImage": "async function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n{% if billTo %}\n\t\t\t\t\"X-HF-Bill-To\": \"{{ billTo }}\",\n{% endif %}\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n\tconst result = await response.blob();\n\treturn result;\n}\n\n\nquery({ {{ providerInputs.asTsString }} }).then((response) => {\n    // Use image\n});",
+      "textToSpeech": "{% if model.library_name == \"transformers\" %}\nasync function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n{% if billTo %}\n\t\t\t\t\"X-HF-Bill-To\": \"{{ billTo }}\",\n{% endif %}\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n\tconst result = await response.blob();\n    return result;\n}\n\nquery({ text: {{ inputs.asObj.inputs }} }).then((response) => {\n    // Returns a byte object of the Audio wavform. Use it directly!\n});\n{% else %}\nasync function query(data) {\n\tconst response = await fetch(\n\t\t\"{{ fullUrl }}\",\n\t\t{\n\t\t\theaders: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tmethod: \"POST\",\n\t\t\tbody: JSON.stringify(data),\n\t\t}\n\t);\n    const result = await response.json();\n    return result;\n}\n\nquery({ text: {{ inputs.asObj.inputs }} }).then((response) => {\n    console.log(JSON.stringify(response));\n});\n{% endif %} ",
       "zeroShotClassification": "async function query(data) {\n    const response = await fetch(\n\t\t\"{{ fullUrl }}\",\n        {\n            headers: {\n\t\t\t\tAuthorization: \"{{ authorizationHeader }}\",\n                \"Content-Type\": \"application/json\",\n{% if billTo %}\n                \"X-HF-Bill-To\": \"{{ billTo }}\",\n{% endif %}         },\n            method: \"POST\",\n            body: JSON.stringify(data),\n        }\n    );\n    const result = await response.json();\n    return result;\n}\n\nquery({\n    inputs: {{ providerInputs.asObj.inputs }},\n    parameters: { candidate_labels: [\"refund\", \"legal\", \"faq\"] }\n}).then((response) => {\n    console.log(JSON.stringify(response));\n});"
     },
     "huggingface.js": {
@@ -16,7 +17,8 @@ export const templates: Record<string, Record<string, Record<string, string>>> =
       "conversational": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nconst chatCompletion = await client.chatCompletion({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n{{ inputs.asTsString }}\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n\nconsole.log(chatCompletion.choices[0].message);",
       "conversationalStream": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nlet out = \"\";\n\nconst stream = client.chatCompletionStream({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n{{ inputs.asTsString }}\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n\nfor await (const chunk of stream) {\n\tif (chunk.choices && chunk.choices.length > 0) {\n\t\tconst newContent = chunk.choices[0].delta.content;\n\t\tout += newContent;\n\t\tconsole.log(newContent);\n\t}\n}",
       "textToImage": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nconst image = await client.textToImage({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n\tinputs: {{ inputs.asObj.inputs }},\n\tparameters: { num_inference_steps: 5 },\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n/// Use the generated image (it's a Blob)",
-      "textToVideo": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nconst image = await client.textToVideo({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n\tinputs: {{ inputs.asObj.inputs }},\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n// Use the generated video (it's a Blob)"
+      "textToSpeech": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nconst audio = await client.textToSpeech({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n\tinputs: {{ inputs.asObj.inputs }},\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n// Use the generated audio (it's a Blob)",
+      "textToVideo": "import { InferenceClient } from \"@huggingface/inference\";\n\nconst client = new InferenceClient(\"{{ accessToken }}\");\n\nconst video = await client.textToVideo({\n    provider: \"{{ provider }}\",\n    model: \"{{ model.id }}\",\n\tinputs: {{ inputs.asObj.inputs }},\n}{% if billTo %}, {\n    billTo: \"{{ billTo }}\",\n}{% endif %});\n// Use the generated video (it's a Blob)"
     },
     "openai": {
       "conversational": "import { OpenAI } from \"openai\";\n\nconst client = new OpenAI({\n\tbaseURL: \"{{ baseUrl }}\",\n\tapiKey: \"{{ accessToken }}\",\n{% if billTo %}\n\tdefaultHeaders: {\n\t\t\"X-HF-Bill-To\": \"{{ billTo }}\" \n\t}\n{% endif %}\n});\n\nconst chatCompletion = await client.chat.completions.create({\n\tmodel: \"{{ providerModelId }}\",\n{{ inputs.asTsString }}\n});\n\nconsole.log(chatCompletion.choices[0].message);",
@@ -25,7 +27,7 @@ export const templates: Record<string, Record<string, Record<string, string>>> =
   },
   "python": {
     "fal_client": {
-      "textToImage": "{% if provider == \"fal-ai\" %}\nimport fal_client\n\nresult = fal_client.subscribe(\n    \"{{ providerModelId }}\",\n    arguments={\n        \"prompt\": {{ inputs.asObj.inputs }},\n    },\n)\nprint(result)\n{% endif %} "
+      "textToImage": "{% if provider == \"fal-ai\" %}\nimport fal_client\n\n{% if providerInputs.asObj.loras is defined and providerInputs.asObj.loras != none %}\nresult = fal_client.subscribe(\n    \"{{ providerModelId }}\",\n    arguments={\n        \"prompt\": {{ inputs.asObj.inputs }},\n        \"loras\":{{ providerInputs.asObj.loras | tojson }},\n    },\n)\n{% else %}\nresult = fal_client.subscribe(\n    \"{{ providerModelId }}\",\n    arguments={\n        \"prompt\": {{ inputs.asObj.inputs }},\n    },\n)\n{% endif %} \nprint(result)\n{% endif %} "
     },
     "huggingface_hub": {
       "basic": "result = client.{{ methodName }}(\n    inputs={{ inputs.asObj.inputs }},\n    model=\"{{ model.id }}\",\n)",
@@ -37,6 +39,7 @@ export const templates: Record<string, Record<string, Record<string, string>>> =
       "imageToImage": "# output is a PIL.Image object\nimage = client.image_to_image(\n    \"{{ inputs.asObj.inputs }}\",\n    prompt=\"{{ inputs.asObj.parameters.prompt }}\",\n    model=\"{{ model.id }}\",\n) ",
       "importInferenceClient": "from huggingface_hub import InferenceClient\n\nclient = InferenceClient(\n    provider=\"{{ provider }}\",\n    api_key=\"{{ accessToken }}\",\n{% if billTo %}\n    bill_to=\"{{ billTo }}\",\n{% endif %}\n)",
       "textToImage": "# output is a PIL.Image object\nimage = client.text_to_image(\n    {{ inputs.asObj.inputs }},\n    model=\"{{ model.id }}\",\n) ",
+      "textToSpeech": "# audio is returned as bytes\naudio = client.text_to_speech(\n    {{ inputs.asObj.inputs }},\n    model=\"{{ model.id }}\",\n) \n",
       "textToVideo": "video = client.text_to_video(\n    {{ inputs.asObj.inputs }},\n    model=\"{{ model.id }}\",\n) "
     },
     "openai": {
@@ -53,8 +56,9 @@ export const templates: Record<string, Record<string, Record<string, string>>> =
       "imageToImage": "def query(payload):\n    with open(payload[\"inputs\"], \"rb\") as f:\n        img = f.read()\n        payload[\"inputs\"] = base64.b64encode(img).decode(\"utf-8\")\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\nimage_bytes = query({\n{{ providerInputs.asJsonString }}\n})\n\n# You can access the image with PIL.Image for example\nimport io\nfrom PIL import Image\nimage = Image.open(io.BytesIO(image_bytes)) ",
       "importRequests": "{% if importBase64 %}\nimport base64\n{% endif %}\n{% if importJson %}\nimport json\n{% endif %}\nimport requests\n\nAPI_URL = \"{{ fullUrl }}\"\nheaders = {\n    \"Authorization\": \"{{ authorizationHeader }}\",\n{% if billTo %}\n    \"X-HF-Bill-To\": \"{{ billTo }}\"\n{% endif %}\n}",
       "tabular": "def query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\nresponse = query({\n    \"inputs\": {\n        \"data\": {{ providerInputs.asObj.inputs }}\n    },\n}) ",
-      "textToAudio": "{% if model.library_name == \"transformers\" %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\naudio_bytes = query({\n    \"inputs\": {{ providerInputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio_bytes)\n{% else %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.json()\n\naudio, sampling_rate = query({\n    \"inputs\": {{ providerInputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio, rate=sampling_rate)\n{% endif %} ",
+      "textToAudio": "{% if model.library_name == \"transformers\" %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\naudio_bytes = query({\n    \"inputs\": {{ inputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio_bytes)\n{% else %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.json()\n\naudio, sampling_rate = query({\n    \"inputs\": {{ inputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio, rate=sampling_rate)\n{% endif %} ",
       "textToImage": "{% if provider == \"hf-inference\" %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\nimage_bytes = query({\n    \"inputs\": {{ providerInputs.asObj.inputs }},\n})\n\n# You can access the image with PIL.Image for example\nimport io\nfrom PIL import Image\nimage = Image.open(io.BytesIO(image_bytes))\n{% endif %}",
+      "textToSpeech": "{% if model.library_name == \"transformers\" %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.content\n\naudio_bytes = query({\n    \"text\": {{ inputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio_bytes)\n{% else %}\ndef query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.json()\n\naudio, sampling_rate = query({\n    \"text\": {{ inputs.asObj.inputs }},\n})\n# You can access the audio with IPython.display for example\nfrom IPython.display import Audio\nAudio(audio, rate=sampling_rate)\n{% endif %} ",
       "zeroShotClassification": "def query(payload):\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.json()\n\noutput = query({\n    \"inputs\": {{ providerInputs.asObj.inputs }},\n    \"parameters\": {\"candidate_labels\": [\"refund\", \"legal\", \"faq\"]},\n}) ",
       "zeroShotImageClassification": "def query(data):\n    with open(data[\"image_path\"], \"rb\") as f:\n        img = f.read()\n    payload={\n        \"parameters\": data[\"parameters\"],\n        \"inputs\": base64.b64encode(img).decode(\"utf-8\")\n    }\n    response = requests.post(API_URL, headers=headers, json=payload)\n    return response.json()\n\noutput = query({\n    \"image_path\": {{ providerInputs.asObj.inputs }},\n    \"parameters\": {\"candidate_labels\": [\"cat\", \"dog\", \"llama\"]},\n}) "
     }

package/src/tasks/audio/audioClassification.ts

@@ -1,4 +1,5 @@
 import type { AudioClassificationInput, AudioClassificationOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -15,7 +16,8 @@ export async function audioClassification(
 	args: AudioClassificationArgs,
 	options?: Options
 ): Promise<AudioClassificationOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "audio-classification");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "audio-classification");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<AudioClassificationOutput>(payload, providerHelper, {
 		...options,

package/src/tasks/audio/audioToAudio.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -36,7 +37,9 @@ export interface AudioToAudioOutput {
  * Example model: speechbrain/sepformer-wham does audio source separation.
  */
 export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioOutput[]> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "audio-to-audio");
+	const model = "inputs" in args ? args.model : undefined;
+	const provider = await resolveProvider(args.provider, model);
+	const providerHelper = getProviderHelper(provider, "audio-to-audio");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<AudioToAudioOutput>(payload, providerHelper, {
 		...options,

package/src/tasks/audio/automaticSpeechRecognition.ts

@@ -1,4 +1,5 @@
 import type { AutomaticSpeechRecognitionInput, AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import { FAL_AI_SUPPORTED_BLOB_TYPES } from "../../providers/fal-ai";
@@ -18,7 +19,8 @@ export async function automaticSpeechRecognition(
 	args: AutomaticSpeechRecognitionArgs,
 	options?: Options
 ): Promise<AutomaticSpeechRecognitionOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "automatic-speech-recognition");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "automatic-speech-recognition");
 	const payload = await buildPayload(args);
 	const { data: res } = await innerRequest<AutomaticSpeechRecognitionOutput>(payload, providerHelper, {
 		...options,

package/src/tasks/audio/textToSpeech.ts

@@ -1,4 +1,5 @@
 import type { TextToSpeechInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -12,7 +13,7 @@ interface OutputUrlTextToSpeechGeneration {
  * Recommended model: espnet/kan-bayashi_ljspeech_vits
  */
 export async function textToSpeech(args: TextToSpeechArgs, options?: Options): Promise<Blob> {
-	const provider = args.provider ?? "hf-inference";
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
 	const providerHelper = getProviderHelper(provider, "text-to-speech");
 	const { data: res } = await innerRequest<Blob | OutputUrlTextToSpeechGeneration>(args, providerHelper, {
 		...options,

package/src/tasks/custom/request.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -16,7 +17,8 @@ export async function request<T>(
 	console.warn(
 		"The request method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
 	);
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", options?.task);
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, options?.task);
 	const result = await innerRequest<T>(args, providerHelper, options);
 	return result.data;
 }

package/src/tasks/custom/streamingRequest.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { innerStreamingRequest } from "../../utils/request";
@@ -16,6 +17,7 @@ export async function* streamingRequest<T>(
 	console.warn(
 		"The streamingRequest method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
 	);
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", options?.task);
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, options?.task);
 	yield* innerStreamingRequest(args, providerHelper, options);
 }

package/src/tasks/cv/imageClassification.ts

@@ -1,4 +1,5 @@
 import type { ImageClassificationInput, ImageClassificationOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -14,7 +15,8 @@ export async function imageClassification(
 	args: ImageClassificationArgs,
 	options?: Options
 ): Promise<ImageClassificationOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "image-classification");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-classification");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<ImageClassificationOutput>(payload, providerHelper, {
 		...options,

package/src/tasks/cv/imageSegmentation.ts

@@ -1,4 +1,5 @@
 import type { ImageSegmentationInput, ImageSegmentationOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -14,7 +15,8 @@ export async function imageSegmentation(
 	args: ImageSegmentationArgs,
 	options?: Options
 ): Promise<ImageSegmentationOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "image-segmentation");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-segmentation");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<ImageSegmentationOutput>(payload, providerHelper, {
 		...options,

package/src/tasks/cv/imageToImage.ts

@@ -1,4 +1,5 @@
 import type { ImageToImageInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options, RequestArgs } from "../../types";
 import { base64FromBytes } from "../../utils/base64FromBytes";
@@ -11,7 +12,8 @@ export type ImageToImageArgs = BaseArgs & ImageToImageInput;
  * Recommended model: lllyasviel/sd-controlnet-depth
  */
 export async function imageToImage(args: ImageToImageArgs, options?: Options): Promise<Blob> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "image-to-image");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-to-image");
 	let reqArgs: RequestArgs;
 	if (!args.parameters) {
 		reqArgs = {

package/src/tasks/cv/imageToText.ts

@@ -1,4 +1,5 @@
 import type { ImageToTextInput, ImageToTextOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -10,7 +11,8 @@ export type ImageToTextArgs = BaseArgs & (ImageToTextInput | LegacyImageInput);
  * This task reads some image input and outputs the text caption.
  */
 export async function imageToText(args: ImageToTextArgs, options?: Options): Promise<ImageToTextOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "image-to-text");
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-to-text");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<[ImageToTextOutput]>(payload, providerHelper, {
 		...options,