@huggingface/inference 3.7.0 → 3.8.0

package/src/lib/makeRequestOptions.ts

@@ -1,23 +1,10 @@
-import { HF_HUB_URL, HF_ROUTER_URL, HF_HEADER_X_BILL_TO } from "../config";
-import { BLACK_FOREST_LABS_CONFIG } from "../providers/black-forest-labs";
-import { CEREBRAS_CONFIG } from "../providers/cerebras";
-import { COHERE_CONFIG } from "../providers/cohere";
-import { FAL_AI_CONFIG } from "../providers/fal-ai";
-import { FIREWORKS_AI_CONFIG } from "../providers/fireworks-ai";
-import { HF_INFERENCE_CONFIG } from "../providers/hf-inference";
-import { HYPERBOLIC_CONFIG } from "../providers/hyperbolic";
-import { NEBIUS_CONFIG } from "../providers/nebius";
-import { NOVITA_CONFIG } from "../providers/novita";
-import { REPLICATE_CONFIG } from "../providers/replicate";
-import { SAMBANOVA_CONFIG } from "../providers/sambanova";
-import { TOGETHER_CONFIG } from "../providers/together";
-import { OPENAI_CONFIG } from "../providers/openai";
-import type { InferenceProvider, InferenceTask, Options, ProviderConfig, RequestArgs } from "../types";
+import { name as packageName, version as packageVersion } from "../../package.json";
+import { HF_HEADER_X_BILL_TO, HF_HUB_URL } from "../config";
+import type { InferenceTask, Options, RequestArgs } from "../types";
+import type { InferenceProviderModelMapping } from "./getInferenceProviderMapping";
+import { getInferenceProviderMapping } from "./getInferenceProviderMapping";
+import type { getProviderHelper } from "./getProviderHelper";
 import { isUrl } from "./isUrl";
-import { version as packageVersion, name as packageName } from "../../package.json";
-import { getProviderModelId } from "./getProviderModelId";
-
-const HF_HUB_INFERENCE_PROXY_TEMPLATE = `${HF_ROUTER_URL}/{{PROVIDER}}`;
 
 /**
  * Lazy-loaded from huggingface.co/api/tasks when needed
@@ -25,25 +12,6 @@ const HF_HUB_INFERENCE_PROXY_TEMPLATE = `${HF_ROUTER_URL}/{{PROVIDER}}`;
  */
 let tasks: Record<string, { models: { id: string }[] }> | null = null;
 
-/**
- * Config to define how to serialize requests for each provider
- */
-const providerConfigs: Record<InferenceProvider, ProviderConfig> = {
-	"black-forest-labs": BLACK_FOREST_LABS_CONFIG,
-	cerebras: CEREBRAS_CONFIG,
-	cohere: COHERE_CONFIG,
-	"fal-ai": FAL_AI_CONFIG,
-	"fireworks-ai": FIREWORKS_AI_CONFIG,
-	"hf-inference": HF_INFERENCE_CONFIG,
-	hyperbolic: HYPERBOLIC_CONFIG,
-	openai: OPENAI_CONFIG,
-	nebius: NEBIUS_CONFIG,
-	novita: NOVITA_CONFIG,
-	replicate: REPLICATE_CONFIG,
-	sambanova: SAMBANOVA_CONFIG,
-	together: TOGETHER_CONFIG,
-};
-
 /**
  * Helper that prepares request arguments.
  * This async version handle the model ID resolution step.
@@ -53,16 +21,15 @@ export async function makeRequestOptions(
 		data?: Blob | ArrayBuffer;
 		stream?: boolean;
 	},
+	providerHelper: ReturnType<typeof getProviderHelper>,
 	options?: Options & {
 		/** In most cases (unless we pass a endpointUrl) we know the task */
 		task?: InferenceTask;
-		chatCompletion?: boolean;
 	}
 ): Promise<{ url: string; info: RequestInit }> {
 	const { provider: maybeProvider, model: maybeModel } = args;
 	const provider = maybeProvider ?? "hf-inference";
-	const providerConfig = providerConfigs[provider];
-	const { task, chatCompletion } = options ?? {};
+	const { task } = options ?? {};
 
 	// Validate inputs
 	if (args.endpointUrl && provider !== "hf-inference") {
@@ -71,29 +38,61 @@ export async function makeRequestOptions(
 	if (maybeModel && isUrl(maybeModel)) {
 		throw new Error(`Model URLs are no longer supported. Use endpointUrl instead.`);
 	}
+
+	if (args.endpointUrl) {
+		// No need to have maybeModel, or to load default model for a task
+		return makeRequestOptionsFromResolvedModel(
+			maybeModel ?? args.endpointUrl,
+			providerHelper,
+			args,
+			undefined,
+			options
+		);
+	}
+
 	if (!maybeModel && !task) {
 		throw new Error("No model provided, and no task has been specified.");
 	}
-	if (!providerConfig) {
-		throw new Error(`No provider config found for provider ${provider}`);
-	}
-	if (providerConfig.clientSideRoutingOnly && !maybeModel) {
-		throw new Error(`Provider ${provider} requires a model ID to be passed directly.`);
-	}
 
 	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 	const hfModel = maybeModel ?? (await loadDefaultModel(task!));
-	const resolvedModel = providerConfig.clientSideRoutingOnly
-		? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		  removeProviderPrefix(maybeModel!, provider)
-		: await getProviderModelId({ model: hfModel, provider }, args, {
-				task,
-				chatCompletion,
-				fetch: options?.fetch,
-		  });
+
+	if (providerHelper.clientSideRoutingOnly && !maybeModel) {
+		throw new Error(`Provider ${provider} requires a model ID to be passed directly.`);
+	}
+
+	const inferenceProviderMapping = providerHelper.clientSideRoutingOnly
+		? ({
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				providerId: removeProviderPrefix(maybeModel!, provider),
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				hfModelId: maybeModel!,
+				status: "live",
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				task: task!,
+		  } satisfies InferenceProviderModelMapping)
+		: await getInferenceProviderMapping(
+				{
+					modelId: hfModel,
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					task: task!,
+					provider,
+					accessToken: args.accessToken,
+				},
+				{ fetch: options?.fetch }
+		  );
+	if (!inferenceProviderMapping) {
+		throw new Error(`We have not been able to find inference provider information for model ${hfModel}.`);
+	}
 
 	// Use the sync version with the resolved model
-	return makeRequestOptionsFromResolvedModel(resolvedModel, args, options);
+	return makeRequestOptionsFromResolvedModel(
+		inferenceProviderMapping.providerId,
+		providerHelper,
+		args,
+		inferenceProviderMapping,
+		options
+	);
 }
 
 /**
@@ -102,25 +101,24 @@ export async function makeRequestOptions(
  */
 export function makeRequestOptionsFromResolvedModel(
 	resolvedModel: string,
+	providerHelper: ReturnType<typeof getProviderHelper>,
 	args: RequestArgs & {
 		data?: Blob | ArrayBuffer;
 		stream?: boolean;
 	},
+	mapping: InferenceProviderModelMapping | undefined,
 	options?: Options & {
 		task?: InferenceTask;
-		chatCompletion?: boolean;
 	}
 ): { url: string; info: RequestInit } {
 	const { accessToken, endpointUrl, provider: maybeProvider, model, ...remainingArgs } = args;
 	void model;
 
 	const provider = maybeProvider ?? "hf-inference";
-	const providerConfig = providerConfigs[provider];
-
-	const { includeCredentials, task, chatCompletion, signal, billTo } = options ?? {};
 
+	const { includeCredentials, task, signal, billTo } = options ?? {};
 	const authMethod = (() => {
-		if (providerConfig.clientSideRoutingOnly) {
+		if (providerHelper.clientSideRoutingOnly) {
 			// Closed-source providers require an accessToken (cannot be routed).
 			if (accessToken && accessToken.startsWith("hf_")) {
 				throw new Error(`Provider ${provider} is closed-source and does not support HF tokens.`);
@@ -138,36 +136,25 @@ export function makeRequestOptionsFromResolvedModel(
 	})();
 
 	// Make URL
-	const url = endpointUrl
-		? chatCompletion
-			? endpointUrl + `/v1/chat/completions`
-			: endpointUrl
-		: providerConfig.makeUrl({
-				authMethod,
-				baseUrl:
-					authMethod !== "provider-key"
-						? HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider)
-						: providerConfig.makeBaseUrl(task),
-				model: resolvedModel,
-				chatCompletion,
-				task,
-		  });
 
-	// Make headers
-	const binary = "data" in args && !!args.data;
-	const headers = providerConfig.makeHeaders({
-		accessToken,
+	const modelId = endpointUrl ?? resolvedModel;
+	const url = providerHelper.makeUrl({
 		authMethod,
+		model: modelId,
+		task,
 	});
+	// Make headers
+	const headers = providerHelper.prepareHeaders(
+		{
+			accessToken,
+			authMethod,
+		},
+		"data" in args && !!args.data
+	);
 	if (billTo) {
 		headers[HF_HEADER_X_BILL_TO] = billTo;
 	}
 
-	// Add content-type to headers
-	if (!binary) {
-		headers["Content-Type"] = "application/json";
-	}
-
 	// Add user-agent to headers
 	// e.g. @huggingface/inference/3.1.3
 	const ownUserAgent = `${packageName}/${packageVersion}`;
@@ -177,17 +164,12 @@ export function makeRequestOptionsFromResolvedModel(
 	headers["User-Agent"] = userAgent;
 
 	// Make body
-	const body = binary
-		? args.data
-		: JSON.stringify(
-				providerConfig.makeBody({
-					args: remainingArgs as Record<string, unknown>,
-					model: resolvedModel,
-					task,
-					chatCompletion,
-				})
-		  );
-
+	const body = providerHelper.makeBody({
+		args: remainingArgs as Record<string, unknown>,
+		model: resolvedModel,
+		task,
+		mapping,
+	});
 	/**
 	 * For edge runtimes, leave 'credentials' undefined, otherwise cloudflare workers will error
 	 */
@@ -201,11 +183,10 @@ export function makeRequestOptionsFromResolvedModel(
 	const info: RequestInit = {
 		headers,
 		method: "POST",
-		body,
+		body: body,
 		...(credentials ? { credentials } : undefined),
 		signal,
 	};
-
 	return { url, info };
 }
 

package/src/providers/black-forest-labs.ts

@@ -14,33 +14,84 @@
  *
  * Thanks!
  */
-import type { BodyParams, HeaderParams, ProviderConfig, UrlParams } from "../types";
+import { InferenceOutputError } from "../lib/InferenceOutputError";
+import type { BodyParams, HeaderParams, UrlParams } from "../types";
+import { delay } from "../utils/delay";
+import { omit } from "../utils/omit";
+import { TaskProviderHelper, type TextToImageTaskHelper } from "./providerHelper";
 
 const BLACK_FOREST_LABS_AI_API_BASE_URL = "https://api.us1.bfl.ai";
+interface BlackForestLabsResponse {
+	id: string;
+	polling_url: string;
+}
 
-const makeBaseUrl = (): string => {
-	return BLACK_FOREST_LABS_AI_API_BASE_URL;
-};
+export class BlackForestLabsTextToImageTask extends TaskProviderHelper implements TextToImageTaskHelper {
+	constructor() {
+		super("black-forest-labs", BLACK_FOREST_LABS_AI_API_BASE_URL);
+	}
 
-const makeBody = (params: BodyParams): Record<string, unknown> => {
-	return params.args;
-};
+	preparePayload(params: BodyParams): Record<string, unknown> {
+		return {
+			...omit(params.args, ["inputs", "parameters"]),
+			...(params.args.parameters as Record<string, unknown>),
+			prompt: params.args.inputs,
+		};
+	}
 
-const makeHeaders = (params: HeaderParams): Record<string, string> => {
-	if (params.authMethod === "provider-key") {
-		return { "X-Key": `${params.accessToken}` };
-	} else {
-		return { Authorization: `Bearer ${params.accessToken}` };
+	override prepareHeaders(params: HeaderParams, binary: boolean): Record<string, string> {
+		const headers: Record<string, string> = {
+			Authorization:
+				params.authMethod !== "provider-key" ? `Bearer ${params.accessToken}` : `X-Key ${params.accessToken}`,
+		};
+		if (!binary) {
+			headers["Content-Type"] = "application/json";
+		}
+		return headers;
 	}
-};
 
-const makeUrl = (params: UrlParams): string => {
-	return `${params.baseUrl}/v1/${params.model}`;
-};
+	makeRoute(params: UrlParams): string {
+		if (!params) {
+			throw new Error("Params are required");
+		}
+		return `/v1/${params.model}`;
+	}
 
-export const BLACK_FOREST_LABS_CONFIG: ProviderConfig = {
-	makeBaseUrl,
-	makeBody,
-	makeHeaders,
-	makeUrl,
-};
+	async getResponse(
+		response: BlackForestLabsResponse,
+		url?: string,
+		headers?: HeadersInit,
+		outputType?: "url" | "blob"
+	): Promise<string | Blob> {
+		const urlObj = new URL(response.polling_url);
+		for (let step = 0; step < 5; step++) {
+			await delay(1000);
+			console.debug(`Polling Black Forest Labs API for the result... ${step + 1}/5`);
+			urlObj.searchParams.set("attempt", step.toString(10));
+			const resp = await fetch(urlObj, { headers: { "Content-Type": "application/json" } });
+			if (!resp.ok) {
+				throw new InferenceOutputError("Failed to fetch result from black forest labs API");
+			}
+			const payload = await resp.json();
+			if (
+				typeof payload === "object" &&
+				payload &&
+				"status" in payload &&
+				typeof payload.status === "string" &&
+				payload.status === "Ready" &&
+				"result" in payload &&
+				typeof payload.result === "object" &&
+				payload.result &&
+				"sample" in payload.result &&
+				typeof payload.result.sample === "string"
+			) {
+				if (outputType === "url") {
+					return payload.result.sample;
+				}
+				const image = await fetch(payload.result.sample);
+				return await image.blob();
+			}
+		}
+		throw new InferenceOutputError("Failed to fetch result from black forest labs API");
+	}
+}

package/src/providers/cerebras.ts

@@ -14,32 +14,11 @@
  *
  * Thanks!
  */
-import type { BodyParams, HeaderParams, ProviderConfig, UrlParams } from "../types";
 
-const CEREBRAS_API_BASE_URL = "https://api.cerebras.ai";
+import { BaseConversationalTask } from "./providerHelper";
 
-const makeBaseUrl = (): string => {
-	return CEREBRAS_API_BASE_URL;
-};
-
-const makeBody = (params: BodyParams): Record<string, unknown> => {
-	return {
-		...params.args,
-		model: params.model,
-	};
-};
-
-const makeHeaders = (params: HeaderParams): Record<string, string> => {
-	return { Authorization: `Bearer ${params.accessToken}` };
-};
-
-const makeUrl = (params: UrlParams): string => {
-	return `${params.baseUrl}/v1/chat/completions`;
-};
-
-export const CEREBRAS_CONFIG: ProviderConfig = {
-	makeBaseUrl,
-	makeBody,
-	makeHeaders,
-	makeUrl,
-};
+export class CerebrasConversationalTask extends BaseConversationalTask {
+	constructor() {
+		super("cerebras", "https://api.cerebras.ai");
+	}
+}

package/src/providers/cohere.ts

@@ -14,32 +14,13 @@
  *
  * Thanks!
  */
-import type { BodyParams, HeaderParams, ProviderConfig, UrlParams } from "../types";
+import { BaseConversationalTask } from "./providerHelper";
 
-const COHERE_API_BASE_URL = "https://api.cohere.com";
-
-const makeBaseUrl = (): string => {
-	return COHERE_API_BASE_URL;
-};
-
-const makeBody = (params: BodyParams): Record<string, unknown> => {
-	return {
-		...params.args,
-		model: params.model,
-	};
-};
-
-const makeHeaders = (params: HeaderParams): Record<string, string> => {
-	return { Authorization: `Bearer ${params.accessToken}` };
-};
-
-const makeUrl = (params: UrlParams): string => {
-	return `${params.baseUrl}/compatibility/v1/chat/completions`;
-};
-
-export const COHERE_CONFIG: ProviderConfig = {
-	makeBaseUrl,
-	makeBody,
-	makeHeaders,
-	makeUrl,
-};
+export class CohereConversationalTask extends BaseConversationalTask {
+	constructor() {
+		super("cohere", "https://api.cohere.com");
+	}
+	override makeRoute(): string {
+		return "/compatibility/v1/chat/completions";
+	}
+}

package/src/providers/consts.ts

@@ -1,7 +1,7 @@
+import type { InferenceProviderModelMapping } from "../lib/getInferenceProviderMapping";
 import type { InferenceProvider } from "../types";
 import { type ModelId } from "../types";
 
-type ProviderId = string;
 /**
  * If you want to try to run inference for a new model locally before it's registered on huggingface.co
  * for a given Inference Provider,
@@ -9,7 +9,10 @@ type ProviderId = string;
  *
  * We also inject into this dictionary from tests.
  */
-export const HARDCODED_MODEL_ID_MAPPING: Record<InferenceProvider, Record<ModelId, ProviderId>> = {
+export const HARDCODED_MODEL_INFERENCE_MAPPING: Record<
+	InferenceProvider,
+	Record<ModelId, InferenceProviderModelMapping>
+> = {
 	/**
 	 * "HF model ID" => "Model ID on Inference Provider's side"
 	 *