@huggingface/inference 3.15.0 → 4.0.1

package/src/providers/nscale.ts

@@ -15,10 +15,10 @@
  * Thanks!
  */
 import type { TextToImageInput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
 import type { BodyParams } from "../types.js";
 import { omit } from "../utils/omit.js";
 import { BaseConversationalTask, TaskProviderHelper, type TextToImageTaskHelper } from "./providerHelper.js";
+import { InferenceClientProviderOutputError } from "../errors.js";
 
 const NSCALE_API_BASE_URL = "https://inference.api.nscale.com";
 
@@ -74,6 +74,6 @@ export class NscaleTextToImageTask extends TaskProviderHelper implements TextToI
 			return fetch(`data:image/jpeg;base64,${base64Data}`).then((res) => res.blob());
 		}
 
-		throw new InferenceOutputError("Expected Nscale text-to-image response format");
+		throw new InferenceClientProviderOutputError("Received malformed response from Nscale text-to-image API");
 	}
 }

package/src/providers/ovhcloud.ts

@@ -17,10 +17,10 @@
 
 import { BaseConversationalTask, BaseTextGenerationTask } from "./providerHelper.js";
 import type { ChatCompletionOutput, TextGenerationOutput, TextGenerationOutputFinishReason } from "@huggingface/tasks";
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
 import type { BodyParams } from "../types.js";
 import { omit } from "../utils/omit.js";
 import type { TextGenerationInput } from "@huggingface/tasks";
+import { InferenceClientProviderOutputError } from "../errors.js";
 
 const OVHCLOUD_API_BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net";
 
@@ -70,6 +70,6 @@ export class OvhCloudTextGenerationTask extends BaseTextGenerationTask {
 				generated_text: completion.text,
 			};
 		}
-		throw new InferenceOutputError("Expected OVHcloud text generation response format");
+		throw new InferenceClientProviderOutputError("Received malformed response from OVHcloud text generation API");
 	}
 }

package/src/providers/providerHelper.ts

@@ -46,7 +46,7 @@ import type {
 	ZeroShotImageClassificationOutput,
 } from "@huggingface/tasks";
 import { HF_ROUTER_URL } from "../config.js";
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
+import { InferenceClientProviderOutputError } from "../errors.js";
 import type { AudioToAudioOutput } from "../tasks/audio/audioToAudio.js";
 import type { BaseArgs, BodyParams, HeaderParams, InferenceProvider, RequestArgs, UrlParams } from "../types.js";
 import { toArray } from "../utils/toArray.js";
@@ -320,7 +320,7 @@ export class BaseConversationalTask extends TaskProviderHelper implements Conver
 			return response;
 		}
 
-		throw new InferenceOutputError("Expected ChatCompletionOutput");
+		throw new InferenceClientProviderOutputError("Expected ChatCompletionOutput");
 	}
 }
 
@@ -353,6 +353,6 @@ export class BaseTextGenerationTask extends TaskProviderHelper implements TextGe
 			return res[0];
 		}
 
-		throw new InferenceOutputError("Expected Array<{generated_text: string}>");
+		throw new InferenceClientProviderOutputError("Expected Array<{generated_text: string}>");
 	}
 }

package/src/providers/replicate.ts

@@ -14,7 +14,7 @@
  *
  * Thanks!
  */
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
+import { InferenceClientProviderOutputError } from "../errors.js";
 import { isUrl } from "../lib/isUrl.js";
 import type { BodyParams, HeaderParams, UrlParams } from "../types.js";
 import { omit } from "../utils/omit.js";
@@ -99,7 +99,7 @@ export class ReplicateTextToImageTask extends ReplicateTask implements TextToIma
 			return await urlResponse.blob();
 		}
 
-		throw new InferenceOutputError("Expected Replicate text-to-image response format");
+		throw new InferenceClientProviderOutputError("Received malformed response from Replicate text-to-image API");
 	}
 }
 
@@ -132,7 +132,7 @@ export class ReplicateTextToSpeechTask extends ReplicateTask {
 				}
 			}
 		}
-		throw new InferenceOutputError("Expected Blob or object with output");
+		throw new InferenceClientProviderOutputError("Received malformed response from Replicate text-to-speech API");
 	}
 }
 
@@ -149,6 +149,6 @@ export class ReplicateTextToVideoTask extends ReplicateTask implements TextToVid
 			return await urlResponse.blob();
 		}
 
-		throw new InferenceOutputError("Expected { output: string }");
+		throw new InferenceClientProviderOutputError("Received malformed response from Replicate text-to-video API");
 	}
 }

package/src/providers/sambanova.ts

@@ -14,12 +14,11 @@
  *
  * Thanks!
  */
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
-
 import type { FeatureExtractionOutput } from "@huggingface/tasks";
 import type { BodyParams } from "../types.js";
 import type { FeatureExtractionTaskHelper } from "./providerHelper.js";
 import { BaseConversationalTask, TaskProviderHelper } from "./providerHelper.js";
+import { InferenceClientProviderOutputError } from "../errors.js";
 
 export class SambanovaConversationalTask extends BaseConversationalTask {
 	constructor() {
@@ -40,8 +39,8 @@ export class SambanovaFeatureExtractionTask extends TaskProviderHelper implement
 		if (typeof response === "object" && "data" in response && Array.isArray(response.data)) {
 			return response.data.map((item) => item.embedding);
 		}
-		throw new InferenceOutputError(
-			"Expected Sambanova feature-extraction (embeddings) response format to be {'data' : list of {'embedding' : number[]}}"
+		throw new InferenceClientProviderOutputError(
+			"Received malformed response from Sambanova feature-extraction (embeddings) API"
 		);
 	}
 

package/src/providers/together.ts

@@ -15,7 +15,6 @@
  * Thanks!
  */
 import type { ChatCompletionOutput, TextGenerationOutput, TextGenerationOutputFinishReason } from "@huggingface/tasks";
-import { InferenceOutputError } from "../lib/InferenceOutputError.js";
 import type { BodyParams } from "../types.js";
 import { omit } from "../utils/omit.js";
 import {
@@ -24,6 +23,7 @@ import {
 	TaskProviderHelper,
 	type TextToImageTaskHelper,
 } from "./providerHelper.js";
+import { InferenceClientProviderOutputError } from "../errors.js";
 
 const TOGETHER_API_BASE_URL = "https://api.together.xyz";
 
@@ -74,7 +74,7 @@ export class TogetherTextGenerationTask extends BaseTextGenerationTask {
 				generated_text: completion.text,
 			};
 		}
-		throw new InferenceOutputError("Expected Together text generation response format");
+		throw new InferenceClientProviderOutputError("Received malformed response from Together text generation API");
 	}
 }
 
@@ -113,6 +113,6 @@ export class TogetherTextToImageTask extends TaskProviderHelper implements TextT
 			return fetch(`data:image/jpeg;base64,${base64Data}`).then((res) => res.blob());
 		}
 
-		throw new InferenceOutputError("Expected Together text-to-image response format");
+		throw new InferenceClientProviderOutputError("Received malformed response from Together text-to-image API");
 	}
 }

package/src/snippets/getInferenceSnippets.ts

@@ -14,7 +14,10 @@ import { makeRequestOptionsFromResolvedModel } from "../lib/makeRequestOptions.j
 import type { InferenceProviderOrPolicy, InferenceTask, RequestArgs } from "../types.js";
 import { templates } from "./templates.exported.js";
 
-export type InferenceSnippetOptions = { streaming?: boolean; billTo?: string } & Record<string, unknown>;
+export type InferenceSnippetOptions = { streaming?: boolean; billTo?: string; accessToken?: string } & Record<
+	string,
+	unknown
+>;
 
 const PYTHON_CLIENTS = ["huggingface_hub", "fal_client", "requests", "openai"] as const;
 const JS_CLIENTS = ["fetch", "huggingface.js", "openai"] as const;
@@ -121,11 +124,12 @@ const HF_JS_METHODS: Partial<Record<WidgetType, string>> = {
 	translation: "translation",
 };
 
+const ACCESS_TOKEN_PLACEHOLDER = "<ACCESS_TOKEN>"; // Placeholder to replace with env variable in snippets
+
 // Snippet generators
 const snippetGenerator = (templateName: string, inputPreparationFn?: InputPreparationFn) => {
 	return (
 		model: ModelDataMinimal,
-		accessToken: string,
 		provider: InferenceProviderOrPolicy,
 		inferenceProviderMapping?: InferenceProviderModelMapping,
 		opts?: InferenceSnippetOptions
@@ -149,13 +153,15 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 			console.error(`Failed to get provider helper for ${provider} (${task})`, e);
 			return [];
 		}
+		const accessTokenOrPlaceholder = opts?.accessToken ?? ACCESS_TOKEN_PLACEHOLDER;
+
 		/// Prepare inputs + make request
 		const inputs = inputPreparationFn ? inputPreparationFn(model, opts) : { inputs: getModelInputSnippet(model) };
 		const request = makeRequestOptionsFromResolvedModel(
 			providerModelId,
 			providerHelper,
 			{
-				accessToken,
+				accessToken: accessTokenOrPlaceholder,
 				provider,
 				...inputs,
 			} as RequestArgs,
@@ -180,7 +186,7 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 
 		/// Prepare template injection data
 		const params: TemplateParams = {
-			accessToken,
+			accessToken: accessTokenOrPlaceholder,
 			authorizationHeader: (request.info.headers as Record<string, string>)?.Authorization,
 			baseUrl: removeSuffix(request.url, "/chat/completions"),
 			fullUrl: request.url,
@@ -248,6 +254,11 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 							snippet = `${importSection}\n\n${snippet}`;
 						}
 
+						/// Replace access token placeholder
+						if (snippet.includes(ACCESS_TOKEN_PLACEHOLDER)) {
+							snippet = replaceAccessTokenPlaceholder(snippet, language, provider);
+						}
+
 						/// Snippet is ready!
 						return { language, client: client as string, content: snippet };
 					})
@@ -299,7 +310,6 @@ const snippets: Partial<
 		PipelineType,
 		(
 			model: ModelDataMinimal,
-			accessToken: string,
 			provider: InferenceProviderOrPolicy,
 			inferenceProviderMapping?: InferenceProviderModelMapping,
 			opts?: InferenceSnippetOptions
@@ -339,13 +349,12 @@ const snippets: Partial<
 
 export function getInferenceSnippets(
 	model: ModelDataMinimal,
-	accessToken: string,
 	provider: InferenceProviderOrPolicy,
 	inferenceProviderMapping?: InferenceProviderModelMapping,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	return model.pipeline_tag && model.pipeline_tag in snippets
-		? snippets[model.pipeline_tag]?.(model, accessToken, provider, inferenceProviderMapping, opts) ?? []
+		? snippets[model.pipeline_tag]?.(model, provider, inferenceProviderMapping, opts) ?? []
 		: [];
 }
 
@@ -420,3 +429,56 @@ function indentString(str: string): string {
 function removeSuffix(str: string, suffix: string) {
 	return str.endsWith(suffix) ? str.slice(0, -suffix.length) : str;
 }
+
+function replaceAccessTokenPlaceholder(
+	snippet: string,
+	language: InferenceSnippetLanguage,
+	provider: InferenceProviderOrPolicy
+): string {
+	// If "opts.accessToken" is not set, the snippets are generated with a placeholder.
+	// Once snippets are rendered, we replace the placeholder with code to fetch the access token from an environment variable.
+
+	// Determine if HF_TOKEN or specific provider token should be used
+	const accessTokenEnvVar =
+		!snippet.includes("https://") || // no URL provided => using a client => use $HF_TOKEN
+		snippet.includes("https://router.huggingface.co") || // explicit routed request => use $HF_TOKEN
+		provider == "hf-inference" // hf-inference provider => use $HF_TOKEN
+			? "HF_TOKEN"
+			: provider.toUpperCase().replace("-", "_") + "_API_KEY"; // e.g. "REPLICATE_API_KEY"
+
+	// Replace the placeholder with the env variable
+	if (language === "sh") {
+		snippet = snippet.replace(
+			`'Authorization: Bearer ${ACCESS_TOKEN_PLACEHOLDER}'`,
+			`"Authorization: Bearer $${accessTokenEnvVar}"` // e.g. "Authorization: Bearer $HF_TOKEN"
+		);
+	} else if (language === "python") {
+		snippet = "import os\n" + snippet;
+		snippet = snippet.replace(
+			`"${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`os.environ["${accessTokenEnvVar}"]` // e.g. os.environ["HF_TOKEN")
+		);
+		snippet = snippet.replace(
+			`"Bearer ${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`f"Bearer {os.environ['${accessTokenEnvVar}']}"` // e.g. f"Bearer {os.environ['HF_TOKEN']}"
+		);
+		snippet = snippet.replace(
+			`"Key ${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`f"Key {os.environ['${accessTokenEnvVar}']}"` // e.g. f"Key {os.environ['FAL_AI_API_KEY']}"
+		);
+	} else if (language === "js") {
+		snippet = snippet.replace(
+			`"${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`process.env.${accessTokenEnvVar}` // e.g. process.env.HF_TOKEN
+		);
+		snippet = snippet.replace(
+			`Authorization: "Bearer ${ACCESS_TOKEN_PLACEHOLDER}",`,
+			`Authorization: \`Bearer $\{process.env.${accessTokenEnvVar}}\`,` // e.g. Authorization: `Bearer ${process.env.HF_TOKEN}`,
+		);
+		snippet = snippet.replace(
+			`Authorization: "Key ${ACCESS_TOKEN_PLACEHOLDER}",`,
+			`Authorization: \`Key $\{process.env.${accessTokenEnvVar}}\`,` // e.g. Authorization: `Key ${process.env.FAL_AI_API_KEY}`,
+		);
+	}
+	return snippet;
+}

package/src/tasks/audio/automaticSpeechRecognition.ts

@@ -1,10 +1,10 @@
 import type { AutomaticSpeechRecognitionInput, AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
 import { resolveProvider } from "../../lib/getInferenceProviderMapping.js";
 import { getProviderHelper } from "../../lib/getProviderHelper.js";
-import { InferenceOutputError } from "../../lib/InferenceOutputError.js";
 import type { BaseArgs, Options } from "../../types.js";
 import { innerRequest } from "../../utils/request.js";
 import type { LegacyAudioInput } from "./utils.js";
+import { InferenceClientProviderOutputError } from "../../errors.js";
 
 export type AutomaticSpeechRecognitionArgs = BaseArgs & (AutomaticSpeechRecognitionInput | LegacyAudioInput);
 /**
@@ -24,7 +24,7 @@ export async function automaticSpeechRecognition(
 	});
 	const isValidOutput = typeof res?.text === "string";
 	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected {text: string}");
+		throw new InferenceClientProviderOutputError("Received malformed response from automatic-speech-recognition API");
 	}
 	return providerHelper.getResponse(res);
 }

package/src/utils/request.ts

@@ -3,6 +3,8 @@ import { makeRequestOptions } from "../lib/makeRequestOptions.js";
 import type { InferenceTask, Options, RequestArgs } from "../types.js";
 import type { EventSourceMessage } from "../vendor/fetch-event-source/parse.js";
 import { getLines, getMessages } from "../vendor/fetch-event-source/parse.js";
+import { InferenceClientProviderApiError } from "../errors.js";
+import type { JsonObject } from "../vendor/type-fest/basic.js";
 
 export interface ResponseWrapper<T> {
 	data: T;
@@ -12,6 +14,17 @@ export interface ResponseWrapper<T> {
 	};
 }
 
+function requestArgsToJson(args: RequestArgs): JsonObject {
+	// Convert the entire args object to a JSON-serializable format
+	const argsWithData = args as RequestArgs & { data?: Blob | ArrayBuffer };
+	return JSON.parse(
+		JSON.stringify({
+			...argsWithData,
+			data: argsWithData.data ? "[Blob or ArrayBuffer]" : null,
+		})
+	) as JsonObject;
+}
+
 /**
  * Primitive to make custom calls to the inference provider
  */
@@ -39,18 +52,54 @@ export async function innerRequest<T>(
 		if (["application/json", "application/problem+json"].some((ct) => contentType?.startsWith(ct))) {
 			const output = await response.json();
 			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
-				throw new Error(
-					`Server ${args.model} does not seem to support chat completion. Error: ${JSON.stringify(output.error)}`
+				throw new InferenceClientProviderApiError(
+					`Provider ${args.provider} does not seem to support chat completion for model ${
+						args.model
+					} . Error: ${JSON.stringify(output.error)}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
 				);
 			}
-			if (output.error || output.detail) {
-				throw new Error(JSON.stringify(output.error ?? output.detail));
+			if (typeof output.error === "string" || typeof output.detail === "string" || typeof output.message === "string") {
+				throw new InferenceClientProviderApiError(
+					`Failed to perform inference: ${output.error ?? output.detail ?? output.message}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			} else {
-				throw new Error(output);
+				throw new InferenceClientProviderApiError(
+					`Failed to perform inference: an HTTP error occurred when requesting the provider.`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			}
 		}
 		const message = contentType?.startsWith("text/plain;") ? await response.text() : undefined;
-		throw new Error(message ?? "An error occurred while fetching the blob");
+		throw new InferenceClientProviderApiError(
+			`Failed to perform inference: ${message ?? "an HTTP error occurred when requesting the provider"}`,
+			{
+				url,
+				method: info.method ?? "GET",
+				headers: info.headers as Record<string, string>,
+				body: requestArgsToJson(args),
+			},
+			{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: message ?? "" }
+		);
 	}
 
 	if (response.headers.get("Content-Type")?.startsWith("application/json")) {
@@ -85,26 +134,81 @@ export async function* innerStreamingRequest<T>(
 		if (response.headers.get("Content-Type")?.startsWith("application/json")) {
 			const output = await response.json();
 			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
-				throw new Error(`Server ${args.model} does not seem to support chat completion. Error: ${output.error}`);
+				throw new InferenceClientProviderApiError(
+					`Provider ${args.provider} does not seem to support chat completion for model ${
+						args.model
+					} . Error: ${JSON.stringify(output.error)}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			}
 			if (typeof output.error === "string") {
-				throw new Error(output.error);
+				throw new InferenceClientProviderApiError(
+					`Failed to perform inference: ${output.error}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			}
 			if (output.error && "message" in output.error && typeof output.error.message === "string") {
 				/// OpenAI errors
-				throw new Error(output.error.message);
+				throw new InferenceClientProviderApiError(
+					`Failed to perform inference: ${output.error.message}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			}
 			// Sambanova errors
 			if (typeof output.message === "string") {
-				throw new Error(output.message);
+				throw new InferenceClientProviderApiError(
+					`Failed to perform inference: ${output.message}`,
+					{
+						url,
+						method: info.method ?? "GET",
+						headers: info.headers as Record<string, string>,
+						body: requestArgsToJson(args),
+					},
+					{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
+				);
 			}
 		}
 
-		throw new Error(`Server response contains error: ${response.status}`);
+		throw new InferenceClientProviderApiError(
+			`Failed to perform inference: an HTTP error occurred when requesting the provider.`,
+			{
+				url,
+				method: info.method ?? "GET",
+				headers: info.headers as Record<string, string>,
+				body: requestArgsToJson(args),
+			},
+			{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: "" }
+		);
 	}
 	if (!response.headers.get("content-type")?.startsWith("text/event-stream")) {
-		throw new Error(
-			`Server does not support event stream content type, it returned ` + response.headers.get("content-type")
+		throw new InferenceClientProviderApiError(
+			`Failed to perform inference: server does not support event stream content type, it returned ` +
+				response.headers.get("content-type"),
+			{
+				url,
+				method: info.method ?? "GET",
+				headers: info.headers as Record<string, string>,
+				body: requestArgsToJson(args),
+			},
+			{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: "" }
 		);
 	}
 
@@ -151,7 +255,16 @@ export async function* innerStreamingRequest<T>(
 								    typeof data.error.message === "string"
 								  ? data.error.message
 								  : JSON.stringify(data.error);
-						throw new Error(`Error forwarded from backend: ` + errorStr);
+						throw new InferenceClientProviderApiError(
+							`Failed to perform inference: an occurred while streaming the response: ${errorStr}`,
+							{
+								url,
+								method: info.method ?? "GET",
+								headers: info.headers as Record<string, string>,
+								body: requestArgsToJson(args),
+							},
+							{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: data }
+						);
 					}
 					yield data as T;
 				}

package/src/vendor/type-fest/basic.ts

@@ -0,0 +1,31 @@
+/**
+Matches a JSON object.
+
+This type can be useful to enforce some input to be JSON-compatible or as a super-type to be extended from. Don't use this as a direct return type as the user would have to double-cast it: `jsonObject as unknown as CustomResponse`. Instead, you could extend your CustomResponse type from it to ensure your type only uses JSON-compatible types: `interface CustomResponse extends JsonObject { … }`.
+
+@category JSON
+*/
+export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined };
+
+/**
+Matches a JSON array.
+
+@category JSON
+*/
+export type JsonArray = JsonValue[] | readonly JsonValue[];
+
+/**
+Matches any valid JSON primitive value.
+
+@category JSON
+*/
+export type JsonPrimitive = string | number | boolean | null;
+
+/**
+Matches any valid JSON value.
+
+@see `Jsonify` if you need to transform a type to one that is assignable to `JsonValue`.
+
+@category JSON
+*/
+export type JsonValue = JsonPrimitive | JsonObject | JsonArray;