@huggingface/inference 3.6.1 → 3.7.0

package/src/tasks/tabular/tabularClassification.ts

@@ -1,6 +1,6 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TabularClassificationArgs = BaseArgs & {
 	inputs: {
@@ -25,7 +25,7 @@ export async function tabularClassification(
 	args: TabularClassificationArgs,
 	options?: Options
 ): Promise<TabularClassificationOutput> {
-	const res = await request<TabularClassificationOutput>(args, {
+	const { data: res } = await innerRequest<TabularClassificationOutput>(args, {
 		...options,
 		task: "tabular-classification",
 	});

package/src/tasks/tabular/tabularRegression.ts

@@ -1,6 +1,6 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TabularRegressionArgs = BaseArgs & {
 	inputs: {
@@ -25,7 +25,7 @@ export async function tabularRegression(
 	args: TabularRegressionArgs,
 	options?: Options
 ): Promise<TabularRegressionOutput> {
-	const res = await request<TabularRegressionOutput>(args, {
+	const { data: res } = await innerRequest<TabularRegressionOutput>(args, {
 		...options,
 		task: "tabular-regression",
 	});

package/src/types.ts

@@ -24,6 +24,14 @@ export interface Options {
 	 * (Default: "same-origin"). String | Boolean. Credentials to use for the request. If this is a string, it will be passed straight on. If it's a boolean, true will be "include" and false will not send credentials at all.
 	 */
 	includeCredentials?: string | boolean;
+
+	/**
+	 * The billing account to use for the requests.
+	 *
+	 * By default the requests are billed on the user's account.
+	 * Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub.
+	 */
+	billTo?: string;
 }
 
 export type InferenceTask = Exclude<PipelineType, "other">;
@@ -94,19 +102,22 @@ export type RequestArgs = BaseArgs &
 	};
 
 export interface ProviderConfig {
-	baseUrl: string;
+	makeBaseUrl: ((task?: InferenceTask) => string) | (() => string);
 	makeBody: (params: BodyParams) => Record<string, unknown>;
 	makeHeaders: (params: HeaderParams) => Record<string, string>;
 	makeUrl: (params: UrlParams) => string;
 	clientSideRoutingOnly?: boolean;
 }
 
+export type AuthMethod = "none" | "hf-token" | "credentials-include" | "provider-key";
+
 export interface HeaderParams {
 	accessToken?: string;
-	authMethod: "none" | "hf-token" | "credentials-include" | "provider-key";
+	authMethod: AuthMethod;
 }
 
 export interface UrlParams {
+	authMethod: AuthMethod;
 	baseUrl: string;
 	model: string;
 	task?: InferenceTask;

package/src/utils/request.ts

@@ -0,0 +1,161 @@
+import { makeRequestOptions } from "../lib/makeRequestOptions";
+import type { InferenceTask, Options, RequestArgs } from "../types";
+import type { EventSourceMessage } from "../vendor/fetch-event-source/parse";
+import { getLines, getMessages } from "../vendor/fetch-event-source/parse";
+
+export interface ResponseWrapper<T> {
+	data: T;
+	requestContext: {
+		url: string;
+		info: RequestInit;
+	};
+}
+
+/**
+ * Primitive to make custom calls to the inference provider
+ */
+export async function innerRequest<T>(
+	args: RequestArgs,
+	options?: Options & {
+		/** In most cases (unless we pass a endpointUrl) we know the task */
+		task?: InferenceTask;
+		/** Is chat completion compatible */
+		chatCompletion?: boolean;
+	}
+): Promise<ResponseWrapper<T>> {
+	const { url, info } = await makeRequestOptions(args, options);
+	const response = await (options?.fetch ?? fetch)(url, info);
+
+	const requestContext: ResponseWrapper<T>["requestContext"] = { url, info };
+
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return innerRequest(args, options);
+	}
+
+	if (!response.ok) {
+		const contentType = response.headers.get("Content-Type");
+		if (["application/json", "application/problem+json"].some((ct) => contentType?.startsWith(ct))) {
+			const output = await response.json();
+			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
+				throw new Error(
+					`Server ${args.model} does not seem to support chat completion. Error: ${JSON.stringify(output.error)}`
+				);
+			}
+			if (output.error || output.detail) {
+				throw new Error(JSON.stringify(output.error ?? output.detail));
+			} else {
+				throw new Error(output);
+			}
+		}
+		const message = contentType?.startsWith("text/plain;") ? await response.text() : undefined;
+		throw new Error(message ?? "An error occurred while fetching the blob");
+	}
+
+	if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+		const data = (await response.json()) as T;
+		return { data, requestContext };
+	}
+
+	const blob = (await response.blob()) as T;
+	return { data: blob as unknown as T, requestContext };
+}
+
+/**
+ * Primitive to make custom inference calls that expect server-sent events, and returns the response through a generator
+ */
+export async function* innerStreamingRequest<T>(
+	args: RequestArgs,
+	options?: Options & {
+		/** In most cases (unless we pass a endpointUrl) we know the task */
+		task?: InferenceTask;
+		/** Is chat completion compatible */
+		chatCompletion?: boolean;
+	}
+): AsyncGenerator<T> {
+	const { url, info } = await makeRequestOptions({ ...args, stream: true }, options);
+	const response = await (options?.fetch ?? fetch)(url, info);
+
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return yield* innerStreamingRequest(args, options);
+	}
+	if (!response.ok) {
+		if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+			const output = await response.json();
+			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
+				throw new Error(`Server ${args.model} does not seem to support chat completion. Error: ${output.error}`);
+			}
+			if (typeof output.error === "string") {
+				throw new Error(output.error);
+			}
+			if (output.error && "message" in output.error && typeof output.error.message === "string") {
+				/// OpenAI errors
+				throw new Error(output.error.message);
+			}
+			// Sambanova errors
+			if (typeof output.message === "string") {
+				throw new Error(output.message);
+			}
+		}
+
+		throw new Error(`Server response contains error: ${response.status}`);
+	}
+	if (!response.headers.get("content-type")?.startsWith("text/event-stream")) {
+		throw new Error(
+			`Server does not support event stream content type, it returned ` + response.headers.get("content-type")
+		);
+	}
+
+	if (!response.body) {
+		return;
+	}
+
+	const reader = response.body.getReader();
+	let events: EventSourceMessage[] = [];
+
+	const onEvent = (event: EventSourceMessage) => {
+		// accumulate events in array
+		events.push(event);
+	};
+
+	const onChunk = getLines(
+		getMessages(
+			() => {},
+			() => {},
+			onEvent
+		)
+	);
+
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) {
+				return;
+			}
+			onChunk(value);
+			for (const event of events) {
+				if (event.data.length > 0) {
+					if (event.data === "[DONE]") {
+						return;
+					}
+					const data = JSON.parse(event.data);
+					if (typeof data === "object" && data !== null && "error" in data) {
+						const errorStr =
+							typeof data.error === "string"
+								? data.error
+								: typeof data.error === "object" &&
+								    data.error &&
+								    "message" in data.error &&
+								    typeof data.error.message === "string"
+								  ? data.error.message
+								  : JSON.stringify(data.error);
+						throw new Error(`Error forwarded from backend: ` + errorStr);
+					}
+					yield data as T;
+				}
+			}
+			events = [];
+		}
+	} finally {
+		reader.releaseLock();
+	}
+}