@huggingface/inference 3.6.2 → 3.7.1

package/src/tasks/nlp/summarization.ts

@@ -1,7 +1,7 @@
 import type { SummarizationInput, SummarizationOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type SummarizationArgs = BaseArgs & SummarizationInput;
 
@@ -9,13 +9,10 @@ export type SummarizationArgs = BaseArgs & SummarizationInput;
  * This task is well known to summarize longer text into shorter text. Be careful, some models have a maximum length of input. That means that the summary cannot handle full books for instance. Be careful when choosing your model.
  */
 export async function summarization(args: SummarizationArgs, options?: Options): Promise<SummarizationOutput> {
-	const res = await request<SummarizationOutput[]>(args, {
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "summarization");
+	const { data: res } = await innerRequest<SummarizationOutput[]>(args, {
 		...options,
 		task: "summarization",
 	});
-	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x?.summary_text === "string");
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Array<{summary_text: string}>");
-	}
-	return res?.[0];
+	return providerHelper.getResponse(res);
 }

package/src/tasks/nlp/tableQuestionAnswering.ts

@@ -1,7 +1,7 @@
 import type { TableQuestionAnsweringInput, TableQuestionAnsweringOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TableQuestionAnsweringArgs = BaseArgs & TableQuestionAnsweringInput;
 
@@ -12,34 +12,10 @@ export async function tableQuestionAnswering(
 	args: TableQuestionAnsweringArgs,
 	options?: Options
 ): Promise<TableQuestionAnsweringOutput[number]> {
-	const res = await request<TableQuestionAnsweringOutput | TableQuestionAnsweringOutput[number]>(args, {
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "table-question-answering");
+	const { data: res } = await innerRequest<TableQuestionAnsweringOutput | TableQuestionAnsweringOutput[number]>(args, {
 		...options,
 		task: "table-question-answering",
 	});
-	const isValidOutput = Array.isArray(res) ? res.every((elem) => validate(elem)) : validate(res);
-	if (!isValidOutput) {
-		throw new InferenceOutputError(
-			"Expected {aggregator: string, answer: string, cells: string[], coordinates: number[][]}"
-		);
-	}
-	return Array.isArray(res) ? res[0] : res;
-}
-
-function validate(elem: unknown): elem is TableQuestionAnsweringOutput[number] {
-	return (
-		typeof elem === "object" &&
-		!!elem &&
-		"aggregator" in elem &&
-		typeof elem.aggregator === "string" &&
-		"answer" in elem &&
-		typeof elem.answer === "string" &&
-		"cells" in elem &&
-		Array.isArray(elem.cells) &&
-		elem.cells.every((x: unknown): x is string => typeof x === "string") &&
-		"coordinates" in elem &&
-		Array.isArray(elem.coordinates) &&
-		elem.coordinates.every(
-			(coord: unknown): coord is number[] => Array.isArray(coord) && coord.every((x) => typeof x === "number")
-		)
-	);
+	return providerHelper.getResponse(res);
 }

package/src/tasks/nlp/textClassification.ts

@@ -1,7 +1,7 @@
 import type { TextClassificationInput, TextClassificationOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TextClassificationArgs = BaseArgs & TextClassificationInput;
 
@@ -12,16 +12,10 @@ export async function textClassification(
 	args: TextClassificationArgs,
 	options?: Options
 ): Promise<TextClassificationOutput> {
-	const res = (
-		await request<TextClassificationOutput>(args, {
-			...options,
-			task: "text-classification",
-		})
-	)?.[0];
-	const isValidOutput =
-		Array.isArray(res) && res.every((x) => typeof x?.label === "string" && typeof x.score === "number");
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Array<{label: string, score: number}>");
-	}
-	return res;
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "text-classification");
+	const { data: res } = await innerRequest<TextClassificationOutput>(args, {
+		...options,
+		task: "text-classification",
+	});
+	return providerHelper.getResponse(res);
 }

package/src/tasks/nlp/textGeneration.ts

@@ -1,33 +1,11 @@
-import type {
-	ChatCompletionOutput,
-	TextGenerationInput,
-	TextGenerationOutput,
-	TextGenerationOutputFinishReason,
-} from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import type { TextGenerationInput, TextGenerationOutput } from "@huggingface/tasks";
+import { getProviderHelper } from "../../lib/getProviderHelper";
+import type { HyperbolicTextCompletionOutput } from "../../providers/hyperbolic";
 import type { BaseArgs, Options } from "../../types";
-import { toArray } from "../../utils/toArray";
-import { request } from "../custom/request";
-import { omit } from "../../utils/omit";
+import { innerRequest } from "../../utils/request";
 
 export type { TextGenerationInput, TextGenerationOutput };
 
-interface TogeteherTextCompletionOutput extends Omit<ChatCompletionOutput, "choices"> {
-	choices: Array<{
-		text: string;
-		finish_reason: TextGenerationOutputFinishReason;
-		seed: number;
-		logprobs: unknown;
-		index: number;
-	}>;
-}
-
-interface HyperbolicTextCompletionOutput extends Omit<ChatCompletionOutput, "choices"> {
-	choices: Array<{
-		message: { content: string };
-	}>;
-}
-
 /**
  * Use to continue text from a prompt. This is a very generic task. Recommended model: gpt2 (it’s a simple model, but fun to play with).
  */
@@ -35,58 +13,13 @@ export async function textGeneration(
 	args: BaseArgs & TextGenerationInput,
 	options?: Options
 ): Promise<TextGenerationOutput> {
-	if (args.provider === "together") {
-		args.prompt = args.inputs;
-		const raw = await request<TogeteherTextCompletionOutput>(args, {
-			...options,
-			task: "text-generation",
-		});
-		const isValidOutput =
-			typeof raw === "object" && "choices" in raw && Array.isArray(raw?.choices) && typeof raw?.model === "string";
-		if (!isValidOutput) {
-			throw new InferenceOutputError("Expected ChatCompletionOutput");
-		}
-		const completion = raw.choices[0];
-		return {
-			generated_text: completion.text,
-		};
-	} else if (args.provider === "hyperbolic") {
-		const payload = {
-			messages: [{ content: args.inputs, role: "user" }],
-			...(args.parameters
-				? {
-						max_tokens: args.parameters.max_new_tokens,
-						...omit(args.parameters, "max_new_tokens"),
-				  }
-				: undefined),
-			...omit(args, ["inputs", "parameters"]),
-		};
-		const raw = await request<HyperbolicTextCompletionOutput>(payload, {
-			...options,
-			task: "text-generation",
-		});
-		const isValidOutput =
-			typeof raw === "object" && "choices" in raw && Array.isArray(raw?.choices) && typeof raw?.model === "string";
-		if (!isValidOutput) {
-			throw new InferenceOutputError("Expected ChatCompletionOutput");
-		}
-		const completion = raw.choices[0];
-		return {
-			generated_text: completion.message.content,
-		};
-	} else {
-		const res = toArray(
-			await request<TextGenerationOutput | TextGenerationOutput[]>(args, {
-				...options,
-				task: "text-generation",
-			})
-		);
-
-		const isValidOutput =
-			Array.isArray(res) && res.every((x) => "generated_text" in x && typeof x?.generated_text === "string");
-		if (!isValidOutput) {
-			throw new InferenceOutputError("Expected Array<{generated_text: string}>");
-		}
-		return (res as TextGenerationOutput[])?.[0];
-	}
+	const provider = args.provider ?? "hf-inference";
+	const providerHelper = getProviderHelper(provider, "text-generation");
+	const { data: response } = await innerRequest<
+		HyperbolicTextCompletionOutput | TextGenerationOutput | TextGenerationOutput[]
+	>(args, {
+		...options,
+		task: "text-generation",
+	});
+	return providerHelper.getResponse(response);
 }

package/src/tasks/nlp/textGenerationStream.ts

@@ -1,6 +1,6 @@
 import type { TextGenerationInput } from "@huggingface/tasks";
 import type { BaseArgs, Options } from "../../types";
-import { streamingRequest } from "../custom/streamingRequest";
+import { innerStreamingRequest } from "../../utils/request";
 
 export interface TextGenerationStreamToken {
 	/** Token ID from the model tokenizer */
@@ -89,7 +89,7 @@ export async function* textGenerationStream(
 	args: BaseArgs & TextGenerationInput,
 	options?: Options
 ): AsyncGenerator<TextGenerationStreamOutput> {
-	yield* streamingRequest<TextGenerationStreamOutput>(args, {
+	yield* innerStreamingRequest<TextGenerationStreamOutput>(args, {
 		...options,
 		task: "text-generation",
 	});

package/src/tasks/nlp/tokenClassification.ts

@@ -1,8 +1,8 @@
 import type { TokenClassificationInput, TokenClassificationOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
+import { innerRequest } from "../../utils/request";
 import { toArray } from "../../utils/toArray";
-import { request } from "../custom/request";
 
 export type TokenClassificationArgs = BaseArgs & TokenClassificationInput;
 
@@ -13,26 +13,10 @@ export async function tokenClassification(
 	args: TokenClassificationArgs,
 	options?: Options
 ): Promise<TokenClassificationOutput> {
-	const res = toArray(
-		await request<TokenClassificationOutput[number] | TokenClassificationOutput>(args, {
-			...options,
-			task: "token-classification",
-		})
-	);
-	const isValidOutput =
-		Array.isArray(res) &&
-		res.every(
-			(x) =>
-				typeof x.end === "number" &&
-				typeof x.entity_group === "string" &&
-				typeof x.score === "number" &&
-				typeof x.start === "number" &&
-				typeof x.word === "string"
-		);
-	if (!isValidOutput) {
-		throw new InferenceOutputError(
-			"Expected Array<{end: number, entity_group: string, score: number, start: number, word: string}>"
-		);
-	}
-	return res;
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "token-classification");
+	const { data: res } = await innerRequest<TokenClassificationOutput[number] | TokenClassificationOutput>(args, {
+		...options,
+		task: "token-classification",
+	});
+	return providerHelper.getResponse(res);
 }

package/src/tasks/nlp/translation.ts

@@ -1,20 +1,17 @@
 import type { TranslationInput, TranslationOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TranslationArgs = BaseArgs & TranslationInput;
 /**
  * This task is well known to translate text from one language to another. Recommended model: Helsinki-NLP/opus-mt-ru-en.
  */
 export async function translation(args: TranslationArgs, options?: Options): Promise<TranslationOutput> {
-	const res = await request<TranslationOutput>(args, {
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "translation");
+	const { data: res } = await innerRequest<TranslationOutput>(args, {
 		...options,
 		task: "translation",
 	});
-	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x?.translation_text === "string");
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected type Array<{translation_text: string}>");
-	}
-	return res?.length === 1 ? res?.[0] : res;
+	return providerHelper.getResponse(res);
 }

package/src/tasks/nlp/zeroShotClassification.ts

@@ -1,8 +1,8 @@
 import type { ZeroShotClassificationInput, ZeroShotClassificationOutput } from "@huggingface/tasks";
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
+import { innerRequest } from "../../utils/request";
 import { toArray } from "../../utils/toArray";
-import { request } from "../custom/request";
 
 export type ZeroShotClassificationArgs = BaseArgs & ZeroShotClassificationInput;
 
@@ -13,24 +13,10 @@ export async function zeroShotClassification(
 	args: ZeroShotClassificationArgs,
 	options?: Options
 ): Promise<ZeroShotClassificationOutput> {
-	const res = toArray(
-		await request<ZeroShotClassificationOutput[number] | ZeroShotClassificationOutput>(args, {
-			...options,
-			task: "zero-shot-classification",
-		})
-	);
-	const isValidOutput =
-		Array.isArray(res) &&
-		res.every(
-			(x) =>
-				Array.isArray(x.labels) &&
-				x.labels.every((_label) => typeof _label === "string") &&
-				Array.isArray(x.scores) &&
-				x.scores.every((_score) => typeof _score === "number") &&
-				typeof x.sequence === "string"
-		);
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected Array<{labels: string[], scores: number[], sequence: string}>");
-	}
-	return res;
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "zero-shot-classification");
+	const { data: res } = await innerRequest<ZeroShotClassificationOutput[number] | ZeroShotClassificationOutput>(args, {
+		...options,
+		task: "zero-shot-classification",
+	});
+	return providerHelper.getResponse(res);
 }

package/src/tasks/tabular/tabularClassification.ts

@@ -1,6 +1,6 @@
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TabularClassificationArgs = BaseArgs & {
 	inputs: {
@@ -25,13 +25,10 @@ export async function tabularClassification(
 	args: TabularClassificationArgs,
 	options?: Options
 ): Promise<TabularClassificationOutput> {
-	const res = await request<TabularClassificationOutput>(args, {
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "tabular-classification");
+	const { data: res } = await innerRequest<TabularClassificationOutput>(args, {
 		...options,
 		task: "tabular-classification",
 	});
-	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected number[]");
-	}
-	return res;
+	return providerHelper.getResponse(res);
 }

package/src/tasks/tabular/tabularRegression.ts

@@ -1,6 +1,6 @@
-import { InferenceOutputError } from "../../lib/InferenceOutputError";
+import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
-import { request } from "../custom/request";
+import { innerRequest } from "../../utils/request";
 
 export type TabularRegressionArgs = BaseArgs & {
 	inputs: {
@@ -25,13 +25,10 @@ export async function tabularRegression(
 	args: TabularRegressionArgs,
 	options?: Options
 ): Promise<TabularRegressionOutput> {
-	const res = await request<TabularRegressionOutput>(args, {
+	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "tabular-regression");
+	const { data: res } = await innerRequest<TabularRegressionOutput>(args, {
 		...options,
 		task: "tabular-regression",
 	});
-	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");
-	if (!isValidOutput) {
-		throw new InferenceOutputError("Expected number[]");
-	}
-	return res;
+	return providerHelper.getResponse(res);
 }

package/src/types.ts

@@ -24,9 +24,17 @@ export interface Options {
 	 * (Default: "same-origin"). String | Boolean. Credentials to use for the request. If this is a string, it will be passed straight on. If it's a boolean, true will be "include" and false will not send credentials at all.
 	 */
 	includeCredentials?: string | boolean;
+
+	/**
+	 * The billing account to use for the requests.
+	 *
+	 * By default the requests are billed on the user's account.
+	 * Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub.
+	 */
+	billTo?: string;
 }
 
-export type InferenceTask = Exclude<PipelineType, "other">;
+export type InferenceTask = Exclude<PipelineType, "other"> | "conversational";
 
 export const INFERENCE_PROVIDERS = [
 	"black-forest-labs",
@@ -93,14 +101,6 @@ export type RequestArgs = BaseArgs &
 		parameters?: Record<string, unknown>;
 	};
 
-export interface ProviderConfig {
-	makeBaseUrl: ((task?: InferenceTask) => string) | (() => string);
-	makeBody: (params: BodyParams) => Record<string, unknown>;
-	makeHeaders: (params: HeaderParams) => Record<string, string>;
-	makeUrl: (params: UrlParams) => string;
-	clientSideRoutingOnly?: boolean;
-}
-
 export type AuthMethod = "none" | "hf-token" | "credentials-include" | "provider-key";
 
 export interface HeaderParams {
@@ -110,15 +110,12 @@ export interface HeaderParams {
 
 export interface UrlParams {
 	authMethod: AuthMethod;
-	baseUrl: string;
 	model: string;
 	task?: InferenceTask;
-	chatCompletion?: boolean;
 }
 
-export interface BodyParams {
-	args: Record<string, unknown>;
-	chatCompletion?: boolean;
+export interface BodyParams<T extends Record<string, unknown> = Record<string, unknown>> {
+	args: T;
 	model: string;
 	task?: InferenceTask;
 }

package/src/utils/request.ts

@@ -0,0 +1,161 @@
+import { makeRequestOptions } from "../lib/makeRequestOptions";
+import type { InferenceTask, Options, RequestArgs } from "../types";
+import type { EventSourceMessage } from "../vendor/fetch-event-source/parse";
+import { getLines, getMessages } from "../vendor/fetch-event-source/parse";
+
+export interface ResponseWrapper<T> {
+	data: T;
+	requestContext: {
+		url: string;
+		info: RequestInit;
+	};
+}
+
+/**
+ * Primitive to make custom calls to the inference provider
+ */
+export async function innerRequest<T>(
+	args: RequestArgs,
+	options?: Options & {
+		/** In most cases (unless we pass a endpointUrl) we know the task */
+		task?: InferenceTask;
+		/** Is chat completion compatible */
+		chatCompletion?: boolean;
+	}
+): Promise<ResponseWrapper<T>> {
+	const { url, info } = await makeRequestOptions(args, options);
+	const response = await (options?.fetch ?? fetch)(url, info);
+
+	const requestContext: ResponseWrapper<T>["requestContext"] = { url, info };
+
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return innerRequest(args, options);
+	}
+
+	if (!response.ok) {
+		const contentType = response.headers.get("Content-Type");
+		if (["application/json", "application/problem+json"].some((ct) => contentType?.startsWith(ct))) {
+			const output = await response.json();
+			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
+				throw new Error(
+					`Server ${args.model} does not seem to support chat completion. Error: ${JSON.stringify(output.error)}`
+				);
+			}
+			if (output.error || output.detail) {
+				throw new Error(JSON.stringify(output.error ?? output.detail));
+			} else {
+				throw new Error(output);
+			}
+		}
+		const message = contentType?.startsWith("text/plain;") ? await response.text() : undefined;
+		throw new Error(message ?? "An error occurred while fetching the blob");
+	}
+
+	if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+		const data = (await response.json()) as T;
+		return { data, requestContext };
+	}
+
+	const blob = (await response.blob()) as T;
+	return { data: blob as unknown as T, requestContext };
+}
+
+/**
+ * Primitive to make custom inference calls that expect server-sent events, and returns the response through a generator
+ */
+export async function* innerStreamingRequest<T>(
+	args: RequestArgs,
+	options?: Options & {
+		/** In most cases (unless we pass a endpointUrl) we know the task */
+		task?: InferenceTask;
+		/** Is chat completion compatible */
+		chatCompletion?: boolean;
+	}
+): AsyncGenerator<T> {
+	const { url, info } = await makeRequestOptions({ ...args, stream: true }, options);
+	const response = await (options?.fetch ?? fetch)(url, info);
+
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return yield* innerStreamingRequest(args, options);
+	}
+	if (!response.ok) {
+		if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+			const output = await response.json();
+			if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
+				throw new Error(`Server ${args.model} does not seem to support chat completion. Error: ${output.error}`);
+			}
+			if (typeof output.error === "string") {
+				throw new Error(output.error);
+			}
+			if (output.error && "message" in output.error && typeof output.error.message === "string") {
+				/// OpenAI errors
+				throw new Error(output.error.message);
+			}
+			// Sambanova errors
+			if (typeof output.message === "string") {
+				throw new Error(output.message);
+			}
+		}
+
+		throw new Error(`Server response contains error: ${response.status}`);
+	}
+	if (!response.headers.get("content-type")?.startsWith("text/event-stream")) {
+		throw new Error(
+			`Server does not support event stream content type, it returned ` + response.headers.get("content-type")
+		);
+	}
+
+	if (!response.body) {
+		return;
+	}
+
+	const reader = response.body.getReader();
+	let events: EventSourceMessage[] = [];
+
+	const onEvent = (event: EventSourceMessage) => {
+		// accumulate events in array
+		events.push(event);
+	};
+
+	const onChunk = getLines(
+		getMessages(
+			() => {},
+			() => {},
+			onEvent
+		)
+	);
+
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) {
+				return;
+			}
+			onChunk(value);
+			for (const event of events) {
+				if (event.data.length > 0) {
+					if (event.data === "[DONE]") {
+						return;
+					}
+					const data = JSON.parse(event.data);
+					if (typeof data === "object" && data !== null && "error" in data) {
+						const errorStr =
+							typeof data.error === "string"
+								? data.error
+								: typeof data.error === "object" &&
+								    data.error &&
+								    "message" in data.error &&
+								    typeof data.error.message === "string"
+								  ? data.error.message
+								  : JSON.stringify(data.error);
+						throw new Error(`Error forwarded from backend: ` + errorStr);
+					}
+					yield data as T;
+				}
+			}
+			events = [];
+		}
+	} finally {
+		reader.releaseLock();
+	}
+}