@huggingface/inference 1.6.2 → 1.7.0

package/README.md

@@ -76,6 +76,13 @@ await hf.textGeneration({
   inputs: 'The answer to the universe is'
 })
 
+for await const (output of hf.textGenerationStream({
+  model: "google/flan-t5-xxl",
+  inputs: 'repeat "one two three four"'
+})) {
+  console.log(output.token.text, output.generated_text);
+}
+
 await hf.tokenClassification({
   model: 'dbmdz/bert-large-cased-finetuned-conll03-english',
   inputs: 'My name is Sarah Jessica Parker but you can call me Jessica'

package/dist/index.d.ts

@@ -206,6 +206,80 @@ interface TextGenerationReturn {
      */
     generated_text: string;
 }
+interface TextGenerationStreamToken {
+    /** Token ID from the model tokenizer */
+    id: number;
+    /** Token text */
+    text: string;
+    /** Logprob */
+    logprob: number;
+    /**
+     * Is the token a special token
+     * Can be used to ignore tokens when concatenating
+     */
+    special: boolean;
+}
+interface TextGenerationStreamPrefillToken {
+    /** Token ID from the model tokenizer */
+    id: number;
+    /** Token text */
+    text: string;
+    /**
+     * Logprob
+     * Optional since the logprob of the first token cannot be computed
+     */
+    logprob?: number;
+}
+interface TextGenerationStreamBestOfSequence {
+    /** Generated text */
+    generated_text: string;
+    /** Generation finish reason */
+    finish_reason: TextGenerationStreamFinishReason;
+    /** Number of generated tokens */
+    generated_tokens: number;
+    /** Sampling seed if sampling was activated */
+    seed?: number;
+    /** Prompt tokens */
+    prefill: TextGenerationStreamPrefillToken[];
+    /** Generated tokens */
+    tokens: TextGenerationStreamToken[];
+}
+declare enum TextGenerationStreamFinishReason {
+    /** number of generated tokens == `max_new_tokens` */
+    Length = "length",
+    /** the model generated its end of sequence token */
+    EndOfSequenceToken = "eos_token",
+    /** the model generated a text included in `stop_sequences` */
+    StopSequence = "stop_sequence"
+}
+interface TextGenerationStreamDetails {
+    /** Generation finish reason */
+    finish_reason: TextGenerationStreamFinishReason;
+    /** Number of generated tokens */
+    generated_tokens: number;
+    /** Sampling seed if sampling was activated */
+    seed?: number;
+    /** Prompt tokens */
+    prefill: TextGenerationStreamPrefillToken[];
+    /** */
+    tokens: TextGenerationStreamToken[];
+    /** Additional sequences when using the `best_of` parameter */
+    best_of_sequences?: TextGenerationStreamBestOfSequence[];
+}
+interface TextGenerationStreamReturn {
+    /** Generated token, one at a time */
+    token: TextGenerationStreamToken;
+    /**
+     * Complete generated text
+     * Only available when the generation is finished
+     */
+    generated_text?: string;
+    /**
+     * Generation details
+     * Only available when the generation is finished
+     */
+    details?: TextGenerationStreamDetails;
+}
 type TokenClassificationArgs = Args & {
     /**
      * A string to be classified
@@ -486,6 +560,10 @@ declare class HfInference {
      * Use to continue text from a prompt. This is a very generic task. Recommended model: gpt2 (it’s a simple model, but fun to play with).
      */
     textGeneration(args: TextGenerationArgs, options?: Options): Promise<TextGenerationReturn>;
+    /**
+     * Use to continue text from a prompt. Same as `textGeneration` but returns generator that can be read one token at a time
+     */
+    textGenerationStream(args: TextGenerationArgs, options?: Options): AsyncGenerator<TextGenerationStreamReturn>;
     /**
      * Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. Recommended model: dbmdz/bert-large-cased-finetuned-conll03-english
      */
@@ -537,6 +615,10 @@ declare class HfInference {
      * Recommended model: stabilityai/stable-diffusion-2
      */
     textToImage(args: TextToImageArgs, options?: Options): Promise<TextToImageReturn>;
+    /**
+     * Helper that prepares request arguments
+     */
+    private makeRequestOptions;
     request<T>(args: Args & {
         data?: Blob | ArrayBuffer;
     }, options?: Options & {
@@ -545,6 +627,17 @@ declare class HfInference {
         /** For internal HF use, which is why it's not exposed in {@link Options} */
         includeCredentials?: boolean;
     }): Promise<T>;
+    /**
+     * Make request that uses server-sent events and returns response as a generator
+     */
+    streamingRequest<T>(args: Args & {
+        data?: Blob | ArrayBuffer;
+    }, options?: Options & {
+        binary?: boolean;
+        blob?: boolean;
+        /** For internal HF use, which is why it's not exposed in {@link Options} */
+        includeCredentials?: boolean;
+    }): AsyncGenerator<T>;
 }
 
-export { Args, AudioClassificationArgs, AudioClassificationReturn, AudioClassificationReturnValue, AutomaticSpeechRecognitionArgs, AutomaticSpeechRecognitionReturn, ConversationalArgs, ConversationalReturn, FeatureExtractionArgs, FeatureExtractionReturn, FillMaskArgs, FillMaskReturn, HfInference, ImageClassificationArgs, ImageClassificationReturn, ImageClassificationReturnValue, ImageSegmentationArgs, ImageSegmentationReturn, ImageSegmentationReturnValue, ObjectDetectionArgs, ObjectDetectionReturn, ObjectDetectionReturnValue, Options, QuestionAnswerArgs, QuestionAnswerReturn, SummarizationArgs, SummarizationReturn, TableQuestionAnswerArgs, TableQuestionAnswerReturn, TextClassificationArgs, TextClassificationReturn, TextGenerationArgs, TextGenerationReturn, TextToImageArgs, TextToImageReturn, TokenClassificationArgs, TokenClassificationReturn, TokenClassificationReturnValue, TranslationArgs, TranslationReturn, ZeroShotClassificationArgs, ZeroShotClassificationReturn, ZeroShotClassificationReturnValue };
+export { Args, AudioClassificationArgs, AudioClassificationReturn, AudioClassificationReturnValue, AutomaticSpeechRecognitionArgs, AutomaticSpeechRecognitionReturn, ConversationalArgs, ConversationalReturn, FeatureExtractionArgs, FeatureExtractionReturn, FillMaskArgs, FillMaskReturn, HfInference, ImageClassificationArgs, ImageClassificationReturn, ImageClassificationReturnValue, ImageSegmentationArgs, ImageSegmentationReturn, ImageSegmentationReturnValue, ObjectDetectionArgs, ObjectDetectionReturn, ObjectDetectionReturnValue, Options, QuestionAnswerArgs, QuestionAnswerReturn, SummarizationArgs, SummarizationReturn, TableQuestionAnswerArgs, TableQuestionAnswerReturn, TextClassificationArgs, TextClassificationReturn, TextGenerationArgs, TextGenerationReturn, TextGenerationStreamBestOfSequence, TextGenerationStreamDetails, TextGenerationStreamFinishReason, TextGenerationStreamPrefillToken, TextGenerationStreamReturn, TextGenerationStreamToken, TextToImageArgs, TextToImageReturn, TokenClassificationArgs, TokenClassificationReturn, TokenClassificationReturnValue, TranslationArgs, TranslationReturn, ZeroShotClassificationArgs, ZeroShotClassificationReturn, ZeroShotClassificationReturnValue };

package/dist/index.js

@@ -19,7 +19,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
-  HfInference: () => HfInference
+  HfInference: () => HfInference,
+  TextGenerationStreamFinishReason: () => TextGenerationStreamFinishReason
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -31,7 +32,113 @@ function toArray(obj) {
   return [obj];
 }
 
+// src/vendor/fetch-event-source/parse.ts
+function getLines(onLine) {
+  let buffer;
+  let position;
+  let fieldLength;
+  let discardTrailingNewline = false;
+  return function onChunk(arr) {
+    if (buffer === void 0) {
+      buffer = arr;
+      position = 0;
+      fieldLength = -1;
+    } else {
+      buffer = concat(buffer, arr);
+    }
+    const bufLength = buffer.length;
+    let lineStart = 0;
+    while (position < bufLength) {
+      if (discardTrailingNewline) {
+        if (buffer[position] === 10 /* NewLine */) {
+          lineStart = ++position;
+        }
+        discardTrailingNewline = false;
+      }
+      let lineEnd = -1;
+      for (; position < bufLength && lineEnd === -1; ++position) {
+        switch (buffer[position]) {
+          case 58 /* Colon */:
+            if (fieldLength === -1) {
+              fieldLength = position - lineStart;
+            }
+            break;
+          case 13 /* CarriageReturn */:
+            discardTrailingNewline = true;
+          case 10 /* NewLine */:
+            lineEnd = position;
+            break;
+        }
+      }
+      if (lineEnd === -1) {
+        break;
+      }
+      onLine(buffer.subarray(lineStart, lineEnd), fieldLength);
+      lineStart = position;
+      fieldLength = -1;
+    }
+    if (lineStart === bufLength) {
+      buffer = void 0;
+    } else if (lineStart !== 0) {
+      buffer = buffer.subarray(lineStart);
+      position -= lineStart;
+    }
+  };
+}
+function getMessages(onId, onRetry, onMessage) {
+  let message = newMessage();
+  const decoder = new TextDecoder();
+  return function onLine(line, fieldLength) {
+    if (line.length === 0) {
+      onMessage?.(message);
+      message = newMessage();
+    } else if (fieldLength > 0) {
+      const field = decoder.decode(line.subarray(0, fieldLength));
+      const valueOffset = fieldLength + (line[fieldLength + 1] === 32 /* Space */ ? 2 : 1);
+      const value = decoder.decode(line.subarray(valueOffset));
+      switch (field) {
+        case "data":
+          message.data = message.data ? message.data + "\n" + value : value;
+          break;
+        case "event":
+          message.event = value;
+          break;
+        case "id":
+          onId(message.id = value);
+          break;
+        case "retry":
+          const retry = parseInt(value, 10);
+          if (!isNaN(retry)) {
+            onRetry(message.retry = retry);
+          }
+          break;
+      }
+    }
+  };
+}
+function concat(a, b) {
+  const res = new Uint8Array(a.length + b.length);
+  res.set(a);
+  res.set(b, a.length);
+  return res;
+}
+function newMessage() {
+  return {
+    data: "",
+    event: "",
+    id: "",
+    retry: void 0
+  };
+}
+
 // src/HfInference.ts
+var HF_INFERENCE_API_BASE_URL = "https://api-inference.huggingface.co/models/";
+var TextGenerationStreamFinishReason = /* @__PURE__ */ ((TextGenerationStreamFinishReason2) => {
+  TextGenerationStreamFinishReason2["Length"] = "length";
+  TextGenerationStreamFinishReason2["EndOfSequenceToken"] = "eos_token";
+  TextGenerationStreamFinishReason2["StopSequence"] = "stop_sequence";
+  return TextGenerationStreamFinishReason2;
+})(TextGenerationStreamFinishReason || {});
 var HfInference = class {
   apiKey;
   defaultOptions;
@@ -43,132 +150,246 @@ var HfInference = class {
    * Tries to fill in a hole with a missing word (token to be precise). That’s the base task for BERT models.
    */
   async fillMask(args, options) {
-    return this.request(args, options);
+    const res = await this.request(args, options);
+    const isValidOutput = Array.isArray(res) && res.every(
+      (x) => typeof x.score === "number" && typeof x.sequence === "string" && typeof x.token === "number" && typeof x.token_str === "string"
+    );
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type Array<score: number, sequence:string, token:number, token_str:string>"
+      );
+    }
+    return res;
   }
   /**
    * This task is well known to summarize longer text into shorter text. Be careful, some models have a maximum length of input. That means that the summary cannot handle full books for instance. Be careful when choosing your model.
    */
   async summarization(args, options) {
-    return (await this.request(args, options))?.[0];
+    const res = await this.request(args, options);
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.summary_text === "string");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<summary_text: string>");
+    }
+    return res?.[0];
   }
   /**
    * Want to have a nice know-it-all bot that can answer any question?. Recommended model: deepset/roberta-base-squad2
    */
   async questionAnswer(args, options) {
-    return await this.request(args, options);
+    const res = await this.request(args, options);
+    const isValidOutput = typeof res.answer === "string" && typeof res.end === "number" && typeof res.score === "number" && typeof res.start === "number";
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type <answer: string, end: number, score: number, start: number>"
+      );
+    }
+    return res;
   }
   /**
    * Don’t know SQL? Don’t want to dive into a large spreadsheet? Ask questions in plain english! Recommended model: google/tapas-base-finetuned-wtq.
    */
   async tableQuestionAnswer(args, options) {
-    return await this.request(args, options);
+    const res = await this.request(args, options);
+    const isValidOutput = typeof res.aggregator === "string" && typeof res.answer === "string" && Array.isArray(res.cells) && res.cells.every((x) => typeof x === "string") && Array.isArray(res.coordinates) && res.coordinates.every((coord) => Array.isArray(coord) && coord.every((x) => typeof x === "number"));
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type <aggregator: string, answer: string, cells: string[], coordinates: number[][]>"
+      );
+    }
+    return res;
   }
   /**
    * Usually used for sentiment-analysis this will output the likelihood of classes of an input. Recommended model: distilbert-base-uncased-finetuned-sst-2-english
    */
   async textClassification(args, options) {
-    return (await this.request(args, options))?.[0];
+    const res = (await this.request(args, options))?.[0];
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<label: string, score: number>");
+    }
+    return res;
   }
   /**
    * Use to continue text from a prompt. This is a very generic task. Recommended model: gpt2 (it’s a simple model, but fun to play with).
    */
   async textGeneration(args, options) {
-    return (await this.request(args, options))?.[0];
+    const res = await this.request(args, options);
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.generated_text === "string");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<generated_text: string>");
+    }
+    return res?.[0];
+  }
+  /**
+   * Use to continue text from a prompt. Same as `textGeneration` but returns generator that can be read one token at a time
+   */
+  async *textGenerationStream(args, options) {
+    yield* this.streamingRequest(args, options);
   }
   /**
    * Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. Recommended model: dbmdz/bert-large-cased-finetuned-conll03-english
    */
   async tokenClassification(args, options) {
-    return toArray(await this.request(args, options));
+    const res = toArray(await this.request(args, options));
+    const isValidOutput = Array.isArray(res) && res.every(
+      (x) => typeof x.end === "number" && typeof x.entity_group === "string" && typeof x.score === "number" && typeof x.start === "number" && typeof x.word === "string"
+    );
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type Array<end: number, entity_group: string, score: number, start: number, word: string>"
+      );
+    }
+    return res;
   }
   /**
    * This task is well known to translate text from one language to another. Recommended model: Helsinki-NLP/opus-mt-ru-en.
    */
   async translation(args, options) {
-    return (await this.request(args, options))?.[0];
+    const res = await this.request(args, options);
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.translation_text === "string");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<translation_text: string>");
+    }
+    return res?.[0];
   }
   /**
    * This task is super useful to try out classification with zero code, you simply pass a sentence/paragraph and the possible labels for that sentence, and you get a result. Recommended model: facebook/bart-large-mnli.
    */
   async zeroShotClassification(args, options) {
-    return toArray(
+    const res = toArray(
       await this.request(args, options)
     );
+    const isValidOutput = Array.isArray(res) && res.every(
+      (x) => Array.isArray(x.labels) && x.labels.every((_label) => typeof _label === "string") && Array.isArray(x.scores) && x.scores.every((_score) => typeof _score === "number") && typeof x.sequence === "string"
+    );
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type Array<labels: string[], scores: number[], sequence: string>"
+      );
+    }
+    return res;
   }
   /**
    * This task corresponds to any chatbot like structure. Models tend to have shorter max_length, so please check with caution when using a given model if you need long range dependency or not. Recommended model: microsoft/DialoGPT-large.
    *
    */
   async conversational(args, options) {
-    return await this.request(args, options);
+    const res = await this.request(args, options);
+    const isValidOutput = Array.isArray(res.conversation.generated_responses) && res.conversation.generated_responses.every((x) => typeof x === "string") && Array.isArray(res.conversation.past_user_inputs) && res.conversation.past_user_inputs.every((x) => typeof x === "string") && typeof res.generated_text === "string" && Array.isArray(res.warnings) && res.warnings.every((x) => typeof x === "string");
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type <conversation: {generated_responses: string[], past_user_inputs: string[]}, generated_text: string, warnings: string[]>"
+      );
+    }
+    return res;
   }
   /**
    * This task reads some text and outputs raw float values, that are usually consumed as part of a semantic database/semantic search.
    */
   async featureExtraction(args, options) {
-    return await this.request(args, options);
+    const res = await this.request(args, options);
+    return res;
   }
   /**
    * This task reads some audio input and outputs the said words within the audio files.
    * Recommended model (english language): facebook/wav2vec2-large-960h-lv60-self
    */
   async automaticSpeechRecognition(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       binary: true
     });
+    const isValidOutput = typeof res.text === "string";
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type <text: string>");
+    }
+    return res;
   }
   /**
    * This task reads some audio input and outputs the likelihood of classes.
    * Recommended model:  superb/hubert-large-superb-er
    */
   async audioClassification(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       binary: true
     });
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<label: string, score: number>");
+    }
+    return res;
   }
   /**
    * This task reads some image input and outputs the likelihood of classes.
    * Recommended model: google/vit-base-patch16-224
    */
   async imageClassification(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       binary: true
     });
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.score === "number");
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type Array<label: string, score: number>");
+    }
+    return res;
   }
   /**
    * This task reads some image input and outputs the likelihood of classes & bounding boxes of detected objects.
    * Recommended model: facebook/detr-resnet-50
    */
   async objectDetection(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       binary: true
     });
+    const isValidOutput = Array.isArray(res) && res.every(
+      (x) => typeof x.label === "string" && typeof x.score === "number" && typeof x.box.xmin === "number" && typeof x.box.ymin === "number" && typeof x.box.xmax === "number" && typeof x.box.ymax === "number"
+    );
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type Array<{label:string; score:number; box:{xmin:number; ymin:number; xmax:number; ymax:number}}>"
+      );
+    }
+    return res;
   }
   /**
    * This task reads some image input and outputs the likelihood of classes & bounding boxes of detected objects.
    * Recommended model: facebook/detr-resnet-50-panoptic
    */
   async imageSegmentation(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       binary: true
     });
+    const isValidOutput = Array.isArray(res) && res.every((x) => typeof x.label === "string" && typeof x.mask === "string" && typeof x.score === "number");
+    if (!isValidOutput) {
+      throw new TypeError(
+        "Invalid inference output: output must be of type Array<label: string, mask: string, score: number>"
+      );
+    }
+    return res;
   }
   /**
    * This task reads some text input and outputs an image.
    * Recommended model: stabilityai/stable-diffusion-2
    */
   async textToImage(args, options) {
-    return await this.request(args, {
+    const res = await this.request(args, {
       ...options,
       blob: true
     });
+    const isValidOutput = res && res instanceof Blob;
+    if (!isValidOutput) {
+      throw new TypeError("Invalid inference output: output must be of type object & of instance Blob");
+    }
+    return res;
   }
-  async request(args, options) {
+  /**
+   * Helper that prepares request arguments
+   */
+  makeRequestOptions(args, options) {
     const mergedOptions = { ...this.defaultOptions, ...options };
     const { model, ...otherArgs } = args;
     const headers = {};
@@ -189,7 +410,8 @@ var HfInference = class {
         headers["X-Load-Model"] = "0";
       }
     }
-    const response = await fetch(`https://api-inference.huggingface.co/models/${model}`, {
+    const url = `${HF_INFERENCE_API_BASE_URL}${model}`;
+    const info = {
       headers,
       method: "POST",
       body: options?.binary ? args.data : JSON.stringify({
@@ -197,7 +419,12 @@ var HfInference = class {
         options: mergedOptions
       }),
       credentials: options?.includeCredentials ? "include" : "same-origin"
-    });
+    };
+    return { url, info, mergedOptions };
+  }
+  async request(args, options) {
+    const { url, info, mergedOptions } = this.makeRequestOptions(args, options);
+    const response = await fetch(url, info);
     if (mergedOptions.retry_on_error !== false && response.status === 503 && !mergedOptions.wait_for_model) {
       return this.request(args, {
         ...mergedOptions,
@@ -216,8 +443,58 @@ var HfInference = class {
     }
     return output;
   }
+  /**
+   * Make request that uses server-sent events and returns response as a generator
+   */
+  async *streamingRequest(args, options) {
+    const { url, info, mergedOptions } = this.makeRequestOptions({ ...args, stream: true }, options);
+    const response = await fetch(url, info);
+    if (mergedOptions.retry_on_error !== false && response.status === 503 && !mergedOptions.wait_for_model) {
+      return this.streamingRequest(args, {
+        ...mergedOptions,
+        wait_for_model: true
+      });
+    }
+    if (!response.ok) {
+      throw new Error(`Server response contains error: ${response.status}`);
+    }
+    if (response.headers.get("content-type") !== "text/event-stream") {
+      throw new Error(`Server does not support event stream content type`);
+    }
+    const reader = response.body.getReader();
+    const events = [];
+    const onEvent = (event) => {
+      events.push(event);
+    };
+    const onChunk = getLines(
+      getMessages(
+        () => {
+        },
+        () => {
+        },
+        onEvent
+      )
+    );
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+          return;
+        onChunk(value);
+        while (events.length > 0) {
+          const event = events.shift();
+          if (event.data.length > 0) {
+            yield JSON.parse(event.data);
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  HfInference
+  HfInference,
+  TextGenerationStreamFinishReason
 });