@huggingface/transformers 3.5.1 → 3.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/transformers",
-  "version": "3.5.1",
+  "version": "3.5.2",
   "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!",
   "main": "./src/transformers.js",
   "types": "./types/transformers.d.ts",

package/src/env.js

@@ -26,7 +26,7 @@ import fs from 'fs';
 import path from 'path';
 import url from 'url';
 
-const VERSION = '3.5.1';
+const VERSION = '3.5.2';
 
 // Check if various APIs are available (depends on environment)
 const IS_BROWSER_ENV = typeof window !== "undefined" && typeof window.document !== "undefined";

package/src/generation/configuration_utils.js

@@ -77,7 +77,7 @@ export class GenerationConfig {
 
     /**
      * Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
-     * See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/1610.02424) for more details.
      * @type {number}
      * @default 1
      */
@@ -122,7 +122,7 @@ export class GenerationConfig {
     /**
      * Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated.
      * If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation.
-     * See [this paper](https://arxiv.org/pdf/2202.00666.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/2202.00666) for more details.
      * @type {number}
      * @default 1.0
      */
@@ -131,7 +131,7 @@ export class GenerationConfig {
     /**
      * If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled.
      * In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model.
-     * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
+     * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
      * @type {number}
      * @default 0.0
      */
@@ -141,7 +141,7 @@ export class GenerationConfig {
      * Eta sampling is a hybrid of locally typical sampling and epsilon sampling.
      * If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`.
      * The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
-     * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
+     * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
      * @type {number}
      * @default 0.0
      */
@@ -157,7 +157,7 @@ export class GenerationConfig {
 
     /**
      * The parameter for repetition penalty. 1.0 means no penalty.
-     * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/1909.05858) for more details.
      * @type {number}
      * @default 1.0
      */

package/src/generation/logits_process.js

@@ -410,7 +410,7 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
  * This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs,
  * the considered tokens include the prompt.
  * 
- * In the original [paper](https://arxiv.org/pdf/1909.05858.pdf), the authors suggest the use of a
+ * In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a
  * penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition.
  * To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes
  * more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where
@@ -580,7 +580,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor {
  * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a
  * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`.
  * 
- * See [the paper](https://arxiv.org/abs/2306.05284) for more information.
+ * See [the paper](https://huggingface.co/papers/2306.05284) for more information.
  */
 export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor {
 

package/src/generation/streamers.js

@@ -208,7 +208,11 @@ export class WhisperTextStreamer extends TextStreamer {
                     this.on_chunk_start?.(time);
                 }
                 this.waiting_for_timestamp = !this.waiting_for_timestamp; // Toggle
-                value = [[]]; // Skip timestamp
+
+                // NOTE: Timestamp tokens should not be printed. Although, since they
+                // aren't classified as "special tokens", we need to handle them here.
+                this.token_callback_function?.(tokens);
+                return;
             }
         }
         return super.put(value);

package/src/models.js

@@ -7053,7 +7053,7 @@ export class DecisionTransformerPreTrainedModel extends PreTrainedModel { }
 
 /**
  * The model builds upon the GPT2 architecture to perform autoregressive prediction of actions in an offline RL setting.
- * Refer to the paper for more details: https://arxiv.org/abs/2106.01345
+ * Refer to the paper for more details: https://huggingface.co/papers/2106.01345
  */
 export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel { }
 

package/src/pipelines.js

@@ -1912,7 +1912,7 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options
         for (const aud of preparedAudios) {
             const inputs = await this.processor(aud);
 
-            // According to the [paper](https://arxiv.org/pdf/2410.15608):
+            // According to the [paper](https://huggingface.co/papers/2410.15608):
             // "We use greedy decoding, with a heuristic limit of 6 output tokens
             // per second of audio to avoid repeated output sequences."
             const max_new_tokens = Math.floor(aud.length / sampling_rate) * 6;

package/src/tokenizers.js

@@ -366,15 +366,19 @@ export class TokenizerModel extends Callable {
                 return new BPE(config);
 
             default:
-                // Some older tokenizers, like `google-t5/t5-small` and `distilbert/distilbert-base-uncased`, do not have a `type` field.
+                // Some older tokenizers, like `google-t5/t5-small`, `openai-community/gpt2`, and `distilbert/distilbert-base-uncased`, do not have a `type` field.
                 // In this case, we can infer the tokenizer type based on the structure of the `vocab` field and other properties.
                 if (config.vocab) {
                     if (Array.isArray(config.vocab)) {
                         // config.vocab is of type `[string, number][]`
                         // @ts-ignore
                         return new Unigram(config, ...args);
-                    } else if (typeof config.vocab === 'object' && config.continuing_subword_prefix && config.unk_token) {
-                        return new WordPieceTokenizer(config);
+                    } else if (Object.hasOwn(config, 'continuing_subword_prefix') && Object.hasOwn(config, 'unk_token')) {
+                        if (Object.hasOwn(config, 'merges')) {
+                            return new BPE(config);
+                        } else {
+                            return new WordPieceTokenizer(config);
+                        }
                     } else {
                         // @ts-ignore
                         return new LegacyTokenizerModel(config, ...args);
@@ -3515,7 +3519,7 @@ function _build_translation_inputs(self, raw_inputs, tokenizer_options, generate
  * between any pair of 200+ languages — including low-resource languages like Asturian,
  * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere,
  * regardless of their language preferences. For more information, check out their
- * [paper](https://arxiv.org/abs/2207.04672).
+ * [paper](https://huggingface.co/papers/2207.04672).
  * 
  * For a list of supported languages (along with their language codes),
  * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200}
@@ -3546,7 +3550,7 @@ export class NllbTokenizer extends PreTrainedTokenizer {
  * The M2M100Tokenizer class is used to tokenize text for M2M100 ("Many-to-Many") models.
  * 
  * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many
- * multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125)
+ * multilingual translation. It was introduced in this [paper](https://huggingface.co/papers/2010.11125)
  * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository.
  * 
  * For a list of supported languages (along with their language codes),

package/src/transformers.js

@@ -40,3 +40,11 @@ export * from './models/auto/processing_auto.js';
 export * from './generation/streamers.js';
 export * from './generation/stopping_criteria.js';
 export * from './generation/logits_process.js';
+
+// Expose common types used across the library for developers to access
+/**
+ * @typedef {import('./utils/hub.js').PretrainedModelOptions} PretrainedModelOptions
+ * @typedef {import('./base/processing_utils.js').PretrainedProcessorOptions} PretrainedProcessorOptions
+ * @typedef {import('./utils/dtypes.js').DataType} DataType
+ * @typedef {import('./utils/devices.js').DeviceType} DeviceType
+ */

package/types/generation/configuration_utils.d.ts

@@ -65,7 +65,7 @@ export class GenerationConfig {
     num_beams: number;
     /**
      * Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
-     * See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/1610.02424) for more details.
      * @type {number}
      * @default 1
      */
@@ -103,7 +103,7 @@ export class GenerationConfig {
     /**
      * Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated.
      * If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation.
-     * See [this paper](https://arxiv.org/pdf/2202.00666.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/2202.00666) for more details.
      * @type {number}
      * @default 1.0
      */
@@ -111,7 +111,7 @@ export class GenerationConfig {
     /**
      * If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled.
      * In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model.
-     * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
+     * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
      * @type {number}
      * @default 0.0
      */
@@ -120,7 +120,7 @@ export class GenerationConfig {
      * Eta sampling is a hybrid of locally typical sampling and epsilon sampling.
      * If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`.
      * The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
-     * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
+     * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
      * @type {number}
      * @default 0.0
      */
@@ -134,7 +134,7 @@ export class GenerationConfig {
     diversity_penalty: number;
     /**
      * The parameter for repetition penalty. 1.0 means no penalty.
-     * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
+     * See [this paper](https://huggingface.co/papers/1909.05858) for more details.
      * @type {number}
      * @default 1.0
      */

package/types/generation/logits_process.d.ts

@@ -191,7 +191,7 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
  * This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs,
  * the considered tokens include the prompt.
  *
- * In the original [paper](https://arxiv.org/pdf/1909.05858.pdf), the authors suggest the use of a
+ * In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a
  * penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition.
  * To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes
  * more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where
@@ -279,7 +279,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor {
  * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a
  * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`.
  *
- * See [the paper](https://arxiv.org/abs/2306.05284) for more information.
+ * See [the paper](https://huggingface.co/papers/2306.05284) for more information.
  */
 export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor {
     /**

package/types/generation/streamers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streamers.d.ts","sourceRoot":"","sources":["../../src/generation/streamers.js"],"names":[],"mappings":"AASA;IACI;;;OAGG;IACH,WAFW,MAAM,EAAE,EAAE,QAIpB;IAED;;OAEG;IACH,YAEC;CACJ;AAMD;;GAEG;AACH;IACI;;;;;;;;;OASG;IACH,uBARW,OAAO,kBAAkB,EAAE,mBAAmB,+GAEtD;QAA0B,WAAW,GAA7B,OAAO;QACW,mBAAmB,GAArC,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACP,aAAa;KACxC,EAoBA;IAVG,0DAA0B;IAC1B,qBAA8B;IAC9B,oCAA0D;IAC1D,gCAfgB,MAAM,EAAE,KAAG,IAAI,CAeuB;IACtD,mBAAyE;IAGzE,mBAAqB;IACrB,kBAAkB;IAClB,gCAAkC;IA8DtC;;;;OAIG;IACH,wBAHW,MAAM,cACN,OAAO,QASjB;CACJ;AAED;;;;;;;GAOG;AACH;IACI;;;;;;;;;;;;OAYG;IACH,uBAZW,OAAO,kBAAkB,EAAE,gBAAgB,gKAEnD;QAA0B,WAAW,GAA7B,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACS,cAAc,GAA/C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACW,YAAY,GAA7C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACK,WAAW,GAAtC,MAAY,IAAI;QACC,cAAc,GAA/B,MAAM;QACY,mBAAmB,GAArC,OAAO;QACU,aAAa;KACxC,EA4BA;IATG,wBAAgD;IAEhD,uBA3BgB,MAAM,KAAG,IAAI,CA2BO;IACpC,qBA3BgB,MAAM,KAAG,IAAI,CA2BG;IAChC,mBA3BmB,IAAI,CA2BO;IAE9B,uBAAoC;IAEpC,+BAAkC;CAiCzC"}
+{"version":3,"file":"streamers.d.ts","sourceRoot":"","sources":["../../src/generation/streamers.js"],"names":[],"mappings":"AASA;IACI;;;OAGG;IACH,WAFW,MAAM,EAAE,EAAE,QAIpB;IAED;;OAEG;IACH,YAEC;CACJ;AAMD;;GAEG;AACH;IACI;;;;;;;;;OASG;IACH,uBARW,OAAO,kBAAkB,EAAE,mBAAmB,+GAEtD;QAA0B,WAAW,GAA7B,OAAO;QACW,mBAAmB,GAArC,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACP,aAAa;KACxC,EAoBA;IAVG,0DAA0B;IAC1B,qBAA8B;IAC9B,oCAA0D;IAC1D,gCAfgB,MAAM,EAAE,KAAG,IAAI,CAeuB;IACtD,mBAAyE;IAGzE,mBAAqB;IACrB,kBAAkB;IAClB,gCAAkC;IA8DtC;;;;OAIG;IACH,wBAHW,MAAM,cACN,OAAO,QASjB;CACJ;AAED;;;;;;;GAOG;AACH;IACI;;;;;;;;;;;;OAYG;IACH,uBAZW,OAAO,kBAAkB,EAAE,gBAAgB,gKAEnD;QAA0B,WAAW,GAA7B,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACS,cAAc,GAA/C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACW,YAAY,GAA7C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACK,WAAW,GAAtC,MAAY,IAAI;QACC,cAAc,GAA/B,MAAM;QACY,mBAAmB,GAArC,OAAO;QACU,aAAa;KACxC,EA4BA;IATG,wBAAgD;IAEhD,uBA3BgB,MAAM,KAAG,IAAI,CA2BO;IACpC,qBA3BgB,MAAM,KAAG,IAAI,CA2BG;IAChC,mBA3BmB,IAAI,CA2BO;IAE9B,uBAAoC;IAEpC,+BAAkC;CAqCzC"}

package/types/models.d.ts

@@ -3588,7 +3588,7 @@ export class DecisionTransformerPreTrainedModel extends PreTrainedModel {
 }
 /**
  * The model builds upon the GPT2 architecture to perform autoregressive prediction of actions in an offline RL setting.
- * Refer to the paper for more details: https://arxiv.org/abs/2106.01345
+ * Refer to the paper for more details: https://huggingface.co/papers/2106.01345
  */
 export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel {
 }

package/types/tokenizers.d.ts

@@ -482,7 +482,7 @@ export class Grok1Tokenizer extends PreTrainedTokenizer {
  * between any pair of 200+ languages — including low-resource languages like Asturian,
  * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere,
  * regardless of their language preferences. For more information, check out their
- * [paper](https://arxiv.org/abs/2207.04672).
+ * [paper](https://huggingface.co/papers/2207.04672).
  *
  * For a list of supported languages (along with their language codes),
  * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200}
@@ -505,7 +505,7 @@ export class NllbTokenizer extends PreTrainedTokenizer {
  * The M2M100Tokenizer class is used to tokenize text for M2M100 ("Many-to-Many") models.
  *
  * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many
- * multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125)
+ * multilingual translation. It was introduced in this [paper](https://huggingface.co/papers/2010.11125)
  * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository.
  *
  * For a list of supported languages (along with their language codes),

package/types/tokenizers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.js"],"names":[],"mappings":"AAmNA;;;;;;;;;;;;;GAaG;AACH,oCAHW,MAAM,GAAC,MAAM,GACX,OAAO,CAanB;;KAxO0E,GAC1E;UAA0B,GAAE;;AA0T7B;;;;GAIG;AACH;IA0BI;;;;;;OAMG;IACH,wCAJc,GAAC,EAAA,GACF,cAAc,CA8B1B;IA3DD;;;OAGG;IACH,yBAmBC;IAjBG,YAAoB;IAEpB,uBAAuB;IACvB,OADW,MAAM,EAAE,CACJ;IAEf;;;OAGG;IACH,eAFU,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEC;IAE9B,kBAA6B;IAC7B,eAA0B;IAC1B,wBAAmC;IAEnC,uFAAuF;IACvF,UADW,OAAO,CAC2B;IAuCjD;;;;OAIG;IACH,cAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;IAED;;;;;OAKG;IACH,eAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;IAED;;;;OAIG;IACH,8BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;IAED;;;;OAIG;IACH,2BAHW,MAAM,EAAE,GAAC,MAAM,EAAE,GACf,MAAM,EAAE,CAIpB;CACJ;;KAva0E,GAC1E;UAA0B,GAAE;;AAw/E7B;;;;GAIG;AAEH;IA0II;;;;;;;;OAQG;IACH,sDANW,MAAM,kFACN,0BAA0B,GAGxB,OAAO,CAAC,mBAAmB,CAAC,CAsBxC;IAnKD;;;;OAIG;IACH,sDAqGC;IA7GD,+BAA8B;IAE9B,qBAAuB;IASnB,uBAAwC;IAGxC,uBAAiE;IACjE,4BAAyE;IACzE,sBAA4E;IAC5E,8BAA4E;IAC5E,iBAAwD;IAGxD,sBAAwB;IACxB,0BAAyB;IAEzB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IAetB,+BAAgF;IAehF,0CAEC;IAED,sCAAsC;IACtC,kBADW,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CACyC;IAG3E,mBAA6C;IAC7C,sBAAkE;IAElE,kBAAwD;IACxD,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,sBAAwD;IAExD,6HAA6H;IAC7H,cADW,OAAO,CAC8B;IAEhD,kCAAwF;IACxF,oCAA6F;IAM7F,gBAAmB;IAEnB,mBAA0D;IAa1D,wCAAyC;IAG7C;;;;;;OAMG;IACH,iBAiBC;IAiCD;;;;;;;OAOG;IAEH;;;;;;;;;;;;OAYG;IACH,YAXW,MAAM,GAAC,MAAM,EAAE,8GAEvB;QAAkC,SAAS,GAAnC,MAAM,GAAC,MAAM,EAAE;QACgB,OAAO,GAAtC,OAAO,GAAC,YAAY;QACF,kBAAkB,GAApC,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QACY,aAAa,GAA/B,OAAO;QACW,qBAAqB,GAAvC,OAAO;KACf;;;;;;;;;;;;;MAgKF;IAED;;;;;OAKG;IACH,mBAHW,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,GAAC,IAAI,CAsDzB;IAED;;;;;;;;;;OAUG;IACH,qBAkBC;IAED;;;;;;;OAOG;IACH,uBANW,MAAM,kCAEd;QAAyB,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;KACf,GAAU;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;KAAC,CAYzD;IAED;;;;;;;OAOG;IACH,eANW,MAAM,kCAEd;QAAyB,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;KACf,GAAU,MAAM,EAAE,CAOpB;IAED;;;;;;;;;OASG;IACH,aAPW,MAAM,8DAEd;QAAyB,SAAS,GAA1B,MAAM;QACY,kBAAkB,GAApC,OAAO;QACW,qBAAqB,GAAvC,OAAO;KACf,GAAU,MAAM,EAAE,CAYpB;IAED;;;;;OAKG;IACH,oBAJW,MAAM,EAAE,EAAE,GAAC,MAAM,sBAEf,MAAM,EAAE,CAOpB;IAED;;;;;;;;;;OAUG;IACH,kBARW,MAAM,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM,gBAEhC;QAA8B,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;KAEf,GAAU,MAAM,CAgBlB;IAED;;;;;;;;OAQG;IACH,yBAPW,MAAM,EAAE,GAAC,MAAM,EAAE,0DAEzB;QAA8B,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;KAEf,GAAU,MAAM,CAiClB;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,8CAZG;QAAyB,aAAa,GAA9B,MAAM;QAIa,KAAK,GAAxB,KAAQ;KAMhB,GAAU,MAAM,CA0ClB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyDG;IACH,kCA/BW,OAAO,EAAE,mKAGjB;QAAyB,aAAa,GAA9B,MAAM;QAEa,KAAK,GAAxB,KAAQ;QAM2B,SAAS,GAA5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;QAMN,qBAAqB,GAAvC,OAAO;QAIW,QAAQ,GAA1B,OAAO;QACW,OAAO,GAAzB,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QAEY,aAAa,GAA/B,OAAO;QACW,WAAW,GAA7B,OAAO;QACU,gBAAgB;KACzC,GAAU,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,GAAE,MAAM,EAAE,EAAE;;;;;;;;;;;;;KAAc,CA4DhE;CACJ;AAED;;;GAGG;AACH;CAEC;AACD;;;GAGG;AACH;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAAgE;AAChE;CAA+D;AAC/D;IAGI,sDAGC;CACJ;AACD;CAEC;AAED;CAAwD;AACxD;CAA0D;AAC1D;CAA0D;AAC1D;IACI,sDAMC;IAHG,sBAA0C;IAC1C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AACD;CAAwD;AAExD;CAA6D;AAE7D;CAA2D;AAI3D;IAII,sDAaC;IAVG,YAA4C;CA+BnD;AACD;CAA+D;AAE/D;CAAgE;AAChE;CAA2D;AAE3D;CAA4D;AAE5D;CAA6D;AAE7D;CAAyD;AAEzD;CAA2D;AAE3D;CAA2D;AAE3D;CAA2D;AAqD3D;;;;;;;;;;;;GAYG;AACH;IAEI,sDAMC;IAHG,sBAA+C;IAC/C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;;;;;;;GASG;AACH;IACI,sDAQC;IALG,sBAAuC;IACvC,sBAE6B;IAC7B,kCAAoC;IAGxC;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;GAGG;AACH;IAEI,8BAEC;IAED;;;;;OAKG;IACH,uBAJW,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAC,CAAC,uFAEtE,KAAK,CAAC,MAAM,GAAC;QAAC,MAAM,CAAC,EAAE,SAAS,GAAC,KAAK,CAAC;YAAC,QAAQ,EAAE,MAAM,GAAC,IAAI,CAAC;YAAC,SAAS,EAAE,KAAK,CAAC,MAAM,GAAC,IAAI,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAC,CAAC,CAAA;KAAC,CAAC,CAyS1H;IAED;;;;;;OAMG;IACH,kCAsGC;IAED,eAAe;IACf,8BAgBC;IAED;;;;;;;;;OASG;IACH,+BAaC;IAwBD;;;;OAIG;IACH,6BAqBC;IAED;;;;;OAKG;IACH,6BAqCC;IAED;;;;OAIG;IACH,4BAoCC;IAED;;;;;;;;OAQG;IACH,0BA+CC;CACJ;AACD;CAA6D;AAC7D;CAA0D;AAC1D;CAA4D;AAE5D;;;GAGG;AACH;IASQ,sBAAqC;IAErC,mCAEC;IAKL;;;;;;;OAOG;IACH,mBAHW,MAAM,GAAC,IAAI,SAsBrB;CAEJ;AAED;CAAiE;AAEjE;CAAgE;AAChE;CAAqE;AAErE;CAA8D;AAE9D;CAA4D;AAE5D;IAEI,sDAKC;CACJ;AAED;CAA4D;AAE5D;CAA4D;AAE5D;;;;;;GAMG;AACH;IACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAiDC;IAGD;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,kFAKN,0BAA0B,GAExB,OAAO,CAAC,mBAAmB,CAAC,CA6BxC;CACJ;;;;;;;;aAhxIa,OAAO;;yCACR,OAAO,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB;;;;;WAqmB/D,MAAM;;;;UACN,MAAM;;;;YACN,MAAM;;;;WACN,OAAO;;;;WACP,OAAO;;qCA00BR,SAAS,GAAC,UAAU,GAAC,oBAAoB,GAAC,gBAAgB,GAAC,YAAY;;;;;YAuGtE,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;eAMR,MAAM,EAAE;;;;oBACR,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;UAm6BR,MAAM;;;;aACN,MAAM;;;KA5/EuD,GAC1E;UAA0B,GAAE;;AAm9B7B;;;GAGG;AACH;IASI;;;;;;OAMG;IACH,gCAHa,UAAU,CAiCtB;IA7CD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA0CxB;;;;;;OAMG;IACH,gBAJW,MAAM,GACJ,MAAM,CAKlB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,MAAM,CAIlB;CAEJ;;KA5hC0E,GAC1E;UAA0B,GAAE;;AA6zC7B;;;;GAIG;AACH;IACI;;;;;;;KAOC;IACD,gCAHW,YAAY,CA+BtB;IAED;;;;;;;;OAQG;IACH,wBALW,MAAM,kBAEJ,MAAM,EAAE,CAKpB;IAED;;;;;OAKG;IACH,mBAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAOpB;IAED;;;;;OAKG;IACH,YAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAIpB;CACJ;;KA74C0E,GAC1E;UAA0B,GAAE;;AA4kD7B;;;;GAIG;AAGH;;;;;GAKG;AAGH;;GAEG;AACH;IAUI;;;;;;OAMG;IACH,gCAHa,aAAa,CAsBzB;IAlCD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA+BxB;;;;;;;OAOG;IACH,qCAJc,GAAC,EAAA,GACF,mBAAmB,CAK/B;IAED;;;;;OAKG;IACH,8BAHc,GAAC,EAAA,GACF,mBAAmB,CAI/B;CACJ;;KA1pD0E,GAC1E;UAA0B,GAAE;;AA6zD7B;;;GAGG;AACH;IAiBI;;;;;;KAMC;IACD,gCAHW,OAAO,CAgCjB;IAnDD;;;;MAIE;IACF,yBAQC;IANG,YAAoB;IAEpB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IACtB,wBAA8B;IAC9B,kBAAuC;IAyC3C;;;;;MAKE;IACF,cAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;MAIE;IACF,eAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;;;OAMG;IACH,qBAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;CAEJ;AAznDD;;;;;GAKG;AACH;IACI;;;;;;;;;;OAUG;IACH,oBARG;QAAuB,OAAO,EAAtB,MAAM;QACS,EAAE,EAAjB,MAAM;QACW,WAAW,GAA5B,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,UAAU,GAA3B,OAAO;QACU,OAAO,GAAxB,OAAO;KACjB,EASA;IAPG,gBAA6B;IAC7B,WAAmB;IACnB,qBAA8C;IAC9C,gBAAoC;IACpC,gBAAoC;IACpC,iBAAsC;IACtC,oBAA2C;CAElD;mCA/QM,4BAA4B;uBARZ,mBAAmB"}
+{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.js"],"names":[],"mappings":"AAmNA;;;;;;;;;;;;;GAaG;AACH,oCAHW,MAAM,GAAC,MAAM,GACX,OAAO,CAanB;;KAxO0E,GAC1E;UAA0B,GAAE;;AA0T7B;;;;GAIG;AACH;IA0BI;;;;;;OAMG;IACH,wCAJc,GAAC,EAAA,GACF,cAAc,CAkC1B;IA/DD;;;OAGG;IACH,yBAmBC;IAjBG,YAAoB;IAEpB,uBAAuB;IACvB,OADW,MAAM,EAAE,CACJ;IAEf;;;OAGG;IACH,eAFU,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEC;IAE9B,kBAA6B;IAC7B,eAA0B;IAC1B,wBAAmC;IAEnC,uFAAuF;IACvF,UADW,OAAO,CAC2B;IA2CjD;;;;OAIG;IACH,cAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;IAED;;;;;OAKG;IACH,eAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;IAED;;;;OAIG;IACH,8BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;IAED;;;;OAIG;IACH,2BAHW,MAAM,EAAE,GAAC,MAAM,EAAE,GACf,MAAM,EAAE,CAIpB;CACJ;;KA3a0E,GAC1E;UAA0B,GAAE;;AA4/E7B;;;;GAIG;AAEH;IA0II;;;;;;;;OAQG;IACH,sDANW,MAAM,kFACN,0BAA0B,GAGxB,OAAO,CAAC,mBAAmB,CAAC,CAsBxC;IAnKD;;;;OAIG;IACH,sDAqGC;IA7GD,+BAA8B;IAE9B,qBAAuB;IASnB,uBAAwC;IAGxC,uBAAiE;IACjE,4BAAyE;IACzE,sBAA4E;IAC5E,8BAA4E;IAC5E,iBAAwD;IAGxD,sBAAwB;IACxB,0BAAyB;IAEzB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IAetB,+BAAgF;IAehF,0CAEC;IAED,sCAAsC;IACtC,kBADW,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CACyC;IAG3E,mBAA6C;IAC7C,sBAAkE;IAElE,kBAAwD;IACxD,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,sBAAwD;IAExD,6HAA6H;IAC7H,cADW,OAAO,CAC8B;IAEhD,kCAAwF;IACxF,oCAA6F;IAM7F,gBAAmB;IAEnB,mBAA0D;IAa1D,wCAAyC;IAG7C;;;;;;OAMG;IACH,iBAiBC;IAiCD;;;;;;;OAOG;IAEH;;;;;;;;;;;;OAYG;IACH,YAXW,MAAM,GAAC,MAAM,EAAE,8GAEvB;QAAkC,SAAS,GAAnC,MAAM,GAAC,MAAM,EAAE;QACgB,OAAO,GAAtC,OAAO,GAAC,YAAY;QACF,kBAAkB,GAApC,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QACY,aAAa,GAA/B,OAAO;QACW,qBAAqB,GAAvC,OAAO;KACf;;;;;;;;;;;;;MAgKF;IAED;;;;;OAKG;IACH,mBAHW,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,GAAC,IAAI,CAsDzB;IAED;;;;;;;;;;OAUG;IACH,qBAkBC;IAED;;;;;;;OAOG;IACH,uBANW,MAAM,kCAEd;QAAyB,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;KACf,GAAU;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;KAAC,CAYzD;IAED;;;;;;;OAOG;IACH,eANW,MAAM,kCAEd;QAAyB,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;KACf,GAAU,MAAM,EAAE,CAOpB;IAED;;;;;;;;;OASG;IACH,aAPW,MAAM,8DAEd;QAAyB,SAAS,GAA1B,MAAM;QACY,kBAAkB,GAApC,OAAO;QACW,qBAAqB,GAAvC,OAAO;KACf,GAAU,MAAM,EAAE,CAYpB;IAED;;;;;OAKG;IACH,oBAJW,MAAM,EAAE,EAAE,GAAC,MAAM,sBAEf,MAAM,EAAE,CAOpB;IAED;;;;;;;;;;OAUG;IACH,kBARW,MAAM,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM,gBAEhC;QAA8B,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;KAEf,GAAU,MAAM,CAgBlB;IAED;;;;;;;;OAQG;IACH,yBAPW,MAAM,EAAE,GAAC,MAAM,EAAE,0DAEzB;QAA8B,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;KAEf,GAAU,MAAM,CAiClB;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,8CAZG;QAAyB,aAAa,GAA9B,MAAM;QAIa,KAAK,GAAxB,KAAQ;KAMhB,GAAU,MAAM,CA0ClB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyDG;IACH,kCA/BW,OAAO,EAAE,mKAGjB;QAAyB,aAAa,GAA9B,MAAM;QAEa,KAAK,GAAxB,KAAQ;QAM2B,SAAS,GAA5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;QAMN,qBAAqB,GAAvC,OAAO;QAIW,QAAQ,GAA1B,OAAO;QACW,OAAO,GAAzB,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QAEY,aAAa,GAA/B,OAAO;QACW,WAAW,GAA7B,OAAO;QACU,gBAAgB;KACzC,GAAU,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,GAAE,MAAM,EAAE,EAAE;;;;;;;;;;;;;KAAc,CA4DhE;CACJ;AAED;;;GAGG;AACH;CAEC;AACD;;;GAGG;AACH;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAAgE;AAChE;CAA+D;AAC/D;IAGI,sDAGC;CACJ;AACD;CAEC;AAED;CAAwD;AACxD;CAA0D;AAC1D;CAA0D;AAC1D;IACI,sDAMC;IAHG,sBAA0C;IAC1C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AACD;CAAwD;AAExD;CAA6D;AAE7D;CAA2D;AAI3D;IAII,sDAaC;IAVG,YAA4C;CA+BnD;AACD;CAA+D;AAE/D;CAAgE;AAChE;CAA2D;AAE3D;CAA4D;AAE5D;CAA6D;AAE7D;CAAyD;AAEzD;CAA2D;AAE3D;CAA2D;AAE3D;CAA2D;AAqD3D;;;;;;;;;;;;GAYG;AACH;IAEI,sDAMC;IAHG,sBAA+C;IAC/C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;;;;;;;GASG;AACH;IACI,sDAQC;IALG,sBAAuC;IACvC,sBAE6B;IAC7B,kCAAoC;IAGxC;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;GAGG;AACH;IAEI,8BAEC;IAED;;;;;OAKG;IACH,uBAJW,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAC,CAAC,uFAEtE,KAAK,CAAC,MAAM,GAAC;QAAC,MAAM,CAAC,EAAE,SAAS,GAAC,KAAK,CAAC;YAAC,QAAQ,EAAE,MAAM,GAAC,IAAI,CAAC;YAAC,SAAS,EAAE,KAAK,CAAC,MAAM,GAAC,IAAI,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAC,CAAC,CAAA;KAAC,CAAC,CAyS1H;IAED;;;;;;OAMG;IACH,kCAsGC;IAED,eAAe;IACf,8BAgBC;IAED;;;;;;;;;OASG;IACH,+BAaC;IAwBD;;;;OAIG;IACH,6BAqBC;IAED;;;;;OAKG;IACH,6BAqCC;IAED;;;;OAIG;IACH,4BAoCC;IAED;;;;;;;;OAQG;IACH,0BA+CC;CACJ;AACD;CAA6D;AAC7D;CAA0D;AAC1D;CAA4D;AAE5D;;;GAGG;AACH;IASQ,sBAAqC;IAErC,mCAEC;IAKL;;;;;;;OAOG;IACH,mBAHW,MAAM,GAAC,IAAI,SAsBrB;CAEJ;AAED;CAAiE;AAEjE;CAAgE;AAChE;CAAqE;AAErE;CAA8D;AAE9D;CAA4D;AAE5D;IAEI,sDAKC;CACJ;AAED;CAA4D;AAE5D;CAA4D;AAE5D;;;;;;GAMG;AACH;IACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAiDC;IAGD;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,kFAKN,0BAA0B,GAExB,OAAO,CAAC,mBAAmB,CAAC,CA6BxC;CACJ;;;;;;;;aApxIa,OAAO;;yCACR,OAAO,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB;;;;;WAymB/D,MAAM;;;;UACN,MAAM;;;;YACN,MAAM;;;;WACN,OAAO;;;;WACP,OAAO;;qCA00BR,SAAS,GAAC,UAAU,GAAC,oBAAoB,GAAC,gBAAgB,GAAC,YAAY;;;;;YAuGtE,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;eAMR,MAAM,EAAE;;;;oBACR,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;UAm6BR,MAAM;;;;aACN,MAAM;;;KAhgFuD,GAC1E;UAA0B,GAAE;;AAu9B7B;;;GAGG;AACH;IASI;;;;;;OAMG;IACH,gCAHa,UAAU,CAiCtB;IA7CD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA0CxB;;;;;;OAMG;IACH,gBAJW,MAAM,GACJ,MAAM,CAKlB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,MAAM,CAIlB;CAEJ;;KAhiC0E,GAC1E;UAA0B,GAAE;;AAi0C7B;;;;GAIG;AACH;IACI;;;;;;;KAOC;IACD,gCAHW,YAAY,CA+BtB;IAED;;;;;;;;OAQG;IACH,wBALW,MAAM,kBAEJ,MAAM,EAAE,CAKpB;IAED;;;;;OAKG;IACH,mBAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAOpB;IAED;;;;;OAKG;IACH,YAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAIpB;CACJ;;KAj5C0E,GAC1E;UAA0B,GAAE;;AAglD7B;;;;GAIG;AAGH;;;;;GAKG;AAGH;;GAEG;AACH;IAUI;;;;;;OAMG;IACH,gCAHa,aAAa,CAsBzB;IAlCD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA+BxB;;;;;;;OAOG;IACH,qCAJc,GAAC,EAAA,GACF,mBAAmB,CAK/B;IAED;;;;;OAKG;IACH,8BAHc,GAAC,EAAA,GACF,mBAAmB,CAI/B;CACJ;;KA9pD0E,GAC1E;UAA0B,GAAE;;AAi0D7B;;;GAGG;AACH;IAiBI;;;;;;KAMC;IACD,gCAHW,OAAO,CAgCjB;IAnDD;;;;MAIE;IACF,yBAQC;IANG,YAAoB;IAEpB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IACtB,wBAA8B;IAC9B,kBAAuC;IAyC3C;;;;;MAKE;IACF,cAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;MAIE;IACF,eAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;;;OAMG;IACH,qBAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;CAEJ;AA7nDD;;;;;GAKG;AACH;IACI;;;;;;;;;;OAUG;IACH,oBARG;QAAuB,OAAO,EAAtB,MAAM;QACS,EAAE,EAAjB,MAAM;QACW,WAAW,GAA5B,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,UAAU,GAA3B,OAAO;QACU,OAAO,GAAxB,OAAO;KACjB,EASA;IAPG,gBAA6B;IAC7B,WAAmB;IACnB,qBAA8C;IAC9C,gBAAoC;IACpC,gBAAoC;IACpC,iBAAsC;IACtC,oBAA2C;CAElD;mCA/QM,4BAA4B;uBARZ,mBAAmB"}

package/types/transformers.d.ts

@@ -20,4 +20,8 @@ export * from "./generation/logits_process.js";
 export { FeatureExtractor } from "./base/feature_extraction_utils.js";
 export { ImageProcessor } from "./base/image_processors_utils.js";
 export { Processor } from "./base/processing_utils.js";
+export type PretrainedModelOptions = import("./utils/hub.js").PretrainedModelOptions;
+export type PretrainedProcessorOptions = import("./base/processing_utils.js").PretrainedProcessorOptions;
+export type DataType = import("./utils/dtypes.js").DataType;
+export type DeviceType = import("./utils/devices.js").DeviceType;
 //# sourceMappingURL=transformers.d.ts.map

package/types/transformers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transformers.d.ts","sourceRoot":"","sources":["../src/transformers.js"],"names":[],"mappings":""}
+{"version":3,"file":"transformers.d.ts","sourceRoot":"","sources":["../src/transformers.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;qCA6Ca,OAAO,gBAAgB,EAAE,sBAAsB;yCAC/C,OAAO,4BAA4B,EAAE,0BAA0B;uBAC/D,OAAO,mBAAmB,EAAE,QAAQ;yBACpC,OAAO,oBAAoB,EAAE,UAAU"}