@huggingface/transformers 3.5.0 → 3.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +129 -127
  2. package/dist/transformers.js +408 -54
  3. package/dist/transformers.js.map +1 -1
  4. package/dist/transformers.min.js +1 -1
  5. package/dist/transformers.min.js.map +1 -1
  6. package/dist/transformers.node.cjs +401 -53
  7. package/dist/transformers.node.cjs.map +1 -1
  8. package/dist/transformers.node.min.cjs +1 -1
  9. package/dist/transformers.node.min.cjs.map +1 -1
  10. package/dist/transformers.node.min.mjs +1 -1
  11. package/dist/transformers.node.min.mjs.map +1 -1
  12. package/dist/transformers.node.mjs +408 -54
  13. package/dist/transformers.node.mjs.map +1 -1
  14. package/dist/transformers.web.js +408 -54
  15. package/dist/transformers.web.js.map +1 -1
  16. package/dist/transformers.web.min.js +1 -1
  17. package/dist/transformers.web.min.js.map +1 -1
  18. package/package.json +2 -2
  19. package/src/base/image_processors_utils.js +6 -3
  20. package/src/configs.js +1 -0
  21. package/src/env.js +1 -1
  22. package/src/generation/configuration_utils.js +5 -5
  23. package/src/generation/logits_process.js +2 -2
  24. package/src/generation/streamers.js +5 -1
  25. package/src/models.js +58 -9
  26. package/src/pipelines.js +1 -1
  27. package/src/tokenizers.js +27 -16
  28. package/src/transformers.js +8 -0
  29. package/src/utils/hub.js +17 -12
  30. package/types/base/image_processors_utils.d.ts +2 -0
  31. package/types/base/image_processors_utils.d.ts.map +1 -1
  32. package/types/configs.d.ts.map +1 -1
  33. package/types/generation/configuration_utils.d.ts +5 -5
  34. package/types/generation/logits_process.d.ts +2 -2
  35. package/types/generation/streamers.d.ts.map +1 -1
  36. package/types/models/auto/image_processing_auto.d.ts.map +1 -1
  37. package/types/models.d.ts +23 -1
  38. package/types/models.d.ts.map +1 -1
  39. package/types/tokenizers.d.ts +2 -2
  40. package/types/tokenizers.d.ts.map +1 -1
  41. package/types/transformers.d.ts +4 -0
  42. package/types/transformers.d.ts.map +1 -1
  43. package/types/utils/hub.d.ts.map +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@huggingface/transformers",
3
- "version": "3.5.0",
3
+ "version": "3.5.2",
4
4
  "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!",
5
5
  "main": "./src/transformers.js",
6
6
  "types": "./types/transformers.d.ts",
@@ -55,7 +55,7 @@
55
55
  },
56
56
  "homepage": "https://github.com/huggingface/transformers.js#readme",
57
57
  "dependencies": {
58
- "@huggingface/jinja": "^0.3.4",
58
+ "@huggingface/jinja": "^0.4.1",
59
59
  "onnxruntime-node": "1.21.0",
60
60
  "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4",
61
61
  "sharp": "^0.34.1"
@@ -619,6 +619,10 @@ export class ImageProcessor extends Callable {
619
619
  this.pad_size = config.pad_size;
620
620
  // @ts-expect-error TS2339
621
621
  this.do_pad = config.do_pad;
622
+ // @ts-expect-error TS2339
623
+ this.min_pixels = config.min_pixels;
624
+ // @ts-expect-error TS2339
625
+ this.max_pixels = config.max_pixels;
622
626
 
623
627
  if (this.do_pad && !this.pad_size && this.size && this.size.width !== undefined && this.size.height !== undefined) {
624
628
  // Should pad, but no pad size specified
@@ -892,12 +896,11 @@ export class ImageProcessor extends Callable {
892
896
 
893
897
  } else if (this.size_divisibility !== undefined) {
894
898
  return enforce_size_divisibility([srcWidth, srcHeight], this.size_divisibility);
895
- } else if (size.min_pixels !== undefined && size.max_pixels !== undefined) {
899
+ } else if (this.min_pixels !== undefined && this.max_pixels !== undefined) {
896
900
  // Custom resize logic for Qwen2-VL models
897
- const { min_pixels, max_pixels } = size;
898
901
  // @ts-expect-error TS2339
899
902
  const factor = this.config.patch_size * this.config.merge_size;
900
- return smart_resize(srcHeight, srcWidth, factor, min_pixels, max_pixels);
903
+ return smart_resize(srcHeight, srcWidth, factor, this.min_pixels, this.max_pixels);
901
904
  } else {
902
905
  throw new Error(`Could not resize image due to unsupported \`this.size\` option in config: ${JSON.stringify(size)}`);
903
906
  }
package/src/configs.js CHANGED
@@ -125,6 +125,7 @@ function getNormalizedConfig(config) {
125
125
  mapping['hidden_size'] = 'hidden_size';
126
126
  mapping['num_attention_heads'] = 'num_attention_heads';
127
127
  break;
128
+ case 'qwen3':
128
129
  case 'gemma':
129
130
  case 'gemma2':
130
131
  case 'gemma3_text':
package/src/env.js CHANGED
@@ -26,7 +26,7 @@ import fs from 'fs';
26
26
  import path from 'path';
27
27
  import url from 'url';
28
28
 
29
- const VERSION = '3.5.0';
29
+ const VERSION = '3.5.2';
30
30
 
31
31
  // Check if various APIs are available (depends on environment)
32
32
  const IS_BROWSER_ENV = typeof window !== "undefined" && typeof window.document !== "undefined";
@@ -77,7 +77,7 @@ export class GenerationConfig {
77
77
 
78
78
  /**
79
79
  * Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
80
- * See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
80
+ * See [this paper](https://huggingface.co/papers/1610.02424) for more details.
81
81
  * @type {number}
82
82
  * @default 1
83
83
  */
@@ -122,7 +122,7 @@ export class GenerationConfig {
122
122
  /**
123
123
  * Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated.
124
124
  * If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation.
125
- * See [this paper](https://arxiv.org/pdf/2202.00666.pdf) for more details.
125
+ * See [this paper](https://huggingface.co/papers/2202.00666) for more details.
126
126
  * @type {number}
127
127
  * @default 1.0
128
128
  */
@@ -131,7 +131,7 @@ export class GenerationConfig {
131
131
  /**
132
132
  * If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled.
133
133
  * In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model.
134
- * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
134
+ * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
135
135
  * @type {number}
136
136
  * @default 0.0
137
137
  */
@@ -141,7 +141,7 @@ export class GenerationConfig {
141
141
  * Eta sampling is a hybrid of locally typical sampling and epsilon sampling.
142
142
  * If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`.
143
143
  * The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
144
- * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
144
+ * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
145
145
  * @type {number}
146
146
  * @default 0.0
147
147
  */
@@ -157,7 +157,7 @@ export class GenerationConfig {
157
157
 
158
158
  /**
159
159
  * The parameter for repetition penalty. 1.0 means no penalty.
160
- * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
160
+ * See [this paper](https://huggingface.co/papers/1909.05858) for more details.
161
161
  * @type {number}
162
162
  * @default 1.0
163
163
  */
@@ -410,7 +410,7 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
410
410
  * This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs,
411
411
  * the considered tokens include the prompt.
412
412
  *
413
- * In the original [paper](https://arxiv.org/pdf/1909.05858.pdf), the authors suggest the use of a
413
+ * In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a
414
414
  * penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition.
415
415
  * To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes
416
416
  * more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where
@@ -580,7 +580,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor {
580
580
  * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a
581
581
  * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`.
582
582
  *
583
- * See [the paper](https://arxiv.org/abs/2306.05284) for more information.
583
+ * See [the paper](https://huggingface.co/papers/2306.05284) for more information.
584
584
  */
585
585
  export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor {
586
586
 
@@ -208,7 +208,11 @@ export class WhisperTextStreamer extends TextStreamer {
208
208
  this.on_chunk_start?.(time);
209
209
  }
210
210
  this.waiting_for_timestamp = !this.waiting_for_timestamp; // Toggle
211
- value = [[]]; // Skip timestamp
211
+
212
+ // NOTE: Timestamp tokens should not be printed. Although, since they
213
+ // aren't classified as "special tokens", we need to handle them here.
214
+ this.token_callback_function?.(tokens);
215
+ return;
212
216
  }
213
217
  }
214
218
  return super.put(value);
package/src/models.js CHANGED
@@ -237,6 +237,7 @@ async function getSession(pretrained_model_name_or_path, fileName, options) {
237
237
  const session_config = {
238
238
  dtype: selectedDtype,
239
239
  kv_cache_dtype,
240
+ device: selectedDevice,
240
241
  }
241
242
 
242
243
  // Construct the model file name
@@ -417,6 +418,10 @@ function validateInputs(session, inputs) {
417
418
  return checkedInputs;
418
419
  }
419
420
 
421
+ // Currently, Transformers.js doesn't support simultaneous execution of sessions in WASM/WebGPU.
422
+ // For this reason, we need to chain the inference calls (otherwise we get "Error: Session already started").
423
+ let webInferenceChain = Promise.resolve();
424
+
420
425
  /**
421
426
  * Executes an InferenceSession using the specified inputs.
422
427
  * NOTE: `inputs` must contain at least the input names of the model.
@@ -433,17 +438,28 @@ async function sessionRun(session, inputs) {
433
438
  try {
434
439
  // pass the original ort tensor
435
440
  const ortFeed = Object.fromEntries(Object.entries(checkedInputs).map(([k, v]) => [k, v.ort_tensor]));
436
- let output = await session.run(ortFeed);
437
- output = replaceTensors(output);
438
- return output;
441
+ const run = () => session.run(ortFeed);
442
+ const output = await ((apis.IS_BROWSER_ENV || apis.IS_WEBWORKER_ENV)
443
+ ? (webInferenceChain = webInferenceChain.then(run))
444
+ : run());
445
+ return replaceTensors(output);
439
446
  } catch (e) {
440
447
  // Error messages can be long (nested) and uninformative. For this reason,
441
448
  // we apply minor formatting to show the most important information
442
449
  const formatted = Object.fromEntries(Object.entries(checkedInputs)
443
- .map(([k, { type, dims, data }]) => [k, {
450
+ .map(([k, tensor]) => {
444
451
  // Extract these properties from the underlying ORT tensor
445
- type, dims, data,
446
- }]));
452
+ const unpacked = {
453
+ type: tensor.type,
454
+ dims: tensor.dims,
455
+ location: tensor.location,
456
+ }
457
+ if (unpacked.location !== "gpu-buffer") {
458
+ // Only return the data if it's not a GPU buffer
459
+ unpacked.data = tensor.data;
460
+ }
461
+ return [k, unpacked];
462
+ }));
447
463
 
448
464
  // This usually occurs when the inputs are of the wrong type.
449
465
  console.error(`An error occurred during model execution: "${e}".`);
@@ -4572,6 +4588,22 @@ export class Qwen2Model extends Qwen2PreTrainedModel { }
4572
4588
  export class Qwen2ForCausalLM extends Qwen2PreTrainedModel { }
4573
4589
  //////////////////////////////////////////////////
4574
4590
 
4591
+
4592
+ //////////////////////////////////////////////////
4593
+ // Qwen3 models
4594
+
4595
+ /**
4596
+ * The bare Qwen3 Model outputting raw hidden-states without any specific head on top.
4597
+ */
4598
+ export class Qwen3PreTrainedModel extends PreTrainedModel { }
4599
+ /**
4600
+ * The bare Qwen3 Model outputting raw hidden-states without any specific head on top.
4601
+ */
4602
+ export class Qwen3Model extends Qwen3PreTrainedModel { }
4603
+
4604
+ export class Qwen3ForCausalLM extends Qwen3PreTrainedModel { }
4605
+ //////////////////////////////////////////////////
4606
+
4575
4607
  export class Qwen2VLPreTrainedModel extends PreTrainedModel {
4576
4608
  forward_params = [
4577
4609
  // Text inputs
@@ -5207,7 +5239,7 @@ export class RTDetrV2ForObjectDetection extends RTDetrV2PreTrainedModel {
5207
5239
  }
5208
5240
  }
5209
5241
 
5210
- export class RTDetrV2ObjectDetectionOutput extends RTDetrObjectDetectionOutput {}
5242
+ export class RTDetrV2ObjectDetectionOutput extends RTDetrObjectDetectionOutput { }
5211
5243
  //////////////////////////////////////////////////
5212
5244
 
5213
5245
  //////////////////////////////////////////////////
@@ -5222,7 +5254,20 @@ export class RFDetrForObjectDetection extends RFDetrPreTrainedModel {
5222
5254
  }
5223
5255
  }
5224
5256
 
5225
- export class RFDetrObjectDetectionOutput extends RTDetrObjectDetectionOutput {}
5257
+ export class RFDetrObjectDetectionOutput extends RTDetrObjectDetectionOutput { }
5258
+ //////////////////////////////////////////////////
5259
+
5260
+ //////////////////////////////////////////////////
5261
+ export class DFinePreTrainedModel extends PreTrainedModel { }
5262
+ export class DFineModel extends DFinePreTrainedModel { }
5263
+ export class DFineForObjectDetection extends DFinePreTrainedModel {
5264
+ /**
5265
+ * @param {any} model_inputs
5266
+ */
5267
+ async _call(model_inputs) {
5268
+ return new RTDetrObjectDetectionOutput(await super._call(model_inputs));
5269
+ }
5270
+ }
5226
5271
  //////////////////////////////////////////////////
5227
5272
 
5228
5273
  //////////////////////////////////////////////////
@@ -7008,7 +7053,7 @@ export class DecisionTransformerPreTrainedModel extends PreTrainedModel { }
7008
7053
 
7009
7054
  /**
7010
7055
  * The model builds upon the GPT2 architecture to perform autoregressive prediction of actions in an offline RL setting.
7011
- * Refer to the paper for more details: https://arxiv.org/abs/2106.01345
7056
+ * Refer to the paper for more details: https://huggingface.co/papers/2106.01345
7012
7057
  */
7013
7058
  export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel { }
7014
7059
 
@@ -7534,6 +7579,7 @@ const MODEL_MAPPING_NAMES_ENCODER_ONLY = new Map([
7534
7579
  ['rt_detr', ['RTDetrModel', RTDetrModel]],
7535
7580
  ['rt_detr_v2', ['RTDetrV2Model', RTDetrV2Model]],
7536
7581
  ['rf_detr', ['RFDetrModel', RFDetrModel]],
7582
+ ['d_fine', ['DFineModel', DFineModel]],
7537
7583
  ['table-transformer', ['TableTransformerModel', TableTransformerModel]],
7538
7584
  ['vit', ['ViTModel', ViTModel]],
7539
7585
  ['ijepa', ['IJepaModel', IJepaModel]],
@@ -7621,6 +7667,7 @@ const MODEL_MAPPING_NAMES_DECODER_ONLY = new Map([
7621
7667
  ['glm', ['GlmModel', GlmModel]],
7622
7668
  ['openelm', ['OpenELMModel', OpenELMModel]],
7623
7669
  ['qwen2', ['Qwen2Model', Qwen2Model]],
7670
+ ['qwen3', ['Qwen3Model', Qwen3Model]],
7624
7671
  ['phi', ['PhiModel', PhiModel]],
7625
7672
  ['phi3', ['Phi3Model', Phi3Model]],
7626
7673
  ['mpt', ['MptModel', MptModel]],
@@ -7721,6 +7768,7 @@ const MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = new Map([
7721
7768
  ['glm', ['GlmForCausalLM', GlmForCausalLM]],
7722
7769
  ['openelm', ['OpenELMForCausalLM', OpenELMForCausalLM]],
7723
7770
  ['qwen2', ['Qwen2ForCausalLM', Qwen2ForCausalLM]],
7771
+ ['qwen3', ['Qwen3ForCausalLM', Qwen3ForCausalLM]],
7724
7772
  ['phi', ['PhiForCausalLM', PhiForCausalLM]],
7725
7773
  ['phi3', ['Phi3ForCausalLM', Phi3ForCausalLM]],
7726
7774
  ['mpt', ['MptForCausalLM', MptForCausalLM]],
@@ -7835,6 +7883,7 @@ const MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES = new Map([
7835
7883
  ['rt_detr', ['RTDetrForObjectDetection', RTDetrForObjectDetection]],
7836
7884
  ['rt_detr_v2', ['RTDetrV2ForObjectDetection', RTDetrV2ForObjectDetection]],
7837
7885
  ['rf_detr', ['RFDetrForObjectDetection', RFDetrForObjectDetection]],
7886
+ ['d_fine', ['DFineForObjectDetection', DFineForObjectDetection]],
7838
7887
  ['table-transformer', ['TableTransformerForObjectDetection', TableTransformerForObjectDetection]],
7839
7888
  ['yolos', ['YolosForObjectDetection', YolosForObjectDetection]],
7840
7889
  ]);
package/src/pipelines.js CHANGED
@@ -1912,7 +1912,7 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options
1912
1912
  for (const aud of preparedAudios) {
1913
1913
  const inputs = await this.processor(aud);
1914
1914
 
1915
- // According to the [paper](https://arxiv.org/pdf/2410.15608):
1915
+ // According to the [paper](https://huggingface.co/papers/2410.15608):
1916
1916
  // "We use greedy decoding, with a heuristic limit of 6 output tokens
1917
1917
  // per second of audio to avoid repeated output sequences."
1918
1918
  const max_new_tokens = Math.floor(aud.length / sampling_rate) * 6;
package/src/tokenizers.js CHANGED
@@ -366,15 +366,19 @@ export class TokenizerModel extends Callable {
366
366
  return new BPE(config);
367
367
 
368
368
  default:
369
- // Some older tokenizers, like `google-t5/t5-small` and `distilbert/distilbert-base-uncased`, do not have a `type` field.
369
+ // Some older tokenizers, like `google-t5/t5-small`, `openai-community/gpt2`, and `distilbert/distilbert-base-uncased`, do not have a `type` field.
370
370
  // In this case, we can infer the tokenizer type based on the structure of the `vocab` field and other properties.
371
371
  if (config.vocab) {
372
372
  if (Array.isArray(config.vocab)) {
373
373
  // config.vocab is of type `[string, number][]`
374
374
  // @ts-ignore
375
375
  return new Unigram(config, ...args);
376
- } else if (typeof config.vocab === 'object' && config.continuing_subword_prefix && config.unk_token) {
377
- return new WordPieceTokenizer(config);
376
+ } else if (Object.hasOwn(config, 'continuing_subword_prefix') && Object.hasOwn(config, 'unk_token')) {
377
+ if (Object.hasOwn(config, 'merges')) {
378
+ return new BPE(config);
379
+ } else {
380
+ return new WordPieceTokenizer(config);
381
+ }
378
382
  } else {
379
383
  // @ts-ignore
380
384
  return new LegacyTokenizerModel(config, ...args);
@@ -2806,22 +2810,29 @@ export class PreTrainedTokenizer extends Callable {
2806
2810
  // For single input, we just wrap in an array, and then unwrap later.
2807
2811
  encodedTokens = [this._encode_plus(text, { text_pair, add_special_tokens, return_token_type_ids })];
2808
2812
  }
2809
- // At this point, tokens is batched: [batch_size, tokens]
2810
- // However, array may be jagged. So, we pad to max_length
2811
-
2813
+ // At this point, `encodedTokens` is batched, of shape [batch_size, tokens].
2814
+ // However, array may be jagged. So, we may need pad to max_length.
2812
2815
  if (max_length === null) {
2813
- if (padding === 'max_length') {
2816
+ max_length = this.model_max_length;
2817
+ } else if (truncation === null) {
2818
+ if (padding === true) {
2819
+ console.warn(
2820
+ "`max_length` is ignored when `padding: true` and there is no truncation strategy. " +
2821
+ "To pad to max length, use `padding: 'max_length'`."
2822
+ )
2814
2823
  max_length = this.model_max_length;
2815
- } else {
2816
- // Calculate max length from sequences
2817
- max_length = max(encodedTokens.map(x => x.input_ids.length))[0];
2818
- }
2819
- } else {
2820
- if (!truncation) {
2821
- console.warn(`Truncation was not explicitly activated but \`max_length\` is provided a specific value, please use \`truncation=true\` to explicitly truncate examples to max length.`)
2824
+ } else if (padding === false) {
2825
+ console.warn("Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation: true` to explicitly truncate examples to max length.");
2826
+ truncation = true;
2822
2827
  }
2823
2828
  }
2824
2829
 
2830
+ // padding: 'max_length' doesn't require any additional calculation
2831
+ // but padding: true has to calculate max_length from the sequences
2832
+ if (padding === true) {
2833
+ max_length = Math.min(max(encodedTokens.map(x => x.input_ids.length))[0], max_length ?? Infinity);
2834
+ }
2835
+
2825
2836
  // Ensure it is less than model max length
2826
2837
  max_length = Math.min(max_length, this.model_max_length ?? Infinity);
2827
2838
 
@@ -3508,7 +3519,7 @@ function _build_translation_inputs(self, raw_inputs, tokenizer_options, generate
3508
3519
  * between any pair of 200+ languages — including low-resource languages like Asturian,
3509
3520
  * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere,
3510
3521
  * regardless of their language preferences. For more information, check out their
3511
- * [paper](https://arxiv.org/abs/2207.04672).
3522
+ * [paper](https://huggingface.co/papers/2207.04672).
3512
3523
  *
3513
3524
  * For a list of supported languages (along with their language codes),
3514
3525
  * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200}
@@ -3539,7 +3550,7 @@ export class NllbTokenizer extends PreTrainedTokenizer {
3539
3550
  * The M2M100Tokenizer class is used to tokenize text for M2M100 ("Many-to-Many") models.
3540
3551
  *
3541
3552
  * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many
3542
- * multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125)
3553
+ * multilingual translation. It was introduced in this [paper](https://huggingface.co/papers/2010.11125)
3543
3554
  * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository.
3544
3555
  *
3545
3556
  * For a list of supported languages (along with their language codes),
@@ -40,3 +40,11 @@ export * from './models/auto/processing_auto.js';
40
40
  export * from './generation/streamers.js';
41
41
  export * from './generation/stopping_criteria.js';
42
42
  export * from './generation/logits_process.js';
43
+
44
+ // Expose common types used across the library for developers to access
45
+ /**
46
+ * @typedef {import('./utils/hub.js').PretrainedModelOptions} PretrainedModelOptions
47
+ * @typedef {import('./base/processing_utils.js').PretrainedProcessorOptions} PretrainedProcessorOptions
48
+ * @typedef {import('./utils/dtypes.js').DataType} DataType
49
+ * @typedef {import('./utils/devices.js').DeviceType} DeviceType
50
+ */
package/src/utils/hub.js CHANGED
@@ -1,7 +1,7 @@
1
1
 
2
2
  /**
3
3
  * @file Utility functions to interact with the Hugging Face Hub (https://huggingface.co/models)
4
- *
4
+ *
5
5
  * @module utils/hub
6
6
  */
7
7
 
@@ -19,7 +19,7 @@ import { dispatchCallback } from './core.js';
19
19
  export const MAX_EXTERNAL_DATA_CHUNKS = 100;
20
20
 
21
21
  /**
22
- * @typedef {Object} PretrainedOptions Options for loading a pretrained model.
22
+ * @typedef {Object} PretrainedOptions Options for loading a pretrained model.
23
23
  * @property {import('./core.js').ProgressCallback} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates.
24
24
  * @property {import('../configs.js').PretrainedConfig} [config=null] Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when:
25
25
  * - The model is a model provided by the library (loaded with the *model id* string of a pretrained model).
@@ -158,7 +158,7 @@ class FileResponse {
158
158
  /**
159
159
  * Reads the contents of the file specified by the filePath property and returns a Promise that
160
160
  * resolves with a parsed JavaScript object containing the file's contents.
161
- *
161
+ *
162
162
  * @returns {Promise<Object>} A Promise that resolves with a parsed JavaScript object containing the file's contents.
163
163
  * @throws {Error} If the file cannot be read.
164
164
  */
@@ -195,7 +195,7 @@ const REPO_ID_REGEX = /^(\b[\w\-.]+\b\/)?\b[\w\-.]{1,96}\b$/;
195
195
  /**
196
196
  * Tests whether a string is a valid Hugging Face model ID or not.
197
197
  * Adapted from https://github.com/huggingface/huggingface_hub/blob/6378820ebb03f071988a96c7f3268f5bdf8f9449/src/huggingface_hub/utils/_validators.py#L119-L170
198
- *
198
+ *
199
199
  * @param {string} string The string to test
200
200
  * @returns {boolean} True if the string is a valid model ID, false otherwise.
201
201
  */
@@ -214,9 +214,14 @@ function isValidHfModelId(string) {
214
214
  */
215
215
  export async function getFile(urlOrPath) {
216
216
 
217
- if (env.useFS && !isValidUrl(urlOrPath, ['http:', 'https:', 'blob:'])) {
218
- return new FileResponse(urlOrPath.toString());
219
-
217
+ if (env.useFS && !isValidUrl(urlOrPath, ["http:", "https:", "blob:"])) {
218
+ return new FileResponse(
219
+ urlOrPath instanceof URL
220
+ ? urlOrPath.protocol === "file:"
221
+ ? urlOrPath.pathname
222
+ : urlOrPath.toString()
223
+ : urlOrPath,
224
+ );
220
225
  } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {
221
226
  const IS_CI = !!process.env?.TESTING_REMOTELY;
222
227
  const version = env.version;
@@ -280,7 +285,7 @@ function handleError(status, remoteURL, fatal) {
280
285
  class FileCache {
281
286
  /**
282
287
  * Instantiate a `FileCache` object.
283
- * @param {string} path
288
+ * @param {string} path
284
289
  */
285
290
  constructor(path) {
286
291
  this.path = path;
@@ -288,7 +293,7 @@ class FileCache {
288
293
 
289
294
  /**
290
295
  * Checks whether the given request is in the cache.
291
- * @param {string} request
296
+ * @param {string} request
292
297
  * @returns {Promise<FileResponse | undefined>}
293
298
  */
294
299
  async match(request) {
@@ -305,8 +310,8 @@ class FileCache {
305
310
 
306
311
  /**
307
312
  * Adds the given response to the cache.
308
- * @param {string} request
309
- * @param {Response} response
313
+ * @param {string} request
314
+ * @param {Response} response
310
315
  * @param {(data: {progress: number, loaded: number, total: number}) => void} [progress_callback] Optional.
311
316
  * The function to call with progress updates
312
317
  * @returns {Promise<void>}
@@ -364,7 +369,7 @@ class FileCache {
364
369
  }
365
370
 
366
371
  /**
367
- *
372
+ *
368
373
  * @param {FileCache|Cache} cache The cache to search
369
374
  * @param {string[]} names The names of the item to search for
370
375
  * @returns {Promise<FileResponse|Response|undefined>} The item from the cache, or undefined if not found.
@@ -131,6 +131,8 @@ export class ImageProcessor extends ImageProcessor_base {
131
131
  do_crop_margin: any;
132
132
  pad_size: any;
133
133
  do_pad: any;
134
+ min_pixels: any;
135
+ max_pixels: any;
134
136
  do_flip_channel_order: boolean;
135
137
  config: ImageProcessorConfig;
136
138
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"image_processors_utils.d.ts","sourceRoot":"","sources":["../../src/base/image_processors_utils.js"],"names":[],"mappings":"AAgEA;;;;;GAKG;AACH,4EAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;AAED;;;;;;;;;GASG;AACH,uDAPG;IAAwB,MAAM,EAAtB,MAAM;IACU,UAAU,EAA1B,MAAM;CACd,cAAQ,MAAM,iBACN,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,iBAClB,OAAO,GACN,KAAQ,CAwEnB;AAGD;;;;;;GAMG;AACH,4DALW,GAAC,iBACD,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAEhB;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAC,EAAE,CAwDtD;AAkPD;;;;;;;;;GASG;AACH,4DARW,GAAC,cACD,MAAM,mBACN,MAAM,gCACN,MAAM,sBACN,GAAG,CAAC,MAAM,CAAC,iBACX,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAChB,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,KAAK,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAA;CAAC,CAAC,CAuE/G;AAGD;;;;;;;GAOG;AACH,4DANW,GAAC,cACD,MAAM,iBACN,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAEhB,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,KAAK,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAA;CAAC,CAAC,CAI/G;;KA3iBsC,GAAG;UAAyB,GACnE;;AA6iBA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH;IAgfI;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,WAKN,OAAO,iBAAiB,EAAE,iBAAiB,GAEzC,OAAO,CAAC,cAAc,CAAC,CAKnC;IAhgBD;;;OAGG;IACH,oBAFW,oBAAoB,EAyC9B;IApCG,qBAAkD;IAClD,oBAA+C;IAE/C,iBAAoC;IACpC,oBAA2C;IAC3C,uBAAwD;IACxD,sBAAuC;IAEvC,sBAAuC;IACvC,UAA4C;IAC5C,mBAA8D;IAE9D,uBAAwE;IAExE,wBAA2C;IAE3C,eAAiC;IAEjC,oBAAmD;IAEnD,oBAA2C;IAG3C,cAA+B;IAE/B,YAA2B;IAQ3B,+BAAkE;IAElE,6BAAoB;IAGxB;;;;;;;OAOG;IACH,iBALW,QAAQ,QACR;QAAC,MAAM,EAAC,MAAM,CAAC;QAAC,KAAK,EAAC,MAAM,CAAA;KAAC,aAC7B,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAC5B,OAAO,CAAC,QAAQ,CAAC,CAsB7B;IAGD;;;;;OAKG;IACH,mBAJW,QAAQ,mBACR,MAAM,GACJ,OAAO,CAAC,QAAQ,CAAC,CAiC7B;IAED;;;;;;;;;;OAUG;IACH,qBATW,YAAY,WACZ,MAAM,EAAE,WACR;QAAC,KAAK,EAAC,MAAM,CAAC;QAAC,MAAM,EAAC,MAAM,CAAA;KAAC,GAAC,MAAM,GAAC,QAAQ,uCAErD;QAAyC,IAAI,GAArC,UAAU,GAAC,WAAW;QACJ,MAAM,GAAxB,OAAO;QACmB,eAAe,GAAzC,MAAM,GAAC,MAAM,EAAE;KACvB,GAAU,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CA+EpC;IAED;;;;OAIG;IACH,mBAHW,YAAY,GACV,IAAI,CAMhB;IAED;;;;;;OAMG;IACH,oCAJW,QAAQ,QACR,GAAG,GACD,CAAC,MAAM,EAAE,MAAM,CAAC,CA8F5B;IAED;;;;OAIG;IACH,cAHW,QAAQ,GACN,OAAO,CAAC,QAAQ,CAAC,CAQ7B;IAED;;;;;OAKG;IAEH;;;;;;OAMG;IACH,kBAJW,QAAQ,iGAEN,OAAO;;;;uBAVN,WAAW;;;;6BACX,WAAW;;;;sBACX,MAAM;MAQmB,CAwHtC;IAED;;;;;;;OAOG;IACH,cAJW,QAAQ,EAAE,WACP,GAAG,EAAA,GACJ,OAAO,CAAC,oBAAoB,CAAC,CAqBzC;CAsBJ;;;;;0BAlkCY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;;;;;kBAM9B,MAAM;;;;oBACN,WAAW,EAAE;;;;0BACb,WAAW,EAAE;;;;;;;;;;;;;iBAgiBb,MAAM,EAAE;;;;gBACR,MAAM,EAAE;;;;iBACR,OAAO;;;;qBACP,MAAM;;;;mBACN,OAAO;;;;gBACP,OAAO;;;;eACP,MAAM;;;;WACN,MAAM,MAAO;;;;iBACb,MAAM,MAAO;;;;;4BACb,OAAO;;;;;qBAEP,OAAO;;;;mBAEP,OAAO;;;;;wBACP,OAAO;;;;;yBAEP,MAAM;;;;WAGN,MAAM,EAAE;;;;UACR,MAAM,EAAE;;uBAtkBqB,oBAAoB;yBAEtC,mBAAmB"}
1
+ {"version":3,"file":"image_processors_utils.d.ts","sourceRoot":"","sources":["../../src/base/image_processors_utils.js"],"names":[],"mappings":"AAgEA;;;;;GAKG;AACH,4EAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;AAED;;;;;;;;;GASG;AACH,uDAPG;IAAwB,MAAM,EAAtB,MAAM;IACU,UAAU,EAA1B,MAAM;CACd,cAAQ,MAAM,iBACN,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,iBAClB,OAAO,GACN,KAAQ,CAwEnB;AAGD;;;;;;GAMG;AACH,4DALW,GAAC,iBACD,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAEhB;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAC,EAAE,CAwDtD;AAkPD;;;;;;;;;GASG;AACH,4DARW,GAAC,cACD,MAAM,mBACN,MAAM,gCACN,MAAM,sBACN,GAAG,CAAC,MAAM,CAAC,iBACX,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAChB,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,KAAK,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAA;CAAC,CAAC,CAuE/G;AAGD;;;;;;;GAOG;AACH,4DANW,GAAC,cACD,MAAM,iBACN,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAEhB,KAAK,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,KAAK,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAA;CAAC,CAAC,CAI/G;;KA3iBsC,GAAG;UAAyB,GACnE;;AA6iBA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH;IAmfI;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,WAKN,OAAO,iBAAiB,EAAE,iBAAiB,GAEzC,OAAO,CAAC,cAAc,CAAC,CAKnC;IAngBD;;;OAGG;IACH,oBAFW,oBAAoB,EA6C9B;IAxCG,qBAAkD;IAClD,oBAA+C;IAE/C,iBAAoC;IACpC,oBAA2C;IAC3C,uBAAwD;IACxD,sBAAuC;IAEvC,sBAAuC;IACvC,UAA4C;IAC5C,mBAA8D;IAE9D,uBAAwE;IAExE,wBAA2C;IAE3C,eAAiC;IAEjC,oBAAmD;IAEnD,oBAA2C;IAG3C,cAA+B;IAE/B,YAA2B;IAE3B,gBAAmC;IAEnC,gBAAmC;IAQnC,+BAAkE;IAElE,6BAAoB;IAGxB;;;;;;;OAOG;IACH,iBALW,QAAQ,QACR;QAAC,MAAM,EAAC,MAAM,CAAC;QAAC,KAAK,EAAC,MAAM,CAAA;KAAC,aAC7B,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAC5B,OAAO,CAAC,QAAQ,CAAC,CAsB7B;IAGD;;;;;OAKG;IACH,mBAJW,QAAQ,mBACR,MAAM,GACJ,OAAO,CAAC,QAAQ,CAAC,CAiC7B;IAED;;;;;;;;;;OAUG;IACH,qBATW,YAAY,WACZ,MAAM,EAAE,WACR;QAAC,KAAK,EAAC,MAAM,CAAC;QAAC,MAAM,EAAC,MAAM,CAAA;KAAC,GAAC,MAAM,GAAC,QAAQ,uCAErD;QAAyC,IAAI,GAArC,UAAU,GAAC,WAAW;QACJ,MAAM,GAAxB,OAAO;QACmB,eAAe,GAAzC,MAAM,GAAC,MAAM,EAAE;KACvB,GAAU,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CA+EpC;IAED;;;;OAIG;IACH,mBAHW,YAAY,GACV,IAAI,CAMhB;IAED;;;;;;OAMG;IACH,oCAJW,QAAQ,QACR,GAAG,GACD,CAAC,MAAM,EAAE,MAAM,CAAC,CA6F5B;IAED;;;;OAIG;IACH,cAHW,QAAQ,GACN,OAAO,CAAC,QAAQ,CAAC,CAQ7B;IAED;;;;;OAKG;IAEH;;;;;;OAMG;IACH,kBAJW,QAAQ,iGAEN,OAAO;;;;uBAVN,WAAW;;;;6BACX,WAAW;;;;sBACX,MAAM;MAQmB,CAwHtC;IAED;;;;;;;OAOG;IACH,cAJW,QAAQ,EAAE,WACP,GAAG,EAAA,GACJ,OAAO,CAAC,oBAAoB,CAAC,CAqBzC;CAsBJ;;;;;0BArkCY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;;;;;kBAM9B,MAAM;;;;oBACN,WAAW,EAAE;;;;0BACb,WAAW,EAAE;;;;;;;;;;;;;iBAgiBb,MAAM,EAAE;;;;gBACR,MAAM,EAAE;;;;iBACR,OAAO;;;;qBACP,MAAM;;;;mBACN,OAAO;;;;gBACP,OAAO;;;;eACP,MAAM;;;;WACN,MAAM,MAAO;;;;iBACb,MAAM,MAAO;;;;;4BACb,OAAO;;;;;qBAEP,OAAO;;;;mBAEP,OAAO;;;;;wBACP,OAAO;;;;;yBAEP,MAAM;;;;WAGN,MAAM,EAAE;;;;UACR,MAAM,EAAE;;uBAtkBqB,oBAAoB;yBAEtC,mBAAmB"}
@@ -1 +1 @@
1
- {"version":3,"file":"configs.d.ts","sourceRoot":"","sources":["../src/configs.js"],"names":[],"mappings":"AA4PA;;;;GAIG;AACH,0CAHW,gBAAgB;;;IACd,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA2EpC;AACD;;;GAGG;AACH;IAwBI;;;;;;;;OAQG;IACH,sDANW,MAAM,0EACN,iBAAiB,GAGf,OAAO,CAAC,gBAAgB,CAAC,CAqBrC;IArCD;;;OAGG;IACH,6BAGC;IAnBD,0BAA0B;IAC1B,YADW,MAAM,GAAC,IAAI,CACJ;IAElB,sBAAsB;IACtB,oBADW,OAAO,CACS;IAE3B,qBAAqB;IACrB,yBADW,MAAM,CACO;IAExB,mCAAmC;IACnC,0BADW,oBAAoB,CACN;IAQrB,uBAAkD;CAgCzD;AAED;;;;;GAKG;AACH;IArCI;;;;;;;;OAQG;IACH,sDANW,MAAM,0EACN,iBAAiB,GAGf,OAAO,CAAC,gBAAgB,CAAC,CAqBrC;CAcJ;gCA9WY,OAAO,gBAAgB,EAAE,iBAAiB;+BAI1C,OAAO,iBAAiB,EAAE,gBAAgB;2BAI1C,OAAO,iBAAiB,EAAE,YAAY;;;;;;;;oBA2WrC,MAAM,CAAC,OAAO,oBAAoB,EAAE,UAAU,EAAE,YAAY,CAAC;;;;qBAC7D,OAAO,mBAAmB,EAAE,QAAQ,GAAC,MAAM,CAAC,OAAO,mBAAmB,EAAE,QAAQ,EAAE,OAAO,mBAAmB,EAAE,QAAQ,CAAC;;;;;;+BACvH,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;;;;aAGtB,OAAO,oBAAoB,EAAE,UAAU;;;;YACvC,OAAO,mBAAmB,EAAE,QAAQ,GAAC,MAAM,CAAC,MAAM,EAAE,OAAO,mBAAmB,EAAE,QAAQ,CAAC;;;;+BACzF,OAAO,gBAAgB,EAAE,YAAY,GAAC,MAAM,CAAC,MAAM,EAAE,OAAO,gBAAgB,EAAE,YAAY,CAAC;;;;;2BAK5F,IAAI,CAAC,oBAAoB,EAAE,QAAQ,GAAG,eAAe,CAAC"}
1
+ {"version":3,"file":"configs.d.ts","sourceRoot":"","sources":["../src/configs.js"],"names":[],"mappings":"AA6PA;;;;GAIG;AACH,0CAHW,gBAAgB;;;IACd,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA2EpC;AACD;;;GAGG;AACH;IAwBI;;;;;;;;OAQG;IACH,sDANW,MAAM,0EACN,iBAAiB,GAGf,OAAO,CAAC,gBAAgB,CAAC,CAqBrC;IArCD;;;OAGG;IACH,6BAGC;IAnBD,0BAA0B;IAC1B,YADW,MAAM,GAAC,IAAI,CACJ;IAElB,sBAAsB;IACtB,oBADW,OAAO,CACS;IAE3B,qBAAqB;IACrB,yBADW,MAAM,CACO;IAExB,mCAAmC;IACnC,0BADW,oBAAoB,CACN;IAQrB,uBAAkD;CAgCzD;AAED;;;;;GAKG;AACH;IArCI;;;;;;;;OAQG;IACH,sDANW,MAAM,0EACN,iBAAiB,GAGf,OAAO,CAAC,gBAAgB,CAAC,CAqBrC;CAcJ;gCA/WY,OAAO,gBAAgB,EAAE,iBAAiB;+BAI1C,OAAO,iBAAiB,EAAE,gBAAgB;2BAI1C,OAAO,iBAAiB,EAAE,YAAY;;;;;;;;oBA4WrC,MAAM,CAAC,OAAO,oBAAoB,EAAE,UAAU,EAAE,YAAY,CAAC;;;;qBAC7D,OAAO,mBAAmB,EAAE,QAAQ,GAAC,MAAM,CAAC,OAAO,mBAAmB,EAAE,QAAQ,EAAE,OAAO,mBAAmB,EAAE,QAAQ,CAAC;;;;;;+BACvH,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;;;;aAGtB,OAAO,oBAAoB,EAAE,UAAU;;;;YACvC,OAAO,mBAAmB,EAAE,QAAQ,GAAC,MAAM,CAAC,MAAM,EAAE,OAAO,mBAAmB,EAAE,QAAQ,CAAC;;;;+BACzF,OAAO,gBAAgB,EAAE,YAAY,GAAC,MAAM,CAAC,MAAM,EAAE,OAAO,gBAAgB,EAAE,YAAY,CAAC;;;;;2BAK5F,IAAI,CAAC,oBAAoB,EAAE,QAAQ,GAAG,eAAe,CAAC"}
@@ -65,7 +65,7 @@ export class GenerationConfig {
65
65
  num_beams: number;
66
66
  /**
67
67
  * Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
68
- * See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
68
+ * See [this paper](https://huggingface.co/papers/1610.02424) for more details.
69
69
  * @type {number}
70
70
  * @default 1
71
71
  */
@@ -103,7 +103,7 @@ export class GenerationConfig {
103
103
  /**
104
104
  * Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated.
105
105
  * If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation.
106
- * See [this paper](https://arxiv.org/pdf/2202.00666.pdf) for more details.
106
+ * See [this paper](https://huggingface.co/papers/2202.00666) for more details.
107
107
  * @type {number}
108
108
  * @default 1.0
109
109
  */
@@ -111,7 +111,7 @@ export class GenerationConfig {
111
111
  /**
112
112
  * If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled.
113
113
  * In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model.
114
- * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
114
+ * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
115
115
  * @type {number}
116
116
  * @default 0.0
117
117
  */
@@ -120,7 +120,7 @@ export class GenerationConfig {
120
120
  * Eta sampling is a hybrid of locally typical sampling and epsilon sampling.
121
121
  * If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`.
122
122
  * The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
123
- * See [Truncation Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more details.
123
+ * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
124
124
  * @type {number}
125
125
  * @default 0.0
126
126
  */
@@ -134,7 +134,7 @@ export class GenerationConfig {
134
134
  diversity_penalty: number;
135
135
  /**
136
136
  * The parameter for repetition penalty. 1.0 means no penalty.
137
- * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
137
+ * See [this paper](https://huggingface.co/papers/1909.05858) for more details.
138
138
  * @type {number}
139
139
  * @default 1.0
140
140
  */
@@ -191,7 +191,7 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
191
191
  * This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs,
192
192
  * the considered tokens include the prompt.
193
193
  *
194
- * In the original [paper](https://arxiv.org/pdf/1909.05858.pdf), the authors suggest the use of a
194
+ * In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a
195
195
  * penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition.
196
196
  * To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes
197
197
  * more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where
@@ -279,7 +279,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor {
279
279
  * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a
280
280
  * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`.
281
281
  *
282
- * See [the paper](https://arxiv.org/abs/2306.05284) for more information.
282
+ * See [the paper](https://huggingface.co/papers/2306.05284) for more information.
283
283
  */
284
284
  export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor {
285
285
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"streamers.d.ts","sourceRoot":"","sources":["../../src/generation/streamers.js"],"names":[],"mappings":"AASA;IACI;;;OAGG;IACH,WAFW,MAAM,EAAE,EAAE,QAIpB;IAED;;OAEG;IACH,YAEC;CACJ;AAMD;;GAEG;AACH;IACI;;;;;;;;;OASG;IACH,uBARW,OAAO,kBAAkB,EAAE,mBAAmB,+GAEtD;QAA0B,WAAW,GAA7B,OAAO;QACW,mBAAmB,GAArC,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACP,aAAa;KACxC,EAoBA;IAVG,0DAA0B;IAC1B,qBAA8B;IAC9B,oCAA0D;IAC1D,gCAfgB,MAAM,EAAE,KAAG,IAAI,CAeuB;IACtD,mBAAyE;IAGzE,mBAAqB;IACrB,kBAAkB;IAClB,gCAAkC;IA8DtC;;;;OAIG;IACH,wBAHW,MAAM,cACN,OAAO,QASjB;CACJ;AAED;;;;;;;GAOG;AACH;IACI;;;;;;;;;;;;OAYG;IACH,uBAZW,OAAO,kBAAkB,EAAE,gBAAgB,gKAEnD;QAA0B,WAAW,GAA7B,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACS,cAAc,GAA/C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACW,YAAY,GAA7C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACK,WAAW,GAAtC,MAAY,IAAI;QACC,cAAc,GAA/B,MAAM;QACY,mBAAmB,GAArC,OAAO;QACU,aAAa;KACxC,EA4BA;IATG,wBAAgD;IAEhD,uBA3BgB,MAAM,KAAG,IAAI,CA2BO;IACpC,qBA3BgB,MAAM,KAAG,IAAI,CA2BG;IAChC,mBA3BmB,IAAI,CA2BO;IAE9B,uBAAoC;IAEpC,+BAAkC;CAiCzC"}
1
+ {"version":3,"file":"streamers.d.ts","sourceRoot":"","sources":["../../src/generation/streamers.js"],"names":[],"mappings":"AASA;IACI;;;OAGG;IACH,WAFW,MAAM,EAAE,EAAE,QAIpB;IAED;;OAEG;IACH,YAEC;CACJ;AAMD;;GAEG;AACH;IACI;;;;;;;;;OASG;IACH,uBARW,OAAO,kBAAkB,EAAE,mBAAmB,+GAEtD;QAA0B,WAAW,GAA7B,OAAO;QACW,mBAAmB,GAArC,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACP,aAAa;KACxC,EAoBA;IAVG,0DAA0B;IAC1B,qBAA8B;IAC9B,oCAA0D;IAC1D,gCAfgB,MAAM,EAAE,KAAG,IAAI,CAeuB;IACtD,mBAAyE;IAGzE,mBAAqB;IACrB,kBAAkB;IAClB,gCAAkC;IA8DtC;;;;OAIG;IACH,wBAHW,MAAM,cACN,OAAO,QASjB;CACJ;AAED;;;;;;;GAOG;AACH;IACI;;;;;;;;;;;;OAYG;IACH,uBAZW,OAAO,kBAAkB,EAAE,gBAAgB,gKAEnD;QAA0B,WAAW,GAA7B,OAAO;QAC0B,iBAAiB,GAAlD,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACa,uBAAuB,GAA1D,CAAS,IAAQ,EAAR,MAAM,EAAE,KAAG,IAAI;QACS,cAAc,GAA/C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACW,YAAY,GAA7C,CAAS,IAAM,EAAN,MAAM,KAAG,IAAI;QACK,WAAW,GAAtC,MAAY,IAAI;QACC,cAAc,GAA/B,MAAM;QACY,mBAAmB,GAArC,OAAO;QACU,aAAa;KACxC,EA4BA;IATG,wBAAgD;IAEhD,uBA3BgB,MAAM,KAAG,IAAI,CA2BO;IACpC,qBA3BgB,MAAM,KAAG,IAAI,CA2BG;IAChC,mBA3BmB,IAAI,CA2BO;IAE9B,uBAAoC;IAEpC,+BAAkC;CAqCzC"}
@@ -1 +1 @@
1
- {"version":3,"file":"image_processing_auto.d.ts","sourceRoot":"","sources":["../../../src/models/auto/image_processing_auto.js"],"names":[],"mappings":"AAMA;kFAuBq9yC,oBAAiB;CADr+yC;+BAzB8B,sCAAsC"}
1
+ {"version":3,"file":"image_processing_auto.d.ts","sourceRoot":"","sources":["../../../src/models/auto/image_processing_auto.js"],"names":[],"mappings":"AAMA;kFAuB0kzC,oBAAiB;CAD1lzC;+BAzB8B,sCAAsC"}