@huggingface/transformers 3.0.0-alpha.16 → 3.0.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/ort-wasm-simd-threaded.jsep.wasm +0 -0
- package/dist/transformers.cjs +12 -7
- package/dist/transformers.cjs.map +1 -1
- package/dist/transformers.js +571 -635
- package/dist/transformers.js.map +1 -1
- package/dist/transformers.min.cjs +8 -8
- package/dist/transformers.min.cjs.map +1 -1
- package/dist/transformers.min.js +37 -37
- package/dist/transformers.min.js.map +1 -1
- package/dist/transformers.min.mjs +2 -2
- package/dist/transformers.min.mjs.map +1 -1
- package/dist/transformers.mjs +12 -7
- package/dist/transformers.mjs.map +1 -1
- package/package.json +2 -2
- package/src/env.js +1 -1
- package/src/tokenizers.js +11 -6
- package/types/tokenizers.d.ts.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@huggingface/transformers",
|
|
3
|
-
"version": "3.0.0-alpha.
|
|
3
|
+
"version": "3.0.0-alpha.18",
|
|
4
4
|
"description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!",
|
|
5
5
|
"main": "./src/transformers.js",
|
|
6
6
|
"types": "./types/transformers.d.ts",
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
"dependencies": {
|
|
64
64
|
"@huggingface/jinja": "^0.3.0",
|
|
65
65
|
"onnxruntime-node": "1.19.2",
|
|
66
|
-
"onnxruntime-web": "1.20.0-dev.
|
|
66
|
+
"onnxruntime-web": "1.20.0-dev.20240928-1bda91fc57",
|
|
67
67
|
"sharp": "^0.33.5"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
package/src/env.js
CHANGED
|
@@ -26,7 +26,7 @@ import fs from 'fs';
|
|
|
26
26
|
import path from 'path';
|
|
27
27
|
import url from 'url';
|
|
28
28
|
|
|
29
|
-
const VERSION = '3.0.0-alpha.
|
|
29
|
+
const VERSION = '3.0.0-alpha.18';
|
|
30
30
|
|
|
31
31
|
// Check if various APIs are available (depends on environment)
|
|
32
32
|
const IS_BROWSER_ENV = typeof self !== 'undefined';
|
package/src/tokenizers.js
CHANGED
|
@@ -671,7 +671,7 @@ class BPE extends TokenizerModel {
|
|
|
671
671
|
* Create a BPE instance.
|
|
672
672
|
* @param {Object} config The configuration object for BPE.
|
|
673
673
|
* @param {Object} config.vocab A mapping of tokens to ids.
|
|
674
|
-
* @param {string[]} config.merges An array of BPE merges as strings.
|
|
674
|
+
* @param {string[]|[string, string][]} config.merges An array of BPE merges as strings.
|
|
675
675
|
* @param {string} config.unk_token The unknown token used for out of vocabulary words.
|
|
676
676
|
* @param {string} config.end_of_word_suffix The suffix to place at the end of each word.
|
|
677
677
|
* @param {string} [config.continuing_subword_suffix] The suffix to insert between words.
|
|
@@ -681,8 +681,6 @@ class BPE extends TokenizerModel {
|
|
|
681
681
|
constructor(config) {
|
|
682
682
|
super(config);
|
|
683
683
|
|
|
684
|
-
this.BPE_SPLIT_TOKEN = ' ';
|
|
685
|
-
|
|
686
684
|
/** @type {Map<string, number>} */
|
|
687
685
|
this.tokens_to_ids = objectToMap(config.vocab);
|
|
688
686
|
|
|
@@ -694,8 +692,15 @@ class BPE extends TokenizerModel {
|
|
|
694
692
|
this.vocab[value] = key;
|
|
695
693
|
}
|
|
696
694
|
|
|
697
|
-
|
|
698
|
-
|
|
695
|
+
// Tokenizers >= 0.20.0 serializes BPE merges as a [string, string][] instead of a string[],
|
|
696
|
+
// which resolves the ambiguity for merges containing spaces.
|
|
697
|
+
const use_new_merge_format = Array.isArray(config.merges[0]);
|
|
698
|
+
|
|
699
|
+
/** @type {[string, string][]} */
|
|
700
|
+
this.merges = use_new_merge_format
|
|
701
|
+
? /** @type {[string, string][]} */(config.merges)
|
|
702
|
+
: (/** @type {string[]} */(config.merges)).map(x => /** @type {[string, string]} */(x.split(' ', 2)));
|
|
703
|
+
this.bpe_ranks = new Map(this.merges.map((x, i) => [JSON.stringify(x), i]));
|
|
699
704
|
|
|
700
705
|
this.end_of_word_suffix = config.end_of_word_suffix;
|
|
701
706
|
|
|
@@ -855,7 +860,7 @@ class BPE extends TokenizerModel {
|
|
|
855
860
|
// `score` is a measure of the merge priority: lower means higher priority
|
|
856
861
|
// We use the BPE rank as a measure of priority (i.e., the local of the merge in the merges list)
|
|
857
862
|
// We also add a fractional component to the score to break ties (with the earlier character having higher priority)
|
|
858
|
-
const rank = this.bpe_ranks.get(node.token
|
|
863
|
+
const rank = this.bpe_ranks.get(JSON.stringify([node.token, node.next.token]));
|
|
859
864
|
if (rank !== undefined) {
|
|
860
865
|
node.score = rank + node.bias;
|
|
861
866
|
queue.push(node);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.js"],"names":[],"mappings":"AAmNA;;;;;;;;;;;;;GAaG;AACH,oCAHW,MAAM,GAAC,MAAM,GACX,OAAO,CAanB;;;;;AA8ED;;;;GAIG;AACH;IA0BI;;;;;;OAMG;IACH,gDAHa,cAAc,CAqB1B;IAlDD;;;OAGG;IACH,yBAmBC;IAjBG,YAAoB;IAEpB,uBAAuB;IACvB,OADW,MAAM,EAAE,CACJ;IAEf;;;OAGG;IACH,eAFU,IAAI,MAAM,EAAE,MAAM,CAAC,CAEC;IAE9B,kBAA6B;IAC7B,eAA0B;IAC1B,wBAAmC;IAEnC,uFAAuF;IACvF,UADW,OAAO,CAC2B;IA8BjD;;;;OAIG;IACH,cAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;IAED;;;;;OAKG;IACH,eAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;IAED;;;;OAIG;IACH,8BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;IAED;;;;OAIG;IACH,2BAHW,MAAM,EAAE,GAAC,MAAM,EAAE,GACf,MAAM,EAAE,CAIpB;CACJ;;;;;
|
|
1
|
+
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.js"],"names":[],"mappings":"AAmNA;;;;;;;;;;;;;GAaG;AACH,oCAHW,MAAM,GAAC,MAAM,GACX,OAAO,CAanB;;;;;AA8ED;;;;GAIG;AACH;IA0BI;;;;;;OAMG;IACH,gDAHa,cAAc,CAqB1B;IAlDD;;;OAGG;IACH,yBAmBC;IAjBG,YAAoB;IAEpB,uBAAuB;IACvB,OADW,MAAM,EAAE,CACJ;IAEf;;;OAGG;IACH,eAFU,IAAI,MAAM,EAAE,MAAM,CAAC,CAEC;IAE9B,kBAA6B;IAC7B,eAA0B;IAC1B,wBAAmC;IAEnC,uFAAuF;IACvF,UADW,OAAO,CAC2B;IA8BjD;;;;OAIG;IACH,cAHW,MAAM,EAAE,GACN,MAAM,EAAE,CASpB;IAED;;;;;OAKG;IACH,eAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;IAED;;;;OAIG;IACH,8BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;IAED;;;;OAIG;IACH,2BAHW,MAAM,EAAE,GAAC,MAAM,EAAE,GACf,MAAM,EAAE,CAIpB;CACJ;;;;;AAijED;;;;GAIG;AAEH;IAqII;;;;;;;;OAQG;IACH,sDANW,MAAM,kFACN,0BAA0B,GAGxB,QAAQ,mBAAmB,CAAC,CAsBxC;IA9JD;;;;OAIG;IACH,sDAgGC;IAxGD,+BAA8B;IAE9B,qBAAuB;IASnB,uBAAwC;IAGxC,uBAAiE;IACjE,4BAAyE;IACzE,sBAA4E;IAC5E,8BAA4E;IAC5E,iBAAwD;IAGxD,sBAAwB;IACxB,0BAAyB;IAEzB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IAetB,+BAAgF;IAehF,2BAMQ;IAGR,mBAA6C;IAC7C,sBAAkE;IAElE,kBAAwD;IACxD,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,kBAA2C;IAC3C,qBAAgE;IAEhE,sBAAwD;IAExD,6HAA6H;IAC7H,cADW,OAAO,CAC8B;IAEhD,kCAAwF;IACxF,oCAA6F;IAM7F,gBAAmB;IAEnB,mBAA0D;IAa1D,wCAAyC;IAG7C;;;;;;OAMG;IACH,iBAiBC;IAiCD;;;;;;;OAOG;IAEH;;;;;;;;;;;;OAYG;IACH,YAXW,MAAM,GAAC,MAAM,EAAE;QAEW,SAAS,GAAnC,MAAM,GAAC,MAAM,EAAE;QACgB,OAAO,GAAtC,OAAO,GAAC,YAAY;QACF,kBAAkB,GAApC,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QACY,aAAa,GAA/B,OAAO;QACW,qBAAqB,GAAvC,OAAO;;;;;;;;;;;;;;MA0JjB;IAED;;;;;OAKG;IACH,mBAHW,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,GAAC,IAAI,CA4CzB;IAED;;;;;;;;;;OAUG;IACH,qBAkBC;IAED;;;;;;;OAOG;IACH,uBANW,MAAM;QAEW,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;;gBACI,MAAM,EAAE;yBAAmB,MAAM,EAAE;MAYxD;IAED;;;;;;;OAOG;IACH,eANW,MAAM;QAEW,IAAI,GAArB,MAAM;QACY,kBAAkB,GAApC,OAAO;QACL,MAAM,EAAE,CAOpB;IAED;;;;;;;;;OASG;IACH,aAPW,MAAM;QAEW,SAAS,GAA1B,MAAM;QACY,kBAAkB,GAApC,OAAO;QACW,qBAAqB,GAAvC,OAAO;QACL,MAAM,EAAE,CAYpB;IAED;;;;;OAKG;IACH,oBAJW,MAAM,EAAE,EAAE,GAAC,MAAM,sBAEf,MAAM,EAAE,CAOpB;IAED;;;;;;;;;;OAUG;IACH,kBARW,MAAM,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM;QAEF,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;QAEL,MAAM,CAgBlB;IAED;;;;;;;;OAQG;IACH,yBAPW,MAAM,EAAE,GAAC,MAAM,EAAE;QAEK,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;QAEL,MAAM,CAiClB;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH;QAZ4B,aAAa,GAA9B,MAAM;QAIa,KAAK,GAAxB,KAAQ;QAMN,MAAM,CA0ClB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyDG;IACH,kCA/BW,OAAO,EAAE;QAGQ,aAAa,GAA9B,MAAM;QAEa,KAAK,GAAxB,KAAQ;QAM2B,SAAS,GAA5C,OAAO,MAAM,EAAE,MAAM,CAAC,EAAE;QAMN,qBAAqB,GAAvC,OAAO;QAIW,QAAQ,GAA1B,OAAO;QACW,OAAO,GAAzB,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QAEY,aAAa,GAA/B,OAAO;QACW,WAAW,GAA7B,OAAO;QACU,gBAAgB;;;;;;;;;;;;;;MA6D3C;CACJ;AAED;;;GAGG;AACH;CAEC;AACD;;;GAGG;AACH;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAEC;AACD;CAAgE;AAChE;CAA+D;AAC/D;IAGI,sDAGC;CACJ;AACD;CAEC;AAED;CAAwD;AACxD;CAA0D;AAC1D;CAA0D;AAC1D;IACI,sDAMC;IAHG,sBAA0C;IAC1C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AACD;CAAwD;AAExD;CAA6D;AAE7D;IAEI,sDASC;CACJ;AAID;IAII,sDAaC;IAVG,YAA4C;CA+BnD;AACD;CAA+D;AAE/D;CAAgE;AAChE;CAA2D;AAE3D;CAA4D;AAE5D;CAA6D;AAE7D;CAAyD;AAEzD;CAA2D;AAE3D;CAA2D;AAE3D;CAA2D;AAqD3D;;;;;;;;;;;;GAYG;AACH;IAEI,sDAMC;IAHG,sBAA+C;IAC/C,sBAAiF;IACjF,+BAA2B;IAG/B;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;;;;;;;GASG;AACH;IACI,sDAQC;IALG,sBAAuC;IACvC,sBAE6B;IAC7B,kCAAoC;IAGxC;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAOzB;CACJ;AAED;;;GAGG;AACH;IAEI,8BAEC;IAED;;;;;OAKG;IACH,uBAJW,MAAM;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAC,CAAC;iBAE/C,SAAS,GAAC,MAAM;YAAC,QAAQ,EAAE,MAAM,GAAC,IAAI,CAAC;YAAC,SAAS,EAAE,MAAM,MAAM,GAAC,IAAI,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAC,CAAC;SAoSxH;IAED;;;;;;OAMG;IACH,kCAsGC;IAED,eAAe;IACf,8BAgBC;IAED;;;;;;;;;OASG;IACH,+BAaC;IAwBD;;;;OAIG;IACH,6BAqBC;IAED;;;;;OAKG;IACH,6BAqCC;IAED;;;;OAIG;IACH,4BAoCC;IAED;;;;;;;;OAQG;IACH,0BA+CC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH;QAV4B,QAAQ,GAAzB,MAAM;QAIW,IAAI,GAArB,MAAM;QAGY,aAAa,GAA/B,OAAO;QACL,MAAM,EAAE,EAAE,CAsDtB;CACJ;AACD;CAA6D;AAC7D;CAA0D;AAC1D;CAA4D;AAE5D;;;GAGG;AACH;IASQ,sBAAqC;IAErC,mCAEC;IAKL;;;;;;;OAOG;IACH,mBAHW,MAAM,GAAC,IAAI,SAsBrB;CAEJ;AAED;CAAiE;AAEjE;CAAgE;AAChE;CAAqE;AAErE;CAA8D;AAE9D;CAA4D;AAE5D;IAEI,sDAKC;CACJ;AAED;CAA4D;AAE5D;;;;;;GAMG;AACH;IACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAgDC;IAGD;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,kFAKN,0BAA0B,GAExB,QAAQ,mBAAmB,CAAC,CA6BxC;CACJ;;;;;;;;aA9xIa,OAAO;;yCACR,OAAO,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB;;;;;WAwlB/D,MAAM;;;;UACN,MAAM;;;;YACN,MAAM;;;;WACN,OAAO;;;;WACP,OAAO;;qCA0yBR,SAAS,GAAC,UAAU,GAAC,oBAAoB,GAAC,gBAAgB,GAAC,YAAY;;;;;YAqGtE,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;eAMR,MAAM,EAAE;;;;oBACR,MAAM,EAAE;;;;qBACR,MAAM,EAAE;;;;;;UAm6BR,MAAM;;;;aACN,MAAM;;;;;;AAxhDpB;;;GAGG;AACH;IASI;;;;;;OAMG;IACH,gCAHa,UAAU,CA+BtB;IA3CD;;OAEG;IACH,yBAGC;IADG,YAAoB;IAwCxB;;;;;;OAMG;IACH,gBAJW,MAAM,GACJ,MAAM,CAKlB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,MAAM,CAIlB;CAEJ;;;;;AAsRD;;;;GAIG;AACH;IACI;;;;;;;KAOC;IACD,gCAHW,YAAY,CA+BtB;IAED;;;;;;;;OAQG;IACH,wBALW,MAAM,kBAEJ,MAAM,EAAE,CAKpB;IAED;;;;;OAKG;IACH,mBAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAOpB;IAED;;;;;OAKG;IACH,YAJW,MAAM,GAAC,MAAM,EAAE,kBAEb,MAAM,EAAE,CAIpB;CACJ;;;;;AA8LD;;;;GAIG;AAGH;;;;;GAKG;AAGH;;GAEG;AACH;IAUI;;;;;;OAMG;IACH,gCAHa,aAAa,CAsBzB;IAlCD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA+BxB;;;;;;;OAOG;IACH,6CAHa,mBAAmB,CAK/B;IAED;;;;;OAKG;IACH,sCAFa,mBAAmB,CAI/B;CACJ;;;;;AAoKD;;;GAGG;AACH;IAiBI;;;;;;KAMC;IACD,gCAHW,OAAO,CAgCjB;IAnDD;;;;MAIE;IACF,yBAQC;IANG,YAAoB;IAEpB,2BAA2B;IAC3B,cADW,UAAU,EAAE,CACD;IACtB,wBAA8B;IAC9B,kBAAuC;IAyC3C;;;;;MAKE;IACF,cAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;MAIE;IACF,eAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;;;OAMG;IACH,qBAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;CAEJ;AA/kDD;;;;;GAKG;AACH;IACI;;;;;;;;;;OAUG;IACH;QAR0B,OAAO,EAAtB,MAAM;QACS,EAAE,EAAjB,MAAM;QACW,WAAW,GAA5B,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,MAAM,GAAvB,OAAO;QACU,UAAU,GAA3B,OAAO;QACU,OAAO,GAAxB,OAAO;OAUjB;IAPG,gBAA6B;IAC7B,WAAmB;IACnB,qBAA8C;IAC9C,gBAAoC;IACpC,gBAAoC;IACpC,iBAAsC;IACtC,oBAA2C;CAElD;uBAlRsB,mBAAmB"}
|