gpt-tokenizer 2.0.0-beta.1 → 2.0.0

package/cjs/modelParams.js

@@ -1,18 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getModelParamsAsync = exports.getModelParams = void 0;
-const EndOfText = '<|endoftext|>';
-const FimPrefix = '<|fim_prefix|>';
-const FimMiddle = '<|fim_middle|>';
-const FimSuffix = '<|fim_suffix|>';
-const EndOfPrompt = '<|endofprompt|>';
+const specialTokens_js_1 = require("./specialTokens.js");
 const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
 function R50KBase(mergeableBytePairRanks) {
     return {
         expectedVocabularySize: 50257,
         tokenSplitRegex,
         mergeableBytePairRanks,
-        specialTokenMapping: new Map([[EndOfText, 50256]]),
+        specialTokenMapping: new Map([[specialTokens_js_1.EndOfText, 50256]]),
     };
 }
 function P50KBase(mergeableBytePairRanks) {
@@ -20,15 +16,15 @@ function P50KBase(mergeableBytePairRanks) {
         expectedVocabularySize: 50281,
         tokenSplitRegex,
         mergeableBytePairRanks,
-        specialTokenMapping: new Map([[EndOfText, 50256]]),
+        specialTokenMapping: new Map([[specialTokens_js_1.EndOfText, 50256]]),
     };
 }
 function P50KEdit(mergeableBytePairRanks) {
     const specialTokenMapping = new Map([
-        [EndOfText, 50256],
-        [FimPrefix, 50281],
-        [FimMiddle, 50282],
-        [FimSuffix, 50283],
+        [specialTokens_js_1.EndOfText, 50256],
+        [specialTokens_js_1.FimPrefix, 50281],
+        [specialTokens_js_1.FimMiddle, 50282],
+        [specialTokens_js_1.FimSuffix, 50283],
     ]);
     return {
         tokenSplitRegex,
@@ -38,11 +34,11 @@ function P50KEdit(mergeableBytePairRanks) {
 }
 function Cl100KBase(mergeableBytePairRanks) {
     const specialTokenMapping = new Map([
-        [EndOfText, 100257],
-        [FimPrefix, 100258],
-        [FimMiddle, 100259],
-        [FimSuffix, 100260],
-        [EndOfPrompt, 100276],
+        [specialTokens_js_1.EndOfText, 100257],
+        [specialTokens_js_1.FimPrefix, 100258],
+        [specialTokens_js_1.FimMiddle, 100259],
+        [specialTokens_js_1.FimSuffix, 100260],
+        [specialTokens_js_1.EndOfPrompt, 100276],
     ]);
     return {
         tokenSplitRegex: /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,

package/cjs/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAqBA,MAAM,SAAS,GAAG,eAAe,CAAA;AACjC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,WAAW,GAAG,iBAAiB,CAAA;AAErC,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,wCAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC;AAND,kDAMC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAGA,yDAM2B;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,8BAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,wCAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC;AAND,kDAMC"}

package/cjs/specialTokens.d.ts

@@ -0,0 +1,5 @@
+export declare const EndOfText = "<|endoftext|>";
+export declare const FimPrefix = "<|fim_prefix|>";
+export declare const FimMiddle = "<|fim_middle|>";
+export declare const FimSuffix = "<|fim_suffix|>";
+export declare const EndOfPrompt = "<|endofprompt|>";

package/cjs/specialTokens.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EndOfPrompt = exports.FimSuffix = exports.FimMiddle = exports.FimPrefix = exports.EndOfText = void 0;
+exports.EndOfText = '<|endoftext|>';
+exports.FimPrefix = '<|fim_prefix|>';
+exports.FimMiddle = '<|fim_middle|>';
+exports.FimSuffix = '<|fim_suffix|>';
+exports.EndOfPrompt = '<|endofprompt|>';
+//# sourceMappingURL=specialTokens.js.map

package/cjs/specialTokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":";;;AAAa,QAAA,SAAS,GAAG,eAAe,CAAA;AAC3B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,WAAW,GAAG,iBAAiB,CAAA"}