gpt-tokenizer 2.0.0 → 2.1.1

package/cjs/modelParams.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getModelParamsAsync = exports.getModelParams = void 0;
+exports.getModelParamsAsync = exports.getEncodingParams = void 0;
 const specialTokens_js_1 = require("./specialTokens.js");
 const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
 function R50KBase(mergeableBytePairRanks) {
@@ -38,6 +38,9 @@ function Cl100KBase(mergeableBytePairRanks) {
         [specialTokens_js_1.FimPrefix, 100258],
         [specialTokens_js_1.FimMiddle, 100259],
         [specialTokens_js_1.FimSuffix, 100260],
+        [specialTokens_js_1.ImStart, 100264],
+        [specialTokens_js_1.ImEnd, 100265],
+        [specialTokens_js_1.ImSep, 100266],
         [specialTokens_js_1.EndOfPrompt, 100276],
     ]);
     return {
@@ -46,7 +49,7 @@ function Cl100KBase(mergeableBytePairRanks) {
         specialTokenMapping,
     };
 }
-function getModelParams(encodingName, getMergeableRanks) {
+function getEncodingParams(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = getMergeableRanks(encodingName);
     switch (encodingName.toLowerCase()) {
         case 'r50k_base':
@@ -61,10 +64,10 @@ function getModelParams(encodingName, getMergeableRanks) {
             throw new Error(`Unknown encoding name: ${encodingName}`);
     }
 }
-exports.getModelParams = getModelParams;
+exports.getEncodingParams = getEncodingParams;
 async function getModelParamsAsync(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = await getMergeableRanks(encodingName);
-    return getModelParams(encodingName, () => mergeableBytePairRanks);
+    return getEncodingParams(encodingName, () => mergeableBytePairRanks);
 }
 exports.getModelParamsAsync = getModelParamsAsync;
 //# sourceMappingURL=modelParams.js.map

package/cjs/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAGA,yDAM2B;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,8BAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,wCAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC;AAND,kDAMC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAGA,yDAS2B;AAoB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,0BAAO,EAAE,MAAO,CAAC;QAClB,CAAC,wBAAK,EAAE,MAAO,CAAC;QAChB,CAAC,wBAAK,EAAE,MAAO,CAAC;QAChB,CAAC,8BAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,8CAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC;AAND,kDAMC"}

package/cjs/specialTokens.d.ts

@@ -2,4 +2,7 @@ export declare const EndOfText = "<|endoftext|>";
 export declare const FimPrefix = "<|fim_prefix|>";
 export declare const FimMiddle = "<|fim_middle|>";
 export declare const FimSuffix = "<|fim_suffix|>";
+export declare const ImStart = "<|im_start|>";
+export declare const ImEnd = "<|im_end|>";
+export declare const ImSep = "<|im_sep|>";
 export declare const EndOfPrompt = "<|endofprompt|>";

package/cjs/specialTokens.js

@@ -1,9 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.EndOfPrompt = exports.FimSuffix = exports.FimMiddle = exports.FimPrefix = exports.EndOfText = void 0;
+exports.EndOfPrompt = exports.ImSep = exports.ImEnd = exports.ImStart = exports.FimSuffix = exports.FimMiddle = exports.FimPrefix = exports.EndOfText = void 0;
 exports.EndOfText = '<|endoftext|>';
 exports.FimPrefix = '<|fim_prefix|>';
 exports.FimMiddle = '<|fim_middle|>';
 exports.FimSuffix = '<|fim_suffix|>';
+exports.ImStart = '<|im_start|>'; // 100264
+exports.ImEnd = '<|im_end|>'; // 100265
+exports.ImSep = '<|im_sep|>'; // 100266
 exports.EndOfPrompt = '<|endofprompt|>';
 //# sourceMappingURL=specialTokens.js.map

package/cjs/specialTokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":";;;AAAa,QAAA,SAAS,GAAG,eAAe,CAAA;AAC3B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,WAAW,GAAG,iBAAiB,CAAA"}
+{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":";;;AAAa,QAAA,SAAS,GAAG,eAAe,CAAA;AAC3B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,SAAS,GAAG,gBAAgB,CAAA;AAC5B,QAAA,OAAO,GAAG,cAAc,CAAA,CAAC,SAAS;AAClC,QAAA,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC9B,QAAA,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC9B,QAAA,WAAW,GAAG,iBAAiB,CAAA"}

package/cjs/util.d.ts

@@ -0,0 +1,2 @@
+export declare function getMaxValueFromMap(map: Map<unknown, number>): number;
+export declare function getSpecialTokenRegex(tokens: Set<string>): RegExp;

package/cjs/util.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSpecialTokenRegex = exports.getMaxValueFromMap = void 0;
+const escapeRegExp_js_1 = require("./escapeRegExp.js");
+function getMaxValueFromMap(map) {
+    let max = 0;
+    map.forEach((val) => {
+        max = Math.max(max, val);
+    });
+    return max;
+}
+exports.getMaxValueFromMap = getMaxValueFromMap;
+function getSpecialTokenRegex(tokens) {
+    const escapedTokens = [...tokens].map(escapeRegExp_js_1.escapeRegExp);
+    const inner = escapedTokens.join('|');
+    return new RegExp(`(${inner})`);
+}
+exports.getSpecialTokenRegex = getSpecialTokenRegex;
+//# sourceMappingURL=util.js.map

package/cjs/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;AAAA,uDAAgD;AAEhD,SAAgB,kBAAkB,CAAC,GAAyB;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;IACF,OAAO,GAAG,CAAA;AACZ,CAAC;AAND,gDAMC;AAED,SAAgB,oBAAoB,CAAC,MAAmB;IACtD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,8BAAY,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;AACjC,CAAC;AAJD,oDAIC"}