gpt-tokenizer 2.6.0 → 2.6.2

package/cjs/modelParams.d.ts

@@ -13,10 +13,8 @@ export interface EncodingParams extends BytePairEncodingConfig {
      * It's complex due to its need to deal with a wide variety of cases in text processing.
      */
     tokenSplitRegex: RegExp;
-    specialTokenMapping: Map<string, number>;
+    specialTokensEncoder: Map<string, number>;
     modelName?: ModelName;
-    /** increases memory consumption, but speeds up subsequent decoding */
-    enableCache?: boolean;
 }
 export declare const tokenSplitRegex: RegExp;
 export type GetMergeableRanksFn = (encodingName: EncodingName) => RawBytePairRanks;

package/cjs/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAyCA,8CAwBC;AAED,kDAMC;AArED,kEAA2D;AAC3D,gEAAyD;AACzD,8DAAuD;AACvD,8DAAuD;AACvD,8DAAuD;AAuB1C,QAAA,eAAe,GAC1B,8EAA8E,CAAA;AAShF,SAAgB,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,IAAA,0BAAU,EAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,IAAA,wBAAS,EAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAsCA,8CAwBC;AAED,kDAMC;AAlED,kEAA2D;AAC3D,gEAAyD;AACzD,8DAAuD;AACvD,8DAAuD;AACvD,8DAAuD;AAoB1C,QAAA,eAAe,GAC1B,8EAA8E,CAAA;AAShF,SAAgB,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,IAAA,sBAAQ,EAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,IAAA,0BAAU,EAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,IAAA,wBAAS,EAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}

package/cjs/models.d.ts

@@ -1,4 +1,4 @@
-import type { EncodingName } from './mapping';
+import type { EncodingName } from './mapping.js';
 export interface Model {
     humanName: string;
     encoding: EncodingName;