gpt-tokenizer 2.1.1 → 2.1.2

package/esm/GptEncoding.js

@@ -39,6 +39,8 @@ class GptEncoding {
         this.decodeGenerator = this.decodeGenerator.bind(this);
         this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this);
         this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
+        this.encodeChat = this.encodeChat.bind(this);
+        this.encodeChatGenerator = this.encodeChatGenerator.bind(this);
         this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {

package/esm/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAGL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEpE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAiBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACjC,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACM;QACf,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,kBAAkB,CAAC,sBAAsB,CAAC,EAC1C,kBAAkB,CAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,EACE,cAAc,GAAG,IAAI,GAAG,EAAU,EAClC,iBAAiB,GAAG,IAAI,GAAG,CAAS,CAAC,kBAAkB,CAAC,CAAC,MACxC,EAAE;QAErB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC7C,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC1C,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,gBAA+B,EAAE;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;SACF;QACD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE;YACzE,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;SAC3D;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE;YAC5D,IAAI,OAAO,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;aAC7D;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnC,MAAM,oBAAoB,CAAA;aAC3B;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;SAC9B;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,MAAM,oBAAoB,CAAA;SAC3B;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;SA1RU,WAAW"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAGL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEpE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAiBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACjC,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACM;QACf,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,kBAAkB,CAAC,sBAAsB,CAAC,EAC1C,kBAAkB,CAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,EACE,cAAc,GAAG,IAAI,GAAG,EAAU,EAClC,iBAAiB,GAAG,IAAI,GAAG,CAAS,CAAC,kBAAkB,CAAC,CAAC,MACxC,EAAE;QAErB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC7C,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC1C,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,gBAA+B,EAAE;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;SACF;QACD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE;YACzE,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;SAC3D;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE;YAC5D,IAAI,OAAO,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;aAC7D;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnC,MAAM,oBAAoB,CAAA;aAC3B;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;SAC9B;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,MAAM,oBAAoB,CAAA;SAC3B;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;SA5RU,WAAW"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",

package/src/GptEncoding.ts

@@ -97,6 +97,8 @@ export class GptEncoding {
     this.decodeGenerator = this.decodeGenerator.bind(this)
     this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this)
     this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this)
+    this.encodeChat = this.encodeChat.bind(this)
+    this.encodeChatGenerator = this.encodeChatGenerator.bind(this)
     this.modelName = modelName
   }