gpt-tokenizer 2.6.2 → 2.7.0

package/esm/GptEncoding.d.ts

@@ -64,6 +64,11 @@ export declare class GptEncoding {
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
     isWithinTokenLimit(input: string | Iterable<ChatMessage>, tokenLimit: number): false | number;
+    /**
+     * Counts the number of tokens in the input.
+     * @returns {number} The number of tokens.
+     */
+    countTokens(input: string | Iterable<ChatMessage>): number;
     decode(inputTokensToDecode: Iterable<number>): string;
     decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void, void>;
     decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;

package/esm/GptEncoding.js

@@ -51,6 +51,7 @@ export class GptEncoding {
         this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
         this.encodeChat = this.encodeChat.bind(this);
         this.encodeChatGenerator = this.encodeChatGenerator.bind(this);
+        this.countTokens = this.countTokens.bind(this);
         this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {
@@ -203,6 +204,20 @@ export class GptEncoding {
         }
         return count;
     }
+    /**
+     * Counts the number of tokens in the input.
+     * @returns {number} The number of tokens.
+     */
+    countTokens(input) {
+        const tokenGenerator = typeof input === 'string'
+            ? this.encodeGenerator(input)
+            : this.encodeChatGenerator(input);
+        let count = 0;
+        for (const tokens of tokenGenerator) {
+            count += tokens.length;
+        }
+        return count;
+    }
     decode(inputTokensToDecode) {
         return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode);
     }

package/esm/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACnD,OAAO,EAKL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAgCpE,MAAM,OAAO,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,kBAAkB,CAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,kBAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,kBAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,kBAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,kBAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,eAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACnD,OAAO,EAKL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAgCpE,MAAM,OAAO,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,kBAAkB,CAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,kBAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,kBAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,kBAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,kBAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,eAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,KAAqC;QAC/C,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC"}

package/esm/encoding/cl100k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/encoding/cl100k_base.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('cl100k_base', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=cl100k_base.js.map

package/esm/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,WAAW,EACX,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/encoding/o200k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/encoding/o200k_base.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('o200k_base', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=o200k_base.js.map

package/esm/encoding/o200k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,WAAW,EACX,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/encoding/p50k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>) => number, vocabularySize: number;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/encoding/p50k_base.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('p50k_base', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, } = api;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=p50k_base.js.map

package/esm/encoding/p50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,WAAW,EACX,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/encoding/p50k_edit.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>) => number, vocabularySize: number;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/encoding/p50k_edit.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('p50k_edit', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, } = api;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=p50k_edit.js.map

package/esm/encoding/p50k_edit.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,WAAW,EACX,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/encoding/r50k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>) => number, vocabularySize: number;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/encoding/r50k_base.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, } = api;
+export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=r50k_base.js.map

package/esm/encoding/r50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,WAAW,EACX,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.6.2",
+  "version": "2.7.0",
   "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",

package/src/GptEncoding.ts

@@ -128,6 +128,7 @@ export class GptEncoding {
     this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this)
     this.encodeChat = this.encodeChat.bind(this)
     this.encodeChatGenerator = this.encodeChatGenerator.bind(this)
+    this.countTokens = this.countTokens.bind(this)
     this.modelName = modelName
   }
 
@@ -349,6 +350,22 @@ export class GptEncoding {
     return count
   }
 
+  /**
+   * Counts the number of tokens in the input.
+   * @returns {number} The number of tokens.
+   */
+  countTokens(input: string | Iterable<ChatMessage>): number {
+    const tokenGenerator =
+      typeof input === 'string'
+        ? this.encodeGenerator(input)
+        : this.encodeChatGenerator(input)
+    let count = 0
+    for (const tokens of tokenGenerator) {
+      count += tokens.length
+    }
+    return count
+  }
+
   decode(inputTokensToDecode: Iterable<number>): string {
     return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode)
   }

package/src/encoding/cl100k_base.ts

@@ -13,11 +13,13 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   encodeChat,
   encodeChatGenerator,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/o200k_base.ts

@@ -13,11 +13,13 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   encodeChat,
   encodeChatGenerator,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/p50k_base.ts

@@ -13,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/p50k_edit.ts

@@ -13,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/r50k_base.ts

@@ -13,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,