gpt-tokenizer 2.6.0 → 2.6.1

package/LICENSE

@@ -1,7 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Dmitry Brazhenko
-Copyright (c) 2023 Bazyli Brzoska
+Copyright (c) 2023-2024 Bazyli Brzoska
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -2,14 +2,22 @@
 
 [![Play with gpt-tokenizer](https://codesandbox.io/static/img/play-codesandbox.svg)](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark)
 
-`gpt-tokenizer` is a Token Byte Pair Encoder/Decoder supporting all OpenAI's models (including those used by GPT-2, GPT-3, GPT-3.5, GPT-4 and GPT-4o).
+`gpt-tokenizer` is a Token Byte Pair Encoder/Decoder supporting all OpenAI's models (including GPT-3.5, GPT-4, GPT-4o, and o1).
 It's the [_fastest, smallest and lowest footprint_](#benchmarks) GPT tokenizer available for all JavaScript environments. It's written in TypeScript.
 
-This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional features sprinkled on top.
+This library has been trusted by:
 
-OpenAI's GPT models utilize byte pair encoding to transform text into a sequence of integers before feeding them into the model.
+- [CodeRabbit](https://www.coderabbit.ai/) (sponsor 🩷)
+- Microsoft ([Teams](https://github.com/microsoft/teams-ai), [GenAIScript](https://github.com/microsoft/genaiscript/))
+- Elastic ([Kibana](https://github.com/elastic/kibana))
+- [Effect TS](https://effect.website/)
+- [Rivet](https://github.com/Ironclad/rivet) by Ironclad
 
-As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. It implements some unique features, such as:
+Please consider [🩷 sponsoring](https://github.com/sponsors/niieani) the project if you find it useful.
+
+#### Features
+
+As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional, unique features sprinkled on top:
 
 - Support for easily tokenizing chats thanks to the `encodeChat` function
 - Support for all current OpenAI models (available encodings: `r50k_base`, `p50k_base`, `p50k_edit`, `cl100k_base` and `o200k_base`)
@@ -22,10 +30,6 @@ As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. I
 - Type-safe (written in TypeScript)
 - Works in the browser out-of-the-box
 
-Thanks to @dmitry-brazhenko's [SharpToken](https://github.com/dmitry-brazhenko/SharpToken), whose code was served as a reference for the port.
-
-Historical note: This package started off as a fork of [latitudegames/GPT-3-Encoder](https://github.com/latitudegames/GPT-3-Encoder), but version 2.0 was rewritten from scratch.
-
 ## Installation
 
 ### As NPM package
@@ -47,7 +51,7 @@ npm install gpt-tokenizer
 
 If you wish to use a custom encoding, fetch the relevant script.
 
-- https://unpkg.com/gpt-tokenizer/dist/o200k_base.js (for `gpt-4o`)
+- https://unpkg.com/gpt-tokenizer/dist/o200k_base.js (for `gpt-4o` and `o1`)
 - https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js (for `gpt-4-*` and `gpt-3.5-turbo`)
 - https://unpkg.com/gpt-tokenizer/dist/p50k_base.js
 - https://unpkg.com/gpt-tokenizer/dist/p50k_edit.js
@@ -61,7 +65,7 @@ Refer to [supported models and their encodings](#Supported-models-and-their-enco
 
 The playground is published under a memorable URL: https://gpt-tokenizer.dev/
 
-You can play with the package in the browser using the [Playground](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark).
+You can play with the package in the browser using the CodeSandbox [Playground](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark).
 
 [![GPT Tokenizer Playground](./docs/gpt-tokenizer.png)](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark)
 
@@ -69,6 +73,8 @@ The playground mimics the official [OpenAI Tokenizer](https://platform.openai.co
 
 ## Usage
 
+The library provides various functions to transform text into (and from) a sequence of integers (tokens) that can be fed into an LLM model. The transformation is done using a Byte Pair Encoding (BPE) algorithm used by OpenAI.
+
 ```typescript
 import {
   encode,
@@ -176,13 +182,14 @@ import {
 
 ### Supported models and their encodings
 
+- `o1-*` (`o200k_base`)
 - `gpt-4o` (`o200k_base`)
 - `gpt-4-*` (`cl100k_base`)
 - `gpt-3.5-turbo` (`cl100k_base`)
 - `text-davinci-003` (`p50k_base`)
 - `text-davinci-002` (`p50k_base`)
 - `text-davinci-001` (`r50k_base`)
-- ...and many other models, see [mapping](./src/mapping.ts) for an up-to-date list of supported models and their encodings.
+- ...and many other models, see [models.ts](./src/models.ts) for an up-to-date list of supported models and their encodings.
 
 Note: if you're using `gpt-3.5-*` or `gpt-4-*` and don't see the model you're looking for, use the `cl100k_base` encoding directly.
 
@@ -381,4 +388,8 @@ MIT
 
 Contributions are welcome! Please open a pull request or an issue to discuss your bug reports, or use the discussions feature for ideas or any other inquiries.
 
+## Thanks
+
+Thanks to @dmitry-brazhenko's [SharpToken](https://github.com/dmitry-brazhenko/SharpToken), whose code was served as a reference for the port.
+
 Hope you find the `gpt-tokenizer` useful in your projects!

package/cjs/GptEncoding.d.ts

@@ -25,6 +25,7 @@ export declare class GptEncoding {
     private specialTokensSet;
     private allSpecialTokenRegex;
     private defaultSpecialTokenConfig;
+    readonly vocabularySize: number;
     private constructor();
     static getEncodingApi(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
     static getEncodingApiForModel(modelName: ModelName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;

package/cjs/GptEncoding.js

@@ -22,6 +22,7 @@ class GptEncoding {
     specialTokensSet;
     allSpecialTokenRegex;
     defaultSpecialTokenConfig;
+    vocabularySize;
     constructor({ mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, modelName, ...rest }) {
         this.specialTokenMapping = specialTokenMapping;
         this.specialTokensSet = new Set(this.specialTokenMapping.keys());
@@ -33,11 +34,12 @@ class GptEncoding {
         });
         this.defaultSpecialTokenConfig = this.processSpecialTokens();
         const maxTokenValue = Math.max(mergeableBytePairRanks.length - 1, (0, util_js_1.getMaxValueFromMap)(specialTokenMapping));
+        this.vocabularySize =
+            this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
+                specialTokenMapping.size;
         if (expectedVocabularySize !== undefined) {
-            if (this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
-                specialTokenMapping.size !==
-                expectedVocabularySize) {
-                throw new Error('The number of mergeable tokens and special tokens must be equal to explicit_n_vocab.');
+            if (this.vocabularySize !== expectedVocabularySize) {
+                throw new Error('The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.');
             }
             if (maxTokenValue !== expectedVocabularySize - 1) {
                 throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${maxTokenValue}, expected ${expectedVocabularySize - 1}`);

package/cjs/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,6CAOqB;AACrB,qDAMyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAEvD,QAAA,kBAAkB,GAAG,KAAK,CAAA;AAsBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IACxC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAErD,YAAoB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,sBAAsB;YACtB,mBAAmB;YACnB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IACE,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC3D,mBAAmB,CAAC,IAAI;gBAC1B,sBAAsB,EACtB,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IAAI,cAAc,EAAE,GAAG,CAAC,0BAAkB,CAAC,EAAE,CAAC;YAC5C,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACjD,CAAC;QAED,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,CAAC,0BAAkB,CAAC,EAAE,CAAC;YACpE,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AAhWH,kCAiWC"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,6CAOqB;AACrB,qDAMyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAEvD,QAAA,kBAAkB,GAAG,KAAK,CAAA;AAsBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IACxC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,sBAAsB;YACtB,mBAAmB;YACnB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,mBAAmB,CAAC,CACxC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,mBAAmB,CAAC,IAAI,CAAA;QAE1B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IAAI,cAAc,EAAE,GAAG,CAAC,0BAAkB,CAAC,EAAE,CAAC;YAC5C,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACjD,CAAC;QAED,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,CAAC,0BAAkB,CAAC,EAAE,CAAC;YACpE,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AAnWH,kCAoWC"}

package/cjs/encoding/cl100k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/encoding/cl100k_base.js

@@ -17,13 +17,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const cl100k_base_js_1 = __importDefault(require("../bpeRanks/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', () => cl100k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -32,6 +32,7 @@ exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=cl100k_base.js.map

package/cjs/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAGnB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/o200k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/encoding/o200k_base.js

@@ -17,13 +17,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const o200k_base_js_1 = __importDefault(require("../bpeRanks/o200k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('o200k_base', () => o200k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -32,6 +32,7 @@ exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=o200k_base.js.map

package/cjs/encoding/o200k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,8EAAgD;AAChD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,uBAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,8EAAgD;AAChD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,uBAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAGnB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/encoding/p50k_base.js

@@ -17,19 +17,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const p50k_base_js_1 = __importDefault(require("../bpeRanks/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_base', () => p50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=p50k_base.js.map

package/cjs/encoding/p50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAClB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_edit.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/encoding/p50k_edit.js

@@ -17,19 +17,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const p50k_base_js_1 = __importDefault(require("../bpeRanks/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_edit', () => p50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=p50k_edit.js.map

package/cjs/encoding/p50k_edit.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAClB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/r50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/encoding/r50k_base.js

@@ -17,19 +17,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const r50k_base_js_1 = __importDefault(require("../bpeRanks/r50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('r50k_base', () => r50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=r50k_base.js.map

package/cjs/encoding/r50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAClB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-3.5-turbo-0125.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/model/gpt-3.5-turbo-0125.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const cl100k_base_js_1 = __importDefault(require("../bpeRanks/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 // prettier-ignore
 const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0125', () => cl100k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -33,6 +33,7 @@ exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=gpt-3.5-turbo-0125.js.map

package/cjs/model/gpt-3.5-turbo-0125.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo-0125.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0125.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-3.5-turbo-0125.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0125.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAGnB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-3.5-turbo-0301.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/cjs/model/gpt-3.5-turbo-0301.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.vocabularySize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const cl100k_base_js_1 = __importDefault(require("../bpeRanks/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 // prettier-ignore
 const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => cl100k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -33,6 +33,7 @@ exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
+exports.vocabularySize = vocabularySize;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=gpt-3.5-turbo-0301.js.map

package/cjs/model/gpt-3.5-turbo-0301.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAGnB,wCAAc;AAEhB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}