gpt-tokenizer 2.0.0-beta.1 → 2.0.0

package/README.md

@@ -2,9 +2,9 @@
 
 [![Play with gpt-tokenizer](https://codesandbox.io/static/img/play-codesandbox.svg)](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark)
 
-`gpt-tokenizer` is a highly optimized Token Byte Pair Encoder/Decoder for all OpenAI's models (including those used by GPT-2, GPT-3, GPT-3.5 and GPT-4), written in TypeScript. OpenAI's GPT models utilize byte pair encoding to transform text into a sequence of integers before feeding them into the model.
+`gpt-tokenizer` is a highly optimized Token Byte Pair Encoder/Decoder for all OpenAI's models (including those used by GPT-2, GPT-3, GPT-3.5 and GPT-4). It's written in TypeScript, and is fully compatible with all modern JavaScript environments.
 
-This package started off as a fork of [latitudegames/GPT-3-Encoder](https://github.com/latitudegames/GPT-3-Encoder), but then in version 2.0 was rewritten from scratch by porting @dmitry-brazhenko's [SharpToken](https://github.com/dmitry-brazhenko/SharpToken), and adding additional features.
+OpenAI's GPT models utilize byte pair encoding to transform text into a sequence of integers before feeding them into the model.
 
 As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. It implements some unique features, such as:
 
@@ -14,23 +14,43 @@ As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. I
 - No global cache (no accidental memory leaks, as with the original GPT-3-Encoder implementation)
 - Includes a highly performant `isWithinTokenLimit` function to assess token limit without encoding the entire text
 - Improves overall performance by eliminating transitive arrays
-- Adds type-checking
+- Type-safe (written in TypeScript)
 - Works in the browser out-of-the-box
 
+This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional features sprinkled on top.
+
+Thanks to @dmitry-brazhenko's [SharpToken](https://github.com/dmitry-brazhenko/SharpToken), whose code was served as a reference for the port.
+
+Historical note: This package started off as a fork of [latitudegames/GPT-3-Encoder](https://github.com/latitudegames/GPT-3-Encoder), but version 2.0 was rewritten from scratch.
+
 ## Installation
 
-As NPM package:
+### As NPM package
 
 ```bash
 npm install gpt-tokenizer
 ```
 
-As an UMD module:
+### As a UMD module
 
 ```html
-<script src="https://unpkg.com/gpt-tokenizer" />
+<script src="https://unpkg.com/gpt-tokenizer"></script>
+
+<script>
+  // the package is now available as a global:
+  const { encode, decode } = GPTTokenizer
+</script>
 ```
 
+If you wish to use a custom encoding, fetch the relevant script:
+
+- https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js
+- https://unpkg.com/gpt-tokenizer/dist/p50k_base.js
+- https://unpkg.com/gpt-tokenizer/dist/p50k_edit.js
+- https://unpkg.com/gpt-tokenizer/dist/r50k_base.js
+
+Refer to [supported models and their encodings](#Supported-models-and-their-encodings) section for more information.
+
 ## Playground
 
 You can play with the package in the browser using the [Playground](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark).
@@ -81,7 +101,7 @@ for await (const textChunk of decodeAsyncGenerator(asyncTokens)) {
 }
 ```
 
-By default, importing from `'gpt-tokenizer'` uses `cl100k_base` encoding, used by GPT-3.5 and GPT-4.
+By default, importing from `gpt-tokenizer` uses `cl100k_base` encoding, used by `gpt-3.5-turbo` and `gpt-4`.
 
 To get a tokenizer for a different model, import it directly, for example:
 
@@ -93,62 +113,72 @@ import {
 } from 'gpt-tokenizer/model/text-davinci-003'
 ```
 
-Supported models and their encodings:
+If you're dealing with a resolver that doesn't support package.json `exports` resolution, you might need to import from the respective `cjs` or `esm` directory, e.g.:
+
+```ts
+import {
+  encode,
+  decode,
+  isWithinTokenLimit,
+} from 'gpt-tokenizer/cjs/model/text-davinci-003'
+```
+
+### Supported models and their encodings
 
 chat:
 
-- gpt-4 (cl100k_base)
-- gpt-3.5-turbo (cl100k_base)
+- `gpt-4` (`cl100k_base`)
+- `gpt-3.5-turbo` (`cl100k_base`)
 
 text:
 
-- text-davinci-003 (p50k_base)
-- text-davinci-002 (p50k_base)
-- text-davinci-001 (r50k_base)
-- text-curie-001 (r50k_base)
-- text-babbage-001 (r50k_base)
-- text-ada-001 (r50k_base)
-- davinci (r50k_base)
-- curie (r50k_base)
-- babbage (r50k_base)
-- ada (r50k_base)
+- `text-davinci-003` (`p50k_base`)
+- `text-davinci-002` (`p50k_base`)
+- `text-davinci-001` (`r50k_base`)
+- `text-curie-001` (`r50k_base`)
+- `text-babbage-001` (`r50k_base`)
+- `text-ada-001` (`r50k_base`)
+- `davinci` (`r50k_base`)
+- `curie` (`r50k_base`)
+- `babbage` (`r50k_base`)
+- `ada` (`r50k_base`)
 
 code:
 
-- code-davinci-002 (p50k_base)
-- code-davinci-001 (p50k_base)
-- code-cushman-002 (p50k_base)
-- code-cushman-001 (p50k_base)
-- davinci-codex (p50k_base)
-- cushman-codex (p50k_base)
+- `code-davinci-002` (`p50k_base`)
+- `code-davinci-001` (`p50k_base`)
+- `code-cushman-002` (`p50k_base`)
+- `code-cushman-001` (`p50k_base`)
+- `davinci-codex` (`p50k_base`)
+- `cushman-codex` (`p50k_base`)
 
 edit:
 
-- text-davinci-edit-001 (p50k_edit)
-- code-davinci-edit-001 (p50k_edit)
+- `text-davinci-edit-001` (`p50k_edit`)
+- `code-davinci-edit-001` (`p50k_edit`)
 
 embeddings:
 
-- text-embedding-ada-002 (cl100k_base)
+- `text-embedding-ada-002` (`cl100k_base`)
 
 old embeddings:
 
-- text-similarity-davinci-001 (r50k_base)
-- text-similarity-curie-001 (r50k_base)
-- text-similarity-babbage-001 (r50k_base)
-- text-similarity-ada-001 (r50k_base)
-- text-search-davinci-doc-001 (r50k_base)
-- text-search-curie-doc-001 (r50k_base)
-- text-search-babbage-doc-001 (r50k_base)
-- text-search-ada-doc-001 (r50k_base)
-- code-search-babbage-code-001 (r50k_base)
-- code-search-ada-code-001 (r50k_base)
+- `text-similarity-davinci-001` (`r50k_base`)
+- `text-similarity-curie-001` (`r50k_base`)
+- `text-similarity-babbage-001` (`r50k_base`)
+- `text-similarity-ada-001` (`r50k_base`)
+- `text-search-davinci-doc-001` (`r50k_base`)
+- `text-search-curie-doc-001` (`r50k_base`)
+- `text-search-babbage-doc-001` (`r50k_base`)
+- `text-search-ada-doc-001` (`r50k_base`)
+- `code-search-babbage-code-001` (`r50k_base`)
+- `code-search-ada-code-001` (`r50k_base`)
 
 ## API
 
 ### `encode(text: string): number[]`
 
-Encodes the given text into a sequence of tokens. Use this method when you need to transform a piece of text into the token format that GPT-2 or GPT-3 models can process.
+Encodes the given text into a sequence of tokens. Use this method when you need to transform a piece of text into the token format that the GPT models can process.
 
 Example:
 
@@ -161,7 +191,7 @@ const tokens = encode(text)
 
 ### `decode(tokens: number[]): string`
 
-Decodes a sequence of tokens back into text. Use this method when you want to convert the output tokens from GPT-2 or GPT-3 models back into human-readable text.
+Decodes a sequence of tokens back into text. Use this method when you want to convert the output tokens from GPT models back into human-readable text.
 
 Example:
 
@@ -174,7 +204,7 @@ const text = decode(tokens)
 
 ### `isWithinTokenLimit(text: string, tokenLimit: number): false | number`
 
-Checks if the text is within the token limit. Returns `false` if the limit is exceeded, otherwise returns the number of tokens. Use this method to quickly check if a given text is within the token limit imposed by GPT-2 or GPT-3 models, without encoding the entire text.
+Checks if the text is within the token limit. Returns `false` if the limit is exceeded, otherwise returns the number of tokens. Use this method to quickly check if a given text is within the token limit imposed by GPT models, without encoding the entire text.
 
 Example:
 
@@ -237,10 +267,62 @@ async function processTokens(asyncTokensIterator) {
 }
 ```
 
+## Special tokens
+
+There are a few special tokens that are used by the GPT models.
+Not all models support all of these tokens.
+
+### Custom Allowed Sets
+
+`gpt-tokenizer` allows you to specify custom sets of allowed special tokens when encoding text. To do this, pass a
+`Set` containing the allowed special tokens as a parameter to the `encode` function:
+
+```ts
+import {
+  EndOfPrompt,
+  EndOfText,
+  FimMiddle,
+  FimPrefix,
+  FimSuffix,
+  encode,
+} from 'gpt-tokenizer'
+
+const inputText = `Some Text ${EndOfPrompt}`
+const allowedSpecialTokens = new Set([EndOfPrompt])
+const encoded = encode(inputText, allowedSpecialTokens)
+const expectedEncoded = [8538, 2991, 220, 100276]
+
+expect(encoded).toBe(expectedEncoded)
+```
+
+### Custom Disallowed Sets
+
+Similarly, you can specify custom sets of disallowed special tokens when encoding text. Pass a `Set`
+containing the disallowed special tokens as a parameter to the `encode` function:
+
+```ts
+import { encode } from 'gpt-tokenizer'
+
+const inputText = `Some Text`
+const disallowedSpecial = new Set(['Some'])
+// throws an error:
+const encoded = encode(inputText, undefined, disallowedSpecial)
+```
+
+In this example, an Error is thrown, because the input text contains a disallowed special token.
+
+## Testing and Validation
+
+`gpt-tokenizer` includes a set of test cases in the [TestPlans.txt](./data/TestPlans.txt) file to ensure its compatibility with OpenAI's Python `tiktoken` library. These test cases validate the functionality and behavior of `gpt-tokenizer`, providing a reliable reference for developers.
+
+Running the unit tests and verifying the test cases helps maintain consistency between the library and the original Python implementation.
+
 ## License
 
 MIT
 
 ## Contributing
 
-Contributions are welcome! Please open a pull request or an issue to discuss your ideas, bug reports, or any other inquiries.
+Contributions are welcome! Please open a pull request or an issue to discuss your bug reports, or use the discussions feature for ideas or any other inquiries.
+
+Hope you find the `gpt-tokenizer` useful in your projects!

package/cjs/GptEncoding.d.ts

@@ -2,6 +2,11 @@ import { type EncodingName, type ModelName } from './mapping.js';
 import { type GetMergeableRanksAsyncFn, type GetMergeableRanksFn } from './modelParams.js';
 export declare const DISALLOW_ALL_SPECIAL_TOKENS = "all";
 export declare class GptEncoding {
+    static EndOfPrompt: string;
+    static EndOfText: string;
+    static FimMiddle: string;
+    static FimPrefix: string;
+    static FimSuffix: string;
     decoder: TextDecoder;
     private bytePairEncodingCoreProcessor;
     private specialTokenMapping;

package/cjs/GptEncoding.js

@@ -6,9 +6,15 @@ const BytePairEncodingCore_js_1 = require("./BytePairEncodingCore.js");
 const escapeRegExp_js_1 = require("./escapeRegExp.js");
 const mapping_js_1 = require("./mapping.js");
 const modelParams_js_1 = require("./modelParams.js");
+const specialTokens_js_1 = require("./specialTokens.js");
 const utfUtil_js_1 = require("./utfUtil.js");
 exports.DISALLOW_ALL_SPECIAL_TOKENS = 'all';
 class GptEncoding {
+    static EndOfPrompt = specialTokens_js_1.EndOfPrompt;
+    static EndOfText = specialTokens_js_1.EndOfText;
+    static FimMiddle = specialTokens_js_1.FimMiddle;
+    static FimPrefix = specialTokens_js_1.FimPrefix;
+    static FimSuffix = specialTokens_js_1.FimSuffix;
     decoder = new TextDecoder('utf8');
     bytePairEncodingCoreProcessor;
     specialTokenMapping;

package/cjs/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,sCAAsC;AACtC,uEAAgE;AAChE,uDAAgD;AAChD,6CAIqB;AACrB,qDAMyB;AACzB,6CAAiE;AAEpD,QAAA,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAa,WAAW;IACtB,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,+BAAc,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,8BAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;CACF;AAvND,kCAuNC"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,sCAAsC;AACtC,uEAAgE;AAChE,uDAAgD;AAChD,6CAIqB;AACrB,qDAMyB;AACzB,yDAM2B;AAC3B,6CAAiE;AAEpD,QAAA,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,+BAAc,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,8BAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;AA5NU,kCAAW"}

package/cjs/GptEncoding.test.js

@@ -1,5 +1,10 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const GptEncoding_js_1 = require("./GptEncoding.js");
 const mapping_js_1 = require("./mapping.js");
 const resolveEncoding_js_1 = require("./resolveEncoding.js");
@@ -38,6 +43,8 @@ const results = {
     p50k_edit: sharedResults,
     r50k_base: sharedResults,
 };
+// eslint-disable-next-line @typescript-eslint/no-use-before-define
+const testPlans = loadTestPlans();
 describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
     const encoding = GptEncoding_js_1.GptEncoding.getEncodingApi(encodingName, resolveEncoding_js_1.resolveEncoding);
     const { decode, decodeGenerator, decodeAsyncGenerator, encode, isWithinTokenLimit, } = encoding;
@@ -116,5 +123,36 @@ describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
             expect(isWithinTokenLimit(str, 300)).toBe(result[str].length);
         });
     });
+    describe('test plan', () => {
+        testPlans[encodingName].forEach(({ sample, encoded }) => {
+            it(`encodes ${sample}`, () => {
+                expect(encode(sample)).toEqual(encoded);
+            });
+            it(`decodes ${sample}`, () => {
+                expect(decode(encoded)).toEqual(sample);
+            });
+        });
+    });
 });
+function loadTestPlans() {
+    const testPlanPath = path_1.default.join(__dirname, '../data/TestPlans.txt');
+    const testPlanData = fs_1.default.readFileSync(testPlanPath, 'utf8');
+    const tests = {
+        cl100k_base: [],
+        p50k_base: [],
+        p50k_edit: [],
+        r50k_base: [],
+    };
+    testPlanData.split('\n\n').forEach((testPlan) => {
+        const [encodingNameLine, sampleLine, encodedLine] = testPlan.split('\n');
+        if (!encodingNameLine || !sampleLine || !encodedLine)
+            return;
+        const encodingName = encodingNameLine.split(': ')[1];
+        tests[encodingName].push({
+            sample: sampleLine.split(': ').slice(1).join(': ') ?? '',
+            encoded: JSON.parse(encodedLine.split(': ')[1] ?? '[]'),
+        });
+    });
+    return tests;
+}
 //# sourceMappingURL=GptEncoding.test.js.map

package/cjs/GptEncoding.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;AAAA,qDAA8C;AAC9C,6CAA+D;AAC/D,6DAAsD;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,QAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
+{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,qDAA8C;AAC9C,6CAA+D;AAC/D,6DAAsD;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}

package/cjs/encoding/cl100k_base.d.ts

@@ -1,2 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
 declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/encoding/cl100k_base.js

@@ -1,4 +1,18 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -8,6 +22,7 @@ exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
 const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
 exports.decode = decode;
@@ -16,4 +31,6 @@ exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=cl100k_base.js.map

package/cjs/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_base.d.ts

@@ -1,2 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
 declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/encoding/p50k_base.js

@@ -1,4 +1,18 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -8,6 +22,7 @@ exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const p50k_base_js_1 = __importDefault(require("../encodings/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(p50k_base_js_1.default));
 const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
 exports.decode = decode;
@@ -16,4 +31,6 @@ exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=p50k_base.js.map

package/cjs/encoding/p50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB"}
+{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_edit.d.ts

@@ -1,2 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
 declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/encoding/p50k_edit.js

@@ -1,4 +1,18 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -8,6 +22,7 @@ exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const p50k_base_js_1 = __importDefault(require("../encodings/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_edit', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(p50k_base_js_1.default));
 const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
 exports.decode = decode;
@@ -16,4 +31,6 @@ exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=p50k_edit.js.map

package/cjs/encoding/p50k_edit.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB"}
+{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/r50k_base.d.ts

@@ -1,2 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
 declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/encoding/r50k_base.js

@@ -1,4 +1,18 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -8,6 +22,7 @@ exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const r50k_base_js_1 = __importDefault(require("../encodings/r50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('r50k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(r50k_base_js_1.default));
 const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
 exports.decode = decode;
@@ -16,4 +31,6 @@ exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=r50k_base.js.map

package/cjs/encoding/r50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB"}
+{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}