gpt-tokenizer 2.8.0 → 2.8.1

package/README.md

@@ -195,10 +195,12 @@ Note: if you're using `gpt-3.5-*` or `gpt-4-*` and don't see the model you're lo
 
 ## API
 
-### `encode(text: string): number[]`
+### `encode(text: string, encodeOptions?: EncodeOptions): number[]`
 
 Encodes the given text into a sequence of tokens. Use this method when you need to transform a piece of text into the token format that the GPT models can process.
 
+The optional `encodeOptions` parameter allows you to specify special token handling (see [special tokens](#special-tokens)).
+
 Example:
 
 ```typescript
@@ -327,7 +329,11 @@ async function processTokens(asyncTokensIterator) {
 ## Special tokens
 
 There are a few special tokens that are used by the GPT models.
-Not all models support all of these tokens.
+Note that not all models support all of these tokens.
+
+By default, **all special tokens are disallowed**.
+
+The `encode`, `encodeGenerator` and `countTokens` functions accept an `EncodeOptions` parameter to customize special token handling:
 
 ### Custom Allowed Sets
 
@@ -349,12 +355,14 @@ import {
 
 const inputText = `Some Text ${EndOfPrompt}`
 const allowedSpecialTokens = new Set([EndOfPrompt])
-const encoded = encode(inputText, allowedSpecialTokens)
+const encoded = encode(inputText, { allowedSpecialTokens })
 const expectedEncoded = [8538, 2991, 220, 100276]
 
 expect(encoded).toBe(expectedEncoded)
 ```
 
+You may also use a special shorthand for either disallowing or allowing all special tokens, by passing in the string `'all'`, e.g. `{ allowedSpecial: 'all' }`.
+
 ### Custom Disallowed Sets
 
 Similarly, you can specify custom sets of disallowed special tokens when encoding text. Pass a `Set`
@@ -366,11 +374,13 @@ import { encode, EndOfText } from 'gpt-tokenizer'
 const inputText = `Some Text ${EndOfText}`
 const disallowedSpecial = new Set([EndOfText])
 // throws an error:
-const encoded = encode(inputText, undefined, disallowedSpecial)
+const encoded = encode(inputText, { disallowedSpecial })
 ```
 
 In this example, an Error is thrown, because the input text contains a disallowed special token.
 
+If both `allowedSpecialTokens` and `disallowedSpecial` are provided, `disallowedSpecial` takes precedence.
+
 ## Performance Optimization
 
 ### LRU Merge Cache
@@ -393,6 +403,14 @@ setMergeCacheSize(5000)
 setMergeCacheSize(0)
 ```
 
+The cache is persisted between encoding calls. To explicitly clear the cache (e.g. to free up memory), use the `clearMergeCache` function:
+
+```ts
+import { clearMergeCache } from 'gpt-tokenizer'
+
+clearMergeCache()
+```
+
 ## Testing and Validation
 
 `gpt-tokenizer` includes a set of test cases in the [TestPlans.txt](./data/TestPlans.txt) file to ensure its compatibility with OpenAI's Python `tiktoken` library. These test cases validate the functionality and behavior of `gpt-tokenizer`, providing a reliable reference for developers.

package/cjs/BytePairEncodingCore.d.ts

@@ -37,6 +37,7 @@ export declare class BytePairEncodingCore {
     private mergeCacheSize;
     constructor({ bytePairRankDecoder, specialTokensEncoder, tokenSplitRegex, mergeCacheSize, }: BytePairEncodingConfig);
     setMergeCacheSize(newSize: number): void;
+    clearMergeCache(): void;
     encodeNativeGenerator(text: string, allowedSpecial?: Set<string>): Generator<number[], number, undefined>;
     encodeNative(text: string, allowedSpecial?: Set<string>): number[];
     countNative(text: string, allowedSpecial?: Set<string>): number;

package/cjs/BytePairEncodingCore.js

@@ -75,6 +75,9 @@ class BytePairEncodingCore {
             this.mergeCache = undefined;
         }
     }
+    clearMergeCache() {
+        this.mergeCache?.clear();
+    }
     *encodeNativeGenerator(text, allowedSpecial) {
         let startIndex = 0;
         let lastTokenLength = 0;

package/cjs/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","sourceRoot":"","sources":["../src/BytePairEncodingCore.ts"],"names":[],"mappings":";AAAA,gCAAgC;;;AAEhC,iDAAyD;AACzD,6CAA8E;AAC9E,uCAAwC;AAiBxC,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;AAExB,QAAA,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;AAE9C,MAAa,oBAAoB;IACtB,0BAA0B,CAAQ;IAC3C;;;;OAIG;IACK,mBAAmB,CAAkB;IACrC,yBAAyB,GAAG,IAAI,GAAG,EAAsB,CAAA;IACzD,2BAA2B,CAAiC;IACpE;;;;OAIG;IACK,yBAAyB,CAAqB;IAC9C,eAAe,CAAQ;IACvB,oBAAoB,CAAqB;IACzC,oBAAoB,CAAqB;IACzC,wBAAwB,CAAQ;IAChC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAC/B,UAAU,CAAwB;IAClC,cAAc,CAAQ;IAE9B,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EACf,cAAc,GAAG,uCAAwB,GAClB;QACvB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;QAC1D,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAA;QAC7B,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,0BAA0B,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAA;QACzE,MAAM,YAAY,GAA2B,EAAE,CAAA;QAC/C,6BAA6B;QAC7B,mBAAmB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAC/C,OAAM;YACR,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACvC,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QACrD,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,2BAA2B,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,IAAA,+BAAkB,EAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/B,CAAA;QACD,IAAI,CAAC,oBAAoB;YACvB,oBAAoB,IAAI,IAAI,GAAG,EAAkB,CAAA;QACnD,IAAI,CAAC,oBAAoB,GAAG,oBAAoB;YAC9C,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxE,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAA;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QAEtC,MAAM,oBAAoB,GAAG,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CACpE,sBAAY,CACb,CAAA;QACD,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC5D,IAAI,CAAC;YACH,IAAI,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAA;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,OAAe;QAC/B,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAA;QAC7B,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAA;QAC7B,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,CAAC,qBAAqB,CACpB,IAAY,EACZ,cAA4B;QAE5B,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,eAAe,GAAG,CAAC,CAAA;oBACnB,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAA;gBAC/B,MAAM,MAAM,CAAA;YACd,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,MAAM,CAAC,iBAAiB,CAAC,CAAA;gBACzB,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;gBACxD,eAAe,GAAG,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,cAA4B;QACrD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,WAAW,GAAa,EAAE,CAAA,CAAC,kCAAkC;QAEnE,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEvB,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;YAC7B,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;gBACnC,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,WAAW,CAAC,IAAY,EAAE,cAA4B;QACpD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,WAAW,GAAG,CAAC,CAAA;QAEnB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,WAAW,EAAE,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAA;YAC9B,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,WAAW,EAAE,CAAA;gBACb,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,CAAC,qBAAqB,CACpB,MAAwB;QAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,UAAU,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAY,CAAC,MAAwB;QACnC,IAAI,OAAO,GAAG,EAAE,CAAA;QAChB,IAAI,SAAS,GAAG,WAAW,CAAA;QAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,mCAAmC,CAAC,CAAA;YACpE,CAAC;YACD,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;oBAC9B,OAAO,IAAI,eAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;oBACtD,SAAS,GAAG,WAAW,CAAA;gBACzB,CAAC;gBACD,OAAO,IAAI,UAAU,CAAA;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;gBACtE,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBACxB,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;gBAC3C,SAAS,GAAG,SAAS,CAAA;YACvB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,eAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACxD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,CAAC,yBAAyB,CAC9B,MAA6B;QAE7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YACrD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,kBAAkB,CAAA;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,GAAW;QACtC,OAAO,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAEO,2BAA2B,CAAC,GAAW;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,EAAE,CAC9D,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,mBAAmB,CAAC,GAAe;QACzC,MAAM,WAAW,GAAG,IAAA,+BAAkB,EAAC,GAAG,CAAC,CAAA;QAE3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QAC/C,CAAC;QAED,2CAA2C;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,0BAA0B,CAAC,GAAe;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,CAAC,QAAQ,EAAE,EAAE,CACzE,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,mCAAmC;IAC3B,YAAY,CAAC,GAAe;QAClC,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,IAAI,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,CAAA;QAEtD,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACnB,sCAAsC;YACtC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAA;YACxD,IAAI,GAAG,GAAG,CAAC,CAAA;YACX,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAA;gBAC1B,IAAI,GAAG,KAAK,CAAC;oBAAE,MAAK;YACtB,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YAClC,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;YACf,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,CAAA;IACX,CAAC;IAEO,oBAAoB,CAC1B,IAAY,EACZ,cAAuC,EACvC,UAAkB;QAElB,IAAI,WAAW,GAAG,UAAU,CAAA;QAE5B,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,wBAAwB,CAAC,SAAS,GAAG,WAAW,CAAA;YACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEjE,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;YAExC,IAAI,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,CAAA;gBACnE,OAAO,CAAC,sBAAsB,EAAE,YAAY,CAAC,CAAA;YAC/C,CAAC;YAED,WAAW,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,GAAG,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,SAAiB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAChE,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,UAAU,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACjD,CAAC;IAEO,eAAe,CAAC,GAAW,EAAE,KAAe;QAClD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAM;QAE5B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,+CAA+C;YAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAA;YACrD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACjC,CAAC;IAEO,cAAc,CAAC,KAAa;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,oBAAO,EAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAA;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;YAC1C,uCAAuC;YACvC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAClC,OAAO,MAAM,CAAA;QACf,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;QAC7C,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACnC,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,aAAa;IACnB,kCAAkC;IAClC,KAAiB;QAEjB,qDAAqD;QACrD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,qDAAqD;QACrD,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,0EAA0E;QAC1E,MAAM,OAAO,GAAG,CACd,UAAkB,EAClB,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,EAC9B,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,EACxB,EAAE;YACV,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,uBAAuB;gBACvB,OAAO,MAAM,CAAC,iBAAiB,CAAA;YACjC,CAAC;YAED,wBAAwB;YACxB,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAE9C,yDAAyD;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAC1C,OAAO,IAAI,IAAI,MAAM,CAAC,iBAAiB,CAAA;QACzC,CAAC,CAAA;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,mDAAmD;gBACnD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;YACtC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,MAAM,CAAC,iBAAiB,CAAA;YACzC,IAAI,oBAAoB,GAAG,CAAC,CAAC,CAAA;YAE7B,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;gBACtB,IAAI,IAAI,GAAG,UAAU,EAAE,CAAC;oBACtB,UAAU,GAAG,IAAI,CAAA;oBACjB,oBAAoB,GAAG,CAAC,CAAA;gBAC1B,CAAC;YACH,CAAC;YAED,mDAAmD;YACnD,IACE,UAAU,KAAK,MAAM,CAAC,iBAAiB;gBACvC,oBAAoB,KAAK,CAAC,CAAC,EAC3B,CAAC;gBACD,MAAK;YACP,CAAC;YAED,4EAA4E;YAC5E,MAAM,CAAC,MAAM,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1C,0CAA0C;YAC1C,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAA;YAErC,wCAAwC;YACxC,KAAK,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAA;YAE3D,qDAAqD;YACrD,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,oBAAoB,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAA;YACrE,CAAC;QACH,CAAC;QAED,uFAAuF;QACvF,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,0BAA0B,CAC9C,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CACnC,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvB,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AApgBD,oDAogBC"}
+{"version":3,"file":"BytePairEncodingCore.js","sourceRoot":"","sources":["../src/BytePairEncodingCore.ts"],"names":[],"mappings":";AAAA,gCAAgC;;;AAEhC,iDAAyD;AACzD,6CAA8E;AAC9E,uCAAwC;AAiBxC,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;AAExB,QAAA,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;AAE9C,MAAa,oBAAoB;IACtB,0BAA0B,CAAQ;IAC3C;;;;OAIG;IACK,mBAAmB,CAAkB;IACrC,yBAAyB,GAAG,IAAI,GAAG,EAAsB,CAAA;IACzD,2BAA2B,CAAiC;IACpE;;;;OAIG;IACK,yBAAyB,CAAqB;IAC9C,eAAe,CAAQ;IACvB,oBAAoB,CAAqB;IACzC,oBAAoB,CAAqB;IACzC,wBAAwB,CAAQ;IAChC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAC/B,UAAU,CAAwB;IAClC,cAAc,CAAQ;IAE9B,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EACf,cAAc,GAAG,uCAAwB,GAClB;QACvB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;QAC1D,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAA;QAC7B,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,0BAA0B,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAA;QACzE,MAAM,YAAY,GAA2B,EAAE,CAAA;QAC/C,6BAA6B;QAC7B,mBAAmB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAC/C,OAAM;YACR,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACvC,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QACrD,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,2BAA2B,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,IAAA,+BAAkB,EAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/B,CAAA;QACD,IAAI,CAAC,oBAAoB;YACvB,oBAAoB,IAAI,IAAI,GAAG,EAAkB,CAAA;QACnD,IAAI,CAAC,oBAAoB,GAAG,oBAAoB;YAC9C,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxE,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAA;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QAEtC,MAAM,oBAAoB,GAAG,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CACpE,sBAAY,CACb,CAAA;QACD,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC5D,IAAI,CAAC;YACH,IAAI,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAA;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,OAAe;QAC/B,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAA;QAC7B,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAA;QAC7B,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,eAAe;QACb,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAA;IAC1B,CAAC;IAED,CAAC,qBAAqB,CACpB,IAAY,EACZ,cAA4B;QAE5B,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,eAAe,GAAG,CAAC,CAAA;oBACnB,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAA;gBAC/B,MAAM,MAAM,CAAA;YACd,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,MAAM,CAAC,iBAAiB,CAAC,CAAA;gBACzB,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;gBACxD,eAAe,GAAG,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,cAA4B;QACrD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,WAAW,GAAa,EAAE,CAAA,CAAC,kCAAkC;QAEnE,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEvB,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;YAC7B,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;gBACnC,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,WAAW,CAAC,IAAY,EAAE,cAA4B;QACpD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,WAAW,GAAG,CAAC,CAAA;QAEnB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAChD,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YACD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEnD,MAAM,QAAQ,GAAG,qBAAqB,IAAI,IAAI,CAAC,MAAM,CAAA;YAErD,MAAM,iBAAiB,GACrB,UAAU,KAAK,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM;gBAC1C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;YAEtC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,WAAW,EAAE,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAA;YAC9B,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,gBAAiB,CAAC,CAAC,CAAC,CAAA;gBACzC,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,WAAW,EAAE,CAAA;gBACb,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,CAAC,qBAAqB,CACpB,MAAwB;QAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,UAAU,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAY,CAAC,MAAwB;QACnC,IAAI,OAAO,GAAG,EAAE,CAAA;QAChB,IAAI,SAAS,GAAG,WAAW,CAAA;QAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,mCAAmC,CAAC,CAAA;YACpE,CAAC;YACD,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;oBAC9B,OAAO,IAAI,eAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;oBACtD,SAAS,GAAG,WAAW,CAAA;gBACzB,CAAC;gBACD,OAAO,IAAI,UAAU,CAAA;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;gBACtE,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBACxB,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;gBAC3C,SAAS,GAAG,SAAS,CAAA;YACvB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,eAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACxD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,CAAC,yBAAyB,CAC9B,MAA6B;QAE7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YACrD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,kBAAkB,CAAA;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,GAAW;QACtC,OAAO,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAEO,2BAA2B,CAAC,GAAW;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,EAAE,CAC9D,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,mBAAmB,CAAC,GAAe;QACzC,MAAM,WAAW,GAAG,IAAA,+BAAkB,EAAC,GAAG,CAAC,CAAA;QAE3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QAC/C,CAAC;QAED,2CAA2C;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,0BAA0B,CAAC,GAAe;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,CAAC,QAAQ,EAAE,EAAE,CACzE,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,mCAAmC;IAC3B,YAAY,CAAC,GAAe;QAClC,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,IAAI,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,CAAA;QAEtD,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACnB,sCAAsC;YACtC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAA;YACxD,IAAI,GAAG,GAAG,CAAC,CAAA;YACX,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAA;gBAC1B,IAAI,GAAG,KAAK,CAAC;oBAAE,MAAK;YACtB,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YAClC,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;YACf,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,CAAA;IACX,CAAC;IAEO,oBAAoB,CAC1B,IAAY,EACZ,cAAuC,EACvC,UAAkB;QAElB,IAAI,WAAW,GAAG,UAAU,CAAA;QAE5B,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,wBAAwB,CAAC,SAAS,GAAG,WAAW,CAAA;YACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEjE,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;YAExC,IAAI,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,CAAA;gBACnE,OAAO,CAAC,sBAAsB,EAAE,YAAY,CAAC,CAAA;YAC/C,CAAC;YAED,WAAW,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,GAAG,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,SAAiB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAChE,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,UAAU,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACjD,CAAC;IAEO,eAAe,CAAC,GAAW,EAAE,KAAe;QAClD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAM;QAE5B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,+CAA+C;YAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAA;YACrD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACjC,CAAC;IAEO,cAAc,CAAC,KAAa;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,oBAAO,EAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAA;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;YAC1C,uCAAuC;YACvC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAClC,OAAO,MAAM,CAAA;QACf,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;QAC7C,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACnC,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,aAAa;IACnB,kCAAkC;IAClC,KAAiB;QAEjB,qDAAqD;QACrD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,qDAAqD;QACrD,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,0EAA0E;QAC1E,MAAM,OAAO,GAAG,CACd,UAAkB,EAClB,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,EAC9B,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,EACxB,EAAE;YACV,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,uBAAuB;gBACvB,OAAO,MAAM,CAAC,iBAAiB,CAAA;YACjC,CAAC;YAED,wBAAwB;YACxB,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAE9C,yDAAyD;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAC1C,OAAO,IAAI,IAAI,MAAM,CAAC,iBAAiB,CAAA;QACzC,CAAC,CAAA;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,mDAAmD;gBACnD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;YACtC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,MAAM,CAAC,iBAAiB,CAAA;YACzC,IAAI,oBAAoB,GAAG,CAAC,CAAC,CAAA;YAE7B,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;gBACtB,IAAI,IAAI,GAAG,UAAU,EAAE,CAAC;oBACtB,UAAU,GAAG,IAAI,CAAA;oBACjB,oBAAoB,GAAG,CAAC,CAAA;gBAC1B,CAAC;YACH,CAAC;YAED,mDAAmD;YACnD,IACE,UAAU,KAAK,MAAM,CAAC,iBAAiB;gBACvC,oBAAoB,KAAK,CAAC,CAAC,EAC3B,CAAC;gBACD,MAAK;YACP,CAAC;YAED,4EAA4E;YAC5E,MAAM,CAAC,MAAM,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1C,0CAA0C;YAC1C,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAA;YAErC,wCAAwC;YACxC,KAAK,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAA;YAE3D,qDAAqD;YACrD,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,oBAAoB,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAA;YACrE,CAAC;QACH,CAAC;QAED,uFAAuF;QACvF,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,0BAA0B,CAC9C,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CACnC,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvB,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AAxgBD,oDAwgBC"}

package/cjs/GptEncoding.d.ts

@@ -68,6 +68,7 @@ export declare class GptEncoding {
      */
     countTokens(input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions): number;
     setMergeCacheSize(size: number): void;
+    clearMergeCache(): void;
     decode(inputTokensToDecode: Iterable<number>): string;
     decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void, void>;
     decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;

package/cjs/GptEncoding.js

@@ -56,6 +56,7 @@ class GptEncoding {
         this.encodeChatGenerator = this.encodeChatGenerator.bind(this);
         this.countTokens = this.countTokens.bind(this);
         this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
+        this.clearMergeCache = this.clearMergeCache.bind(this);
         this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {
@@ -228,6 +229,9 @@ class GptEncoding {
     setMergeCacheSize(size) {
         this.bytePairEncodingCoreProcessor.setMergeCacheSize(size);
     }
+    clearMergeCache() {
+        this.bytePairEncodingCoreProcessor.clearMergeCache();
+    }
     decode(inputTokensToDecode) {
         return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode);
     }

package/cjs/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,iDAAmD;AACnD,6CAOqB;AACrB,qDAIyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAgCpE,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,iCAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,iCAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,iCAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,iCAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,iCAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC/B,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,WAAW,CACT,KAAqC,EACrC,aAA6B;QAE7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,kBAAkB,GAAG,aAAa;gBACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;YAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;gBAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;gBAChE,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,WAAW,CACnD,KAAK,EACL,kBAAkB,CAAC,cAAc,CAClC,CAAA;QACH,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACtD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,iBAAiB,CAAC,IAAY;QAC5B,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AA5YH,kCA6YC"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,iDAAmD;AACnD,6CAOqB;AACrB,qDAIyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAgCpE,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,iCAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,iCAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,iCAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,iCAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,iCAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC/B,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,WAAW,CACT,KAAqC,EACrC,aAA6B;QAE7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,kBAAkB,GAAG,aAAa;gBACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;YAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;gBAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;gBAChE,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,WAAW,CACnD,KAAK,EACL,kBAAkB,CAAC,cAAc,CAClC,CAAA;QACH,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACtD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,iBAAiB,CAAC,IAAY;QAC5B,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED,eAAe;QACb,IAAI,CAAC,6BAA6B,CAAC,eAAe,EAAE,CAAA;IACtD,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AAjZH,kCAkZC"}

package/cjs/encoding/cl100k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void;
-export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/cl100k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const cl100k_base_js_1 = __importDefault(require("../bpeRanks/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', () => cl100k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -36,6 +36,7 @@ exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
+exports.clearMergeCache = clearMergeCache;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=cl100k_base.js.map

package/cjs/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,GAClB,GAAG,GAAG,CAAA;AAGL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AARlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAInB,wCAAc;AADd,8CAAiB;AAGnB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AARlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAInB,wCAAc;AADd,8CAAiB;AAVjB,0CAAe;AAajB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/o200k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void;
-export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/o200k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const o200k_base_js_1 = __importDefault(require("../bpeRanks/o200k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('o200k_base', () => o200k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -36,6 +36,7 @@ exports.encodeChat = encodeChat;
 exports.encodeChatGenerator = encodeChatGenerator;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
+exports.clearMergeCache = clearMergeCache;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=o200k_base.js.map

package/cjs/encoding/o200k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,8EAAgD;AAChD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,uBAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,GAClB,GAAG,GAAG,CAAA;AAGL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AARlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAInB,wCAAc;AADd,8CAAiB;AAGnB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,8EAAgD;AAChD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,uBAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AARlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAInB,wCAAc;AADd,8CAAiB;AAVjB,0CAAe;AAajB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void;
-export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/p50k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const p50k_base_js_1 = __importDefault(require("../bpeRanks/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_base', () => p50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -34,6 +34,7 @@ exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.countTokens = countTokens;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
+exports.clearMergeCache = clearMergeCache;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=p50k_base.js.map

package/cjs/encoding/p50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,GAAG,GAAG,CAAA;AAGL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AAGnB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AARjB,0CAAe;AAWjB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_edit.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void;
-export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/p50k_edit.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const p50k_base_js_1 = __importDefault(require("../bpeRanks/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_edit', () => p50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -34,6 +34,7 @@ exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.countTokens = countTokens;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
+exports.clearMergeCache = clearMergeCache;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=p50k_edit.js.map

package/cjs/encoding/p50k_edit.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,GAAG,GAAG,CAAA;AAGL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AAGnB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AARjB,0CAAe;AAWjB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/r50k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void;
-export { countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/r50k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const r50k_base_js_1 = __importDefault(require("../bpeRanks/r50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('r50k_base', () => r50k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -34,6 +34,7 @@ exports.isWithinTokenLimit = isWithinTokenLimit;
 exports.countTokens = countTokens;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
+exports.clearMergeCache = clearMergeCache;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=r50k_base.js.map

package/cjs/encoding/r50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,GAAG,GAAG,CAAA;AAGL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AAGnB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4EAA+C;AAC/C,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,sBAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AANlB,kCAAW;AAQX,wCAAc;AADd,8CAAiB;AARjB,0CAAe;AAWjB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}