gpt-tokenizer 2.6.1 → 2.6.2

package/src/GptEncoding.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable no-param-reassign */
 import { BytePairEncodingCore, decoder } from './BytePairEncodingCore.js'
+import { ALL_SPECIAL_TOKENS } from './constants.js'
 import {
   type ChatModelName,
   type ChatParameters,
@@ -29,11 +30,19 @@ import {
 import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js'
 import { getMaxValueFromMap, getSpecialTokenRegex } from './util.js'
 
-export const ALL_SPECIAL_TOKENS = 'all'
-
 export interface EncodeOptions {
-  allowedSpecial?: Set<string>
-  disallowedSpecial?: Set<string>
+  /**
+   * A list of special tokens that are allowed in the input.
+   * If set to 'all', all special tokens are allowed except those in disallowedSpecial.
+   * @default undefined
+   */
+  allowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
+  /**
+   * A list of special tokens that are disallowed in the input.
+   * If set to 'all', all special tokens are disallowed except those in allowedSpecial.
+   * @default 'all'
+   */
+  disallowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
 }
 
 export interface ChatMessage {
@@ -60,7 +69,7 @@ export class GptEncoding {
 
   modelName?: ModelName
   private bytePairEncodingCoreProcessor: BytePairEncodingCore
-  private specialTokenMapping: Map<string, number>
+  private specialTokensEncoder: Map<string, number>
   private specialTokensSet: Set<string>
   private allSpecialTokenRegex: RegExp
   private defaultSpecialTokenConfig: SpecialTokenConfig
@@ -68,31 +77,31 @@ export class GptEncoding {
   readonly vocabularySize: number
 
   private constructor({
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder: mergeableBytePairRanks,
+    specialTokensEncoder,
     expectedVocabularySize,
     modelName,
     ...rest
   }: EncodingParams) {
-    this.specialTokenMapping = specialTokenMapping
-    this.specialTokensSet = new Set<string>(this.specialTokenMapping.keys())
+    this.specialTokensEncoder = specialTokensEncoder
+    this.specialTokensSet = new Set<string>(this.specialTokensEncoder.keys())
     this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet)
 
     this.bytePairEncodingCoreProcessor = new BytePairEncodingCore({
-      mergeableBytePairRanks,
-      specialTokenMapping,
+      bytePairRankDecoder: mergeableBytePairRanks,
+      specialTokensEncoder,
       ...rest,
     })
     this.defaultSpecialTokenConfig = this.processSpecialTokens()
 
     const maxTokenValue = Math.max(
       mergeableBytePairRanks.length - 1,
-      getMaxValueFromMap(specialTokenMapping),
+      getMaxValueFromMap(specialTokensEncoder),
     )
 
     this.vocabularySize =
       this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
-      specialTokenMapping.size
+      specialTokensEncoder.size
 
     if (expectedVocabularySize !== undefined) {
       if (this.vocabularySize !== expectedVocabularySize) {
@@ -168,15 +177,37 @@ export class GptEncoding {
   }: EncodeOptions = {}): SpecialTokenConfig {
     let regexPattern: RegExp | undefined
 
-    if (allowedSpecial?.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      allowedSpecial === ALL_SPECIAL_TOKENS ||
+      allowedSpecial?.has(ALL_SPECIAL_TOKENS)
+    ) {
       allowedSpecial = new Set(this.specialTokensSet)
+      const allowedSpecialSet = allowedSpecial
+      if (disallowedSpecial === ALL_SPECIAL_TOKENS) {
+        throw new Error(
+          'allowedSpecial and disallowedSpecial cannot both be set to "all".',
+        )
+      }
+      if (typeof disallowedSpecial === 'object') {
+        // remove any special tokens that are disallowed
+        disallowedSpecial.forEach((val) => allowedSpecialSet.delete(val))
+      } else {
+        // all special tokens are allowed, and no 'disallowedSpecial' is provided
+        disallowedSpecial = new Set()
+      }
     }
 
-    if (!disallowedSpecial || disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      !disallowedSpecial ||
+      disallowedSpecial === ALL_SPECIAL_TOKENS ||
+      disallowedSpecial.has(ALL_SPECIAL_TOKENS)
+    ) {
       // by default, all special tokens are disallowed
       disallowedSpecial = new Set(this.specialTokensSet)
+      const disallowedSpecialSet = disallowedSpecial
       if (allowedSpecial?.size) {
-        allowedSpecial.forEach((val) => disallowedSpecial!.delete(val))
+        allowedSpecial.forEach((val) => disallowedSpecialSet.delete(val))
+        // disallowed takes precedence over allowed
         disallowedSpecial.forEach((val) => allowedSpecial.delete(val))
         regexPattern = getSpecialTokenRegex(disallowedSpecial)
       } else {
@@ -245,8 +276,8 @@ export class GptEncoding {
 
     const params: ChatParameters | undefined =
       chatModelParams[model as ChatModelName]
-    const chatStartToken = this.specialTokenMapping.get(ImStart)
-    const chatEndToken = this.specialTokenMapping.get(ImEnd)
+    const chatStartToken = this.specialTokensEncoder.get(ImStart)
+    const chatEndToken = this.specialTokensEncoder.get(ImEnd)
 
     if (!params || chatStartToken === undefined || chatEndToken === undefined) {
       throw new Error(`Model '${model}' does not support chat.`)

package/src/constants.ts

@@ -0,0 +1 @@
+export const ALL_SPECIAL_TOKENS = 'all'

package/src/encoding/cl100k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('cl100k_base', () => bpeRanks)

package/src/encoding/o200k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('o200k_base', () => bpeRanks)

package/src/encoding/p50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_base', () => bpeRanks)

package/src/encoding/p50k_edit.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_edit', () => bpeRanks)

package/src/encoding/r50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/r50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks)

package/src/encodingParams/Cl100KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function Cl100KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 100_257],
@@ -29,7 +29,7 @@ export function Cl100KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/O200KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function O200KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 199_999],
@@ -29,7 +29,7 @@ export function O200KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/P50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function P50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_281,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/encodingParams/P50KEdit.ts

@@ -4,7 +4,7 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText, FimMiddle, FimPrefix, FimSuffix } from '../specialTokens.js'
 
 export function P50KEdit(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 50_256],
@@ -15,7 +15,7 @@ export function P50KEdit(
 
   return {
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/R50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function R50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_257,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/model/gpt-3.5-turbo-0125.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0125', () => bpeRanks)

package/src/model/gpt-3.5-turbo-0301.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => bpeRanks)

package/src/model/gpt-3.5-turbo-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0613', () => bpeRanks)

package/src/model/gpt-3.5-turbo-1106.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-1106', () => bpeRanks)

package/src/model/gpt-3.5-turbo-16k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k-0613', () => bpeRanks)

package/src/model/gpt-3.5-turbo-16k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k', () => bpeRanks)

package/src/model/gpt-3.5-turbo-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-finetune', () => bpeRanks)

package/src/model/gpt-3.5-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => bpeRanks)

package/src/model/gpt-4-0125-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0125-preview', () => bpeRanks)

package/src/model/gpt-4-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () => bpeRanks)

package/src/model/gpt-4-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0613', () => bpeRanks)

package/src/model/gpt-4-1106-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-preview', () => bpeRanks)

package/src/model/gpt-4-1106-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-vision-preview', () => bpeRanks)

package/src/model/gpt-4-32k-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => bpeRanks)

package/src/model/gpt-4-32k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0613', () => bpeRanks)

package/src/model/gpt-4-32k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () => bpeRanks)

package/src/model/gpt-4-turbo-2024-04-09.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-2024-04-09', () => bpeRanks)

package/src/model/gpt-4-turbo-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-preview', () => bpeRanks)

package/src/model/gpt-4-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo', () => bpeRanks)

package/src/model/gpt-4-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-vision-preview', () => bpeRanks)

package/src/model/gpt-4.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4', () => bpeRanks)

package/src/model/gpt-4o-2024-05-13.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-05-13', () => bpeRanks)

package/src/model/gpt-4o-2024-08-06-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06-finetune', () => bpeRanks)

package/src/model/gpt-4o-2024-08-06.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06', () => bpeRanks)

package/src/model/gpt-4o-audio-preview-2024-10-01.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview-2024-10-01', () => bpeRanks)

package/src/model/gpt-4o-audio-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview', () => bpeRanks)

package/src/model/gpt-4o-mini-2024-07-18-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-2024-07-18-finetune', () => bpeRanks)

package/src/model/gpt-4o-mini-2024-07-18.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-2024-07-18', () => bpeRanks)

package/src/model/gpt-4o-mini-training-2024-07-18.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-training-2024-07-18', () => bpeRanks)

package/src/model/gpt-4o-mini-training.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-training', () => bpeRanks)

package/src/model/gpt-4o-mini.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini', () => bpeRanks)

package/src/model/gpt-4o-realtime-preview-2024-10-01.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview-2024-10-01', () => bpeRanks)

package/src/model/gpt-4o-realtime-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview', () => bpeRanks)

package/src/model/gpt-4o.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o', () => bpeRanks)

package/src/modelParams.ts

@@ -22,7 +22,7 @@ export interface EncodingParams extends BytePairEncodingConfig {
    * It's complex due to its need to deal with a wide variety of cases in text processing.
    */
   tokenSplitRegex: RegExp
-  specialTokenMapping: Map<string, number>
+  specialTokensEncoder: Map<string, number>
   modelName?: ModelName
 }