gpt-tokenizer 2.6.1 → 2.7.0

package/src/GptEncoding.test.ts

@@ -1,5 +1,6 @@
 import fs from 'fs'
 import path from 'path'
+import { ALL_SPECIAL_TOKENS } from './constants.js'
 import { type ChatMessage, GptEncoding } from './GptEncoding.js'
 import {
   type ChatModelName,
@@ -61,6 +62,25 @@ const results = {
   r50k_base: sharedResults,
 } satisfies Record<EncodingName, unknown>
 
+const offsetPrompts = [
+  // Basic prompt with "hello world"
+  'hello world',
+
+  // Basic prompt with special token "<|endoftext|>"
+  'hello world<|endoftext|> green cow',
+
+  // Chinese text: "我非常渴望与人工智能一起工作"
+  '我非常渴望与人工智能一起工作',
+
+  // Contains the interesting tokens b'\xe0\xae\xbf\xe0\xae' and b'\xe0\xaf\x8d\xe0\xae'
+  // in which \xe0 is the start of a 3-byte UTF-8 character
+  'நடிகர் சூர்யா',
+
+  // Contains the interesting token b'\xa0\xe9\x99\xa4'
+  // in which \xe9 is the start of a 3-byte UTF-8 character and \xa0 is a continuation byte
+  ' Ġ除',
+]
+
 // eslint-disable-next-line @typescript-eslint/no-use-before-define
 const testPlans = loadTestPlans()
 
@@ -75,9 +95,16 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
     isWithinTokenLimit,
   } = encoding
 
-  const result = results[encodingName]
+  describe('encode and decode', () => {
+    it.each(offsetPrompts)('offset prompt: %s', (str) => {
+      expect(
+        decode(encode(str, { allowedSpecial: ALL_SPECIAL_TOKENS })),
+      ).toEqual(str)
+    })
+  })
 
   describe('basic functionality', () => {
+    const result = results[encodingName]
     it('empty string', () => {
       const str = ''
       expect(encode(str)).toEqual([])
@@ -131,6 +158,14 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
       })
     })
 
+    it('encodes and decodes special tokens', () => {
+      const str = 'hello <|endoftext|> world'
+      const encoded = encode(str, {
+        allowedSpecial: ALL_SPECIAL_TOKENS,
+      })
+      expect(decode(encoded)).toEqual(str)
+    })
+
     async function* getHelloWorldTokensAsync() {
       const str = 'hello 👋 world 🌍'
       for (const token of result[str]) {

package/src/GptEncoding.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable no-param-reassign */
 import { BytePairEncodingCore, decoder } from './BytePairEncodingCore.js'
+import { ALL_SPECIAL_TOKENS } from './constants.js'
 import {
   type ChatModelName,
   type ChatParameters,
@@ -29,11 +30,19 @@ import {
 import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js'
 import { getMaxValueFromMap, getSpecialTokenRegex } from './util.js'
 
-export const ALL_SPECIAL_TOKENS = 'all'
-
 export interface EncodeOptions {
-  allowedSpecial?: Set<string>
-  disallowedSpecial?: Set<string>
+  /**
+   * A list of special tokens that are allowed in the input.
+   * If set to 'all', all special tokens are allowed except those in disallowedSpecial.
+   * @default undefined
+   */
+  allowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
+  /**
+   * A list of special tokens that are disallowed in the input.
+   * If set to 'all', all special tokens are disallowed except those in allowedSpecial.
+   * @default 'all'
+   */
+  disallowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
 }
 
 export interface ChatMessage {
@@ -60,7 +69,7 @@ export class GptEncoding {
 
   modelName?: ModelName
   private bytePairEncodingCoreProcessor: BytePairEncodingCore
-  private specialTokenMapping: Map<string, number>
+  private specialTokensEncoder: Map<string, number>
   private specialTokensSet: Set<string>
   private allSpecialTokenRegex: RegExp
   private defaultSpecialTokenConfig: SpecialTokenConfig
@@ -68,31 +77,31 @@ export class GptEncoding {
   readonly vocabularySize: number
 
   private constructor({
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder: mergeableBytePairRanks,
+    specialTokensEncoder,
     expectedVocabularySize,
     modelName,
     ...rest
   }: EncodingParams) {
-    this.specialTokenMapping = specialTokenMapping
-    this.specialTokensSet = new Set<string>(this.specialTokenMapping.keys())
+    this.specialTokensEncoder = specialTokensEncoder
+    this.specialTokensSet = new Set<string>(this.specialTokensEncoder.keys())
     this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet)
 
     this.bytePairEncodingCoreProcessor = new BytePairEncodingCore({
-      mergeableBytePairRanks,
-      specialTokenMapping,
+      bytePairRankDecoder: mergeableBytePairRanks,
+      specialTokensEncoder,
       ...rest,
     })
     this.defaultSpecialTokenConfig = this.processSpecialTokens()
 
     const maxTokenValue = Math.max(
       mergeableBytePairRanks.length - 1,
-      getMaxValueFromMap(specialTokenMapping),
+      getMaxValueFromMap(specialTokensEncoder),
     )
 
     this.vocabularySize =
       this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
-      specialTokenMapping.size
+      specialTokensEncoder.size
 
     if (expectedVocabularySize !== undefined) {
       if (this.vocabularySize !== expectedVocabularySize) {
@@ -119,6 +128,7 @@ export class GptEncoding {
     this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this)
     this.encodeChat = this.encodeChat.bind(this)
     this.encodeChatGenerator = this.encodeChatGenerator.bind(this)
+    this.countTokens = this.countTokens.bind(this)
     this.modelName = modelName
   }
 
@@ -168,15 +178,37 @@ export class GptEncoding {
   }: EncodeOptions = {}): SpecialTokenConfig {
     let regexPattern: RegExp | undefined
 
-    if (allowedSpecial?.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      allowedSpecial === ALL_SPECIAL_TOKENS ||
+      allowedSpecial?.has(ALL_SPECIAL_TOKENS)
+    ) {
       allowedSpecial = new Set(this.specialTokensSet)
+      const allowedSpecialSet = allowedSpecial
+      if (disallowedSpecial === ALL_SPECIAL_TOKENS) {
+        throw new Error(
+          'allowedSpecial and disallowedSpecial cannot both be set to "all".',
+        )
+      }
+      if (typeof disallowedSpecial === 'object') {
+        // remove any special tokens that are disallowed
+        disallowedSpecial.forEach((val) => allowedSpecialSet.delete(val))
+      } else {
+        // all special tokens are allowed, and no 'disallowedSpecial' is provided
+        disallowedSpecial = new Set()
+      }
     }
 
-    if (!disallowedSpecial || disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      !disallowedSpecial ||
+      disallowedSpecial === ALL_SPECIAL_TOKENS ||
+      disallowedSpecial.has(ALL_SPECIAL_TOKENS)
+    ) {
       // by default, all special tokens are disallowed
       disallowedSpecial = new Set(this.specialTokensSet)
+      const disallowedSpecialSet = disallowedSpecial
       if (allowedSpecial?.size) {
-        allowedSpecial.forEach((val) => disallowedSpecial!.delete(val))
+        allowedSpecial.forEach((val) => disallowedSpecialSet.delete(val))
+        // disallowed takes precedence over allowed
         disallowedSpecial.forEach((val) => allowedSpecial.delete(val))
         regexPattern = getSpecialTokenRegex(disallowedSpecial)
       } else {
@@ -245,8 +277,8 @@ export class GptEncoding {
 
     const params: ChatParameters | undefined =
       chatModelParams[model as ChatModelName]
-    const chatStartToken = this.specialTokenMapping.get(ImStart)
-    const chatEndToken = this.specialTokenMapping.get(ImEnd)
+    const chatStartToken = this.specialTokensEncoder.get(ImStart)
+    const chatEndToken = this.specialTokensEncoder.get(ImEnd)
 
     if (!params || chatStartToken === undefined || chatEndToken === undefined) {
       throw new Error(`Model '${model}' does not support chat.`)
@@ -318,6 +350,22 @@ export class GptEncoding {
     return count
   }
 
+  /**
+   * Counts the number of tokens in the input.
+   * @returns {number} The number of tokens.
+   */
+  countTokens(input: string | Iterable<ChatMessage>): number {
+    const tokenGenerator =
+      typeof input === 'string'
+        ? this.encodeGenerator(input)
+        : this.encodeChatGenerator(input)
+    let count = 0
+    for (const tokens of tokenGenerator) {
+      count += tokens.length
+    }
+    return count
+  }
+
   decode(inputTokensToDecode: Iterable<number>): string {
     return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode)
   }

package/src/constants.ts

@@ -0,0 +1 @@
+export const ALL_SPECIAL_TOKENS = 'all'

package/src/encoding/cl100k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('cl100k_base', () => bpeRanks)
@@ -12,11 +13,13 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   encodeChat,
   encodeChatGenerator,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/o200k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('o200k_base', () => bpeRanks)
@@ -12,11 +13,13 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   encodeChat,
   encodeChatGenerator,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/p50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_base', () => bpeRanks)
@@ -12,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/p50k_edit.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_edit', () => bpeRanks)
@@ -12,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encoding/r50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/r50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks)
@@ -12,9 +13,11 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  countTokens,
   vocabularySize,
 } = api
 export {
+  countTokens,
   decode,
   decodeAsyncGenerator,
   decodeGenerator,

package/src/encodingParams/Cl100KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function Cl100KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 100_257],
@@ -29,7 +29,7 @@ export function Cl100KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/O200KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function O200KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 199_999],
@@ -29,7 +29,7 @@ export function O200KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/P50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function P50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_281,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/encodingParams/P50KEdit.ts

@@ -4,7 +4,7 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText, FimMiddle, FimPrefix, FimSuffix } from '../specialTokens.js'
 
 export function P50KEdit(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 50_256],
@@ -15,7 +15,7 @@ export function P50KEdit(
 
   return {
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/R50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function R50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_257,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/model/gpt-3.5-turbo-0125.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0125', () => bpeRanks)

package/src/model/gpt-3.5-turbo-0301.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => bpeRanks)

package/src/model/gpt-3.5-turbo-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0613', () => bpeRanks)

package/src/model/gpt-3.5-turbo-1106.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-1106', () => bpeRanks)

package/src/model/gpt-3.5-turbo-16k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k-0613', () => bpeRanks)

package/src/model/gpt-3.5-turbo-16k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k', () => bpeRanks)

package/src/model/gpt-3.5-turbo-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-finetune', () => bpeRanks)

package/src/model/gpt-3.5-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => bpeRanks)

package/src/model/gpt-4-0125-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0125-preview', () => bpeRanks)

package/src/model/gpt-4-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () => bpeRanks)

package/src/model/gpt-4-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0613', () => bpeRanks)

package/src/model/gpt-4-1106-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-preview', () => bpeRanks)

package/src/model/gpt-4-1106-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-vision-preview', () => bpeRanks)

package/src/model/gpt-4-32k-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => bpeRanks)

package/src/model/gpt-4-32k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0613', () => bpeRanks)

package/src/model/gpt-4-32k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () => bpeRanks)

package/src/model/gpt-4-turbo-2024-04-09.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-2024-04-09', () => bpeRanks)

package/src/model/gpt-4-turbo-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-preview', () => bpeRanks)

package/src/model/gpt-4-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo', () => bpeRanks)

package/src/model/gpt-4-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-vision-preview', () => bpeRanks)

package/src/model/gpt-4.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4', () => bpeRanks)

package/src/model/gpt-4o-2024-05-13.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-05-13', () => bpeRanks)

package/src/model/gpt-4o-2024-08-06-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06-finetune', () => bpeRanks)

package/src/model/gpt-4o-2024-08-06.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06', () => bpeRanks)

package/src/model/gpt-4o-audio-preview-2024-10-01.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview-2024-10-01', () => bpeRanks)