gpt-tokenizer 2.8.0 → 2.9.0

package/src/model/gpt-4o-audio-preview-2024-12-17.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview-2024-12-17', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-audio-preview.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-mini-2024-07-18-finetune.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-mini-2024-07-18.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-mini-audio-preview-2024-12-17.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-audio-preview-2024-12-17', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-mini-audio-preview.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-audio-preview', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-mini-realtime-preview-2024-12-17.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-realtime-preview-2024-12-17', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-mini-realtime-preview.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-realtime-preview', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-mini-training-2024-07-18.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-mini-training.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-mini.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-realtime-preview-2024-10-01.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o-realtime-preview-2024-12-17.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview-2024-12-17', () => bpeRanks)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4o-realtime-preview.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/gpt-4o.ts

@@ -18,8 +18,11 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,
@@ -28,6 +31,7 @@ export {
   encodeChat,
   encodeChatGenerator,
   encodeGenerator,
+  estimateCost,
   isWithinTokenLimit,
   setMergeCacheSize,
   vocabularySize,

package/src/model/o1-2024-12-17.ts

@@ -0,0 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/o200k_base.js'
+export * from '../encoding/o200k_base.js'

package/src/model/o1.ts

@@ -0,0 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/o200k_base.js'
+export * from '../encoding/o200k_base.js'

package/src/model/o3-mini.ts

@@ -0,0 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/o200k_base.js'
+export * from '../encoding/o200k_base.js'