gpt-tokenizer 2.1.0 → 2.1.1

package/esm/model/text-similarity-babbage-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-babbage-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-babbage-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-curie-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-curie-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-curie-001.js.map

package/esm/model/text-similarity-curie-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-curie-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-curie-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-curie-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-curie-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-davinci-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-davinci-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-davinci-001.js.map

package/esm/model/text-similarity-davinci-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-davinci-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-davinci-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",
@@ -78,10 +78,10 @@
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
     "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
     "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base",
-    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'export=api' --env 'filename=cl100k_base.js'",
-    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'export=api' --env 'filename=p50k_base.js'",
-    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'export=api' --env 'filename=p50k_edit.js'",
-    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'export=api' --env 'filename=r50k_base.js'",
+    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'",
+    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'",
+    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'filename=p50k_edit.js'",
+    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'filename=r50k_base.js'",
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
     "postinstallDev": "yarn prepare",

package/src/codegen/generateByModel.ts

@@ -23,7 +23,7 @@ await Promise.all(
           `getEncodingApi('cl100k_base'`,
           `getEncodingApiForModel('${modelName}'`,
         )
-      : `export * from '../encoding/${encoding}.js'\n`
+      : `// eslint-disable-next-line no-restricted-exports, import/no-default-export\nexport { default } from '../encoding/${encoding}.js'\nexport * from '../encoding/${encoding}.js'\n`
     await fs.writeFile(
       path.join(__dirname, `../model/${modelName}.ts`),
       content,

package/src/encoding/cl100k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-export const api = GptEncoding.getEncodingApi('cl100k_base', () =>
+const api = GptEncoding.getEncodingApi('cl100k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/encoding/p50k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-export const api = GptEncoding.getEncodingApi('p50k_base', () =>
+const api = GptEncoding.getEncodingApi('p50k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/encoding/p50k_edit.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-export const api = GptEncoding.getEncodingApi('p50k_edit', () =>
+const api = GptEncoding.getEncodingApi('p50k_edit', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/encoding/r50k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-export const api = GptEncoding.getEncodingApi('r50k_base', () =>
+const api = GptEncoding.getEncodingApi('r50k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/main.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from './encoding/cl100k_base.js'
 export * from './encoding/cl100k_base.js'

package/src/model/ada.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/babbage.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/code-cushman-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/code-cushman-002.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/code-davinci-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/code-davinci-002.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/code-davinci-edit-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_edit.js'
 export * from '../encoding/p50k_edit.js'

package/src/model/code-search-ada-code-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/code-search-babbage-code-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/curie.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/cushman-codex.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/davinci-codex.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/davinci.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-ada-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-babbage-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-curie-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-davinci-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-davinci-002.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/text-davinci-003.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js'
 export * from '../encoding/p50k_base.js'

package/src/model/text-davinci-edit-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_edit.js'
 export * from '../encoding/p50k_edit.js'

package/src/model/text-embedding-ada-002.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/cl100k_base.js'
 export * from '../encoding/cl100k_base.js'

package/src/model/text-search-ada-doc-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-search-babbage-doc-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-search-curie-doc-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-search-davinci-doc-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-similarity-ada-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-similarity-babbage-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-similarity-curie-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'

package/src/model/text-similarity-davinci-001.ts

@@ -1 +1,3 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js'
 export * from '../encoding/r50k_base.js'