gpt-tokenizer 2.0.0-beta.2 → 2.1.0

package/src/codegen/generateByModel.ts

@@ -0,0 +1,35 @@
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { fileURLToPath } from 'url'
+import { chatModelParams, modelToEncodingMap } from '../mapping.js'
+
+// eslint-disable-next-line no-underscore-dangle
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+const chatModels = Object.keys(chatModelParams)
+
+const template = await fs.readFile(
+  path.join(__dirname, '../encoding/cl100k_base.ts'),
+  'utf8',
+)
+
+await Promise.all(
+  Object.entries(modelToEncodingMap).map(async ([modelName, encoding]) => {
+    await fs.mkdir(path.join(__dirname, '../model'), { recursive: true })
+    const isChatModel = chatModels.includes(modelName)
+
+    const content = isChatModel
+      ? template.replace(
+          `getEncodingApi('cl100k_base'`,
+          `getEncodingApiForModel('${modelName}'`,
+        )
+      : `export * from '../encoding/${encoding}.js'\n`
+    await fs.writeFile(
+      path.join(__dirname, `../model/${modelName}.ts`),
+      content,
+    )
+
+    // eslint-disable-next-line no-console
+    console.log(`encoding/${modelName}.ts`)
+  }),
+)

package/src/{generateJsModels.ts → codegen/generateJsEncodings.ts}

@@ -28,7 +28,7 @@ const processFilesInDirectory = async (
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 await processFilesInDirectory(
-  path.join(__dirname, '../data'),
+  path.join(__dirname, '../../data'),
   async (filePath) => {
     if (!filePath.endsWith('.tiktoken')) return
 
@@ -40,9 +40,9 @@ await processFilesInDirectory(
       return [token, Number.parseInt(rank!, 10)]
     })
 
-    await fs.mkdir(path.join(__dirname, 'models'), { recursive: true })
+    await fs.mkdir(path.join(__dirname, '../encodings'), { recursive: true })
     await fs.writeFile(
-      path.join(__dirname, `models/${modelName}.js`),
+      path.join(__dirname, `../encodings/${modelName}.js`),
       `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\n/** @type {[string, number][]} */\nconst encoder = ${JSON.stringify(
         encoder,
       )};\nexport default encoder;`,

package/src/encoding/cl100k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-const api = GptEncoding.getEncodingApi('cl100k_base', () =>
+export const api = GptEncoding.getEncodingApi('cl100k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {
@@ -15,12 +15,16 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
 } = api
 export {
   decode,
   decodeAsyncGenerator,
   decodeGenerator,
   encode,
+  encodeChat,
+  encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
 }

package/src/encoding/p50k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-const api = GptEncoding.getEncodingApi('p50k_base', () =>
+export const api = GptEncoding.getEncodingApi('p50k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/encoding/p50k_edit.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-const api = GptEncoding.getEncodingApi('p50k_edit', () =>
+export const api = GptEncoding.getEncodingApi('p50k_edit', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/encoding/r50k_base.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 
 export * from '../specialTokens.js'
 
-const api = GptEncoding.getEncodingApi('r50k_base', () =>
+export const api = GptEncoding.getEncodingApi('r50k_base', () =>
   convertTokenBytePairEncodingFromTuples(encoder),
 )
 const {

package/src/mapping.ts

@@ -1,4 +1,7 @@
 /* eslint-disable camelcase */
+
+import { ImSep } from './specialTokens.js'
+
 export const cl100k_base = 'cl100k_base'
 export const p50k_base = 'p50k_base'
 export const p50k_edit = 'p50k_edit'
@@ -14,7 +17,11 @@ export const encodingNames = [
 export const modelToEncodingMap = {
   // chat
   'gpt-4': cl100k_base,
+  'gpt-4-32k': cl100k_base,
+  'gpt-4-0314': cl100k_base,
+  'gpt-4-32k-0314': cl100k_base,
   'gpt-3.5-turbo': cl100k_base,
+  'gpt-3.5-turbo-0301': cl100k_base,
   // text
   'text-davinci-003': p50k_base,
   'text-davinci-002': p50k_base,
@@ -51,5 +58,40 @@ export const modelToEncodingMap = {
   'code-search-ada-code-001': r50k_base,
 } as const
 
+export interface ChatParameters {
+  messageSeparator: string
+  roleSeparator: string
+}
+
+const internalChatModelParams = {
+  'gpt-3.5-turbo': {
+    messageSeparator: '\n',
+    roleSeparator: '\n',
+  },
+  'gpt-3.5-turbo-0301': {
+    messageSeparator: '\n',
+    roleSeparator: '\n',
+  },
+  'gpt-4': {
+    messageSeparator: '',
+    roleSeparator: ImSep,
+  },
+  'gpt-4-0314': {
+    messageSeparator: '',
+    roleSeparator: ImSep,
+  },
+  'gpt-4-32k': {
+    messageSeparator: '',
+    roleSeparator: ImSep,
+  },
+  'gpt-4-32k-0314': {
+    messageSeparator: '',
+    roleSeparator: ImSep,
+  },
+}
+
+export const chatModelParams: Partial<Record<ModelName, ChatParameters>> =
+  internalChatModelParams
 export type ModelName = keyof typeof modelToEncodingMap
+export type ChatModelName = keyof typeof internalChatModelParams
 export type EncodingName = (typeof modelToEncodingMap)[ModelName]

package/src/model/gpt-3.5-turbo-0301.ts

@@ -0,0 +1,32 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-3.5-turbo.ts

@@ -1 +1,32 @@
-export * from '../encoding/cl100k_base.js'
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4-0314.ts

@@ -0,0 +1,32 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4-32k-0314.ts

@@ -0,0 +1,32 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4-32k.ts

@@ -0,0 +1,32 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/gpt-4.ts

@@ -1 +1,32 @@
-export * from '../encoding/cl100k_base.js'
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js'
+import encoder from '../encodings/cl100k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+
+const api = GptEncoding.getEncodingApiForModel('gpt-4', () =>
+  convertTokenBytePairEncodingFromTuples(encoder),
+)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/modelParams.ts

@@ -1,15 +1,18 @@
 /* eslint-disable no-magic-numbers */
 import { EncoderMap } from './EncoderMap.js'
-import type { EncodingName } from './mapping.js'
+import type { EncodingName, ModelName } from './mapping.js'
 import {
   EndOfPrompt,
   EndOfText,
   FimMiddle,
   FimPrefix,
   FimSuffix,
+  ImEnd,
+  ImSep,
+  ImStart,
 } from './specialTokens.js'
 
-export interface ModelParams {
+export interface EncodingParams {
   /**
    * The expected total number of tokens in the vocabulary, including both regular and special tokens.
    * This parameter is used to ensure that the combined number of regular and special tokens matches the expected total.
@@ -24,12 +27,13 @@ export interface ModelParams {
   tokenSplitRegex: RegExp
   mergeableBytePairRanks: EncoderMap
   specialTokenMapping: Map<string, number>
+  modelName?: ModelName
 }
 
 const tokenSplitRegex =
   /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
 
-function R50KBase(mergeableBytePairRanks: EncoderMap): ModelParams {
+function R50KBase(mergeableBytePairRanks: EncoderMap): EncodingParams {
   return {
     expectedVocabularySize: 50_257,
     tokenSplitRegex,
@@ -38,7 +42,7 @@ function R50KBase(mergeableBytePairRanks: EncoderMap): ModelParams {
   }
 }
 
-function P50KBase(mergeableBytePairRanks: EncoderMap): ModelParams {
+function P50KBase(mergeableBytePairRanks: EncoderMap): EncodingParams {
   return {
     expectedVocabularySize: 50_281,
     tokenSplitRegex,
@@ -47,7 +51,7 @@ function P50KBase(mergeableBytePairRanks: EncoderMap): ModelParams {
   }
 }
 
-function P50KEdit(mergeableBytePairRanks: EncoderMap): ModelParams {
+function P50KEdit(mergeableBytePairRanks: EncoderMap): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 50_256],
     [FimPrefix, 50_281],
@@ -62,12 +66,15 @@ function P50KEdit(mergeableBytePairRanks: EncoderMap): ModelParams {
   }
 }
 
-function Cl100KBase(mergeableBytePairRanks: EncoderMap): ModelParams {
+function Cl100KBase(mergeableBytePairRanks: EncoderMap): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 100_257],
     [FimPrefix, 100_258],
     [FimMiddle, 100_259],
     [FimSuffix, 100_260],
+    [ImStart, 100_264],
+    [ImEnd, 100_265],
+    [ImSep, 100_266],
     [EndOfPrompt, 100_276],
   ])
 
@@ -84,10 +91,10 @@ export type GetMergeableRanksAsyncFn = (
   encodingName: EncodingName,
 ) => Promise<EncoderMap>
 
-export function getModelParams(
+export function getEncodingParams(
   encodingName: EncodingName,
   getMergeableRanks: GetMergeableRanksFn,
-): ModelParams {
+): EncodingParams {
   const mergeableBytePairRanks = getMergeableRanks(encodingName)
   switch (encodingName.toLowerCase()) {
     case 'r50k_base':
@@ -110,7 +117,7 @@ export function getModelParams(
 export async function getModelParamsAsync(
   encodingName: EncodingName,
   getMergeableRanks: GetMergeableRanksAsyncFn,
-): Promise<ModelParams> {
+): Promise<EncodingParams> {
   const mergeableBytePairRanks = await getMergeableRanks(encodingName)
-  return getModelParams(encodingName, () => mergeableBytePairRanks)
+  return getEncodingParams(encodingName, () => mergeableBytePairRanks)
 }

package/src/specialTokens.ts

@@ -2,4 +2,7 @@ export const EndOfText = '<|endoftext|>'
 export const FimPrefix = '<|fim_prefix|>'
 export const FimMiddle = '<|fim_middle|>'
 export const FimSuffix = '<|fim_suffix|>'
+export const ImStart = '<|im_start|>' // 100264
+export const ImEnd = '<|im_end|>' // 100265
+export const ImSep = '<|im_sep|>' // 100266
 export const EndOfPrompt = '<|endofprompt|>'

package/src/util.ts

@@ -0,0 +1,15 @@
+import { escapeRegExp } from './escapeRegExp.js'
+
+export function getMaxValueFromMap(map: Map<unknown, number>): number {
+  let max = 0
+  map.forEach((val) => {
+    max = Math.max(max, val)
+  })
+  return max
+}
+
+export function getSpecialTokenRegex(tokens: Set<string>): RegExp {
+  const escapedTokens = [...tokens].map(escapeRegExp)
+  const inner = escapedTokens.join('|')
+  return new RegExp(`(${inner})`)
+}

package/esm/generateByModel.js

@@ -1,13 +0,0 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
-import { fileURLToPath } from 'url';
-import { modelToEncodingMap } from './mapping.js';
-// eslint-disable-next-line no-underscore-dangle
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-await Promise.all(Object.entries(modelToEncodingMap).map(async ([modelName, encoding]) => {
-    await fs.mkdir(path.join(__dirname, 'model'), { recursive: true });
-    await fs.writeFile(path.join(__dirname, `model/${modelName}.ts`), `export * from '../encoding/${encoding}.js'\n`);
-    // eslint-disable-next-line no-console
-    console.log(`encoding/${modelName}.js`);
-}));
-//# sourceMappingURL=generateByModel.js.map

package/esm/generateByModel.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateByModel.js","sourceRoot":"","sources":["../src/generateByModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAEjD,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAE9D,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE;IACrE,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAClE,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,SAAS,KAAK,CAAC,EAC7C,8BAA8B,QAAQ,QAAQ,CAC/C,CAAA;IAED,sCAAsC;IACtC,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,KAAK,CAAC,CAAA;AACzC,CAAC,CAAC,CACH,CAAA"}

package/esm/generateJsModels.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateJsModels.js","sourceRoot":"","sources":["../src/generateJsModels.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAInC,MAAM,uBAAuB,GAAG,KAAK,EACnC,aAAqB,EACrB,EAAoB,EACL,EAAE;IACjB,IAAI;QACF,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QAEtE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBAAE,SAAQ;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACpD,4CAA4C;YAC5C,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAA;SACnB;KACF;IAAC,OAAO,KAAK,EAAE;QACd,sCAAsC;QACtC,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;KAC3C;AACH,CAAC,CAAA;AAED,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAE9D,MAAM,uBAAuB,CAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,EAC/B,KAAK,EAAE,QAAQ,EAAE,EAAE;IACjB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAM;IAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACtD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAClC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAK,EAAE,EAAE,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnE,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,SAAS,KAAK,CAAC,EAC9C,gHAAgH,IAAI,CAAC,SAAS,CAC5H,OAAO,CACR,4BAA4B,CAC9B,CAAA;IAED,sCAAsC;IACtC,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,CAAC,CAAA;AACtC,CAAC,CACF,CAAA"}

package/src/generateByModel.ts

@@ -1,20 +0,0 @@
-import * as fs from 'fs/promises'
-import * as path from 'path'
-import { fileURLToPath } from 'url'
-import { modelToEncodingMap } from './mapping.js'
-
-// eslint-disable-next-line no-underscore-dangle
-const __dirname = path.dirname(fileURLToPath(import.meta.url))
-
-await Promise.all(
-  Object.entries(modelToEncodingMap).map(async ([modelName, encoding]) => {
-    await fs.mkdir(path.join(__dirname, 'model'), { recursive: true })
-    await fs.writeFile(
-      path.join(__dirname, `model/${modelName}.ts`),
-      `export * from '../encoding/${encoding}.js'\n`,
-    )
-
-    // eslint-disable-next-line no-console
-    console.log(`encoding/${modelName}.js`)
-  }),
-)