gpt-tokenizer 2.6.0 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/LICENSE +1 -2
  2. package/README.md +22 -11
  3. package/cjs/GptEncoding.d.ts +1 -0
  4. package/cjs/GptEncoding.js +6 -4
  5. package/cjs/GptEncoding.js.map +1 -1
  6. package/cjs/encoding/cl100k_base.d.ts +2 -2
  7. package/cjs/encoding/cl100k_base.js +3 -2
  8. package/cjs/encoding/cl100k_base.js.map +1 -1
  9. package/cjs/encoding/o200k_base.d.ts +2 -2
  10. package/cjs/encoding/o200k_base.js +3 -2
  11. package/cjs/encoding/o200k_base.js.map +1 -1
  12. package/cjs/encoding/p50k_base.d.ts +2 -2
  13. package/cjs/encoding/p50k_base.js +3 -2
  14. package/cjs/encoding/p50k_base.js.map +1 -1
  15. package/cjs/encoding/p50k_edit.d.ts +2 -2
  16. package/cjs/encoding/p50k_edit.js +3 -2
  17. package/cjs/encoding/p50k_edit.js.map +1 -1
  18. package/cjs/encoding/r50k_base.d.ts +2 -2
  19. package/cjs/encoding/r50k_base.js +3 -2
  20. package/cjs/encoding/r50k_base.js.map +1 -1
  21. package/cjs/model/gpt-3.5-turbo-0125.d.ts +2 -2
  22. package/cjs/model/gpt-3.5-turbo-0125.js +3 -2
  23. package/cjs/model/gpt-3.5-turbo-0125.js.map +1 -1
  24. package/cjs/model/gpt-3.5-turbo-0301.d.ts +2 -2
  25. package/cjs/model/gpt-3.5-turbo-0301.js +3 -2
  26. package/cjs/model/gpt-3.5-turbo-0301.js.map +1 -1
  27. package/cjs/model/gpt-3.5-turbo-0613.d.ts +2 -2
  28. package/cjs/model/gpt-3.5-turbo-0613.js +3 -2
  29. package/cjs/model/gpt-3.5-turbo-0613.js.map +1 -1
  30. package/cjs/model/gpt-3.5-turbo-1106.d.ts +2 -2
  31. package/cjs/model/gpt-3.5-turbo-1106.js +3 -2
  32. package/cjs/model/gpt-3.5-turbo-1106.js.map +1 -1
  33. package/cjs/model/gpt-3.5-turbo-16k-0613.d.ts +2 -2
  34. package/cjs/model/gpt-3.5-turbo-16k-0613.js +3 -2
  35. package/cjs/model/gpt-3.5-turbo-16k-0613.js.map +1 -1
  36. package/cjs/model/gpt-3.5-turbo-16k.d.ts +2 -2
  37. package/cjs/model/gpt-3.5-turbo-16k.js +3 -2
  38. package/cjs/model/gpt-3.5-turbo-16k.js.map +1 -1
  39. package/cjs/model/gpt-3.5-turbo-finetune.d.ts +2 -2
  40. package/cjs/model/gpt-3.5-turbo-finetune.js +3 -2
  41. package/cjs/model/gpt-3.5-turbo-finetune.js.map +1 -1
  42. package/cjs/model/gpt-3.5-turbo.d.ts +2 -2
  43. package/cjs/model/gpt-3.5-turbo.js +3 -2
  44. package/cjs/model/gpt-3.5-turbo.js.map +1 -1
  45. package/cjs/model/gpt-4-0125-preview.d.ts +2 -2
  46. package/cjs/model/gpt-4-0125-preview.js +3 -2
  47. package/cjs/model/gpt-4-0125-preview.js.map +1 -1
  48. package/cjs/model/gpt-4-0314.d.ts +2 -2
  49. package/cjs/model/gpt-4-0314.js +3 -2
  50. package/cjs/model/gpt-4-0314.js.map +1 -1
  51. package/cjs/model/gpt-4-0613.d.ts +2 -2
  52. package/cjs/model/gpt-4-0613.js +3 -2
  53. package/cjs/model/gpt-4-0613.js.map +1 -1
  54. package/cjs/model/gpt-4-1106-preview.d.ts +2 -2
  55. package/cjs/model/gpt-4-1106-preview.js +3 -2
  56. package/cjs/model/gpt-4-1106-preview.js.map +1 -1
  57. package/cjs/model/gpt-4-1106-vision-preview.d.ts +2 -2
  58. package/cjs/model/gpt-4-1106-vision-preview.js +3 -2
  59. package/cjs/model/gpt-4-1106-vision-preview.js.map +1 -1
  60. package/cjs/model/gpt-4-32k-0314.d.ts +2 -2
  61. package/cjs/model/gpt-4-32k-0314.js +3 -2
  62. package/cjs/model/gpt-4-32k-0314.js.map +1 -1
  63. package/cjs/model/gpt-4-32k-0613.d.ts +2 -2
  64. package/cjs/model/gpt-4-32k-0613.js +3 -2
  65. package/cjs/model/gpt-4-32k-0613.js.map +1 -1
  66. package/cjs/model/gpt-4-32k.d.ts +2 -2
  67. package/cjs/model/gpt-4-32k.js +3 -2
  68. package/cjs/model/gpt-4-32k.js.map +1 -1
  69. package/cjs/model/gpt-4-turbo-2024-04-09.d.ts +2 -2
  70. package/cjs/model/gpt-4-turbo-2024-04-09.js +3 -2
  71. package/cjs/model/gpt-4-turbo-2024-04-09.js.map +1 -1
  72. package/cjs/model/gpt-4-turbo-preview.d.ts +2 -2
  73. package/cjs/model/gpt-4-turbo-preview.js +3 -2
  74. package/cjs/model/gpt-4-turbo-preview.js.map +1 -1
  75. package/cjs/model/gpt-4-turbo.d.ts +2 -2
  76. package/cjs/model/gpt-4-turbo.js +3 -2
  77. package/cjs/model/gpt-4-turbo.js.map +1 -1
  78. package/cjs/model/gpt-4-vision-preview.d.ts +2 -2
  79. package/cjs/model/gpt-4-vision-preview.js +3 -2
  80. package/cjs/model/gpt-4-vision-preview.js.map +1 -1
  81. package/cjs/model/gpt-4.d.ts +2 -2
  82. package/cjs/model/gpt-4.js +3 -2
  83. package/cjs/model/gpt-4.js.map +1 -1
  84. package/cjs/model/gpt-4o-2024-05-13.d.ts +2 -2
  85. package/cjs/model/gpt-4o-2024-05-13.js +3 -2
  86. package/cjs/model/gpt-4o-2024-05-13.js.map +1 -1
  87. package/cjs/model/gpt-4o-2024-08-06-finetune.d.ts +2 -2
  88. package/cjs/model/gpt-4o-2024-08-06-finetune.js +3 -2
  89. package/cjs/model/gpt-4o-2024-08-06-finetune.js.map +1 -1
  90. package/cjs/model/gpt-4o-2024-08-06.d.ts +2 -2
  91. package/cjs/model/gpt-4o-2024-08-06.js +3 -2
  92. package/cjs/model/gpt-4o-2024-08-06.js.map +1 -1
  93. package/cjs/model/gpt-4o-audio-preview-2024-10-01.d.ts +2 -2
  94. package/cjs/model/gpt-4o-audio-preview-2024-10-01.js +3 -2
  95. package/cjs/model/gpt-4o-audio-preview-2024-10-01.js.map +1 -1
  96. package/cjs/model/gpt-4o-audio-preview.d.ts +2 -2
  97. package/cjs/model/gpt-4o-audio-preview.js +3 -2
  98. package/cjs/model/gpt-4o-audio-preview.js.map +1 -1
  99. package/cjs/model/gpt-4o-mini-2024-07-18-finetune.d.ts +2 -2
  100. package/cjs/model/gpt-4o-mini-2024-07-18-finetune.js +3 -2
  101. package/cjs/model/gpt-4o-mini-2024-07-18-finetune.js.map +1 -1
  102. package/cjs/model/gpt-4o-mini-2024-07-18.d.ts +2 -2
  103. package/cjs/model/gpt-4o-mini-2024-07-18.js +3 -2
  104. package/cjs/model/gpt-4o-mini-2024-07-18.js.map +1 -1
  105. package/cjs/model/gpt-4o-mini-training-2024-07-18.d.ts +2 -2
  106. package/cjs/model/gpt-4o-mini-training-2024-07-18.js +3 -2
  107. package/cjs/model/gpt-4o-mini-training-2024-07-18.js.map +1 -1
  108. package/cjs/model/gpt-4o-mini-training.d.ts +2 -2
  109. package/cjs/model/gpt-4o-mini-training.js +3 -2
  110. package/cjs/model/gpt-4o-mini-training.js.map +1 -1
  111. package/cjs/model/gpt-4o-mini.d.ts +2 -2
  112. package/cjs/model/gpt-4o-mini.js +3 -2
  113. package/cjs/model/gpt-4o-mini.js.map +1 -1
  114. package/cjs/model/gpt-4o-realtime-preview-2024-10-01.d.ts +2 -2
  115. package/cjs/model/gpt-4o-realtime-preview-2024-10-01.js +3 -2
  116. package/cjs/model/gpt-4o-realtime-preview-2024-10-01.js.map +1 -1
  117. package/cjs/model/gpt-4o-realtime-preview.d.ts +2 -2
  118. package/cjs/model/gpt-4o-realtime-preview.js +3 -2
  119. package/cjs/model/gpt-4o-realtime-preview.js.map +1 -1
  120. package/cjs/model/gpt-4o.d.ts +2 -2
  121. package/cjs/model/gpt-4o.js +3 -2
  122. package/cjs/model/gpt-4o.js.map +1 -1
  123. package/cjs/modelParams.d.ts +0 -2
  124. package/cjs/modelParams.js.map +1 -1
  125. package/cjs/models.d.ts +1 -1
  126. package/dist/cl100k_base.js +1 -1
  127. package/dist/cl100k_base.js.map +1 -1
  128. package/dist/o200k_base.js +1 -1
  129. package/dist/o200k_base.js.map +1 -1
  130. package/dist/p50k_base.js +1 -1
  131. package/dist/p50k_base.js.map +1 -1
  132. package/dist/p50k_edit.js +1 -1
  133. package/dist/p50k_edit.js.map +1 -1
  134. package/dist/r50k_base.js +1 -1
  135. package/dist/r50k_base.js.map +1 -1
  136. package/esm/GptEncoding.d.ts +1 -0
  137. package/esm/GptEncoding.js +6 -4
  138. package/esm/GptEncoding.js.map +1 -1
  139. package/esm/codegen/generateJsBpe.js +1 -1
  140. package/esm/codegen/generateJsBpe.js.map +1 -1
  141. package/esm/encoding/cl100k_base.d.ts +2 -2
  142. package/esm/encoding/cl100k_base.js +2 -2
  143. package/esm/encoding/cl100k_base.js.map +1 -1
  144. package/esm/encoding/o200k_base.d.ts +2 -2
  145. package/esm/encoding/o200k_base.js +2 -2
  146. package/esm/encoding/o200k_base.js.map +1 -1
  147. package/esm/encoding/p50k_base.d.ts +2 -2
  148. package/esm/encoding/p50k_base.js +2 -2
  149. package/esm/encoding/p50k_base.js.map +1 -1
  150. package/esm/encoding/p50k_edit.d.ts +2 -2
  151. package/esm/encoding/p50k_edit.js +2 -2
  152. package/esm/encoding/p50k_edit.js.map +1 -1
  153. package/esm/encoding/r50k_base.d.ts +2 -2
  154. package/esm/encoding/r50k_base.js +2 -2
  155. package/esm/encoding/r50k_base.js.map +1 -1
  156. package/esm/model/gpt-3.5-turbo-0125.d.ts +2 -2
  157. package/esm/model/gpt-3.5-turbo-0125.js +2 -2
  158. package/esm/model/gpt-3.5-turbo-0125.js.map +1 -1
  159. package/esm/model/gpt-3.5-turbo-0301.d.ts +2 -2
  160. package/esm/model/gpt-3.5-turbo-0301.js +2 -2
  161. package/esm/model/gpt-3.5-turbo-0301.js.map +1 -1
  162. package/esm/model/gpt-3.5-turbo-0613.d.ts +2 -2
  163. package/esm/model/gpt-3.5-turbo-0613.js +2 -2
  164. package/esm/model/gpt-3.5-turbo-0613.js.map +1 -1
  165. package/esm/model/gpt-3.5-turbo-1106.d.ts +2 -2
  166. package/esm/model/gpt-3.5-turbo-1106.js +2 -2
  167. package/esm/model/gpt-3.5-turbo-1106.js.map +1 -1
  168. package/esm/model/gpt-3.5-turbo-16k-0613.d.ts +2 -2
  169. package/esm/model/gpt-3.5-turbo-16k-0613.js +2 -2
  170. package/esm/model/gpt-3.5-turbo-16k-0613.js.map +1 -1
  171. package/esm/model/gpt-3.5-turbo-16k.d.ts +2 -2
  172. package/esm/model/gpt-3.5-turbo-16k.js +2 -2
  173. package/esm/model/gpt-3.5-turbo-16k.js.map +1 -1
  174. package/esm/model/gpt-3.5-turbo-finetune.d.ts +2 -2
  175. package/esm/model/gpt-3.5-turbo-finetune.js +2 -2
  176. package/esm/model/gpt-3.5-turbo-finetune.js.map +1 -1
  177. package/esm/model/gpt-3.5-turbo.d.ts +2 -2
  178. package/esm/model/gpt-3.5-turbo.js +2 -2
  179. package/esm/model/gpt-3.5-turbo.js.map +1 -1
  180. package/esm/model/gpt-4-0125-preview.d.ts +2 -2
  181. package/esm/model/gpt-4-0125-preview.js +2 -2
  182. package/esm/model/gpt-4-0125-preview.js.map +1 -1
  183. package/esm/model/gpt-4-0314.d.ts +2 -2
  184. package/esm/model/gpt-4-0314.js +2 -2
  185. package/esm/model/gpt-4-0314.js.map +1 -1
  186. package/esm/model/gpt-4-0613.d.ts +2 -2
  187. package/esm/model/gpt-4-0613.js +2 -2
  188. package/esm/model/gpt-4-0613.js.map +1 -1
  189. package/esm/model/gpt-4-1106-preview.d.ts +2 -2
  190. package/esm/model/gpt-4-1106-preview.js +2 -2
  191. package/esm/model/gpt-4-1106-preview.js.map +1 -1
  192. package/esm/model/gpt-4-1106-vision-preview.d.ts +2 -2
  193. package/esm/model/gpt-4-1106-vision-preview.js +2 -2
  194. package/esm/model/gpt-4-1106-vision-preview.js.map +1 -1
  195. package/esm/model/gpt-4-32k-0314.d.ts +2 -2
  196. package/esm/model/gpt-4-32k-0314.js +2 -2
  197. package/esm/model/gpt-4-32k-0314.js.map +1 -1
  198. package/esm/model/gpt-4-32k-0613.d.ts +2 -2
  199. package/esm/model/gpt-4-32k-0613.js +2 -2
  200. package/esm/model/gpt-4-32k-0613.js.map +1 -1
  201. package/esm/model/gpt-4-32k.d.ts +2 -2
  202. package/esm/model/gpt-4-32k.js +2 -2
  203. package/esm/model/gpt-4-32k.js.map +1 -1
  204. package/esm/model/gpt-4-turbo-2024-04-09.d.ts +2 -2
  205. package/esm/model/gpt-4-turbo-2024-04-09.js +2 -2
  206. package/esm/model/gpt-4-turbo-2024-04-09.js.map +1 -1
  207. package/esm/model/gpt-4-turbo-preview.d.ts +2 -2
  208. package/esm/model/gpt-4-turbo-preview.js +2 -2
  209. package/esm/model/gpt-4-turbo-preview.js.map +1 -1
  210. package/esm/model/gpt-4-turbo.d.ts +2 -2
  211. package/esm/model/gpt-4-turbo.js +2 -2
  212. package/esm/model/gpt-4-turbo.js.map +1 -1
  213. package/esm/model/gpt-4-vision-preview.d.ts +2 -2
  214. package/esm/model/gpt-4-vision-preview.js +2 -2
  215. package/esm/model/gpt-4-vision-preview.js.map +1 -1
  216. package/esm/model/gpt-4.d.ts +2 -2
  217. package/esm/model/gpt-4.js +2 -2
  218. package/esm/model/gpt-4.js.map +1 -1
  219. package/esm/model/gpt-4o-2024-05-13.d.ts +2 -2
  220. package/esm/model/gpt-4o-2024-05-13.js +2 -2
  221. package/esm/model/gpt-4o-2024-05-13.js.map +1 -1
  222. package/esm/model/gpt-4o-2024-08-06-finetune.d.ts +2 -2
  223. package/esm/model/gpt-4o-2024-08-06-finetune.js +2 -2
  224. package/esm/model/gpt-4o-2024-08-06-finetune.js.map +1 -1
  225. package/esm/model/gpt-4o-2024-08-06.d.ts +2 -2
  226. package/esm/model/gpt-4o-2024-08-06.js +2 -2
  227. package/esm/model/gpt-4o-2024-08-06.js.map +1 -1
  228. package/esm/model/gpt-4o-audio-preview-2024-10-01.d.ts +2 -2
  229. package/esm/model/gpt-4o-audio-preview-2024-10-01.js +2 -2
  230. package/esm/model/gpt-4o-audio-preview-2024-10-01.js.map +1 -1
  231. package/esm/model/gpt-4o-audio-preview.d.ts +2 -2
  232. package/esm/model/gpt-4o-audio-preview.js +2 -2
  233. package/esm/model/gpt-4o-audio-preview.js.map +1 -1
  234. package/esm/model/gpt-4o-mini-2024-07-18-finetune.d.ts +2 -2
  235. package/esm/model/gpt-4o-mini-2024-07-18-finetune.js +2 -2
  236. package/esm/model/gpt-4o-mini-2024-07-18-finetune.js.map +1 -1
  237. package/esm/model/gpt-4o-mini-2024-07-18.d.ts +2 -2
  238. package/esm/model/gpt-4o-mini-2024-07-18.js +2 -2
  239. package/esm/model/gpt-4o-mini-2024-07-18.js.map +1 -1
  240. package/esm/model/gpt-4o-mini-training-2024-07-18.d.ts +2 -2
  241. package/esm/model/gpt-4o-mini-training-2024-07-18.js +2 -2
  242. package/esm/model/gpt-4o-mini-training-2024-07-18.js.map +1 -1
  243. package/esm/model/gpt-4o-mini-training.d.ts +2 -2
  244. package/esm/model/gpt-4o-mini-training.js +2 -2
  245. package/esm/model/gpt-4o-mini-training.js.map +1 -1
  246. package/esm/model/gpt-4o-mini.d.ts +2 -2
  247. package/esm/model/gpt-4o-mini.js +2 -2
  248. package/esm/model/gpt-4o-mini.js.map +1 -1
  249. package/esm/model/gpt-4o-realtime-preview-2024-10-01.d.ts +2 -2
  250. package/esm/model/gpt-4o-realtime-preview-2024-10-01.js +2 -2
  251. package/esm/model/gpt-4o-realtime-preview-2024-10-01.js.map +1 -1
  252. package/esm/model/gpt-4o-realtime-preview.d.ts +2 -2
  253. package/esm/model/gpt-4o-realtime-preview.js +2 -2
  254. package/esm/model/gpt-4o-realtime-preview.js.map +1 -1
  255. package/esm/model/gpt-4o.d.ts +2 -2
  256. package/esm/model/gpt-4o.js +2 -2
  257. package/esm/model/gpt-4o.js.map +1 -1
  258. package/esm/modelParams.d.ts +0 -2
  259. package/esm/modelParams.js.map +1 -1
  260. package/esm/models.d.ts +1 -1
  261. package/package.json +5 -5
  262. package/src/GptEncoding.ts +9 -6
  263. package/src/codegen/generateJsBpe.ts +1 -1
  264. package/src/encoding/cl100k_base.ts +2 -0
  265. package/src/encoding/o200k_base.ts +2 -0
  266. package/src/encoding/p50k_base.ts +2 -0
  267. package/src/encoding/p50k_edit.ts +2 -0
  268. package/src/encoding/r50k_base.ts +2 -0
  269. package/src/model/gpt-3.5-turbo-0125.ts +2 -0
  270. package/src/model/gpt-3.5-turbo-0301.ts +2 -0
  271. package/src/model/gpt-3.5-turbo-0613.ts +2 -0
  272. package/src/model/gpt-3.5-turbo-1106.ts +2 -0
  273. package/src/model/gpt-3.5-turbo-16k-0613.ts +2 -0
  274. package/src/model/gpt-3.5-turbo-16k.ts +2 -0
  275. package/src/model/gpt-3.5-turbo-finetune.ts +2 -0
  276. package/src/model/gpt-3.5-turbo.ts +2 -0
  277. package/src/model/gpt-4-0125-preview.ts +2 -0
  278. package/src/model/gpt-4-0314.ts +2 -0
  279. package/src/model/gpt-4-0613.ts +2 -0
  280. package/src/model/gpt-4-1106-preview.ts +2 -0
  281. package/src/model/gpt-4-1106-vision-preview.ts +2 -0
  282. package/src/model/gpt-4-32k-0314.ts +2 -0
  283. package/src/model/gpt-4-32k-0613.ts +2 -0
  284. package/src/model/gpt-4-32k.ts +2 -0
  285. package/src/model/gpt-4-turbo-2024-04-09.ts +2 -0
  286. package/src/model/gpt-4-turbo-preview.ts +2 -0
  287. package/src/model/gpt-4-turbo.ts +2 -0
  288. package/src/model/gpt-4-vision-preview.ts +2 -0
  289. package/src/model/gpt-4.ts +2 -0
  290. package/src/model/gpt-4o-2024-05-13.ts +2 -0
  291. package/src/model/gpt-4o-2024-08-06-finetune.ts +2 -0
  292. package/src/model/gpt-4o-2024-08-06.ts +2 -0
  293. package/src/model/gpt-4o-audio-preview-2024-10-01.ts +2 -0
  294. package/src/model/gpt-4o-audio-preview.ts +2 -0
  295. package/src/model/gpt-4o-mini-2024-07-18-finetune.ts +2 -0
  296. package/src/model/gpt-4o-mini-2024-07-18.ts +2 -0
  297. package/src/model/gpt-4o-mini-training-2024-07-18.ts +2 -0
  298. package/src/model/gpt-4o-mini-training.ts +2 -0
  299. package/src/model/gpt-4o-mini.ts +2 -0
  300. package/src/model/gpt-4o-realtime-preview-2024-10-01.ts +2 -0
  301. package/src/model/gpt-4o-realtime-preview.ts +2 -0
  302. package/src/model/gpt-4o.ts +2 -0
  303. package/src/modelParams.ts +0 -3
  304. package/src/models.ts +1 -1
package/esm/GptEncoding.d.ts CHANGED
@@ -25,6 +25,7 @@ export declare class GptEncoding {
25
25
  private specialTokensSet;
26
26
  private allSpecialTokenRegex;
27
27
  private defaultSpecialTokenConfig;
28
+ readonly vocabularySize: number;
28
29
  private constructor();
29
30
  static getEncodingApi(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
30
31
  static getEncodingApiForModel(modelName: ModelName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
package/esm/GptEncoding.js CHANGED
@@ -19,6 +19,7 @@ export class GptEncoding {
19
19
  specialTokensSet;
20
20
  allSpecialTokenRegex;
21
21
  defaultSpecialTokenConfig;
22
+ vocabularySize;
22
23
  constructor({ mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, modelName, ...rest }) {
23
24
  this.specialTokenMapping = specialTokenMapping;
24
25
  this.specialTokensSet = new Set(this.specialTokenMapping.keys());
@@ -30,11 +31,12 @@ export class GptEncoding {
30
31
  });
31
32
  this.defaultSpecialTokenConfig = this.processSpecialTokens();
32
33
  const maxTokenValue = Math.max(mergeableBytePairRanks.length - 1, getMaxValueFromMap(specialTokenMapping));
34
+ this.vocabularySize =
35
+ this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
36
+ specialTokenMapping.size;
33
37
  if (expectedVocabularySize !== undefined) {
34
- if (this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
35
- specialTokenMapping.size !==
36
- expectedVocabularySize) {
37
- throw new Error('The number of mergeable tokens and special tokens must be equal to explicit_n_vocab.');
38
+ if (this.vocabularySize !== expectedVocabularySize) {
39
+ throw new Error('The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.');
38
40
  }
39
41
  if (maxTokenValue !== expectedVocabularySize - 1) {
40
42
  throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${maxTokenValue}, expected ${expectedVocabularySize - 1}`);
package/esm/GptEncoding.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AACzE,OAAO,EAKL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEpE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAsBvC,MAAM,OAAO,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IACxC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAErD,YAAoB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,sBAAsB;YACtB,mBAAmB;YACnB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,kBAAkB,CAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IACE,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC3D,mBAAmB,CAAC,IAAI;gBAC1B,sBAAsB,EACtB,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IAAI,cAAc,EAAE,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC5C,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACjD,CAAC;QAED,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACpE,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,eAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC"}
1
+ {"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AACzE,OAAO,EAKL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEpE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAsBvC,MAAM,OAAO,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IACxC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,sBAAsB;YACtB,mBAAmB;YACnB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,kBAAkB,CAAC,mBAAmB,CAAC,CACxC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,mBAAmB,CAAC,IAAI,CAAA;QAE1B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IAAI,cAAc,EAAE,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC5C,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACjD,CAAC;QAED,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACpE,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YACxD,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,eAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC"}
package/esm/codegen/generateJsBpe.js CHANGED
@@ -88,8 +88,8 @@ await processFilesInDirectory(path.join(__dirname, '../../data'), async (filePat
88
88
  const jsCodeToResetHelperArrays = jsCodeConstsForEachChunk.map((_, i) => `c${i}.length = 0;`);
89
89
  await fs.mkdir(path.join(__dirname, '../bpeRanks'), { recursive: true });
90
90
  await fs.writeFile(path.join(__dirname, `../bpeRanks/${modelName}.js`), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore
91
- /** @type {(string | number[])[]} */
92
91
  ${jsCodeConstsForEachChunk.join('\n')}
92
+ /** @type {(string | number[])[]} */
93
93
  const bpe = ${jsCodeBpeArray};
94
94
  ${jsCodeToResetHelperArrays.join('\n')}
95
95
  export default bpe;`);
package/esm/codegen/generateJsBpe.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"generateJsBpe.js","sourceRoot":"","sources":["../../src/codegen/generateJsBpe.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAA;AAE1C,MAAM,uBAAuB,GAAG,KAAK,EACnC,aAAqB,EACrB,EAAoB,EACL,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QAEtE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBAAE,SAAQ;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACpD,4CAA4C;YAC5C,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC;AACH,CAAC,CAAA;AAED,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9D,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAC5D,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAErC,IAAI,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAA;YAC1D,CAAC;YACD,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,OAAO,CAAC,CAAA;IACV,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC;AAED,MAAM,uBAAuB,CAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAClC,KAAK,EAAE,QAAQ,EAAE,EAAE;IACjB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAM;IAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAA;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QAC/C,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAU,CAAA;IACzD,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,iBAAiB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAE1D,MAAM,mBAAmB,GAAa,EAAE,CAAA;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,CAAA;QAE9C,6CAA6C;QAC7C,IAAI,aAAa,GAAG,IAAI,GAAG,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAO,aAAa,EAAE,GAAG,CAAC,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC9B,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAA;QACrD,mBAAmB,CAAC,IAAI,CACtB,UAAU;YACR,CAAC,OAAO,OAAO,KAAK,QAAQ;gBAC1B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBACzB,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAC9B,CAAA;QAED,QAAQ,GAAG,IAAI,CAAA;IACjB,CAAC;IAED,oGAAoG;IACpG,4CAA4C;IAC5C,MAAM,SAAS,GAAG,OAAO,CAAA;IACzB,MAAM,wBAAwB,GAAa,EAAE,CAAA;IAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,GAAG,SAAS,CAAA;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,wBAAwB,CAAC,IAAI,CAC3B,UAAU,CAAC,OAAO,mBAAmB,CAAC,KAAK,CACzC,CAAC,GAAG,SAAS,EACb,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CACpB,GAAG,CACL,CAAA;IACH,CAAC;IACD,0FAA0F;IAC1F,MAAM,cAAc,GAAG,aAAa,wBAAwB;SACzD,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;SAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;IAEhB,gDAAgD;IAChD,MAAM,yBAAyB,GAAG,wBAAwB,CAAC,GAAG,CAC5D,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAC9B,CAAA;IAED,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACxE,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,SAAS,KAAK,CAAC,EACnD;;EAEJ,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;cACvB,cAAc;EAC1B,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,CACf,CAAA;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,CAAC,CAAA;AACtC,CAAC,CACF,CAAA"}
1
+ {"version":3,"file":"generateJsBpe.js","sourceRoot":"","sources":["../../src/codegen/generateJsBpe.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAA;AAE1C,MAAM,uBAAuB,GAAG,KAAK,EACnC,aAAqB,EACrB,EAAoB,EACL,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QAEtE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBAAE,SAAQ;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACpD,4CAA4C;YAC5C,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC;AACH,CAAC,CAAA;AAED,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9D,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAC5D,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAErC,IAAI,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAA;YAC1D,CAAC;YACD,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,OAAO,CAAC,CAAA;IACV,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC;AAED,MAAM,uBAAuB,CAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAClC,KAAK,EAAE,QAAQ,EAAE,EAAE;IACjB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAM;IAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAA;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QAC/C,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAU,CAAA;IACzD,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,iBAAiB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAE1D,MAAM,mBAAmB,GAAa,EAAE,CAAA;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,CAAA;QAE9C,6CAA6C;QAC7C,IAAI,aAAa,GAAG,IAAI,GAAG,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAO,aAAa,EAAE,GAAG,CAAC,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC9B,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAA;QACrD,mBAAmB,CAAC,IAAI,CACtB,UAAU;YACR,CAAC,OAAO,OAAO,KAAK,QAAQ;gBAC1B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBACzB,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAC9B,CAAA;QAED,QAAQ,GAAG,IAAI,CAAA;IACjB,CAAC;IAED,oGAAoG;IACpG,4CAA4C;IAC5C,MAAM,SAAS,GAAG,OAAO,CAAA;IACzB,MAAM,wBAAwB,GAAa,EAAE,CAAA;IAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,GAAG,SAAS,CAAA;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,wBAAwB,CAAC,IAAI,CAC3B,UAAU,CAAC,OAAO,mBAAmB,CAAC,KAAK,CACzC,CAAC,GAAG,SAAS,EACb,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CACpB,GAAG,CACL,CAAA;IACH,CAAC;IACD,0FAA0F;IAC1F,MAAM,cAAc,GAAG,aAAa,wBAAwB;SACzD,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;SAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;IAEhB,gDAAgD;IAChD,MAAM,yBAAyB,GAAG,wBAAwB,CAAC,GAAG,CAC5D,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAC9B,CAAA;IAED,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACxE,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,SAAS,KAAK,CAAC,EACnD;EACJ,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;;cAEvB,cAAc;EAC1B,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClB,CACf,CAAA;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,CAAC,CAAA;AACtC,CAAC,CACF,CAAA"}
package/esm/encoding/cl100k_base.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/encoding/cl100k_base.js CHANGED
@@ -3,8 +3,8 @@ import bpeRanks from '../bpeRanks/cl100k_base.js';
3
3
  import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  const api = GptEncoding.getEncodingApi('cl100k_base', () => bpeRanks);
6
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
7
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
6
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
7
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
8
8
  // eslint-disable-next-line import/no-default-export
9
9
  export default api;
10
10
  //# sourceMappingURL=cl100k_base.js.map
package/esm/encoding/cl100k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/o200k_base.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/encoding/o200k_base.js CHANGED
@@ -3,8 +3,8 @@ import bpeRanks from '../bpeRanks/o200k_base.js';
3
3
  import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  const api = GptEncoding.getEncodingApi('o200k_base', () => bpeRanks);
6
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
7
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
6
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
7
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
8
8
  // eslint-disable-next-line import/no-default-export
9
9
  export default api;
10
10
  //# sourceMappingURL=o200k_base.js.map
package/esm/encoding/o200k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"o200k_base.js","sourceRoot":"","sources":["../../src/encoding/o200k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/p50k_base.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/encoding/p50k_base.js CHANGED
@@ -3,8 +3,8 @@ import bpeRanks from '../bpeRanks/p50k_base.js';
3
3
  import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  const api = GptEncoding.getEncodingApi('p50k_base', () => bpeRanks);
6
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
7
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
8
8
  // eslint-disable-next-line import/no-default-export
9
9
  export default api;
10
10
  //# sourceMappingURL=p50k_base.js.map
package/esm/encoding/p50k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/p50k_edit.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/encoding/p50k_edit.js CHANGED
@@ -3,8 +3,8 @@ import bpeRanks from '../bpeRanks/p50k_base.js';
3
3
  import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  const api = GptEncoding.getEncodingApi('p50k_edit', () => bpeRanks);
6
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
7
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
8
8
  // eslint-disable-next-line import/no-default-export
9
9
  export default api;
10
10
  //# sourceMappingURL=p50k_edit.js.map
package/esm/encoding/p50k_edit.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/r50k_base.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/encoding/r50k_base.js CHANGED
@@ -3,8 +3,8 @@ import bpeRanks from '../bpeRanks/r50k_base.js';
3
3
  import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks);
6
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, } = api;
7
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, vocabularySize, };
8
8
  // eslint-disable-next-line import/no-default-export
9
9
  export default api;
10
10
  //# sourceMappingURL=r50k_base.js.map
package/esm/encoding/r50k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACnE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/model/gpt-3.5-turbo-0125.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/model/gpt-3.5-turbo-0125.js CHANGED
@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  // prettier-ignore
6
6
  const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0125', () => bpeRanks);
7
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
8
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
7
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
8
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
9
9
  // eslint-disable-next-line import/no-default-export
10
10
  export default api;
11
11
  //# sourceMappingURL=gpt-3.5-turbo-0125.js.map
package/esm/model/gpt-3.5-turbo-0125.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"gpt-3.5-turbo-0125.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0125.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"gpt-3.5-turbo-0125.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0125.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/model/gpt-3.5-turbo-0301.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/model/gpt-3.5-turbo-0301.js CHANGED
@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  // prettier-ignore
6
6
  const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => bpeRanks);
7
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
8
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
7
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
8
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
9
9
  // eslint-disable-next-line import/no-default-export
10
10
  export default api;
11
11
  //# sourceMappingURL=gpt-3.5-turbo-0301.js.map
package/esm/model/gpt-3.5-turbo-0301.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/model/gpt-3.5-turbo-0613.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/model/gpt-3.5-turbo-0613.js CHANGED
@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  // prettier-ignore
6
6
  const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0613', () => bpeRanks);
7
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
8
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
7
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
8
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
9
9
  // eslint-disable-next-line import/no-default-export
10
10
  export default api;
11
11
  //# sourceMappingURL=gpt-3.5-turbo-0613.js.map
package/esm/model/gpt-3.5-turbo-0613.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"gpt-3.5-turbo-0613.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0613.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"gpt-3.5-turbo-0613.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0613.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/model/gpt-3.5-turbo-1106.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GptEncoding } from '../GptEncoding.js';
2
2
  export * from '../specialTokens.js';
3
3
  declare const api: GptEncoding;
4
- declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
5
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
4
+ declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
5
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
6
6
  export default api;
package/esm/model/gpt-3.5-turbo-1106.js CHANGED
@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
4
4
  export * from '../specialTokens.js';
5
5
  // prettier-ignore
6
6
  const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-1106', () => bpeRanks);
7
- const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
8
- export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
7
+ const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
8
+ export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
9
9
  // eslint-disable-next-line import/no-default-export
10
10
  export default api;
11
11
  //# sourceMappingURL=gpt-3.5-turbo-1106.js.map
package/esm/model/gpt-3.5-turbo-1106.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"gpt-3.5-turbo-1106.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-1106.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"gpt-3.5-turbo-1106.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-1106.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,4BAA4B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}