@zwa73/utils 1.0.8 → 1.0.9

package/dist/UtilCodecs.js

@@ -3,8 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.decodeTokenDavinci = exports.decodeTokenTurbo = exports.encodeTokenDavinci = exports.encodeTokenTurbo = exports.tokenNumDavinci = exports.tokenNumTurbo = exports.encodeHtmlEntities = exports.decodeHtmlEntities = void 0;
 const he = require("html-entities");
 const tiktoken_1 = require("tiktoken");
-const encoderTurbo = (0, tiktoken_1.get_encoding)("cl100k_base");
-const encoderDavinci = (0, tiktoken_1.get_encoding)("p50k_base");
+let encoderTurbo = null;
+let encoderDavinci = null;
 const textDecoder = new TextDecoder();
 // 定义一个对象，存储常见的HTML实体和对应的字符
 let htmlEntities = {
@@ -46,13 +46,21 @@ exports.encodeHtmlEntities = encodeHtmlEntities;
 //cl100k_base	ChatGPT models, text-embedding-ada-002
 //p50k_base	Code models, text-davinci-002, text-davinci-003
 //r50k_base (or gpt2)	GPT-3 models like davinci
+//避免在nextjs调用时出错
+function initTikTokenEncoder() {
+    if (encoderTurbo != null && encoderDavinci != null)
+        return;
+    encoderTurbo = (0, tiktoken_1.get_encoding)("cl100k_base");
+    encoderDavinci = (0, tiktoken_1.get_encoding)("p50k_base");
+}
 /**token长度计算器 Turbo模型
  * @param {string} str = 所要计算的消息
  * @returns {number} 整数长度结果
  */
 function tokenNumTurbo(str) {
+    initTikTokenEncoder();
     //return encoder.encode(str).length
-    return encoderTurbo.encode(str).length;
+    return encoderTurbo?.encode(str).length;
 }
 exports.tokenNumTurbo = tokenNumTurbo;
 /**token长度计算器 Davinci模型
@@ -60,7 +68,8 @@ exports.tokenNumTurbo = tokenNumTurbo;
  * @returns {number} 整数长度结果
  */
 function tokenNumDavinci(str) {
-    return encoderDavinci.encode(str).length;
+    initTikTokenEncoder();
+    return encoderDavinci?.encode(str).length;
 }
 exports.tokenNumDavinci = tokenNumDavinci;
 /**token编码 Turbo模型
@@ -68,7 +77,8 @@ exports.tokenNumDavinci = tokenNumDavinci;
  * @returns {Array<number>} Token数组
  */
 function encodeTokenTurbo(str) {
-    return encoderTurbo.encode(str);
+    initTikTokenEncoder();
+    return encoderTurbo?.encode(str);
 }
 exports.encodeTokenTurbo = encodeTokenTurbo;
 /**token编码 Davinci模型
@@ -76,7 +86,8 @@ exports.encodeTokenTurbo = encodeTokenTurbo;
  * @returns {Array<number>} Token数组
  */
 function encodeTokenDavinci(str) {
-    return encoderDavinci.encode(str);
+    initTikTokenEncoder();
+    return encoderDavinci?.encode(str);
 }
 exports.encodeTokenDavinci = encodeTokenDavinci;
 /**token解码 Turbo模型
@@ -84,7 +95,8 @@ exports.encodeTokenDavinci = encodeTokenDavinci;
  * @returns {string} 消息字符串
  */
 function decodeTokenTurbo(arr) {
-    return textDecoder.decode(encoderTurbo.decode(arr));
+    initTikTokenEncoder();
+    return textDecoder.decode(encoderTurbo?.decode(arr));
 }
 exports.decodeTokenTurbo = decodeTokenTurbo;
 /**token解码 Davinci模型
@@ -92,6 +104,7 @@ exports.decodeTokenTurbo = decodeTokenTurbo;
  * @returns {string} 消息字符串
  */
 function decodeTokenDavinci(arr) {
-    return textDecoder.decode(encoderDavinci.decode(arr));
+    initTikTokenEncoder();
+    return textDecoder.decode(encoderDavinci?.decode(arr));
 }
 exports.decodeTokenDavinci = decodeTokenDavinci;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zwa73/utils",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "my utils",
   "main": "index.js",
   "scripts": {

package/publish.bat

@@ -0,0 +1,2 @@
+npm publish --access public
+pause

package/src/UtilCodecs.ts

@@ -1,7 +1,7 @@
 import * as he from 'html-entities';
-import {get_encoding} from 'tiktoken';
-const encoderTurbo = get_encoding("cl100k_base");
-const encoderDavinci = get_encoding("p50k_base");
+import {get_encoding,Tiktoken} from 'tiktoken';
+let encoderTurbo:Tiktoken|null = null;
+let encoderDavinci:Tiktoken|null = null;
 const textDecoder = new TextDecoder();
 
 
@@ -48,20 +48,32 @@ export function encodeHtmlEntities(str:string) {
 //p50k_base	Code models, text-davinci-002, text-davinci-003
 //r50k_base (or gpt2)	GPT-3 models like davinci
 
+
+//避免在nextjs调用时出错
+function initTikTokenEncoder (){
+	if(encoderTurbo!=null && encoderDavinci!=null)
+		return;
+
+	encoderTurbo = get_encoding("cl100k_base");
+	encoderDavinci = get_encoding("p50k_base");
+}
+
 /**token长度计算器 Turbo模型
  * @param {string} str = 所要计算的消息
  * @returns {number} 整数长度结果
  */
-export function tokenNumTurbo(str:string){
+export function tokenNumTurbo(str:string):number{
+	initTikTokenEncoder();
 	//return encoder.encode(str).length
-	return encoderTurbo.encode(str).length;
+	return encoderTurbo?.encode(str).length as any as number;
 }
 /**token长度计算器 Davinci模型
  * @param {string} str = 所要计算的消息
  * @returns {number} 整数长度结果
  */
 export function tokenNumDavinci(str:string):number{
-	return encoderDavinci.encode(str).length;
+	initTikTokenEncoder();
+	return encoderDavinci?.encode(str).length as any as number;
 }
 
 /**token编码 Turbo模型
@@ -69,26 +81,30 @@ export function tokenNumDavinci(str:string):number{
  * @returns {Array<number>} Token数组
  */
 export function encodeTokenTurbo(str:string):Uint32Array{
-	return encoderTurbo.encode(str)
+	initTikTokenEncoder();
+	return encoderTurbo?.encode(str) as any as Uint32Array
 }
 /**token编码 Davinci模型
  * @param {string} str = 所要计算的消息
  * @returns {Array<number>} Token数组
  */
 export function encodeTokenDavinci(str:string):Uint32Array{
-	return encoderDavinci.encode(str)
+	initTikTokenEncoder();
+	return encoderDavinci?.encode(str) as any as Uint32Array;
 }
 /**token解码 Turbo模型
  * @param {Array<number>} arr = Token数组
  * @returns {string} 消息字符串
  */
 export function decodeTokenTurbo(arr:Uint32Array):string{
-	return textDecoder.decode(encoderTurbo.decode(arr));
+	initTikTokenEncoder();
+	return textDecoder.decode(encoderTurbo?.decode(arr));
 }
 /**token解码 Davinci模型
  * @param {Array<number>} arr = Token数组
  * @returns {string} 消息字符串
  */
 export function decodeTokenDavinci(arr:Uint32Array):string{
-	return textDecoder.decode(encoderDavinci.decode(arr));
+	initTikTokenEncoder();
+	return textDecoder.decode(encoderDavinci?.decode(arr));
 }