@zwa73/utils 1.0.223 → 1.0.224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{UtilClass → cjs/UtilClass}/Hbs.js +13 -6
- package/dist/{UtilClass → cjs/UtilClass}/index.js +0 -1
- package/dist/{UtilCodecs.js → cjs/UtilCodecs.js} +24 -16
- package/dist/{UtilDecorators.js → cjs/UtilDecorators.js} +0 -1
- package/dist/{UtilFileTools.js → cjs/UtilFileTools.js} +10 -4
- package/dist/{UtilFunctions.js → cjs/UtilFunctions.js} +10 -5
- package/dist/{UtilSymbol.js → cjs/UtilSymbol.js} +0 -1
- package/dist/mjs/QuickExport.mjs +35 -0
- package/dist/mjs/UtilClass/Hbs.mjs +51 -0
- package/dist/mjs/UtilClass/index.mjs +3 -0
- package/dist/mjs/UtilCodecs.mjs +109 -0
- package/dist/mjs/UtilDecorators.mjs +2 -0
- package/dist/mjs/UtilFP.mjs +2 -0
- package/dist/mjs/UtilFileTools.mjs +407 -0
- package/dist/mjs/UtilFunctions.mjs +138 -0
- package/dist/mjs/UtilHttp.mjs +475 -0
- package/dist/mjs/UtilI18n.mjs +206 -0
- package/dist/mjs/UtilInterfaces.mjs +2 -0
- package/dist/mjs/UtilLogger.mjs +361 -0
- package/dist/mjs/UtilSymbol.mjs +2 -0
- package/dist/mjs/index.mjs +17 -0
- package/dist/{UtilClass → types/UtilClass}/Hbs.d.ts +6 -3
- package/dist/types/UtilClass/index.d.ts +3 -0
- package/dist/{UtilCodecs.d.ts → types/UtilCodecs.d.ts} +6 -6
- package/dist/types/UtilDecorators.d.ts +2 -0
- package/dist/{UtilFileTools.d.ts → types/UtilFileTools.d.ts} +11 -3
- package/dist/{UtilFunctions.d.ts → types/UtilFunctions.d.ts} +3 -3
- package/dist/types/UtilInterfaces.d.ts +1 -0
- package/dist/types/UtilSymbol.d.ts +2 -0
- package/package.json +8 -2
- package/dist/UtilClass/index.d.ts +0 -2
- package/dist/UtilDecorators.d.ts +0 -1
- package/dist/UtilInterfaces.d.ts +0 -1
- package/dist/UtilSymbol.d.ts +0 -1
- /package/dist/{QuickExport.js → cjs/QuickExport.js} +0 -0
- /package/dist/{UtilFP.js → cjs/UtilFP.js} +0 -0
- /package/dist/{UtilHttp.js → cjs/UtilHttp.js} +0 -0
- /package/dist/{UtilI18n.js → cjs/UtilI18n.js} +0 -0
- /package/dist/{UtilInterfaces.js → cjs/UtilInterfaces.js} +0 -0
- /package/dist/{UtilLogger.js → cjs/UtilLogger.js} +0 -0
- /package/dist/{index.js → cjs/index.js} +0 -0
- /package/dist/{QuickExport.d.ts → types/QuickExport.d.ts} +0 -0
- /package/dist/{UtilFP.d.ts → types/UtilFP.d.ts} +0 -0
- /package/dist/{UtilHttp.d.ts → types/UtilHttp.d.ts} +0 -0
- /package/dist/{UtilI18n.d.ts → types/UtilI18n.d.ts} +0 -0
- /package/dist/{UtilLogger.d.ts → types/UtilLogger.d.ts} +0 -0
- /package/dist/{index.d.ts → types/index.d.ts} +0 -0
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.Hbs = void 0;
|
|
7
4
|
const UtilLogger_1 = require("../UtilLogger");
|
|
8
|
-
const
|
|
5
|
+
const UtilFunctions_1 = require("../UtilFunctions");
|
|
9
6
|
/**文本模板渲染器
|
|
10
7
|
* @template T - 上下文对象的类型,默认为记录类型
|
|
11
8
|
*/
|
|
12
9
|
class Hbs {
|
|
13
|
-
hbs
|
|
10
|
+
hbs;
|
|
14
11
|
context;
|
|
15
12
|
/**创建一个新的 Hbs 实例
|
|
16
13
|
* @param context - 上下文对象,用于在渲染模板时提供数据
|
|
17
14
|
*/
|
|
18
|
-
constructor(context = {}) {
|
|
15
|
+
constructor(hbs, context = {}) {
|
|
16
|
+
this.hbs = hbs;
|
|
19
17
|
this.context = context;
|
|
20
18
|
this.hbs.registerHelper('def', (name, options) => {
|
|
21
19
|
const value = options.fn(this.context).trim();
|
|
@@ -31,6 +29,15 @@ class Hbs {
|
|
|
31
29
|
return '';
|
|
32
30
|
});
|
|
33
31
|
}
|
|
32
|
+
/**创建一个新的 Hbs 实例
|
|
33
|
+
* @param context - 上下文对象,用于在渲染模板时提供数据
|
|
34
|
+
*/
|
|
35
|
+
static async create(context = {}) {
|
|
36
|
+
const Handlebars = await UtilFunctions_1.UtilFunc.dynamicImport('handlebars');
|
|
37
|
+
//console.log(Handlebars);
|
|
38
|
+
const hbs = Handlebars.create();
|
|
39
|
+
return new Hbs(hbs, context);
|
|
40
|
+
}
|
|
34
41
|
/**渲染模板
|
|
35
42
|
* @param template - 要渲染的 Handlebars 模板字符串
|
|
36
43
|
* @returns 渲染后的字符串
|
|
@@ -16,7 +16,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.Stream = exports.SmartCache = exports.PromiseQueue = exports.Piper = exports.DLinkedList = exports.Bridge = void 0;
|
|
18
18
|
__exportStar(require("./Hbs"), exports);
|
|
19
|
-
//#region UtilClass转导
|
|
20
19
|
var js_utils_1 = require("@zwa73/js-utils");
|
|
21
20
|
Object.defineProperty(exports, "Bridge", { enumerable: true, get: function () { return js_utils_1.Bridge; } });
|
|
22
21
|
Object.defineProperty(exports, "DLinkedList", { enumerable: true, get: function () { return js_utils_1.DLinkedList; } });
|
|
@@ -25,7 +25,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.UtilCodec = void 0;
|
|
27
27
|
const he = __importStar(require("html-entities"));
|
|
28
|
-
const
|
|
28
|
+
const UtilFunctions_1 = require("./UtilFunctions");
|
|
29
|
+
let tiktoken = undefined;
|
|
30
|
+
async function importTikToken() {
|
|
31
|
+
if (tiktoken != null)
|
|
32
|
+
return tiktoken;
|
|
33
|
+
tiktoken = await UtilFunctions_1.UtilFunc.dynamicImport("tiktoken");
|
|
34
|
+
return tiktoken;
|
|
35
|
+
}
|
|
29
36
|
/**编码/解码器 */
|
|
30
37
|
var UtilCodec;
|
|
31
38
|
(function (UtilCodec) {
|
|
@@ -58,18 +65,19 @@ var UtilCodec;
|
|
|
58
65
|
//p50k_base Code models, text-davinci-002, text-davinci-003
|
|
59
66
|
//r50k_base (or gpt2) GPT-3 models like davinci
|
|
60
67
|
//避免在nextjs调用时出错
|
|
61
|
-
function initTikTokenEncoder() {
|
|
68
|
+
async function initTikTokenEncoder() {
|
|
62
69
|
if (encoderCl100kBase != null && encoderP50kBase != null)
|
|
63
70
|
return;
|
|
64
|
-
|
|
65
|
-
|
|
71
|
+
const { get_encoding } = await importTikToken();
|
|
72
|
+
encoderCl100kBase = get_encoding("cl100k_base");
|
|
73
|
+
encoderP50kBase = get_encoding("p50k_base");
|
|
66
74
|
}
|
|
67
75
|
/**token长度计算器 Turbo模型
|
|
68
76
|
* @param str = 所要计算的消息
|
|
69
77
|
* @returns 整数长度结果
|
|
70
78
|
*/
|
|
71
|
-
function tokenNumTurbo(str) {
|
|
72
|
-
initTikTokenEncoder();
|
|
79
|
+
async function tokenNumTurbo(str) {
|
|
80
|
+
await initTikTokenEncoder();
|
|
73
81
|
//return encoder.encode(str).length
|
|
74
82
|
return encoderCl100kBase?.encode(str).length;
|
|
75
83
|
}
|
|
@@ -78,8 +86,8 @@ var UtilCodec;
|
|
|
78
86
|
* @param str = 所要计算的消息
|
|
79
87
|
* @returns 整数长度结果
|
|
80
88
|
*/
|
|
81
|
-
function tokenNumDavinci(str) {
|
|
82
|
-
initTikTokenEncoder();
|
|
89
|
+
async function tokenNumDavinci(str) {
|
|
90
|
+
await initTikTokenEncoder();
|
|
83
91
|
return encoderP50kBase?.encode(str).length;
|
|
84
92
|
}
|
|
85
93
|
UtilCodec.tokenNumDavinci = tokenNumDavinci;
|
|
@@ -87,8 +95,8 @@ var UtilCodec;
|
|
|
87
95
|
* @param str = 所要计算的消息
|
|
88
96
|
* @returns Token数组
|
|
89
97
|
*/
|
|
90
|
-
function encodeTokenTurbo(str) {
|
|
91
|
-
initTikTokenEncoder();
|
|
98
|
+
async function encodeTokenTurbo(str) {
|
|
99
|
+
await initTikTokenEncoder();
|
|
92
100
|
return encoderCl100kBase?.encode(str);
|
|
93
101
|
}
|
|
94
102
|
UtilCodec.encodeTokenTurbo = encodeTokenTurbo;
|
|
@@ -96,8 +104,8 @@ var UtilCodec;
|
|
|
96
104
|
* @param str = 所要计算的消息
|
|
97
105
|
* @returns Token数组
|
|
98
106
|
*/
|
|
99
|
-
function encodeTokenDavinci(str) {
|
|
100
|
-
initTikTokenEncoder();
|
|
107
|
+
async function encodeTokenDavinci(str) {
|
|
108
|
+
await initTikTokenEncoder();
|
|
101
109
|
return encoderP50kBase?.encode(str);
|
|
102
110
|
}
|
|
103
111
|
UtilCodec.encodeTokenDavinci = encodeTokenDavinci;
|
|
@@ -105,8 +113,8 @@ var UtilCodec;
|
|
|
105
113
|
* @param arr = Token数组
|
|
106
114
|
* @returns 消息字符串
|
|
107
115
|
*/
|
|
108
|
-
function decodeTokenTurbo(arr) {
|
|
109
|
-
initTikTokenEncoder();
|
|
116
|
+
async function decodeTokenTurbo(arr) {
|
|
117
|
+
await initTikTokenEncoder();
|
|
110
118
|
if (Array.isArray(arr))
|
|
111
119
|
arr = new Uint32Array(arr);
|
|
112
120
|
return textDecoder.decode(encoderCl100kBase?.decode(arr));
|
|
@@ -116,8 +124,8 @@ var UtilCodec;
|
|
|
116
124
|
* @param arr = Token数组
|
|
117
125
|
* @returns 消息字符串
|
|
118
126
|
*/
|
|
119
|
-
function decodeTokenDavinci(arr) {
|
|
120
|
-
initTikTokenEncoder();
|
|
127
|
+
async function decodeTokenDavinci(arr) {
|
|
128
|
+
await initTikTokenEncoder();
|
|
121
129
|
if (Array.isArray(arr))
|
|
122
130
|
arr = new Uint32Array(arr);
|
|
123
131
|
return textDecoder.decode(encoderP50kBase?.decode(arr));
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AwaitInited = exports.PostProcess = exports.LogTrace = void 0;
|
|
4
|
-
//#region UtilDecorators转导
|
|
5
4
|
var js_utils_1 = require("@zwa73/js-utils");
|
|
6
5
|
Object.defineProperty(exports, "LogTrace", { enumerable: true, get: function () { return js_utils_1.LogTrace; } });
|
|
7
6
|
Object.defineProperty(exports, "PostProcess", { enumerable: true, get: function () { return js_utils_1.PostProcess; } });
|
|
@@ -257,11 +257,13 @@ var UtilFT;
|
|
|
257
257
|
UtilFT.currosizePath = currosizePath;
|
|
258
258
|
/**逐级寻找node_modules文件夹
|
|
259
259
|
* @param basePath - 基础位置
|
|
260
|
+
* @param extpaths - 额外的路径 如 .bin
|
|
261
|
+
* @returns node_modules文件夹路径 包含额外路径
|
|
260
262
|
*/
|
|
261
|
-
async function findNodeModulesDir(basePath) {
|
|
263
|
+
async function findNodeModulesDir(basePath, ...extpaths) {
|
|
262
264
|
let currentDir = basePath;
|
|
263
265
|
while (currentDir) {
|
|
264
|
-
const nodeModulesPath = pathe_1.default.join(currentDir, 'node_modules');
|
|
266
|
+
const nodeModulesPath = pathe_1.default.join(currentDir, 'node_modules', ...extpaths);
|
|
265
267
|
if (await UtilFT.pathExists(nodeModulesPath))
|
|
266
268
|
return nodeModulesPath;
|
|
267
269
|
const parentDir = pathe_1.default.dirname(currentDir);
|
|
@@ -344,7 +346,9 @@ var UtilFT;
|
|
|
344
346
|
* @param dir - 起始目录
|
|
345
347
|
* @param globPattern - glob匹配
|
|
346
348
|
* @param opt - 可选参数
|
|
347
|
-
* @param opt.ignore
|
|
349
|
+
* @param opt.ignore - 忽略的文件
|
|
350
|
+
* @param opt.relative - 忽略目录 默认 true
|
|
351
|
+
* @param opt.nodir - 忽略大小写 默认 跟随系统 mac/win为true 其他false
|
|
348
352
|
* @returns 文件绝对路径数组
|
|
349
353
|
*/
|
|
350
354
|
async function fileSearchGlob(dir, globPattern, opt) {
|
|
@@ -359,7 +363,9 @@ var UtilFT;
|
|
|
359
363
|
* @param dir - 起始目录
|
|
360
364
|
* @param globPattern - glob匹配
|
|
361
365
|
* @param opt - 可选参数
|
|
362
|
-
* @param opt.ignore
|
|
366
|
+
* @param opt.ignore - 忽略的文件
|
|
367
|
+
* @param opt.relative - 忽略目录 默认 true
|
|
368
|
+
* @param opt.nodir - 忽略大小写 默认 跟随系统 mac/win为true 其他false
|
|
363
369
|
* @returns 文件绝对路径数组
|
|
364
370
|
*/
|
|
365
371
|
function fileSearchGlobSync(dir, globPattern, opt) {
|
|
@@ -77,10 +77,10 @@ class _UtilFunc {
|
|
|
77
77
|
return new Promise(async (resolve, reject) => {
|
|
78
78
|
// 创建一个新的环境变量对象,并将项目的 node_modules/.bin 目录添加到 PATH 环境变量中
|
|
79
79
|
const env = Object.create(process.env);
|
|
80
|
-
const penv =
|
|
81
|
-
? UtilFileTools_1.UtilFT.currosizePath(opt.nodeModules)
|
|
82
|
-
: UtilFileTools_1.UtilFT.currosizePath(await exports.UtilFunc.memoize(UtilFileTools_1.UtilFT.findNodeModulesDir)(process.cwd()
|
|
83
|
-
if (_UtilFunc.checkEnv && !await UtilFileTools_1.UtilFT.pathExists(penv)) {
|
|
80
|
+
const penv = opt?.nodeModules
|
|
81
|
+
? path_1.default.join(UtilFileTools_1.UtilFT.currosizePath(opt.nodeModules), '.bin')
|
|
82
|
+
: UtilFileTools_1.UtilFT.currosizePath(await exports.UtilFunc.memoize(UtilFileTools_1.UtilFT.findNodeModulesDir)(process.cwd(), '.bin') ?? '');
|
|
83
|
+
if (_UtilFunc.checkEnv && (penv == "" || !await UtilFileTools_1.UtilFT.pathExists(penv))) {
|
|
84
84
|
UtilLogger_1.SLogger.warn(`UtilFunc.exec 错误, 没有检测到 node_modules/.bin 环境 penv:${penv}`);
|
|
85
85
|
_UtilFunc.checkEnv = false;
|
|
86
86
|
}
|
|
@@ -120,6 +120,8 @@ class _UtilFunc {
|
|
|
120
120
|
static async getPublicIpv4() {
|
|
121
121
|
if (_UtilFunc.publicIp === undefined)
|
|
122
122
|
_UtilFunc.publicIp = await _UtilFunc.dynamicImport('public-ip');
|
|
123
|
+
//if(_UtilFunc.publicIp===undefined)
|
|
124
|
+
// _UtilFunc.publicIp = await import("public-ip");
|
|
123
125
|
return _UtilFunc.publicIp.publicIpv4();
|
|
124
126
|
}
|
|
125
127
|
/**动态导入模块的函数。这个函数是为了在TypeScript的模块系统配置为CommonJS时,防止动态import被转译为require而设计的。
|
|
@@ -139,7 +141,10 @@ class _UtilFunc {
|
|
|
139
141
|
* @since Node.js 13.2,Node.js开始支持动态import。
|
|
140
142
|
*/
|
|
141
143
|
static async dynamicImport(moduleName) {
|
|
142
|
-
|
|
144
|
+
const mod = await eval(`import('${moduleName}')`);
|
|
145
|
+
if ('default' in mod)
|
|
146
|
+
return mod.default;
|
|
147
|
+
return mod;
|
|
143
148
|
}
|
|
144
149
|
}
|
|
145
150
|
const key = '__jsUtils';
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.NoneOut = exports.Timeout = exports.Terminated = exports.Completed = exports.Uninited = exports.None = exports.Failed = exports.Success = void 0;
|
|
4
|
-
//#region UtilSymbol转导
|
|
5
4
|
var js_utils_1 = require("@zwa73/js-utils");
|
|
6
5
|
Object.defineProperty(exports, "Success", { enumerable: true, get: function () { return js_utils_1.Success; } });
|
|
7
6
|
Object.defineProperty(exports, "Failed", { enumerable: true, get: function () { return js_utils_1.Failed; } });
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { UtilFT } from "./UtilFileTools.mjs";
|
|
2
|
+
import { UtilFP } from "./UtilFP.mjs";
|
|
3
|
+
import { UtilFunc } from "./UtilFunctions.mjs";
|
|
4
|
+
export const { outcome, success, failed, match, isSafeNumber, assertType, assertLiteral, deepClone, sleep, stringifyJToken, getTime, mapEntries, dedent, throwError, eitherize, memoize, ivk, l2s, s2l, structEqual, } = UtilFunc;
|
|
5
|
+
export const { stylizePath, posixizePath, win32izePath, currosizePath, } = UtilFT;
|
|
6
|
+
export const { when, map, flow, pipe, chain, alt, tap, curry, } = UtilFP;
|
|
7
|
+
if (false) {
|
|
8
|
+
//let aaser = assertLiteral({a:1});
|
|
9
|
+
//let a:"asd"|"dsa" = null as any;
|
|
10
|
+
//const r = matchProc(a,{
|
|
11
|
+
// "asd":(a)=>"ssa" as const,
|
|
12
|
+
// "dsa":(b)=>"ssb" as const,
|
|
13
|
+
//});
|
|
14
|
+
//let b:Outcome<"a","123">|NoneOut = null as any;
|
|
15
|
+
//const r1 = matchProc(b,{
|
|
16
|
+
// "a":(a,s)=>"ssa" as const,
|
|
17
|
+
// [None]:(b,s)=>"ssb" as const,
|
|
18
|
+
//});
|
|
19
|
+
//let udt = outcome("ssa","None");
|
|
20
|
+
//type test = Outcome<"a"|"b",any> extends Outcome<"a",12>|Outcome<"b",34>
|
|
21
|
+
// ? "suc" : "fal"
|
|
22
|
+
//let assa = isFailed(null as any as Failed);
|
|
23
|
+
//let assb = isFailed(null as any as Failed|Success);
|
|
24
|
+
//let assc = isFailed(null as any as Success);
|
|
25
|
+
//let asssc = isFailed(null);
|
|
26
|
+
//let assa1 = isFailed(null as any as Outcome<Failed,unknown>);
|
|
27
|
+
//let assb1 = isFailed(null as any as Outcome<Failed|Success,unknown>);
|
|
28
|
+
//let assc1 = isFailed(null as any as Outcome<Success,unknown>);
|
|
29
|
+
//let proct = sucesProc(null as any as Outcome<Failed|Success,"123">|Outcome<Failed,"1234">|Outcome<Success,"1s23">,
|
|
30
|
+
// (s)=>"ssa" as const,
|
|
31
|
+
// (f)=>"ssad" as const,
|
|
32
|
+
// ()=>123 as const);
|
|
33
|
+
//let matt = matchProc(null as any as Outcome<symbol,123>,{
|
|
34
|
+
//});
|
|
35
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { SLogger } from '../UtilLogger.mjs';
|
|
2
|
+
import { UtilFunc } from '../UtilFunctions.mjs';
|
|
3
|
+
/**文本模板渲染器
|
|
4
|
+
* @template T - 上下文对象的类型,默认为记录类型
|
|
5
|
+
*/
|
|
6
|
+
export class Hbs {
|
|
7
|
+
hbs;
|
|
8
|
+
context;
|
|
9
|
+
/**创建一个新的 Hbs 实例
|
|
10
|
+
* @param context - 上下文对象,用于在渲染模板时提供数据
|
|
11
|
+
*/
|
|
12
|
+
constructor(hbs, context = {}) {
|
|
13
|
+
this.hbs = hbs;
|
|
14
|
+
this.context = context;
|
|
15
|
+
this.hbs.registerHelper('def', (name, options) => {
|
|
16
|
+
const value = options.fn(this.context).trim();
|
|
17
|
+
this.context[name] = value;
|
|
18
|
+
return '';
|
|
19
|
+
});
|
|
20
|
+
this.hbs.registerHelper('defli', (name, options) => {
|
|
21
|
+
const rawvalue = options.fn(this.context);
|
|
22
|
+
const values = rawvalue
|
|
23
|
+
.split('[[br]]')
|
|
24
|
+
.map(value => value.trim());
|
|
25
|
+
this.context[name] = values;
|
|
26
|
+
return '';
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/**创建一个新的 Hbs 实例
|
|
30
|
+
* @param context - 上下文对象,用于在渲染模板时提供数据
|
|
31
|
+
*/
|
|
32
|
+
static async create(context = {}) {
|
|
33
|
+
const Handlebars = await UtilFunc.dynamicImport('handlebars');
|
|
34
|
+
//console.log(Handlebars);
|
|
35
|
+
const hbs = Handlebars.create();
|
|
36
|
+
return new Hbs(hbs, context);
|
|
37
|
+
}
|
|
38
|
+
/**渲染模板
|
|
39
|
+
* @param template - 要渲染的 Handlebars 模板字符串
|
|
40
|
+
* @returns 渲染后的字符串
|
|
41
|
+
*/
|
|
42
|
+
render(template) {
|
|
43
|
+
try {
|
|
44
|
+
return this.hbs.compile(template)(this.context);
|
|
45
|
+
}
|
|
46
|
+
catch (e) {
|
|
47
|
+
SLogger.error(e);
|
|
48
|
+
return '';
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import * as he from 'html-entities';
|
|
2
|
+
import { UtilFunc } from './UtilFunctions.mjs';
|
|
3
|
+
let tiktoken = undefined;
|
|
4
|
+
async function importTikToken() {
|
|
5
|
+
if (tiktoken != null)
|
|
6
|
+
return tiktoken;
|
|
7
|
+
tiktoken = await UtilFunc.dynamicImport("tiktoken");
|
|
8
|
+
return tiktoken;
|
|
9
|
+
}
|
|
10
|
+
/**编码/解码器 */
|
|
11
|
+
export var UtilCodec;
|
|
12
|
+
(function (UtilCodec) {
|
|
13
|
+
/**gpt-4, gpt-3.5-turbo, text-embedding-ada-002, text-embedding-3-small, text-embedding-3-large
|
|
14
|
+
*/
|
|
15
|
+
let encoderCl100kBase = null;
|
|
16
|
+
/**Codex models, text-davinci-002, text-davinci-003
|
|
17
|
+
*/
|
|
18
|
+
let encoderP50kBase = null;
|
|
19
|
+
const textDecoder = new TextDecoder();
|
|
20
|
+
/**HTML实体解码 将一个字符串中的HTML实体转换为对应的字符
|
|
21
|
+
* @param str - 要转换的字符串
|
|
22
|
+
* @returns 转换后的字符串
|
|
23
|
+
*/
|
|
24
|
+
function decodeHtmlEntities(str) {
|
|
25
|
+
return he.decode(str);
|
|
26
|
+
}
|
|
27
|
+
UtilCodec.decodeHtmlEntities = decodeHtmlEntities;
|
|
28
|
+
/**HTML实体编码 将一个字符串中的 需编码字符转换为 HTML实体
|
|
29
|
+
* @param str - 要转换的字符串
|
|
30
|
+
* @returns 转换后的字符串
|
|
31
|
+
*/
|
|
32
|
+
function encodeHtmlEntities(str) {
|
|
33
|
+
return he.encode(str);
|
|
34
|
+
}
|
|
35
|
+
UtilCodec.encodeHtmlEntities = encodeHtmlEntities;
|
|
36
|
+
//#region LAM
|
|
37
|
+
//token长度计算器
|
|
38
|
+
//cl100k_base ChatGPT models, text-embedding-ada-002
|
|
39
|
+
//p50k_base Code models, text-davinci-002, text-davinci-003
|
|
40
|
+
//r50k_base (or gpt2) GPT-3 models like davinci
|
|
41
|
+
//避免在nextjs调用时出错
|
|
42
|
+
async function initTikTokenEncoder() {
|
|
43
|
+
if (encoderCl100kBase != null && encoderP50kBase != null)
|
|
44
|
+
return;
|
|
45
|
+
const { get_encoding } = await importTikToken();
|
|
46
|
+
encoderCl100kBase = get_encoding("cl100k_base");
|
|
47
|
+
encoderP50kBase = get_encoding("p50k_base");
|
|
48
|
+
}
|
|
49
|
+
/**token长度计算器 Turbo模型
|
|
50
|
+
* @param str = 所要计算的消息
|
|
51
|
+
* @returns 整数长度结果
|
|
52
|
+
*/
|
|
53
|
+
async function tokenNumTurbo(str) {
|
|
54
|
+
await initTikTokenEncoder();
|
|
55
|
+
//return encoder.encode(str).length
|
|
56
|
+
return encoderCl100kBase?.encode(str).length;
|
|
57
|
+
}
|
|
58
|
+
UtilCodec.tokenNumTurbo = tokenNumTurbo;
|
|
59
|
+
/**token长度计算器 Davinci模型
|
|
60
|
+
* @param str = 所要计算的消息
|
|
61
|
+
* @returns 整数长度结果
|
|
62
|
+
*/
|
|
63
|
+
async function tokenNumDavinci(str) {
|
|
64
|
+
await initTikTokenEncoder();
|
|
65
|
+
return encoderP50kBase?.encode(str).length;
|
|
66
|
+
}
|
|
67
|
+
UtilCodec.tokenNumDavinci = tokenNumDavinci;
|
|
68
|
+
/**token编码 Turbo模型
|
|
69
|
+
* @param str = 所要计算的消息
|
|
70
|
+
* @returns Token数组
|
|
71
|
+
*/
|
|
72
|
+
async function encodeTokenTurbo(str) {
|
|
73
|
+
await initTikTokenEncoder();
|
|
74
|
+
return encoderCl100kBase?.encode(str);
|
|
75
|
+
}
|
|
76
|
+
UtilCodec.encodeTokenTurbo = encodeTokenTurbo;
|
|
77
|
+
/**token编码 Davinci模型
|
|
78
|
+
* @param str = 所要计算的消息
|
|
79
|
+
* @returns Token数组
|
|
80
|
+
*/
|
|
81
|
+
async function encodeTokenDavinci(str) {
|
|
82
|
+
await initTikTokenEncoder();
|
|
83
|
+
return encoderP50kBase?.encode(str);
|
|
84
|
+
}
|
|
85
|
+
UtilCodec.encodeTokenDavinci = encodeTokenDavinci;
|
|
86
|
+
/**token解码 Turbo模型
|
|
87
|
+
* @param arr = Token数组
|
|
88
|
+
* @returns 消息字符串
|
|
89
|
+
*/
|
|
90
|
+
async function decodeTokenTurbo(arr) {
|
|
91
|
+
await initTikTokenEncoder();
|
|
92
|
+
if (Array.isArray(arr))
|
|
93
|
+
arr = new Uint32Array(arr);
|
|
94
|
+
return textDecoder.decode(encoderCl100kBase?.decode(arr));
|
|
95
|
+
}
|
|
96
|
+
UtilCodec.decodeTokenTurbo = decodeTokenTurbo;
|
|
97
|
+
/**token解码 Davinci模型
|
|
98
|
+
* @param arr = Token数组
|
|
99
|
+
* @returns 消息字符串
|
|
100
|
+
*/
|
|
101
|
+
async function decodeTokenDavinci(arr) {
|
|
102
|
+
await initTikTokenEncoder();
|
|
103
|
+
if (Array.isArray(arr))
|
|
104
|
+
arr = new Uint32Array(arr);
|
|
105
|
+
return textDecoder.decode(encoderP50kBase?.decode(arr));
|
|
106
|
+
}
|
|
107
|
+
UtilCodec.decodeTokenDavinci = decodeTokenDavinci;
|
|
108
|
+
//#endregion
|
|
109
|
+
})(UtilCodec || (UtilCodec = {}));
|