@shenghuabi/knowledge 1.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/embedding/embedding.service.d.ts +4 -0
  2. package/embedding/index.d.ts +2 -0
  3. package/embedding/type.d.ts +12 -0
  4. package/embedding.mjs +61 -0
  5. package/embedding.mjs.map +7 -0
  6. package/file-parser/const.d.ts +20 -0
  7. package/file-parser/dict/dict-format/dsl/dsl-parse.service.d.ts +6 -0
  8. package/file-parser/dict/dict-format/dsl/dsl.format.d.ts +1 -0
  9. package/file-parser/dict/dict-format/mdict-parse.service.d.ts +20 -0
  10. package/file-parser/dict/dict-format/stardict-parse.service.d.ts +12 -0
  11. package/file-parser/dict/dict-format/yaml-parse.service.d.ts +25 -0
  12. package/file-parser/dict/dict.service.d.ts +10 -0
  13. package/file-parser/dict/index.d.ts +2 -0
  14. package/file-parser/dict/type.d.ts +24 -0
  15. package/file-parser/document-file-parser.service.d.ts +15 -0
  16. package/file-parser/document-loader/pdf-img.loader.d.ts +8 -0
  17. package/file-parser/document-loader/xlsx.loader.d.ts +6 -0
  18. package/file-parser/file-parser.service.d.ts +13 -0
  19. package/file-parser/index.d.ts +6 -0
  20. package/file-parser/text-analyse.d.ts +1 -0
  21. package/file-parser/text-parser.d.ts +3 -0
  22. package/file-parser/vl-parser/markdown.parser.d.ts +8 -0
  23. package/file-parser.mjs +850 -0
  24. package/file-parser.mjs.map +7 -0
  25. package/image/convert.d.ts +25 -0
  26. package/image/extract.d.ts +2 -0
  27. package/image/image-metadata.d.ts +2 -0
  28. package/image/index.d.ts +3 -0
  29. package/image.mjs +134 -0
  30. package/image.mjs.map +7 -0
  31. package/knowledge/article/article.knowledge.service.d.ts +53 -0
  32. package/knowledge/article/define/config.d.ts +60 -0
  33. package/knowledge/article/define/index.d.ts +2 -0
  34. package/knowledge/article/define/payload.d.ts +16 -0
  35. package/knowledge/article/index.d.ts +2 -0
  36. package/knowledge/common/common.knowledge.service.d.ts +240 -0
  37. package/knowledge/common/define/base.d.ts +7 -0
  38. package/knowledge/common/define/chunk.d.ts +14 -0
  39. package/knowledge/common/define/embedding.d.ts +5 -0
  40. package/knowledge/common/define/index.d.ts +3 -0
  41. package/knowledge/common/index.d.ts +1 -0
  42. package/knowledge/common/query.d.ts +7 -0
  43. package/knowledge/const.d.ts +95 -0
  44. package/knowledge/define/index.d.ts +245 -0
  45. package/knowledge/dict/define/config.d.ts +68 -0
  46. package/knowledge/dict/define/index.d.ts +1 -0
  47. package/knowledge/dict/dict.knowledge.service.d.ts +67 -0
  48. package/knowledge/graph/const.d.ts +20 -0
  49. package/knowledge/graph/define/config.d.ts +169 -0
  50. package/knowledge/graph/define/define.d.ts +402 -0
  51. package/knowledge/graph/define/index.d.ts +2 -0
  52. package/knowledge/graph/define/query.d.ts +14 -0
  53. package/knowledge/graph/graph.handle.service.d.ts +28 -0
  54. package/knowledge/graph/graph.knowledge.service.d.ts +40 -0
  55. package/knowledge/graph/graph.local.service.d.ts +85 -0
  56. package/knowledge/graph/graph.query.service.d.ts +160 -0
  57. package/knowledge/graph/graph.service.d.ts +24 -0
  58. package/knowledge/graph/graph.util.service.d.ts +31 -0
  59. package/knowledge/graph/type.d.ts +11 -0
  60. package/knowledge/graph/util/format-attr.d.ts +48 -0
  61. package/knowledge/graph/util/graph-util.d.ts +5 -0
  62. package/knowledge/graph/util.d.ts +1 -0
  63. package/knowledge/graph/vecotr-format.d.ts +11 -0
  64. package/knowledge/index.d.ts +17 -0
  65. package/knowledge/knowledge.manager.service.d.ts +42 -0
  66. package/knowledge/knowledge.util.service.d.ts +21 -0
  67. package/knowledge/normal/define/config.d.ts +60 -0
  68. package/knowledge/normal/define/index.d.ts +1 -0
  69. package/knowledge/normal/normal.knowledge.service.d.ts +49 -0
  70. package/knowledge/template.format.d.ts +6 -0
  71. package/knowledge/type.d.ts +28 -0
  72. package/knowledge.mjs +2856 -0
  73. package/knowledge.mjs.map +7 -0
  74. package/ocr/FileUtils.d.ts +4 -0
  75. package/ocr/ImageRaw.d.ts +11 -0
  76. package/ocr/index.d.ts +2 -0
  77. package/ocr/model-config.d.ts +8 -0
  78. package/ocr/ocr.d.ts +29 -0
  79. package/ocr.mjs +351 -0
  80. package/ocr.mjs.map +7 -0
  81. package/package.json +105 -0
  82. package/qdrant/index.d.ts +3 -0
  83. package/qdrant/qdrant-client.service.d.ts +396 -0
  84. package/qdrant/qdrant-server.service.d.ts +21 -0
  85. package/qdrant/type.d.ts +18 -0
  86. package/qdrant/util.d.ts +1 -0
  87. package/qdrant.mjs +274 -0
  88. package/qdrant.mjs.map +7 -0
  89. package/util/batch-queue.d.ts +6 -0
  90. package/util/cache-queue.d.ts +10 -0
  91. package/util/clone.d.ts +1 -0
  92. package/util/embedding-queue.d.ts +3 -0
  93. package/util/get-hash.d.ts +2 -0
  94. package/util/html-to-text/index.d.ts +5 -0
  95. package/util/index.d.ts +10 -0
  96. package/util/is-truthy.d.ts +1 -0
  97. package/util/log.service.d.ts +6 -0
  98. package/util/promise.d.ts +5 -0
  99. package/util/type.d.ts +1 -0
  100. package/util/uniq-object-key.d.ts +1 -0
  101. package/util.mjs +219 -0
  102. package/util.mjs.map +7 -0
  103. package/worker/custom-cache.d.ts +28 -0
  104. package/worker/ocr/index.d.ts +17 -0
  105. package/worker/ocr.mjs +75 -0
  106. package/worker/ocr.mjs.map +7 -0
  107. package/worker/reranker.mjs +180 -0
  108. package/worker/reranker.mjs.map +7 -0
  109. package/worker/set-transformers-config.d.ts +19 -0
  110. package/worker/text2vec/index.d.ts +9 -0
  111. package/worker/text2vec.mjs +194 -0
  112. package/worker/text2vec.mjs.map +7 -0
@@ -0,0 +1,17 @@
1
+ import { ImageAdjustType } from '@shenghuabi/knowledge/ocr';
2
+ import { MessagePort } from 'worker_threads';
3
+ declare function init(ocrConfig: {
4
+ key: string;
5
+ modelDir: string;
6
+ port: MessagePort;
7
+ }): Promise<void>;
8
+ declare function convert(input: {
9
+ filePath: string | Uint8Array;
10
+ ocrConfig: {
11
+ key: string;
12
+ modelDir: string;
13
+ device?: 'dml' | 'cuda' | 'cpu';
14
+ };
15
+ options?: ImageAdjustType;
16
+ }): Promise<import("@gutenye/ocr-common").Line[]>;
17
+ export { init, convert };
package/worker/ocr.mjs ADDED
@@ -0,0 +1,75 @@
1
+ // packages/worker/ocr/index.ts
2
+ import { Ocr, ModelConfig } from "@shenghuabi/knowledge/ocr";
3
+ import { createNormalizeVfs, path } from "@cyia/vfs2";
4
+ import { getUniqueObjectKey } from "@shenghuabi/knowledge/util";
5
+ import { downloadFile } from "@cyia/dl";
6
+ var key;
7
+ var ocrInstance;
8
+ var BaseUrl = "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/onnx/PP-OCRv4";
9
+ var DictUrl = "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/paddle/PP-OCRv4";
10
+ async function init(ocrConfig) {
11
+ const messageCb = (message) => {
12
+ ocrConfig.port.postMessage({ type: "progress", message });
13
+ };
14
+ const modelConfig = ModelConfig.find((item) => item.key === ocrConfig.key);
15
+ const fs = createNormalizeVfs({ dir: ocrConfig.modelDir });
16
+ const absDetectionPath = path.join(ocrConfig.modelDir, modelConfig.det);
17
+ if (!await fs.exists(modelConfig.det)) {
18
+ await downloadFile(`${BaseUrl}/${modelConfig.det}`, {
19
+ savePath: absDetectionPath,
20
+ message: messageCb
21
+ });
22
+ }
23
+ const absRecognitionPath = path.join(ocrConfig.modelDir, modelConfig.rec);
24
+ if (!await fs.exists(modelConfig.rec)) {
25
+ await downloadFile(`${BaseUrl}/${modelConfig.rec}`, {
26
+ savePath: absRecognitionPath,
27
+ message: messageCb
28
+ });
29
+ }
30
+ const absDictionaryPath = path.join(ocrConfig.modelDir, modelConfig.dict);
31
+ if (!await fs.exists(modelConfig.dict)) {
32
+ await downloadFile(`${DictUrl}/${modelConfig.dict}`, {
33
+ savePath: absDictionaryPath,
34
+ message: messageCb
35
+ });
36
+ }
37
+ }
38
+ async function convert(input) {
39
+ const inputKey = getUniqueObjectKey(input.ocrConfig);
40
+ if (key !== inputKey) {
41
+ const modelConfig = ModelConfig.find(
42
+ (item) => item.key === input.ocrConfig.key
43
+ );
44
+ const absDetectionPath = path.join(
45
+ input.ocrConfig.modelDir,
46
+ modelConfig.det
47
+ );
48
+ const absRecognitionPath = path.join(
49
+ input.ocrConfig.modelDir,
50
+ modelConfig.rec
51
+ );
52
+ const absDictionaryPath = path.join(
53
+ input.ocrConfig.modelDir,
54
+ modelConfig.dict
55
+ );
56
+ ocrInstance = Ocr.create({
57
+ onnxOptions: {
58
+ executionProviders: input.ocrConfig.device ? [input.ocrConfig.device] : ["dml", "cuda", "cpu"],
59
+ executionMode: "parallel"
60
+ },
61
+ models: {
62
+ detectionPath: absDetectionPath,
63
+ recognitionPath: absRecognitionPath,
64
+ dictionaryPath: absDictionaryPath
65
+ }
66
+ });
67
+ key = inputKey;
68
+ }
69
+ return (await ocrInstance).convert(input.filePath, input.options);
70
+ }
71
+ export {
72
+ convert,
73
+ init
74
+ };
75
+ //# sourceMappingURL=ocr.mjs.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../packages/worker/ocr/index.ts"],
4
+ "sourcesContent": ["import { ImageAdjustType, Ocr, ModelConfig } from '@shenghuabi/knowledge/ocr';\nimport { createNormalizeVfs, path } from '@cyia/vfs2';\n// import * as ort from 'onnxruntime-node';\nimport { MessagePort } from 'worker_threads';\nimport { getUniqueObjectKey } from '@shenghuabi/knowledge/util';\nimport { downloadFile } from '@cyia/dl';\nlet key!: string;\nlet ocrInstance: ReturnType<(typeof Ocr)['create']>;\nconst BaseUrl =\n 'https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/onnx/PP-OCRv4';\n\nconst DictUrl =\n 'https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/paddle/PP-OCRv4';\nasync function init(ocrConfig: {\n key: string;\n modelDir: string;\n port: MessagePort;\n}) {\n const messageCb = (message: any) => {\n ocrConfig.port.postMessage({ type: 'progress', message });\n };\n const modelConfig = ModelConfig.find((item) => item.key === ocrConfig.key)!;\n const fs = createNormalizeVfs({ dir: ocrConfig.modelDir });\n // 自动下载模型\n const absDetectionPath = path.join(ocrConfig.modelDir, modelConfig.det);\n if (!(await fs.exists(modelConfig.det))) {\n await downloadFile(`${BaseUrl}/${modelConfig.det}`, {\n savePath: absDetectionPath,\n message: messageCb,\n });\n }\n const absRecognitionPath = path.join(ocrConfig.modelDir, modelConfig.rec);\n if (!(await fs.exists(modelConfig.rec))) {\n await downloadFile(`${BaseUrl}/${modelConfig.rec}`, {\n savePath: absRecognitionPath,\n message: messageCb,\n });\n }\n const absDictionaryPath = path.join(ocrConfig.modelDir, modelConfig.dict);\n if (!(await fs.exists(modelConfig.dict))) {\n await downloadFile(`${DictUrl}/${modelConfig.dict}`, {\n savePath: absDictionaryPath,\n message: messageCb,\n });\n }\n}\n// 改为init和convert\nasync function convert(input: {\n filePath: string | Uint8Array;\n ocrConfig: { key: string; modelDir: string; device?: 'dml' | 'cuda' | 'cpu' };\n options?: ImageAdjustType;\n}) {\n const inputKey = getUniqueObjectKey(input.ocrConfig);\n if (key !== inputKey) {\n const modelConfig = ModelConfig.find(\n (item) => item.key === input.ocrConfig.key,\n )!;\n // 自动下载模型\n const absDetectionPath = path.join(\n input.ocrConfig.modelDir,\n modelConfig.det,\n );\n const absRecognitionPath = path.join(\n input.ocrConfig.modelDir,\n modelConfig.rec,\n );\n const absDictionaryPath = path.join(\n input.ocrConfig.modelDir,\n modelConfig.dict,\n );\n ocrInstance = Ocr.create({\n onnxOptions: {\n executionProviders: input.ocrConfig.device\n ? [input.ocrConfig.device]\n : ['dml', 'cuda', 'cpu'],\n executionMode: 'parallel',\n },\n models: {\n detectionPath: absDetectionPath,\n recognitionPath: absRecognitionPath,\n dictionaryPath: absDictionaryPath,\n },\n });\n key = inputKey;\n }\n return (await ocrInstance).convert(input.filePath, input.options);\n}\n\nexport { init, convert };\n"],
5
+ "mappings": ";AAAA,SAA0B,KAAK,mBAAmB;AAClD,SAAS,oBAAoB,YAAY;AAGzC,SAAS,0BAA0B;AACnC,SAAS,oBAAoB;AAC7B,IAAI;AACJ,IAAI;AACJ,IAAM,UACJ;AAEF,IAAM,UACJ;AACF,eAAe,KAAK,WAIjB;AACD,QAAM,YAAY,CAAC,YAAiB;AAClC,cAAU,KAAK,YAAY,EAAE,MAAM,YAAY,QAAQ,CAAC;AAAA,EAC1D;AACA,QAAM,cAAc,YAAY,KAAK,CAAC,SAAS,KAAK,QAAQ,UAAU,GAAG;AACzE,QAAM,KAAK,mBAAmB,EAAE,KAAK,UAAU,SAAS,CAAC;AAEzD,QAAM,mBAAmB,KAAK,KAAK,UAAU,UAAU,YAAY,GAAG;AACtE,MAAI,CAAE,MAAM,GAAG,OAAO,YAAY,GAAG,GAAI;AACvC,UAAM,aAAa,GAAG,OAAO,IAAI,YAAY,GAAG,IAAI;AAAA,MAClD,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACA,QAAM,qBAAqB,KAAK,KAAK,UAAU,UAAU,YAAY,GAAG;AACxE,MAAI,CAAE,MAAM,GAAG,OAAO,YAAY,GAAG,GAAI;AACvC,UAAM,aAAa,GAAG,OAAO,IAAI,YAAY,GAAG,IAAI;AAAA,MAClD,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACA,QAAM,oBAAoB,KAAK,KAAK,UAAU,UAAU,YAAY,IAAI;AACxE,MAAI,CAAE,MAAM,GAAG,OAAO,YAAY,IAAI,GAAI;AACxC,UAAM,aAAa,GAAG,OAAO,IAAI,YAAY,IAAI,IAAI;AAAA,MACnD,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,eAAe,QAAQ,OAIpB;AACD,QAAM,WAAW,mBAAmB,MAAM,SAAS;AACnD,MAAI,QAAQ,UAAU;AACpB,UAAM,cAAc,YAAY;AAAA,MAC9B,CAAC,SAAS,KAAK,QAAQ,MAAM,UAAU;AAAA,IACzC;AAEA,UAAM,mBAAmB,KAAK;AAAA,MAC5B,MAAM,UAAU;AAAA,MAChB,YAAY;AAAA,IACd;AACA,UAAM,qBAAqB,KAAK;AAAA,MAC9B,MAAM,UAAU;AAAA,MAChB,YAAY;AAAA,IACd;AACA,UAAM,oBAAoB,KAAK;AAAA,MAC7B,MAAM,UAAU;AAAA,MAChB,YAAY;AAAA,IACd;AACA,kBAAc,IAAI,OAAO;AAAA,MACvB,aAAa;AAAA,QACX,oBAAoB,MAAM,UAAU,SAChC,CAAC,MAAM,UAAU,MAAM,IACvB,CAAC,OAAO,QAAQ,KAAK;AAAA,QACzB,eAAe;AAAA,MACjB;AAAA,MACA,QAAQ;AAAA,QACN,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,gBAAgB;AAAA,MAClB;AAAA,IACF,CAAC;AACD,UAAM;AAAA,EACR;AACA,UAAQ,MAAM,aAAa,QAAQ,MAAM,UAAU,MAAM,OAAO;AAClE;",
6
+ "names": []
7
+ }
@@ -0,0 +1,180 @@
1
+ // packages/worker/reranker/index.ts
2
+ import {
3
+ AutoModelForSequenceClassification,
4
+ AutoTokenizer
5
+ } from "@huggingface/transformers";
6
+
7
+ // packages/worker/set-transformers-config.ts
8
+ import { env as env2 } from "@huggingface/transformers";
9
+
10
+ // packages/worker/custom-cache.ts
11
+ import { env } from "@huggingface/transformers";
12
+ import { createNormalizeVfs, path } from "@cyia/vfs2";
13
+ import { downloadFile } from "@cyia/dl";
14
+ import fs from "fs";
15
+ var FileProxyCache = class {
16
+ #path;
17
+ #vfs;
18
+ #sendMessage;
19
+ #modelName;
20
+ #downloadConfig;
21
+ #initOptions;
22
+ constructor(initOptions) {
23
+ this.#modelName = initOptions.modelName;
24
+ this.#sendMessage = (message) => {
25
+ initOptions.port?.postMessage({ type: "progress", message });
26
+ };
27
+ this.#path = initOptions.dir;
28
+ this.#vfs = createNormalizeVfs({ dir: initOptions.dir });
29
+ this.#downloadConfig = initOptions.downloadConfig;
30
+ this.#initOptions = initOptions;
31
+ }
32
+ async match(request) {
33
+ let filePath;
34
+ if (request.startsWith("http")) {
35
+ const data = new URL(request);
36
+ filePath = path.join(
37
+ this.#path,
38
+ data.pathname.replace(
39
+ "/" + env.remotePathTemplate.replaceAll("{model}", this.#modelName).replaceAll("{revision}", encodeURIComponent("main")),
40
+ `/${this.#modelName}/`
41
+ )
42
+ );
43
+ await downloadFile(request, {
44
+ ...this.#downloadConfig,
45
+ savePath: filePath,
46
+ message: this.#sendMessage,
47
+ headers: {
48
+ token: this.#initOptions?.hfToken ?? "",
49
+ "software-bbs": "bbs.shenghuabi.site"
50
+ }
51
+ });
52
+ } else {
53
+ filePath = request;
54
+ }
55
+ const exists = await this.#vfs.exists(filePath);
56
+ if (exists) {
57
+ return new FileResponse(filePath);
58
+ }
59
+ return void 0;
60
+ }
61
+ async put(request, response) {
62
+ throw new Error("no put");
63
+ }
64
+ };
65
+ var decoder = new TextDecoder("utf-8");
66
+ var CONTENT_TYPE_MAP = {
67
+ txt: "text/plain",
68
+ html: "text/html",
69
+ css: "text/css",
70
+ js: "text/javascript",
71
+ json: "application/json",
72
+ png: "image/png",
73
+ jpg: "image/jpeg",
74
+ jpeg: "image/jpeg",
75
+ gif: "image/gif"
76
+ };
77
+ var FileResponse = class _FileResponse {
78
+ filePath;
79
+ headers;
80
+ exists = true;
81
+ status = 200;
82
+ statusText = "OK";
83
+ body;
84
+ constructor(filePath) {
85
+ this.filePath = filePath;
86
+ this.headers = new Headers();
87
+ this.updateContentType();
88
+ this.body = fs.createReadStream(filePath);
89
+ }
90
+ updateContentType() {
91
+ const stats = fs.statSync(this.filePath);
92
+ this.headers.set("content-length", stats.size.toString());
93
+ const extension = this.filePath.toString().split(".").pop().toLowerCase();
94
+ this.headers.set(
95
+ "content-type",
96
+ CONTENT_TYPE_MAP[extension] ?? "application/octet-stream"
97
+ );
98
+ }
99
+ clone() {
100
+ const response = new _FileResponse(this.filePath);
101
+ response.exists = this.exists;
102
+ response.status = this.status;
103
+ response.statusText = this.statusText;
104
+ response.headers = new Headers(this.headers);
105
+ return response;
106
+ }
107
+ async arrayBuffer() {
108
+ return fs.promises.readFile(this.filePath).then((buffer) => buffer.buffer);
109
+ }
110
+ async blob() {
111
+ return new Blob([await this.arrayBuffer()], {
112
+ type: this.headers.get("content-type")
113
+ });
114
+ }
115
+ async text() {
116
+ return decoder.decode(await this.arrayBuffer());
117
+ }
118
+ async json() {
119
+ return JSON.parse(await this.text());
120
+ }
121
+ };
122
+
123
+ // packages/worker/set-transformers-config.ts
124
+ function setTransformersConfig(options) {
125
+ env2.useFS = false;
126
+ env2.localModelPath = options.dir;
127
+ env2.allowLocalModels = false;
128
+ env2.allowRemoteModels = true;
129
+ env2.cacheDir = options.dir;
130
+ env2.customCache = new FileProxyCache(options);
131
+ env2.useBrowserCache = false;
132
+ env2.useFSCache = true;
133
+ env2.useCustomCache = true;
134
+ env2.remoteHost = `https://${options.remoteHost}`;
135
+ }
136
+
137
+ // packages/worker/reranker/index.ts
138
+ var ReRanderService = class {
139
+ init = async (options) => {
140
+ if (!this.model || !this.tokenizer) {
141
+ await this.#downloadOnly(options);
142
+ }
143
+ return true;
144
+ };
145
+ convert = async (input) => {
146
+ const inputs = this.tokenizer(
147
+ new Array(input.docs.length).fill(input.value),
148
+ {
149
+ text_pair: input.docs,
150
+ padding: true,
151
+ truncation: true
152
+ }
153
+ );
154
+ const { logits } = await this.model(inputs);
155
+ return logits.sigmoid().tolist().map(([score], i) => ({
156
+ index: i,
157
+ score
158
+ })).sort((a, b) => b.score - a.score);
159
+ };
160
+ tokenizer;
161
+ model;
162
+ async #downloadOnly(options) {
163
+ setTransformersConfig(options);
164
+ this.tokenizer = await AutoTokenizer.from_pretrained(options.modelName);
165
+ this.model = await AutoModelForSequenceClassification.from_pretrained(
166
+ options.modelName,
167
+ {
168
+ ...options.options
169
+ }
170
+ );
171
+ }
172
+ };
173
+ var instance = new ReRanderService();
174
+ var init = instance.init;
175
+ var convert = instance.convert;
176
+ export {
177
+ convert,
178
+ init
179
+ };
180
+ //# sourceMappingURL=reranker.mjs.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../packages/worker/reranker/index.ts", "../../packages/worker/set-transformers-config.ts", "../../packages/worker/custom-cache.ts"],
4
+ "sourcesContent": ["import {\n AutoModelForSequenceClassification,\n AutoTokenizer,\n} from '@huggingface/transformers';\nimport type {\n Tensor,\n XLMRobertaModel,\n XLMRobertaTokenizer,\n} from '@huggingface/transformers';\n\nimport { InitOptions, setTransformersConfig } from '../set-transformers-config';\n\nclass ReRanderService {\n init = async (options: InitOptions) => {\n if (!this.model || !this.tokenizer) {\n await this.#downloadOnly(options);\n }\n return true;\n };\n convert = async (input: { value: string; docs: string[] }) => {\n const inputs = this.tokenizer!(\n new Array(input.docs.length).fill(input.value),\n {\n text_pair: input.docs,\n padding: true,\n truncation: true,\n },\n );\n const { logits } = await this.model!(inputs);\n return (logits as Tensor)\n .sigmoid()\n .tolist()\n .map(([score], i: number) => ({\n index: i,\n score,\n }))\n .sort((a, b) => b.score - a.score);\n };\n tokenizer?: XLMRobertaTokenizer;\n model?: XLMRobertaModel;\n async #downloadOnly(options: InitOptions) {\n setTransformersConfig(options);\n\n this.tokenizer = await AutoTokenizer.from_pretrained(options.modelName);\n this.model = await AutoModelForSequenceClassification.from_pretrained(\n options.modelName,\n {\n ...options.options,\n } as any,\n );\n }\n}\nconst instance = new ReRanderService();\nconst init = instance.init;\nconst convert = instance.convert;\nexport { init, convert };\n", "import { env, pipeline } from '@huggingface/transformers';\nimport { MessagePort } from 'worker_threads';\nimport { FileProxyCache } from './custom-cache';\nimport type { DownloadConfigType } from '@cyia/external-call';\ntype PipeLineOptions = Partial<NonNullable<Parameters<typeof pipeline>[2]>>;\nexport interface InitOptions {\n /** 文件夹 */\n dir: string;\n /** 模型 */\n modelName: string;\n /** 模型参数 */\n options: PipeLineOptions;\n /**直接链接 */\n remoteHost: string;\n downloadConfig?: DownloadConfigType;\n port?: MessagePort;\n hfToken?: string;\n}\nexport function setTransformersConfig(options: InitOptions) {\n env.useFS = false;\n env.localModelPath = options.dir;\n env.allowLocalModels = false;\n env.allowRemoteModels = true;\n env.cacheDir = options.dir;\n env.customCache = new FileProxyCache(options);\n env.useBrowserCache = false;\n env.useFSCache = true;\n env.useCustomCache = true;\n\n env.remoteHost = `https://${options.remoteHost}`;\n}\n", "import { env } from '@huggingface/transformers';\nimport { createNormalizeVfs, path } from '@cyia/vfs2';\nimport { downloadFile } from '@cyia/dl';\nimport fs from 'fs';\nimport { InitOptions } from './set-transformers-config';\nexport interface NodeProxy {\n match: (request: string) => Promise<ArrayBuffer | undefined>;\n put: (request: string, arraybuffer: ArrayBuffer) => Promise<void>;\n}\nexport class FileProxyCache {\n #path;\n #vfs;\n #sendMessage;\n #modelName;\n #downloadConfig;\n #initOptions;\n constructor(initOptions: InitOptions) {\n this.#modelName = initOptions.modelName;\n this.#sendMessage = (message: any) => {\n initOptions.port?.postMessage({ type: 'progress', message });\n };\n this.#path = initOptions.dir;\n this.#vfs = createNormalizeVfs({ dir: initOptions.dir });\n this.#downloadConfig = initOptions.downloadConfig;\n this.#initOptions = initOptions;\n }\n async match(request: string): Promise<FileResponse | undefined> {\n let filePath;\n if (request.startsWith('http')) {\n const data = new URL(request);\n filePath = path.join(\n this.#path,\n data.pathname.replace(\n '/' +\n env.remotePathTemplate\n .replaceAll('{model}', this.#modelName)\n .replaceAll('{revision}', encodeURIComponent('main')),\n `/${this.#modelName}/`,\n ),\n );\n\n await downloadFile(request, {\n ...this.#downloadConfig,\n savePath: filePath,\n message: this.#sendMessage,\n headers: {\n token: this.#initOptions?.hfToken ?? '',\n 'software-bbs': 'bbs.shenghuabi.site',\n },\n });\n } else {\n filePath = request;\n }\n const exists = await this.#vfs.exists(filePath);\n if (exists) {\n return new FileResponse(filePath);\n }\n return undefined;\n }\n\n async put(request: string, response: Response | FileResponse): Promise<void> {\n throw new Error('no put');\n }\n}\nconst decoder = new TextDecoder('utf-8');\nconst CONTENT_TYPE_MAP = {\n txt: 'text/plain',\n html: 'text/html',\n css: 'text/css',\n js: 'text/javascript',\n json: 'application/json',\n png: 'image/png',\n jpg: 'image/jpeg',\n jpeg: 'image/jpeg',\n gif: 'image/gif',\n};\nclass FileResponse {\n filePath;\n headers;\n exists = true;\n status = 200;\n statusText = 'OK';\n body;\n constructor(filePath: string) {\n this.filePath = filePath;\n this.headers = new Headers();\n\n this.updateContentType();\n\n this.body = fs.createReadStream(filePath);\n }\n\n updateContentType() {\n const stats = fs.statSync(this.filePath);\n this.headers.set('content-length', stats.size.toString());\n\n const extension = this.filePath.toString().split('.').pop()!.toLowerCase();\n this.headers.set(\n 'content-type',\n (CONTENT_TYPE_MAP as any)[extension] ?? 'application/octet-stream',\n );\n }\n\n clone(): FileResponse {\n const response = new FileResponse(this.filePath);\n response.exists = this.exists;\n response.status = this.status;\n response.statusText = this.statusText;\n response.headers = new Headers(this.headers);\n return response;\n }\n\n async arrayBuffer(): Promise<ArrayBuffer> {\n return fs.promises\n .readFile(this.filePath)\n .then((buffer) => buffer.buffer as ArrayBuffer);\n }\n\n async blob(): Promise<Blob> {\n return new Blob([await this.arrayBuffer()], {\n type: this.headers.get('content-type')!,\n });\n }\n\n async text(): Promise<string> {\n return decoder.decode(await this.arrayBuffer());\n }\n\n async json(): Promise<object> {\n return JSON.parse(await this.text());\n }\n}\n"],
5
+ "mappings": ";AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;;;ACHP,SAAS,OAAAA,YAAqB;;;ACA9B,SAAS,WAAW;AACpB,SAAS,oBAAoB,YAAY;AACzC,SAAS,oBAAoB;AAC7B,OAAO,QAAQ;AAMR,IAAM,iBAAN,MAAqB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,aAA0B;AACpC,SAAK,aAAa,YAAY;AAC9B,SAAK,eAAe,CAAC,YAAiB;AACpC,kBAAY,MAAM,YAAY,EAAE,MAAM,YAAY,QAAQ,CAAC;AAAA,IAC7D;AACA,SAAK,QAAQ,YAAY;AACzB,SAAK,OAAO,mBAAmB,EAAE,KAAK,YAAY,IAAI,CAAC;AACvD,SAAK,kBAAkB,YAAY;AACnC,SAAK,eAAe;AAAA,EACtB;AAAA,EACA,MAAM,MAAM,SAAoD;AAC9D,QAAI;AACJ,QAAI,QAAQ,WAAW,MAAM,GAAG;AAC9B,YAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,iBAAW,KAAK;AAAA,QACd,KAAK;AAAA,QACL,KAAK,SAAS;AAAA,UACZ,MACE,IAAI,mBACD,WAAW,WAAW,KAAK,UAAU,EACrC,WAAW,cAAc,mBAAmB,MAAM,CAAC;AAAA,UACxD,IAAI,KAAK,UAAU;AAAA,QACrB;AAAA,MACF;AAEA,YAAM,aAAa,SAAS;AAAA,QAC1B,GAAG,KAAK;AAAA,QACR,UAAU;AAAA,QACV,SAAS,KAAK;AAAA,QACd,SAAS;AAAA,UACP,OAAO,KAAK,cAAc,WAAW;AAAA,UACrC,gBAAgB;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,iBAAW;AAAA,IACb;AACA,UAAM,SAAS,MAAM,KAAK,KAAK,OAAO,QAAQ;AAC9C,QAAI,QAAQ;AACV,aAAO,IAAI,aAAa,QAAQ;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,SAAiB,UAAkD;AAC3E,UAAM,IAAI,MAAM,QAAQ;AAAA,EAC1B;AACF;AACA,IAAM,UAAU,IAAI,YAAY,OAAO;AACvC,IAAM,mBAAmB;AAAA,EACvB,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AAAA,EACL,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,KAAK;AAAA,EACL,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AACP;AACA,IAAM,eAAN,MAAM,cAAa;AAAA,EACjB;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,SAAS;AAAA,EACT,aAAa;AAAA,EACb;AAAA,EACA,YAAY,UAAkB;AAC5B,SAAK,WAAW;AAChB,SAAK,UAAU,IAAI,QAAQ;AAE3B,SAAK,kBAAkB;AAEvB,SAAK,OAAO,GAAG,iBAAiB,QAAQ;AAAA,EAC1C;AAAA,EAEA,oBAAoB;AAClB,UAAM,QAAQ,GAAG,SAAS,KAAK,QAAQ;AACvC,SAAK,QAAQ,IAAI,kBAAkB,MAAM,KAAK,SAAS,CAAC;AAExD,UAAM,YAAY,KAAK,SAAS,SAAS,EAAE,MAAM,GAAG,EAAE,IAAI,EAAG,YAAY;AACzE,SAAK,QAAQ;AAAA,MACX;AAAA,MACC,iBAAyB,SAAS,KAAK;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,QAAsB;AACpB,UAAM,WAAW,IAAI,cAAa,KAAK,QAAQ;AAC/C,aAAS,SAAS,KAAK;AACvB,aAAS,SAAS,KAAK;AACvB,aAAS,aAAa,KAAK;AAC3B,aAAS,UAAU,IAAI,QAAQ,KAAK,OAAO;AAC3C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAAoC;AACxC,WAAO,GAAG,SACP,SAAS,KAAK,QAAQ,EACtB,KAAK,CAAC,WAAW,OAAO,MAAqB;AAAA,EAClD;AAAA,EAEA,MAAM,OAAsB;AAC1B,WAAO,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,CAAC,GAAG;AAAA,MAC1C,MAAM,KAAK,QAAQ,IAAI,cAAc;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAwB;AAC5B,WAAO,QAAQ,OAAO,MAAM,KAAK,YAAY,CAAC;AAAA,EAChD;AAAA,EAEA,MAAM,OAAwB;AAC5B,WAAO,KAAK,MAAM,MAAM,KAAK,KAAK,CAAC;AAAA,EACrC;AACF;;;ADjHO,SAAS,sBAAsB,SAAsB;AAC1D,EAAAC,KAAI,QAAQ;AACZ,EAAAA,KAAI,iBAAiB,QAAQ;AAC7B,EAAAA,KAAI,mBAAmB;AACvB,EAAAA,KAAI,oBAAoB;AACxB,EAAAA,KAAI,WAAW,QAAQ;AACvB,EAAAA,KAAI,cAAc,IAAI,eAAe,OAAO;AAC5C,EAAAA,KAAI,kBAAkB;AACtB,EAAAA,KAAI,aAAa;AACjB,EAAAA,KAAI,iBAAiB;AAErB,EAAAA,KAAI,aAAa,WAAW,QAAQ,UAAU;AAChD;;;ADlBA,IAAM,kBAAN,MAAsB;AAAA,EACpB,OAAO,OAAO,YAAyB;AACrC,QAAI,CAAC,KAAK,SAAS,CAAC,KAAK,WAAW;AAClC,YAAM,KAAK,cAAc,OAAO;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EACA,UAAU,OAAO,UAA6C;AAC5D,UAAM,SAAS,KAAK;AAAA,MAClB,IAAI,MAAM,MAAM,KAAK,MAAM,EAAE,KAAK,MAAM,KAAK;AAAA,MAC7C;AAAA,QACE,WAAW,MAAM;AAAA,QACjB,SAAS;AAAA,QACT,YAAY;AAAA,MACd;AAAA,IACF;AACA,UAAM,EAAE,OAAO,IAAI,MAAM,KAAK,MAAO,MAAM;AAC3C,WAAQ,OACL,QAAQ,EACR,OAAO,EACP,IAAI,CAAC,CAAC,KAAK,GAAG,OAAe;AAAA,MAC5B,OAAO;AAAA,MACP;AAAA,IACF,EAAE,EACD,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA,MAAM,cAAc,SAAsB;AACxC,0BAAsB,OAAO;AAE7B,SAAK,YAAY,MAAM,cAAc,gBAAgB,QAAQ,SAAS;AACtE,SAAK,QAAQ,MAAM,mCAAmC;AAAA,MACpD,QAAQ;AAAA,MACR;AAAA,QACE,GAAG,QAAQ;AAAA,MACb;AAAA,IACF;AAAA,EACF;AACF;AACA,IAAM,WAAW,IAAI,gBAAgB;AACrC,IAAM,OAAO,SAAS;AACtB,IAAM,UAAU,SAAS;",
6
+ "names": ["env", "env"]
7
+ }
@@ -0,0 +1,19 @@
1
+ import { pipeline } from '@huggingface/transformers';
2
+ import { MessagePort } from 'worker_threads';
3
+ import type { DownloadConfigType } from '@cyia/external-call';
4
+ type PipeLineOptions = Partial<NonNullable<Parameters<typeof pipeline>[2]>>;
5
+ export interface InitOptions {
6
+ /** 文件夹 */
7
+ dir: string;
8
+ /** 模型 */
9
+ modelName: string;
10
+ /** 模型参数 */
11
+ options: PipeLineOptions;
12
+ /**直接链接 */
13
+ remoteHost: string;
14
+ downloadConfig?: DownloadConfigType;
15
+ port?: MessagePort;
16
+ hfToken?: string;
17
+ }
18
+ export declare function setTransformersConfig(options: InitOptions): void;
19
+ export {};
@@ -0,0 +1,9 @@
1
+ import { InitOptions } from '../set-transformers-config';
2
+ declare const init: (options: InitOptions) => Promise<boolean>;
3
+ declare const getSize: () => any;
4
+ declare const convert: (input: {
5
+ value: string | string[];
6
+ mode?: "qwen3";
7
+ taskDescription?: string;
8
+ } & InitOptions) => Promise<any>;
9
+ export { init, getSize, convert };
@@ -0,0 +1,194 @@
1
+ // packages/worker/text2vec/index.ts
2
+ import { pipeline as pipeline2 } from "@huggingface/transformers";
3
+
4
+ // packages/worker/set-transformers-config.ts
5
+ import { env as env2 } from "@huggingface/transformers";
6
+
7
+ // packages/worker/custom-cache.ts
8
+ import { env } from "@huggingface/transformers";
9
+ import { createNormalizeVfs, path } from "@cyia/vfs2";
10
+ import { downloadFile } from "@cyia/dl";
11
+ import fs from "fs";
12
+ var FileProxyCache = class {
13
+ #path;
14
+ #vfs;
15
+ #sendMessage;
16
+ #modelName;
17
+ #downloadConfig;
18
+ #initOptions;
19
+ constructor(initOptions) {
20
+ this.#modelName = initOptions.modelName;
21
+ this.#sendMessage = (message) => {
22
+ initOptions.port?.postMessage({ type: "progress", message });
23
+ };
24
+ this.#path = initOptions.dir;
25
+ this.#vfs = createNormalizeVfs({ dir: initOptions.dir });
26
+ this.#downloadConfig = initOptions.downloadConfig;
27
+ this.#initOptions = initOptions;
28
+ }
29
+ async match(request) {
30
+ let filePath;
31
+ if (request.startsWith("http")) {
32
+ const data = new URL(request);
33
+ filePath = path.join(
34
+ this.#path,
35
+ data.pathname.replace(
36
+ "/" + env.remotePathTemplate.replaceAll("{model}", this.#modelName).replaceAll("{revision}", encodeURIComponent("main")),
37
+ `/${this.#modelName}/`
38
+ )
39
+ );
40
+ await downloadFile(request, {
41
+ ...this.#downloadConfig,
42
+ savePath: filePath,
43
+ message: this.#sendMessage,
44
+ headers: {
45
+ token: this.#initOptions?.hfToken ?? "",
46
+ "software-bbs": "bbs.shenghuabi.site"
47
+ }
48
+ });
49
+ } else {
50
+ filePath = request;
51
+ }
52
+ const exists = await this.#vfs.exists(filePath);
53
+ if (exists) {
54
+ return new FileResponse(filePath);
55
+ }
56
+ return void 0;
57
+ }
58
+ async put(request, response) {
59
+ throw new Error("no put");
60
+ }
61
+ };
62
+ var decoder = new TextDecoder("utf-8");
63
+ var CONTENT_TYPE_MAP = {
64
+ txt: "text/plain",
65
+ html: "text/html",
66
+ css: "text/css",
67
+ js: "text/javascript",
68
+ json: "application/json",
69
+ png: "image/png",
70
+ jpg: "image/jpeg",
71
+ jpeg: "image/jpeg",
72
+ gif: "image/gif"
73
+ };
74
+ var FileResponse = class _FileResponse {
75
+ filePath;
76
+ headers;
77
+ exists = true;
78
+ status = 200;
79
+ statusText = "OK";
80
+ body;
81
+ constructor(filePath) {
82
+ this.filePath = filePath;
83
+ this.headers = new Headers();
84
+ this.updateContentType();
85
+ this.body = fs.createReadStream(filePath);
86
+ }
87
+ updateContentType() {
88
+ const stats = fs.statSync(this.filePath);
89
+ this.headers.set("content-length", stats.size.toString());
90
+ const extension = this.filePath.toString().split(".").pop().toLowerCase();
91
+ this.headers.set(
92
+ "content-type",
93
+ CONTENT_TYPE_MAP[extension] ?? "application/octet-stream"
94
+ );
95
+ }
96
+ clone() {
97
+ const response = new _FileResponse(this.filePath);
98
+ response.exists = this.exists;
99
+ response.status = this.status;
100
+ response.statusText = this.statusText;
101
+ response.headers = new Headers(this.headers);
102
+ return response;
103
+ }
104
+ async arrayBuffer() {
105
+ return fs.promises.readFile(this.filePath).then((buffer) => buffer.buffer);
106
+ }
107
+ async blob() {
108
+ return new Blob([await this.arrayBuffer()], {
109
+ type: this.headers.get("content-type")
110
+ });
111
+ }
112
+ async text() {
113
+ return decoder.decode(await this.arrayBuffer());
114
+ }
115
+ async json() {
116
+ return JSON.parse(await this.text());
117
+ }
118
+ };
119
+
120
+ // packages/worker/set-transformers-config.ts
121
+ function setTransformersConfig(options) {
122
+ env2.useFS = false;
123
+ env2.localModelPath = options.dir;
124
+ env2.allowLocalModels = false;
125
+ env2.allowRemoteModels = true;
126
+ env2.cacheDir = options.dir;
127
+ env2.customCache = new FileProxyCache(options);
128
+ env2.useBrowserCache = false;
129
+ env2.useFSCache = true;
130
+ env2.useCustomCache = true;
131
+ env2.remoteHost = `https://${options.remoteHost}`;
132
+ }
133
+
134
+ // packages/worker/text2vec/index.ts
135
+ function qwen3ToVec(extractor, query, description = "Given a web search query, retrieve relevant passages that answer the query") {
136
+ return extractor(
137
+ query.map((item) => `Instruct: ${description}
138
+ Query:${item}`),
139
+ {
140
+ pooling: "last_token",
141
+ normalize: true
142
+ }
143
+ );
144
+ }
145
+ var Text2VecService = class {
146
+ #extractor;
147
+ constructor() {
148
+ }
149
+ init = async (options) => {
150
+ if (this.#extractor) {
151
+ return true;
152
+ }
153
+ this.#extractor = await this.#downloadOnly(options);
154
+ return !!this.#extractor;
155
+ };
156
+ convert = async (input) => {
157
+ if (!this.#extractor) {
158
+ await this.init(input);
159
+ }
160
+ const inputList = typeof input.value === "string" ? [input.value] : input.value;
161
+ let result;
162
+ if (input.mode === "qwen3") {
163
+ result = qwen3ToVec(this.#extractor, inputList, input.taskDescription);
164
+ } else {
165
+ result = this.#extractor(inputList, {
166
+ pooling: "mean",
167
+ normalize: true
168
+ });
169
+ }
170
+ return result.then((result2) => {
171
+ const list = result2.tolist();
172
+ return typeof input.value === "string" ? list[0] : list;
173
+ });
174
+ };
175
+ async #downloadOnly(options) {
176
+ setTransformersConfig(options);
177
+ return await pipeline2(
178
+ "feature-extraction",
179
+ options.modelName,
180
+ options.options
181
+ );
182
+ }
183
+ getSize = () => this.#extractor.model.config.hidden_size;
184
+ };
185
+ var instance = new Text2VecService();
186
+ var init = instance.init;
187
+ var getSize = instance.getSize;
188
+ var convert = instance.convert;
189
+ export {
190
+ convert,
191
+ getSize,
192
+ init
193
+ };
194
+ //# sourceMappingURL=text2vec.mjs.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../packages/worker/text2vec/index.ts", "../../packages/worker/set-transformers-config.ts", "../../packages/worker/custom-cache.ts"],
4
+ "sourcesContent": ["import { pipeline } from '@huggingface/transformers';\nimport type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport { InitOptions, setTransformersConfig } from '../set-transformers-config';\nfunction qwen3ToVec(\n extractor: FeatureExtractionPipeline,\n query: string[],\n description = 'Given a web search query, retrieve relevant passages that answer the query',\n) {\n return extractor!(\n query.map((item) => `Instruct: ${description}\\nQuery:${item}`),\n {\n pooling: 'last_token',\n normalize: true,\n },\n );\n}\n\nclass Text2VecService {\n #extractor!: FeatureExtractionPipeline | undefined;\n constructor() {}\n init = async (options: InitOptions) => {\n if (this.#extractor) {\n return true;\n }\n this.#extractor = await this.#downloadOnly(options);\n return !!this.#extractor;\n };\n convert = async (\n input: {\n value: string | string[];\n mode?: 'qwen3';\n taskDescription?: string;\n } & InitOptions,\n ) => {\n if (!this.#extractor) {\n await this.init(input);\n }\n const inputList =\n typeof input.value === 'string' ? [input.value] : input.value;\n let result;\n if (input.mode === 'qwen3') {\n result = qwen3ToVec(this.#extractor!, inputList, input.taskDescription);\n } else {\n result = this.#extractor!(inputList, {\n pooling: 'mean',\n normalize: true,\n });\n }\n return result.then((result) => {\n const list = result.tolist();\n return typeof input.value === 'string' ? list[0] : list;\n });\n };\n async #downloadOnly(options: InitOptions) {\n setTransformersConfig(options);\n\n return await pipeline(\n 'feature-extraction',\n options.modelName,\n options.options,\n );\n }\n getSize = () => (this.#extractor!.model.config as any).hidden_size;\n}\nconst instance = new Text2VecService();\nconst init = instance.init;\nconst getSize = instance.getSize;\nconst convert = instance.convert;\nexport { init, getSize, convert };\n", "import { env, pipeline } from '@huggingface/transformers';\nimport { MessagePort } from 'worker_threads';\nimport { FileProxyCache } from './custom-cache';\nimport type { DownloadConfigType } from '@cyia/external-call';\ntype PipeLineOptions = Partial<NonNullable<Parameters<typeof pipeline>[2]>>;\nexport interface InitOptions {\n /** 文件夹 */\n dir: string;\n /** 模型 */\n modelName: string;\n /** 模型参数 */\n options: PipeLineOptions;\n /**直接链接 */\n remoteHost: string;\n downloadConfig?: DownloadConfigType;\n port?: MessagePort;\n hfToken?: string;\n}\nexport function setTransformersConfig(options: InitOptions) {\n env.useFS = false;\n env.localModelPath = options.dir;\n env.allowLocalModels = false;\n env.allowRemoteModels = true;\n env.cacheDir = options.dir;\n env.customCache = new FileProxyCache(options);\n env.useBrowserCache = false;\n env.useFSCache = true;\n env.useCustomCache = true;\n\n env.remoteHost = `https://${options.remoteHost}`;\n}\n", "import { env } from '@huggingface/transformers';\nimport { createNormalizeVfs, path } from '@cyia/vfs2';\nimport { downloadFile } from '@cyia/dl';\nimport fs from 'fs';\nimport { InitOptions } from './set-transformers-config';\nexport interface NodeProxy {\n match: (request: string) => Promise<ArrayBuffer | undefined>;\n put: (request: string, arraybuffer: ArrayBuffer) => Promise<void>;\n}\nexport class FileProxyCache {\n #path;\n #vfs;\n #sendMessage;\n #modelName;\n #downloadConfig;\n #initOptions;\n constructor(initOptions: InitOptions) {\n this.#modelName = initOptions.modelName;\n this.#sendMessage = (message: any) => {\n initOptions.port?.postMessage({ type: 'progress', message });\n };\n this.#path = initOptions.dir;\n this.#vfs = createNormalizeVfs({ dir: initOptions.dir });\n this.#downloadConfig = initOptions.downloadConfig;\n this.#initOptions = initOptions;\n }\n async match(request: string): Promise<FileResponse | undefined> {\n let filePath;\n if (request.startsWith('http')) {\n const data = new URL(request);\n filePath = path.join(\n this.#path,\n data.pathname.replace(\n '/' +\n env.remotePathTemplate\n .replaceAll('{model}', this.#modelName)\n .replaceAll('{revision}', encodeURIComponent('main')),\n `/${this.#modelName}/`,\n ),\n );\n\n await downloadFile(request, {\n ...this.#downloadConfig,\n savePath: filePath,\n message: this.#sendMessage,\n headers: {\n token: this.#initOptions?.hfToken ?? '',\n 'software-bbs': 'bbs.shenghuabi.site',\n },\n });\n } else {\n filePath = request;\n }\n const exists = await this.#vfs.exists(filePath);\n if (exists) {\n return new FileResponse(filePath);\n }\n return undefined;\n }\n\n async put(request: string, response: Response | FileResponse): Promise<void> {\n throw new Error('no put');\n }\n}\nconst decoder = new TextDecoder('utf-8');\nconst CONTENT_TYPE_MAP = {\n txt: 'text/plain',\n html: 'text/html',\n css: 'text/css',\n js: 'text/javascript',\n json: 'application/json',\n png: 'image/png',\n jpg: 'image/jpeg',\n jpeg: 'image/jpeg',\n gif: 'image/gif',\n};\nclass FileResponse {\n filePath;\n headers;\n exists = true;\n status = 200;\n statusText = 'OK';\n body;\n constructor(filePath: string) {\n this.filePath = filePath;\n this.headers = new Headers();\n\n this.updateContentType();\n\n this.body = fs.createReadStream(filePath);\n }\n\n updateContentType() {\n const stats = fs.statSync(this.filePath);\n this.headers.set('content-length', stats.size.toString());\n\n const extension = this.filePath.toString().split('.').pop()!.toLowerCase();\n this.headers.set(\n 'content-type',\n (CONTENT_TYPE_MAP as any)[extension] ?? 'application/octet-stream',\n );\n }\n\n clone(): FileResponse {\n const response = new FileResponse(this.filePath);\n response.exists = this.exists;\n response.status = this.status;\n response.statusText = this.statusText;\n response.headers = new Headers(this.headers);\n return response;\n }\n\n async arrayBuffer(): Promise<ArrayBuffer> {\n return fs.promises\n .readFile(this.filePath)\n .then((buffer) => buffer.buffer as ArrayBuffer);\n }\n\n async blob(): Promise<Blob> {\n return new Blob([await this.arrayBuffer()], {\n type: this.headers.get('content-type')!,\n });\n }\n\n async text(): Promise<string> {\n return decoder.decode(await this.arrayBuffer());\n }\n\n async json(): Promise<object> {\n return JSON.parse(await this.text());\n }\n}\n"],
5
+ "mappings": ";AAAA,SAAS,YAAAA,iBAAgB;;;ACAzB,SAAS,OAAAC,YAAqB;;;ACA9B,SAAS,WAAW;AACpB,SAAS,oBAAoB,YAAY;AACzC,SAAS,oBAAoB;AAC7B,OAAO,QAAQ;AAMR,IAAM,iBAAN,MAAqB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,aAA0B;AACpC,SAAK,aAAa,YAAY;AAC9B,SAAK,eAAe,CAAC,YAAiB;AACpC,kBAAY,MAAM,YAAY,EAAE,MAAM,YAAY,QAAQ,CAAC;AAAA,IAC7D;AACA,SAAK,QAAQ,YAAY;AACzB,SAAK,OAAO,mBAAmB,EAAE,KAAK,YAAY,IAAI,CAAC;AACvD,SAAK,kBAAkB,YAAY;AACnC,SAAK,eAAe;AAAA,EACtB;AAAA,EACA,MAAM,MAAM,SAAoD;AAC9D,QAAI;AACJ,QAAI,QAAQ,WAAW,MAAM,GAAG;AAC9B,YAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,iBAAW,KAAK;AAAA,QACd,KAAK;AAAA,QACL,KAAK,SAAS;AAAA,UACZ,MACE,IAAI,mBACD,WAAW,WAAW,KAAK,UAAU,EACrC,WAAW,cAAc,mBAAmB,MAAM,CAAC;AAAA,UACxD,IAAI,KAAK,UAAU;AAAA,QACrB;AAAA,MACF;AAEA,YAAM,aAAa,SAAS;AAAA,QAC1B,GAAG,KAAK;AAAA,QACR,UAAU;AAAA,QACV,SAAS,KAAK;AAAA,QACd,SAAS;AAAA,UACP,OAAO,KAAK,cAAc,WAAW;AAAA,UACrC,gBAAgB;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,iBAAW;AAAA,IACb;AACA,UAAM,SAAS,MAAM,KAAK,KAAK,OAAO,QAAQ;AAC9C,QAAI,QAAQ;AACV,aAAO,IAAI,aAAa,QAAQ;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,SAAiB,UAAkD;AAC3E,UAAM,IAAI,MAAM,QAAQ;AAAA,EAC1B;AACF;AACA,IAAM,UAAU,IAAI,YAAY,OAAO;AACvC,IAAM,mBAAmB;AAAA,EACvB,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AAAA,EACL,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,KAAK;AAAA,EACL,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AACP;AACA,IAAM,eAAN,MAAM,cAAa;AAAA,EACjB;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,SAAS;AAAA,EACT,aAAa;AAAA,EACb;AAAA,EACA,YAAY,UAAkB;AAC5B,SAAK,WAAW;AAChB,SAAK,UAAU,IAAI,QAAQ;AAE3B,SAAK,kBAAkB;AAEvB,SAAK,OAAO,GAAG,iBAAiB,QAAQ;AAAA,EAC1C;AAAA,EAEA,oBAAoB;AAClB,UAAM,QAAQ,GAAG,SAAS,KAAK,QAAQ;AACvC,SAAK,QAAQ,IAAI,kBAAkB,MAAM,KAAK,SAAS,CAAC;AAExD,UAAM,YAAY,KAAK,SAAS,SAAS,EAAE,MAAM,GAAG,EAAE,IAAI,EAAG,YAAY;AACzE,SAAK,QAAQ;AAAA,MACX;AAAA,MACC,iBAAyB,SAAS,KAAK;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,QAAsB;AACpB,UAAM,WAAW,IAAI,cAAa,KAAK,QAAQ;AAC/C,aAAS,SAAS,KAAK;AACvB,aAAS,SAAS,KAAK;AACvB,aAAS,aAAa,KAAK;AAC3B,aAAS,UAAU,IAAI,QAAQ,KAAK,OAAO;AAC3C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAAoC;AACxC,WAAO,GAAG,SACP,SAAS,KAAK,QAAQ,EACtB,KAAK,CAAC,WAAW,OAAO,MAAqB;AAAA,EAClD;AAAA,EAEA,MAAM,OAAsB;AAC1B,WAAO,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,CAAC,GAAG;AAAA,MAC1C,MAAM,KAAK,QAAQ,IAAI,cAAc;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAwB;AAC5B,WAAO,QAAQ,OAAO,MAAM,KAAK,YAAY,CAAC;AAAA,EAChD;AAAA,EAEA,MAAM,OAAwB;AAC5B,WAAO,KAAK,MAAM,MAAM,KAAK,KAAK,CAAC;AAAA,EACrC;AACF;;;ADjHO,SAAS,sBAAsB,SAAsB;AAC1D,EAAAC,KAAI,QAAQ;AACZ,EAAAA,KAAI,iBAAiB,QAAQ;AAC7B,EAAAA,KAAI,mBAAmB;AACvB,EAAAA,KAAI,oBAAoB;AACxB,EAAAA,KAAI,WAAW,QAAQ;AACvB,EAAAA,KAAI,cAAc,IAAI,eAAe,OAAO;AAC5C,EAAAA,KAAI,kBAAkB;AACtB,EAAAA,KAAI,aAAa;AACjB,EAAAA,KAAI,iBAAiB;AAErB,EAAAA,KAAI,aAAa,WAAW,QAAQ,UAAU;AAChD;;;AD1BA,SAAS,WACP,WACA,OACA,cAAc,8EACd;AACA,SAAO;AAAA,IACL,MAAM,IAAI,CAAC,SAAS,aAAa,WAAW;AAAA,QAAW,IAAI,EAAE;AAAA,IAC7D;AAAA,MACE,SAAS;AAAA,MACT,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,IAAM,kBAAN,MAAsB;AAAA,EACpB;AAAA,EACA,cAAc;AAAA,EAAC;AAAA,EACf,OAAO,OAAO,YAAyB;AACrC,QAAI,KAAK,YAAY;AACnB,aAAO;AAAA,IACT;AACA,SAAK,aAAa,MAAM,KAAK,cAAc,OAAO;AAClD,WAAO,CAAC,CAAC,KAAK;AAAA,EAChB;AAAA,EACA,UAAU,OACR,UAKG;AACH,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,KAAK,KAAK,KAAK;AAAA,IACvB;AACA,UAAM,YACJ,OAAO,MAAM,UAAU,WAAW,CAAC,MAAM,KAAK,IAAI,MAAM;AAC1D,QAAI;AACJ,QAAI,MAAM,SAAS,SAAS;AAC1B,eAAS,WAAW,KAAK,YAAa,WAAW,MAAM,eAAe;AAAA,IACxE,OAAO;AACL,eAAS,KAAK,WAAY,WAAW;AAAA,QACnC,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AACA,WAAO,OAAO,KAAK,CAACC,YAAW;AAC7B,YAAM,OAAOA,QAAO,OAAO;AAC3B,aAAO,OAAO,MAAM,UAAU,WAAW,KAAK,CAAC,IAAI;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EACA,MAAM,cAAc,SAAsB;AACxC,0BAAsB,OAAO;AAE7B,WAAO,MAAMC;AAAA,MACX;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,IACV;AAAA,EACF;AAAA,EACA,UAAU,MAAO,KAAK,WAAY,MAAM,OAAe;AACzD;AACA,IAAM,WAAW,IAAI,gBAAgB;AACrC,IAAM,OAAO,SAAS;AACtB,IAAM,UAAU,SAAS;AACzB,IAAM,UAAU,SAAS;",
6
+ "names": ["pipeline", "env", "env", "result", "pipeline"]
7
+ }