nexum-code 0.7.11 → 0.7.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{app-BwutxHqk.mjs → app-CYgSfBVZ.mjs} +3789 -988
- package/dist/{dist-0bMQWc-B.mjs → dist-Dh_F9syl.mjs} +1 -1
- package/dist/{esm-BLHs2dah.mjs → esm-C7p5AwJU.mjs} +39 -39
- package/dist/{from-CKE2n10i.mjs → from-D_EZ04z8.mjs} +1 -1
- package/dist/{index.min-DtYFUEA3.mjs → index.min-Di7B-aJf.mjs} +138 -138
- package/dist/main.mjs +1 -1
- package/dist/{multipart-parser-BxHsVgPe.mjs → multipart-parser-uEw3PR6T.mjs} +2 -2
- package/dist/{chunk-apG1qJts.mjs → rolldown-runtime-CDCXy-ZS.mjs} +9 -2
- package/dist/{src-C6XAfwit.mjs → src-BAMmHkmC.mjs} +6 -6
- package/package.json +3 -1
|
@@ -3,7 +3,7 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { i as __require, t as __commonJSMin } from "./
|
|
6
|
+
import { i as __require, t as __commonJSMin } from "./rolldown-runtime-CDCXy-ZS.mjs";
|
|
7
7
|
//#region ../../node_modules/.pnpm/ms@2.1.3/node_modules/ms/index.js
|
|
8
8
|
var require_ms = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
9
9
|
/**
|
|
@@ -3,11 +3,11 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { a as __toESM, i as __require, t as __commonJSMin } from "./
|
|
7
|
-
import { t as uo } from "./index.min-
|
|
8
|
-
import
|
|
6
|
+
import { a as __toESM, i as __require, t as __commonJSMin } from "./rolldown-runtime-CDCXy-ZS.mjs";
|
|
7
|
+
import { t as uo } from "./index.min-Di7B-aJf.mjs";
|
|
8
|
+
import $ from "fs";
|
|
9
9
|
import https from "https";
|
|
10
|
-
import
|
|
10
|
+
import nr from "path";
|
|
11
11
|
import "fs/promises";
|
|
12
12
|
//#region ../../node_modules/.pnpm/@anush008+tokenizers@0.0.0/node_modules/@anush008/tokenizers/index.js
|
|
13
13
|
var require_tokenizers = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
@@ -867,7 +867,7 @@ var require_tensor_factory_impl = /* @__PURE__ */ __commonJSMin(((exports) => {
|
|
|
867
867
|
var require_tensor_impl_type_mapping = /* @__PURE__ */ __commonJSMin(((exports) => {
|
|
868
868
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
869
869
|
exports.checkTypedArray = exports.NUMERIC_TENSOR_TYPEDARRAY_TO_TYPE_MAP = exports.NUMERIC_TENSOR_TYPE_TO_TYPEDARRAY_MAP = void 0;
|
|
870
|
-
exports.NUMERIC_TENSOR_TYPE_TO_TYPEDARRAY_MAP = new Map([
|
|
870
|
+
exports.NUMERIC_TENSOR_TYPE_TO_TYPEDARRAY_MAP = /* @__PURE__ */ new Map([
|
|
871
871
|
["float32", Float32Array],
|
|
872
872
|
["uint8", Uint8Array],
|
|
873
873
|
["int8", Int8Array],
|
|
@@ -880,7 +880,7 @@ var require_tensor_impl_type_mapping = /* @__PURE__ */ __commonJSMin(((exports)
|
|
|
880
880
|
["int4", Uint8Array],
|
|
881
881
|
["uint4", Uint8Array]
|
|
882
882
|
]);
|
|
883
|
-
exports.NUMERIC_TENSOR_TYPEDARRAY_TO_TYPE_MAP = new Map([
|
|
883
|
+
exports.NUMERIC_TENSOR_TYPEDARRAY_TO_TYPE_MAP = /* @__PURE__ */ new Map([
|
|
884
884
|
[Float32Array, "float32"],
|
|
885
885
|
[Uint8Array, "uint8"],
|
|
886
886
|
[Int8Array, "int8"],
|
|
@@ -2887,7 +2887,7 @@ var XofReader = class {
|
|
|
2887
2887
|
this.counter = counter;
|
|
2888
2888
|
this.blockLen = blockLen;
|
|
2889
2889
|
this.flags = flags | 8;
|
|
2890
|
-
this.outputBlock = new Uint32Array(16);
|
|
2890
|
+
this.outputBlock = /* @__PURE__ */ new Uint32Array(16);
|
|
2891
2891
|
this.outputBlockOffset = 64;
|
|
2892
2892
|
}
|
|
2893
2893
|
/**
|
|
@@ -2937,7 +2937,7 @@ var ChunkState = class {
|
|
|
2937
2937
|
constructor(keyWords, chunkCounter, flags) {
|
|
2938
2938
|
this.chainingValue = new Uint32Array(keyWords);
|
|
2939
2939
|
this.chunkCounter = chunkCounter;
|
|
2940
|
-
this.blockWords = new Uint32Array(16);
|
|
2940
|
+
this.blockWords = /* @__PURE__ */ new Uint32Array(16);
|
|
2941
2941
|
this.blockLen = 0;
|
|
2942
2942
|
this.blocksCompressed = 0;
|
|
2943
2943
|
this.flags = flags;
|
|
@@ -3038,13 +3038,13 @@ var Hasher = class Hasher {
|
|
|
3038
3038
|
this.keyWords = keyWords ? new Uint32Array(keyWords) : new Uint32Array(IV);
|
|
3039
3039
|
this.flags = flags ?? 0;
|
|
3040
3040
|
this.chunkState = new ChunkState(this.keyWords, 0, this.flags);
|
|
3041
|
-
this.cvStack = new Uint32Array(432);
|
|
3041
|
+
this.cvStack = /* @__PURE__ */ new Uint32Array(432);
|
|
3042
3042
|
this.cvStackLen = 0;
|
|
3043
|
-
this.parentBlock = new Uint32Array(16);
|
|
3044
|
-
this.parentCv = new Uint32Array(8);
|
|
3045
|
-
this.chunkCv = new Uint32Array(8);
|
|
3046
|
-
this.outWords = new Uint32Array(16);
|
|
3047
|
-
this.finalizeCv = new Uint32Array(8);
|
|
3043
|
+
this.parentBlock = /* @__PURE__ */ new Uint32Array(16);
|
|
3044
|
+
this.parentCv = /* @__PURE__ */ new Uint32Array(8);
|
|
3045
|
+
this.chunkCv = /* @__PURE__ */ new Uint32Array(8);
|
|
3046
|
+
this.outWords = /* @__PURE__ */ new Uint32Array(16);
|
|
3047
|
+
this.finalizeCv = /* @__PURE__ */ new Uint32Array(8);
|
|
3048
3048
|
}
|
|
3049
3049
|
/**
|
|
3050
3050
|
* Reset the hasher to process a new message with the same key/flags.
|
|
@@ -3062,7 +3062,7 @@ var Hasher = class Hasher {
|
|
|
3062
3062
|
*/
|
|
3063
3063
|
static newKeyed(key) {
|
|
3064
3064
|
if (key.length !== 32) throw new Error(`Key must be 32 bytes, got ${key.length}`);
|
|
3065
|
-
const keyWords = new Uint32Array(8);
|
|
3065
|
+
const keyWords = /* @__PURE__ */ new Uint32Array(8);
|
|
3066
3066
|
if (IS_LITTLE_ENDIAN) {
|
|
3067
3067
|
const view = new Uint32Array(key.buffer, key.byteOffset, 8);
|
|
3068
3068
|
keyWords.set(view);
|
|
@@ -3081,7 +3081,7 @@ var Hasher = class Hasher {
|
|
|
3081
3081
|
const contextBytes = encodeUTF8(context);
|
|
3082
3082
|
const contextHasher = new Hasher(new Uint32Array(IV), 32);
|
|
3083
3083
|
contextHasher.update(contextBytes);
|
|
3084
|
-
const contextKey = new Uint32Array(8);
|
|
3084
|
+
const contextKey = /* @__PURE__ */ new Uint32Array(8);
|
|
3085
3085
|
const output = contextHasher.finalizeOutput();
|
|
3086
3086
|
compress(output.inputCv, 0, output.blockWords, 0, contextKey, 0, false, output.counter, output.blockLen, output.flags | 8);
|
|
3087
3087
|
return new Hasher(contextKey, 64);
|
|
@@ -3275,7 +3275,7 @@ const BLAKE3_NODE_KEY = new Uint8Array([
|
|
|
3275
3275
|
Hasher.newKeyed(BLAKE3_NODE_KEY);
|
|
3276
3276
|
//#endregion
|
|
3277
3277
|
//#region ../../node_modules/.pnpm/@huggingface+xetchunk-wasm@0.0.6/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.js
|
|
3278
|
-
const ZERO_KEY = new Uint8Array(32);
|
|
3278
|
+
const ZERO_KEY = /* @__PURE__ */ new Uint8Array(32);
|
|
3279
3279
|
const VERIFICATION_KEY = new Uint8Array([
|
|
3280
3280
|
127,
|
|
3281
3281
|
24,
|
|
@@ -3420,26 +3420,26 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
3420
3420
|
const modelDir = model === EmbeddingModel.CUSTOM ? modelAbsoluteDirPath : await FlagEmbedding.retrieveModel(model, cacheDir, showDownloadProgress);
|
|
3421
3421
|
const tokenizer = this.loadTokenizer(modelDir, maxLength);
|
|
3422
3422
|
const defaultModelName = model === EmbeddingModel.MLE5Large || model === EmbeddingModel.AllMiniLML6V2 ? "model.onnx" : "model_optimized.onnx";
|
|
3423
|
-
const modelPath =
|
|
3424
|
-
if (
|
|
3423
|
+
const modelPath = nr.join(modelDir.toString(), modelName || defaultModelName);
|
|
3424
|
+
if (!$.existsSync(modelPath)) throw new Error(`Model file not found at ${modelPath}`);
|
|
3425
3425
|
return new FlagEmbedding(tokenizer, await import_dist.InferenceSession.create(modelPath, {
|
|
3426
3426
|
executionProviders,
|
|
3427
3427
|
graphOptimizationLevel: "all"
|
|
3428
3428
|
}), model);
|
|
3429
3429
|
}
|
|
3430
3430
|
static loadTokenizer(modelDir, maxLength) {
|
|
3431
|
-
const tokenizerPath =
|
|
3432
|
-
if (
|
|
3433
|
-
const configPath =
|
|
3434
|
-
if (
|
|
3435
|
-
const config = JSON.parse(
|
|
3436
|
-
const tokenizerFilePath =
|
|
3437
|
-
if (
|
|
3438
|
-
const tokenizerConfig = JSON.parse(
|
|
3431
|
+
const tokenizerPath = nr.join(modelDir.toString(), "tokenizer.json");
|
|
3432
|
+
if (!$.existsSync(tokenizerPath)) throw new Error(`Tokenizer file not found at ${tokenizerPath}`);
|
|
3433
|
+
const configPath = nr.join(modelDir.toString(), "config.json");
|
|
3434
|
+
if (!$.existsSync(configPath)) throw new Error(`Config file not found at ${configPath}`);
|
|
3435
|
+
const config = JSON.parse($.readFileSync(configPath, "utf-8"));
|
|
3436
|
+
const tokenizerFilePath = nr.join(modelDir.toString(), "tokenizer_config.json");
|
|
3437
|
+
if (!$.existsSync(tokenizerFilePath)) throw new Error(`Tokenizer file not found at ${tokenizerFilePath}`);
|
|
3438
|
+
const tokenizerConfig = JSON.parse($.readFileSync(tokenizerFilePath, "utf-8"));
|
|
3439
3439
|
maxLength = Math.min(maxLength, tokenizerConfig["model_max_length"]);
|
|
3440
|
-
const tokensMapPath =
|
|
3441
|
-
if (
|
|
3442
|
-
const tokensMap = JSON.parse(
|
|
3440
|
+
const tokensMapPath = nr.join(modelDir.toString(), "special_tokens_map.json");
|
|
3441
|
+
if (!$.existsSync(tokensMapPath)) throw new Error(`Tokens map file not found at ${tokensMapPath}`);
|
|
3442
|
+
const tokensMap = JSON.parse($.readFileSync(tokensMapPath, "utf-8"));
|
|
3443
3443
|
const tokenizer = import_tokenizers.Tokenizer.fromFile(tokenizerPath);
|
|
3444
3444
|
tokenizer.setTruncation(maxLength);
|
|
3445
3445
|
tokenizer.setPadding({
|
|
@@ -3460,10 +3460,10 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
3460
3460
|
return tokenizer;
|
|
3461
3461
|
}
|
|
3462
3462
|
static async downloadFileFromGCS(outputFilePath, model, showDownloadProgress = true) {
|
|
3463
|
-
if (
|
|
3463
|
+
if ($.existsSync(outputFilePath)) return outputFilePath;
|
|
3464
3464
|
if (model === EmbeddingModel.AllMiniLML6V2) model = "sentence-transformers" + model.substring(model.indexOf("-"));
|
|
3465
3465
|
const url = `https://storage.googleapis.com/qdrant-fastembed/${model}.tar.gz`;
|
|
3466
|
-
const fileStream =
|
|
3466
|
+
const fileStream = $.createWriteStream(outputFilePath);
|
|
3467
3467
|
return new Promise((resolve, reject) => {
|
|
3468
3468
|
https.get(url, { headers: { "User-Agent": "Mozilla/5.0" } }, (response) => {
|
|
3469
3469
|
const totalSizeInBytes = parseInt(response.headers["content-length"] || "0", 10);
|
|
@@ -3490,27 +3490,27 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
3490
3490
|
reject(error);
|
|
3491
3491
|
});
|
|
3492
3492
|
}).on("error", (error) => {
|
|
3493
|
-
|
|
3493
|
+
$.unlink(outputFilePath, () => {
|
|
3494
3494
|
reject(error);
|
|
3495
3495
|
});
|
|
3496
3496
|
});
|
|
3497
3497
|
});
|
|
3498
3498
|
}
|
|
3499
3499
|
static async decompressToCache(targzPath, cacheDir) {
|
|
3500
|
-
if (
|
|
3500
|
+
if (nr.extname(targzPath.toString()) === ".gz") await uo({
|
|
3501
3501
|
file: targzPath,
|
|
3502
3502
|
cwd: cacheDir
|
|
3503
3503
|
});
|
|
3504
3504
|
else throw new Error(`Unsupported file extension: ${targzPath}`);
|
|
3505
3505
|
}
|
|
3506
3506
|
static async retrieveModel(model, cacheDir, showDownloadProgress = true) {
|
|
3507
|
-
if (
|
|
3508
|
-
const modelDir =
|
|
3509
|
-
if (
|
|
3510
|
-
const modelTarGz =
|
|
3507
|
+
if (!$.existsSync(cacheDir)) $.mkdirSync(cacheDir, { mode: 511 });
|
|
3508
|
+
const modelDir = nr.join(cacheDir.toString(), model);
|
|
3509
|
+
if ($.existsSync(modelDir)) return modelDir;
|
|
3510
|
+
const modelTarGz = nr.join(cacheDir.toString(), `${model}.tar.gz`);
|
|
3511
3511
|
await this.downloadFileFromGCS(modelTarGz, model, showDownloadProgress);
|
|
3512
3512
|
await this.decompressToCache(modelTarGz, cacheDir);
|
|
3513
|
-
|
|
3513
|
+
$.unlinkSync(modelTarGz);
|
|
3514
3514
|
return modelDir;
|
|
3515
3515
|
}
|
|
3516
3516
|
async *embed(textStrings, batchSize = 256) {
|
|
@@ -3,7 +3,7 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { i as __require, n as __esmMin, t as __commonJSMin } from "./
|
|
6
|
+
import { i as __require, n as __esmMin, t as __commonJSMin } from "./rolldown-runtime-CDCXy-ZS.mjs";
|
|
7
7
|
import { promises } from "node:fs";
|
|
8
8
|
import "node:path";
|
|
9
9
|
//#region ../../node_modules/.pnpm/web-streams-polyfill@3.3.3/node_modules/web-streams-polyfill/dist/ponyfill.es2018.js
|