@mastra/fastembed 1.3.4 → 1.3.5-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fastembed.d.ts.map +1 -1
- package/dist/index.cjs +48 -24
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +48 -24
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
package/dist/fastembed.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AASxC,oBAAY,iBAAiB;IAC3B,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,KAAK,UAAU;IACf,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,oBAAY,cAAc;IACxB,aAAa,0BAA0B;IACvC,SAAS,qBAAqB;IAC9B,YAAY,0BAA0B;IACtC,UAAU,sBAAsB;IAChC,aAAa,2BAA2B;IACxC,UAAU,2BAA2B;IACrC,SAAS,+BAA+B;IACxC,MAAM,WAAW;CAClB;AAcD,oBAAY,oBAAoB;IAC9B,YAAY,+BAA+B;IAC3C,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,UAAU,SAAS;IACjB,KAAK,EAAE,cAAc,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AA6BD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC;IAC7B,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAElE,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IAChE,KAAK,EAAE,OAAO,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAClE,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;IACnC,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,yBAAyB,GAAG,uBAAuB,CAAC;AAEpF,uBAAe,SAAS;IACtB,QAAQ,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/F,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACvD;AAED,uBAAe,eAAe;IAC5B,QAAQ,CAAC,mBAAmB,IAAI,eAAe,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnG,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3D;AA8ED,qBAAa,aAAc,SAAQ,SAAS;IAExC,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IAHf,OAAO,eAMN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxE,OAAa,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAwCtE;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAgBlH;IAED,OAAa,aAAa,CACxB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,GAAE,OAAc,GACnC,OAAO,CAAC,QAAQ,CAAC,CA2CnB;IAEM,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAyD1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAGpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGjD;IAED,mBAAmB,IAAI,SAAS,EAAE,CAcjC;CACF;AAED,qBAAa,mBAAoB,SAAQ,eAAe;IAEpD,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAFjB,OAAO,eAKN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,OAAa,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAyClF,OAAO,CAAC,MAAM,CAAC,aAAa;mBAKP,aAAa;IAkD3B,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAkF1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAEpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGrD;IAED,mBAAmB,IAAI,eAAe,EAAE,CAQvC;CACF"}
|
package/dist/index.cjs
CHANGED
|
@@ -27,14 +27,14 @@ let os = require("os");
|
|
|
27
27
|
os = __toESM$1(os, 1);
|
|
28
28
|
let path = require("path");
|
|
29
29
|
path = __toESM$1(path, 1);
|
|
30
|
-
let _anush008_tokenizers = require("@anush008/tokenizers");
|
|
31
30
|
let fs = require("fs");
|
|
32
31
|
fs = __toESM$1(fs, 1);
|
|
33
32
|
let stream = require("stream");
|
|
34
33
|
let stream_promises = require("stream/promises");
|
|
34
|
+
let _anush008_tokenizers = require("@anush008/tokenizers");
|
|
35
|
+
let _huggingface_hub = require("@huggingface/hub");
|
|
35
36
|
let onnxruntime_node = require("onnxruntime-node");
|
|
36
37
|
onnxruntime_node = __toESM$1(onnxruntime_node, 1);
|
|
37
|
-
let _huggingface_hub = require("@huggingface/hub");
|
|
38
38
|
//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/zod/4.6.7/4c9f15c99caaca8b83ba8ab68f247a75121c86fb7e498876ed36daccd0f3d3ce/node_modules/zod/v4/core/util.js
|
|
39
39
|
function getEnumValues(entries) {
|
|
40
40
|
const numericValues = Object.values(entries).filter((v) => typeof v === "number");
|
|
@@ -18415,12 +18415,19 @@ function normalize(v) {
|
|
|
18415
18415
|
const epsilon = 1e-12;
|
|
18416
18416
|
return v.map((val) => val / Math.max(norm, epsilon));
|
|
18417
18417
|
}
|
|
18418
|
-
function getEmbeddings(data, dimensions) {
|
|
18419
|
-
const [
|
|
18420
|
-
return
|
|
18421
|
-
const
|
|
18422
|
-
|
|
18423
|
-
|
|
18418
|
+
function getEmbeddings(data, dimensions, masks) {
|
|
18419
|
+
const [batchSize, sequenceLength, hiddenSize] = dimensions;
|
|
18420
|
+
return Array.from({ length: batchSize }, (_, batchIndex) => {
|
|
18421
|
+
const start = batchIndex * sequenceLength * hiddenSize;
|
|
18422
|
+
if (!masks) return data.slice(start, start + hiddenSize);
|
|
18423
|
+
const embedding = new Array(hiddenSize).fill(0);
|
|
18424
|
+
let tokenCount = 0;
|
|
18425
|
+
for (let tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++) {
|
|
18426
|
+
if (masks[batchIndex][tokenIndex] === 0n) continue;
|
|
18427
|
+
tokenCount++;
|
|
18428
|
+
for (let dimension = 0; dimension < hiddenSize; dimension++) embedding[dimension] = embedding[dimension] + data[start + tokenIndex * hiddenSize + dimension];
|
|
18429
|
+
}
|
|
18430
|
+
return embedding.map((value) => value / Math.max(tokenCount, 1));
|
|
18424
18431
|
});
|
|
18425
18432
|
}
|
|
18426
18433
|
var Embedding = class {};
|
|
@@ -18437,7 +18444,7 @@ function loadTokenizerFromDir(modelDir, maxLength) {
|
|
|
18437
18444
|
const tokenizerFilePath = path.default.join(modelDir.toString(), "tokenizer_config.json");
|
|
18438
18445
|
if (!fs.default.existsSync(tokenizerFilePath)) throw new Error(`Tokenizer config file not found at ${tokenizerFilePath}`);
|
|
18439
18446
|
const tokenizerConfig = JSON.parse(fs.default.readFileSync(tokenizerFilePath, "utf-8"));
|
|
18440
|
-
maxLength = Math.min(maxLength, tokenizerConfig["model_max_length"]);
|
|
18447
|
+
maxLength = Math.min(maxLength, tokenizerConfig["model_max_length"] ?? maxLength);
|
|
18441
18448
|
const tokensMapPath = path.default.join(modelDir.toString(), "special_tokens_map.json");
|
|
18442
18449
|
if (!fs.default.existsSync(tokensMapPath)) throw new Error(`Tokens map file not found at ${tokensMapPath}`);
|
|
18443
18450
|
const tokensMap = JSON.parse(fs.default.readFileSync(tokensMapPath, "utf-8"));
|
|
@@ -18572,7 +18579,7 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
18572
18579
|
};
|
|
18573
18580
|
if (this.model === "fast-multilingual-e5-large") delete inputs.token_type_ids;
|
|
18574
18581
|
const lastHiddenState = (await this.session.run(inputs)).last_hidden_state;
|
|
18575
|
-
yield getEmbeddings(lastHiddenState.data, lastHiddenState.dims).map(normalize);
|
|
18582
|
+
yield getEmbeddings(lastHiddenState.data, lastHiddenState.dims, this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
|
|
18576
18583
|
}
|
|
18577
18584
|
}
|
|
18578
18585
|
passageEmbed(texts, batchSize = 256) {
|
|
@@ -18647,29 +18654,46 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
|
|
|
18647
18654
|
return { tokenizer: loadTokenizerFromDir(modelDir, maxLength) };
|
|
18648
18655
|
}
|
|
18649
18656
|
static async retrieveModel(model, cacheDir, _showDownloadProgress = true) {
|
|
18650
|
-
if (!fs.default.existsSync(cacheDir)) fs.default.mkdirSync(cacheDir, {
|
|
18657
|
+
if (!fs.default.existsSync(cacheDir)) fs.default.mkdirSync(cacheDir, {
|
|
18658
|
+
recursive: true,
|
|
18659
|
+
mode: 493
|
|
18660
|
+
});
|
|
18651
18661
|
const modelDir = path.default.join(cacheDir.toString(), model.replace("/", "_"));
|
|
18652
18662
|
if (fs.default.existsSync(modelDir)) return modelDir;
|
|
18653
|
-
|
|
18654
|
-
for (const fileName of [
|
|
18663
|
+
const filesToDownload = [
|
|
18655
18664
|
"onnx/model.onnx",
|
|
18656
18665
|
"tokenizer.json",
|
|
18657
18666
|
"tokenizer_config.json",
|
|
18658
18667
|
"config.json",
|
|
18659
18668
|
"special_tokens_map.json"
|
|
18660
|
-
]
|
|
18661
|
-
|
|
18662
|
-
|
|
18663
|
-
|
|
18669
|
+
];
|
|
18670
|
+
const tmpDir = fs.default.mkdtempSync(`${modelDir}.tmp-`);
|
|
18671
|
+
try {
|
|
18672
|
+
for (const fileName of filesToDownload) {
|
|
18673
|
+
const outputPath = path.default.join(tmpDir, fileName);
|
|
18674
|
+
fs.default.mkdirSync(path.default.dirname(outputPath), {
|
|
18675
|
+
recursive: true,
|
|
18676
|
+
mode: 493
|
|
18677
|
+
});
|
|
18678
|
+
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
18679
|
+
repo: model,
|
|
18680
|
+
path: fileName
|
|
18681
|
+
});
|
|
18682
|
+
if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
|
|
18683
|
+
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(outputPath));
|
|
18684
|
+
}
|
|
18685
|
+
try {
|
|
18686
|
+
fs.default.renameSync(tmpDir, modelDir);
|
|
18687
|
+
} catch (error) {
|
|
18688
|
+
const code = error.code;
|
|
18689
|
+
if ((code === "EEXIST" || code === "ENOTEMPTY") && fs.default.existsSync(modelDir)) return modelDir;
|
|
18690
|
+
throw error;
|
|
18691
|
+
}
|
|
18692
|
+
} finally {
|
|
18693
|
+
fs.default.rmSync(tmpDir, {
|
|
18664
18694
|
recursive: true,
|
|
18665
|
-
|
|
18666
|
-
});
|
|
18667
|
-
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
18668
|
-
repo: model,
|
|
18669
|
-
path: fileName
|
|
18695
|
+
force: true
|
|
18670
18696
|
});
|
|
18671
|
-
if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
|
|
18672
|
-
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(outputPath));
|
|
18673
18697
|
}
|
|
18674
18698
|
return modelDir;
|
|
18675
18699
|
}
|