@mastra/fastembed 1.3.4 → 1.3.5-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fastembed.d.ts.map +1 -1
- package/dist/index.cjs +43 -22
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +43 -22
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/fastembed.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AASxC,oBAAY,iBAAiB;IAC3B,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,KAAK,UAAU;IACf,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,oBAAY,cAAc;IACxB,aAAa,0BAA0B;IACvC,SAAS,qBAAqB;IAC9B,YAAY,0BAA0B;IACtC,UAAU,sBAAsB;IAChC,aAAa,2BAA2B;IACxC,UAAU,2BAA2B;IACrC,SAAS,+BAA+B;IACxC,MAAM,WAAW;CAClB;AAcD,oBAAY,oBAAoB;IAC9B,YAAY,+BAA+B;IAC3C,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,UAAU,SAAS;IACjB,KAAK,EAAE,cAAc,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AA6BD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC;IAC7B,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAElE,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IAChE,KAAK,EAAE,OAAO,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAClE,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;IACnC,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,yBAAyB,GAAG,uBAAuB,CAAC;AAEpF,uBAAe,SAAS;IACtB,QAAQ,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/F,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACvD;AAED,uBAAe,eAAe;IAC5B,QAAQ,CAAC,mBAAmB,IAAI,eAAe,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnG,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3D;AA8ED,qBAAa,aAAc,SAAQ,SAAS;IAExC,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IAHf,OAAO,eAMN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxE,OAAa,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAwCtE;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAgBlH;IAED,OAAa,aAAa,CACxB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,GAAE,OAAc,GACnC,OAAO,CAAC,QAAQ,CAAC,CA2CnB;IAEM,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAyD1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAGpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGjD;IAED,mBAAmB,IAAI,SAAS,EAAE,CAcjC;CACF;AAED,qBAAa,mBAAoB,SAAQ,eAAe;IAEpD,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAFjB,OAAO,eAKN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,OAAa,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAyClF,OAAO,CAAC,MAAM,CAAC,aAAa;mBAKP,aAAa;IAkD3B,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAkF1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAEpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGrD;IAED,mBAAmB,IAAI,eAAe,EAAE,CAQvC;CACF"}
|
package/dist/index.cjs
CHANGED
|
@@ -27,14 +27,14 @@ let os = require("os");
|
|
|
27
27
|
os = __toESM$1(os, 1);
|
|
28
28
|
let path = require("path");
|
|
29
29
|
path = __toESM$1(path, 1);
|
|
30
|
-
let _anush008_tokenizers = require("@anush008/tokenizers");
|
|
31
30
|
let fs = require("fs");
|
|
32
31
|
fs = __toESM$1(fs, 1);
|
|
33
32
|
let stream = require("stream");
|
|
34
33
|
let stream_promises = require("stream/promises");
|
|
34
|
+
let _anush008_tokenizers = require("@anush008/tokenizers");
|
|
35
|
+
let _huggingface_hub = require("@huggingface/hub");
|
|
35
36
|
let onnxruntime_node = require("onnxruntime-node");
|
|
36
37
|
onnxruntime_node = __toESM$1(onnxruntime_node, 1);
|
|
37
|
-
let _huggingface_hub = require("@huggingface/hub");
|
|
38
38
|
//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/zod/4.6.7/4c9f15c99caaca8b83ba8ab68f247a75121c86fb7e498876ed36daccd0f3d3ce/node_modules/zod/v4/core/util.js
|
|
39
39
|
function getEnumValues(entries) {
|
|
40
40
|
const numericValues = Object.values(entries).filter((v) => typeof v === "number");
|
|
@@ -18415,12 +18415,19 @@ function normalize(v) {
|
|
|
18415
18415
|
const epsilon = 1e-12;
|
|
18416
18416
|
return v.map((val) => val / Math.max(norm, epsilon));
|
|
18417
18417
|
}
|
|
18418
|
-
function getEmbeddings(data, dimensions) {
|
|
18419
|
-
const [
|
|
18420
|
-
return
|
|
18421
|
-
const
|
|
18422
|
-
|
|
18423
|
-
|
|
18418
|
+
function getEmbeddings(data, dimensions, masks) {
|
|
18419
|
+
const [batchSize, sequenceLength, hiddenSize] = dimensions;
|
|
18420
|
+
return Array.from({ length: batchSize }, (_, batchIndex) => {
|
|
18421
|
+
const start = batchIndex * sequenceLength * hiddenSize;
|
|
18422
|
+
if (!masks) return data.slice(start, start + hiddenSize);
|
|
18423
|
+
const embedding = new Array(hiddenSize).fill(0);
|
|
18424
|
+
let tokenCount = 0;
|
|
18425
|
+
for (let tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++) {
|
|
18426
|
+
if (masks[batchIndex][tokenIndex] === 0n) continue;
|
|
18427
|
+
tokenCount++;
|
|
18428
|
+
for (let dimension = 0; dimension < hiddenSize; dimension++) embedding[dimension] = embedding[dimension] + data[start + tokenIndex * hiddenSize + dimension];
|
|
18429
|
+
}
|
|
18430
|
+
return embedding.map((value) => value / Math.max(tokenCount, 1));
|
|
18424
18431
|
});
|
|
18425
18432
|
}
|
|
18426
18433
|
var Embedding = class {};
|
|
@@ -18572,7 +18579,7 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
18572
18579
|
};
|
|
18573
18580
|
if (this.model === "fast-multilingual-e5-large") delete inputs.token_type_ids;
|
|
18574
18581
|
const lastHiddenState = (await this.session.run(inputs)).last_hidden_state;
|
|
18575
|
-
yield getEmbeddings(lastHiddenState.data, lastHiddenState.dims).map(normalize);
|
|
18582
|
+
yield getEmbeddings(lastHiddenState.data, lastHiddenState.dims, this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
|
|
18576
18583
|
}
|
|
18577
18584
|
}
|
|
18578
18585
|
passageEmbed(texts, batchSize = 256) {
|
|
@@ -18650,26 +18657,40 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
|
|
|
18650
18657
|
if (!fs.default.existsSync(cacheDir)) fs.default.mkdirSync(cacheDir, { mode: 493 });
|
|
18651
18658
|
const modelDir = path.default.join(cacheDir.toString(), model.replace("/", "_"));
|
|
18652
18659
|
if (fs.default.existsSync(modelDir)) return modelDir;
|
|
18653
|
-
|
|
18654
|
-
for (const fileName of [
|
|
18660
|
+
const filesToDownload = [
|
|
18655
18661
|
"onnx/model.onnx",
|
|
18656
18662
|
"tokenizer.json",
|
|
18657
18663
|
"tokenizer_config.json",
|
|
18658
18664
|
"config.json",
|
|
18659
18665
|
"special_tokens_map.json"
|
|
18660
|
-
]
|
|
18661
|
-
|
|
18662
|
-
|
|
18663
|
-
|
|
18666
|
+
];
|
|
18667
|
+
const tmpDir = fs.default.mkdtempSync(`${modelDir}.tmp-`);
|
|
18668
|
+
try {
|
|
18669
|
+
for (const fileName of filesToDownload) {
|
|
18670
|
+
const outputPath = path.default.join(tmpDir, fileName);
|
|
18671
|
+
fs.default.mkdirSync(path.default.dirname(outputPath), {
|
|
18672
|
+
recursive: true,
|
|
18673
|
+
mode: 493
|
|
18674
|
+
});
|
|
18675
|
+
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
18676
|
+
repo: model,
|
|
18677
|
+
path: fileName
|
|
18678
|
+
});
|
|
18679
|
+
if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
|
|
18680
|
+
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(outputPath));
|
|
18681
|
+
}
|
|
18682
|
+
try {
|
|
18683
|
+
fs.default.renameSync(tmpDir, modelDir);
|
|
18684
|
+
} catch (error) {
|
|
18685
|
+
const code = error.code;
|
|
18686
|
+
if ((code === "EEXIST" || code === "ENOTEMPTY") && fs.default.existsSync(modelDir)) return modelDir;
|
|
18687
|
+
throw error;
|
|
18688
|
+
}
|
|
18689
|
+
} finally {
|
|
18690
|
+
fs.default.rmSync(tmpDir, {
|
|
18664
18691
|
recursive: true,
|
|
18665
|
-
|
|
18666
|
-
});
|
|
18667
|
-
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
18668
|
-
repo: model,
|
|
18669
|
-
path: fileName
|
|
18692
|
+
force: true
|
|
18670
18693
|
});
|
|
18671
|
-
if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
|
|
18672
|
-
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(outputPath));
|
|
18673
18694
|
}
|
|
18674
18695
|
return modelDir;
|
|
18675
18696
|
}
|