@mastra/fastembed 1.3.6-alpha.0 → 1.3.6-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fastembed.d.ts.map +1 -1
- package/dist/index.cjs +6 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/fastembed.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AASxC,oBAAY,iBAAiB;IAC3B,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,KAAK,UAAU;IACf,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,oBAAY,cAAc;IACxB,aAAa,0BAA0B;IACvC,SAAS,qBAAqB;IAC9B,YAAY,0BAA0B;IACtC,UAAU,sBAAsB;IAChC,aAAa,2BAA2B;IACxC,UAAU,2BAA2B;IACrC,SAAS,+BAA+B;IACxC,MAAM,WAAW;CAClB;AAcD,oBAAY,oBAAoB;IAC9B,YAAY,+BAA+B;IAC3C,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,UAAU,SAAS;IACjB,KAAK,EAAE,cAAc,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAgCD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC;IAC7B,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAElE,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IAChE,KAAK,EAAE,OAAO,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAClE,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;IACnC,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,yBAAyB,GAAG,uBAAuB,CAAC;AAEpF,uBAAe,SAAS;IACtB,QAAQ,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/F,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACvD;AAED,uBAAe,eAAe;IAC5B,QAAQ,CAAC,mBAAmB,IAAI,eAAe,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnG,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3D;
|
|
1
|
+
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AASxC,oBAAY,iBAAiB;IAC3B,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,KAAK,UAAU;IACf,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,oBAAY,cAAc;IACxB,aAAa,0BAA0B;IACvC,SAAS,qBAAqB;IAC9B,YAAY,0BAA0B;IACtC,UAAU,sBAAsB;IAChC,aAAa,2BAA2B;IACxC,UAAU,2BAA2B;IACrC,SAAS,+BAA+B;IACxC,MAAM,WAAW;CAClB;AAcD,oBAAY,oBAAoB;IAC9B,YAAY,+BAA+B;IAC3C,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,UAAU,SAAS;IACjB,KAAK,EAAE,cAAc,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAgCD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC;IAC7B,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAElE,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IAChE,KAAK,EAAE,OAAO,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAClE,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;IACnC,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,yBAAyB,GAAG,uBAAuB,CAAC;AAEpF,uBAAe,SAAS;IACtB,QAAQ,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/F,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACvD;AAED,uBAAe,eAAe;IAC5B,QAAQ,CAAC,mBAAmB,IAAI,eAAe,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnG,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3D;AAiHD,qBAAa,aAAc,SAAQ,SAAS;IAExC,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IAHf,OAAO,eAMN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxE,OAAa,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAyCtE;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAgBlH;IAED,OAAa,aAAa,CACxB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,GAAE,OAAc,GACnC,OAAO,CAAC,QAAQ,CAAC,CA2CnB;IAEM,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAyD1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAGpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGjD;IAED,mBAAmB,IAAI,SAAS,EAAE,CAcjC;CACF;AAED,qBAAa,mBAAoB,SAAQ,eAAe;IAEpD,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAFjB,OAAO,eAKN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,OAAa,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IA0ClF,OAAO,CAAC,MAAM,CAAC,aAAa;mBAKP,aAAa;IAkD3B,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAwF1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAEpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGrD;IAED,mBAAmB,IAAI,eAAe,EAAE,CAQvC;CACF"}
|
package/dist/index.cjs
CHANGED
|
@@ -18416,10 +18416,11 @@ function loadTokenizerFromDir(modelDir, maxLength) {
|
|
|
18416
18416
|
const tokensMap = JSON.parse(fs.default.readFileSync(tokensMapPath, "utf-8"));
|
|
18417
18417
|
const tokenizer = _anush008_tokenizers.Tokenizer.fromFile(tokenizerPath);
|
|
18418
18418
|
tokenizer.setTruncation(maxLength);
|
|
18419
|
+
const padToken = tokenizerConfig["pad_token"];
|
|
18419
18420
|
tokenizer.setPadding({
|
|
18420
18421
|
maxLength,
|
|
18421
18422
|
padId: config["pad_token_id"],
|
|
18422
|
-
padToken:
|
|
18423
|
+
padToken: isAddedTokenMap(padToken) ? padToken.content : padToken
|
|
18423
18424
|
});
|
|
18424
18425
|
for (const token of Object.values(tokensMap)) if (typeof token === "string") tokenizer.addSpecialTokens([token]);
|
|
18425
18426
|
else if (isAddedTokenMap(token)) {
|
|
@@ -18444,8 +18445,9 @@ function getRequiredOutput(outputs, outputName) {
|
|
|
18444
18445
|
if (!tensor) throw new Error(`Expected model output "${outputName}" was not found. Available outputs: ${Object.keys(outputs).join(", ") || "(none)"}`);
|
|
18445
18446
|
return tensor;
|
|
18446
18447
|
}
|
|
18447
|
-
function getOutputDimensions(outputName, tensor) {
|
|
18448
|
+
function getOutputDimensions(outputName, tensor, expectedBatchSize) {
|
|
18448
18449
|
if (tensor.dims.length !== 3) throw new Error(`Expected model output "${outputName}" to have 3 dimensions, received [${tensor.dims.join(", ")}]`);
|
|
18450
|
+
if (tensor.dims[0] !== expectedBatchSize) throw new Error(`Expected model output "${outputName}" to have batch size ${expectedBatchSize}, received ${tensor.dims[0]}`);
|
|
18449
18451
|
return tensor.dims;
|
|
18450
18452
|
}
|
|
18451
18453
|
var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
@@ -18558,7 +18560,7 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
18558
18560
|
};
|
|
18559
18561
|
if (this.model === "fast-multilingual-e5-large") delete inputs.token_type_ids;
|
|
18560
18562
|
const lastHiddenState = getRequiredOutput(await this.session.run(inputs), "last_hidden_state");
|
|
18561
|
-
yield getEmbeddings(lastHiddenState.data, getOutputDimensions("last_hidden_state", lastHiddenState), this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
|
|
18563
|
+
yield getEmbeddings(lastHiddenState.data, getOutputDimensions("last_hidden_state", lastHiddenState, batchTexts.length), this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
|
|
18562
18564
|
}
|
|
18563
18565
|
}
|
|
18564
18566
|
passageEmbed(texts, batchSize = 256) {
|
|
@@ -18701,7 +18703,7 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
|
|
|
18701
18703
|
};
|
|
18702
18704
|
const outputTensor = getRequiredOutput(await this.session.run(inputs), "output");
|
|
18703
18705
|
const logits = outputTensor.cpuData;
|
|
18704
|
-
const [currentBatchSize, seqLen, currentVocabSize] = getOutputDimensions("output", outputTensor);
|
|
18706
|
+
const [currentBatchSize, seqLen, currentVocabSize] = getOutputDimensions("output", outputTensor, batchTexts.length);
|
|
18705
18707
|
const sparseVectors = [];
|
|
18706
18708
|
for (let batchIdx = 0; batchIdx < currentBatchSize; batchIdx++) {
|
|
18707
18709
|
const valuesArr = new Float32Array(currentVocabSize).fill(0);
|