@mastra/fastembed 1.3.6-alpha.0 → 1.3.6

package/dist/index.js

@@ -18389,10 +18389,11 @@ function loadTokenizerFromDir(modelDir, maxLength) {
 	const tokensMap = JSON.parse(fs.readFileSync(tokensMapPath, "utf-8"));
 	const tokenizer = Tokenizer.fromFile(tokenizerPath);
 	tokenizer.setTruncation(maxLength);
+	const padToken = tokenizerConfig["pad_token"];
 	tokenizer.setPadding({
 		maxLength,
 		padId: config["pad_token_id"],
-		padToken: tokenizerConfig["pad_token"]
+		padToken: isAddedTokenMap(padToken) ? padToken.content : padToken
 	});
 	for (const token of Object.values(tokensMap)) if (typeof token === "string") tokenizer.addSpecialTokens([token]);
 	else if (isAddedTokenMap(token)) {
@@ -18417,8 +18418,9 @@ function getRequiredOutput(outputs, outputName) {
 	if (!tensor) throw new Error(`Expected model output "${outputName}" was not found. Available outputs: ${Object.keys(outputs).join(", ") || "(none)"}`);
 	return tensor;
 }
-function getOutputDimensions(outputName, tensor) {
+function getOutputDimensions(outputName, tensor, expectedBatchSize) {
 	if (tensor.dims.length !== 3) throw new Error(`Expected model output "${outputName}" to have 3 dimensions, received [${tensor.dims.join(", ")}]`);
+	if (tensor.dims[0] !== expectedBatchSize) throw new Error(`Expected model output "${outputName}" to have batch size ${expectedBatchSize}, received ${tensor.dims[0]}`);
 	return tensor.dims;
 }
 var FlagEmbedding = class FlagEmbedding extends Embedding {
@@ -18531,7 +18533,7 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
 			};
 			if (this.model === "fast-multilingual-e5-large") delete inputs.token_type_ids;
 			const lastHiddenState = getRequiredOutput(await this.session.run(inputs), "last_hidden_state");
-			yield getEmbeddings(lastHiddenState.data, getOutputDimensions("last_hidden_state", lastHiddenState), this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
+			yield getEmbeddings(lastHiddenState.data, getOutputDimensions("last_hidden_state", lastHiddenState, batchTexts.length), this.model === "fast-all-MiniLM-L6-v2" || this.model === "fast-multilingual-e5-large" ? maskArray : void 0).map(normalize);
 		}
 	}
 	passageEmbed(texts, batchSize = 256) {
@@ -18674,7 +18676,7 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
 			};
 			const outputTensor = getRequiredOutput(await this.session.run(inputs), "output");
 			const logits = outputTensor.cpuData;
-			const [currentBatchSize, seqLen, currentVocabSize] = getOutputDimensions("output", outputTensor);
+			const [currentBatchSize, seqLen, currentVocabSize] = getOutputDimensions("output", outputTensor, batchTexts.length);
 			const sparseVectors = [];
 			for (let batchIdx = 0; batchIdx < currentBatchSize; batchIdx++) {
 				const valuesArr = new Float32Array(currentVocabSize).fill(0);