@mastra/fastembed 1.3.3 → 1.3.4-alpha.1

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAI9E,OAAO,EACL,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnF,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAE1E;;;GAGG;AACH,wBAAsB,MAAM,kBAE3B;AAyID,eAAO,MAAM,SAAS,EAAE,gBAAgB,GAAG;IACzC,KAAK,EAAE,gBAAgB,CAAC;IACxB,IAAI,EAAE,gBAAgB,CAAC;IACvB,wBAAwB,EAAE,gBAAgB,CAAC;IAC3C,0BAA0B,EAAE,gBAAgB,CAAC;IAC7C,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,WAAW,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACtC,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;CAUrC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAI9E,OAAO,EACL,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnF,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,YAAY,EAAE,cAAc,IAAI,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAE1E;;;GAGG;AACH,wBAAsB,MAAM,kBAE3B;AAoJD,eAAO,MAAM,SAAS,EAAE,gBAAgB,GAAG;IACzC,KAAK,EAAE,gBAAgB,CAAC;IACxB,IAAI,EAAE,gBAAgB,CAAC;IACvB,wBAAwB,EAAE,gBAAgB,CAAC;IAC3C,0BAA0B,EAAE,gBAAgB,CAAC;IAC7C,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,WAAW,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACtC,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;CAUrC,CAAC"}

package/dist/index.js

@@ -3,8 +3,10 @@ import os from "os";
 import path from "path";
 import { AddedToken, Tokenizer } from "@anush008/tokenizers";
 import fs from "fs";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
 import * as ort from "onnxruntime-node";
-import { downloadFileToCacheDir } from "@huggingface/hub";
+import { downloadFile } from "@huggingface/hub";
 //#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/zod/4.6.5/6448fd814698825170f93bd428080c609bd45ccbc007f4c2dfd5fac1bb52f28d/node_modules/zod/v4/core/util.js
 function getEnumValues(entries) {
 	const numericValues = Object.values(entries).filter((v) => typeof v === "number");
@@ -21288,6 +21290,25 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
 		});
 		return new FlagEmbedding(tokenizer, session, model);
 	}
+	/**
+	* Counts texts longer than the model's token window. `embed()` silently truncates these to `maxLength`.
+	*/
+	async countTruncated(textStrings, batchSize = 256) {
+		let count = 0;
+		let maxLength = 0;
+		for (let i = 0; i < textStrings.length; i += batchSize) {
+			const batch = textStrings.slice(i, i + batchSize);
+			const encodings = await Promise.all(batch.map((textString) => this.tokenizer.encode(textString)));
+			for (const encoding of encodings) if (encoding.getOverflowing().length > 0) {
+				count++;
+				maxLength = encoding.getLength();
+			}
+		}
+		return {
+			count,
+			maxLength
+		};
+	}
 	static async retrieveModel(model, cacheDir, showDownloadProgress = true) {
 		if (model === "custom") throw new Error("Custom models must be loaded from modelAbsoluteDirPath");
 		if (!fs.existsSync(cacheDir)) fs.mkdirSync(cacheDir, {
@@ -21298,21 +21319,16 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
 		if (fs.existsSync(modelDir)) return modelDir;
 		const { repo, files } = HF_MODEL_SOURCES[model];
 		const tmpDir = fs.mkdtempSync(`${modelDir}.tmp-`);
-		const hubCacheDir = path.join(tmpDir, ".hf-hub");
 		try {
-			if (showDownloadProgress) console.info(`Downloading ${model} from Hugging Face (${repo})...`);
+			if (showDownloadProgress) console.error(`Downloading ${model} from Hugging Face (${repo})...`);
 			for (const file of [...COMMON_MODEL_FILES, ...files]) {
-				const downloaded = await downloadFileToCacheDir({
+				const downloaded = await downloadFile({
 					repo,
-					path: file,
-					cacheDir: hubCacheDir
+					path: file
 				});
-				fs.renameSync(fs.realpathSync(downloaded), path.join(tmpDir, file));
+				if (!downloaded) throw new Error(`Model file not found in ${repo}: ${file}`);
+				await pipeline(Readable.fromWeb(downloaded.stream()), fs.createWriteStream(path.join(tmpDir, file)));
 			}
-			fs.rmSync(hubCacheDir, {
-				recursive: true,
-				force: true
-			});
 			try {
 				fs.renameSync(tmpDir, modelDir);
 			} catch (error) {
@@ -21450,11 +21466,12 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
 				recursive: true,
 				mode: 493
 			});
-			const downloaded = await downloadFileToCacheDir({
+			const downloaded = await downloadFile({
 				repo: model,
 				path: fileName
 			});
-			if (downloaded && typeof downloaded === "string") fs.copyFileSync(downloaded, outputPath);
+			if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
+			await pipeline(Readable.fromWeb(downloaded.stream()), fs.createWriteStream(outputPath));
 		}
 		return modelDir;
 	}
@@ -21560,8 +21577,21 @@ async function generateEmbeddings(values, modelType) {
 	if (allResults.length === 0) throw new Error("No embeddings generated");
 	return { embeddings: allResults };
 }
-async function generatePrefixedEmbeddings(values, modelType, prefix) {
-	return generateEmbeddings(values.map((value) => `${prefix}: ${value}`), modelType);
+function applyPrefix(values, prefix) {
+	return prefix ? values.map((value) => `${prefix}: ${value}`) : values;
+}
+async function generateEmbeddingsWithWarnings(values, modelType, prefix) {
+	const inputs = applyPrefix(values, prefix);
+	const result = await generateEmbeddings(inputs, modelType);
+	const { count, maxLength } = await (await getCachedModel(modelType)).countTruncated(inputs);
+	const warnings = count > 0 ? [{
+		type: "other",
+		message: `${count} of ${inputs.length} inputs exceed the model's ${maxLength}-token window and were truncated; text beyond it does not affect the embedding. Use smaller chunks to embed the full text.`
+	}] : [];
+	return {
+		...result,
+		warnings
+	};
 }
 const fastEmbedLegacyProvider = customProvider$1({ textEmbeddingModels: {
 	"bge-small-en-v1.5": {
@@ -21615,10 +21645,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
 		maxEmbeddingsPerCall: 256,
 		supportsParallelCalls: true,
 		async doEmbed({ values }) {
-			return {
-				...await generateEmbeddings(values, "BGESmallENV15"),
-				warnings: []
-			};
+			return generateEmbeddingsWithWarnings(values, "BGESmallENV15");
 		}
 	},
 	"bge-base-en-v1.5": {
@@ -21628,10 +21655,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
 		maxEmbeddingsPerCall: 256,
 		supportsParallelCalls: true,
 		async doEmbed({ values }) {
-			return {
-				...await generateEmbeddings(values, "BGEBaseENV15"),
-				warnings: []
-			};
+			return generateEmbeddingsWithWarnings(values, "BGEBaseENV15");
 		}
 	},
 	"multilingual-e5-large-query": {
@@ -21641,10 +21665,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
 		maxEmbeddingsPerCall: 256,
 		supportsParallelCalls: true,
 		async doEmbed({ values }) {
-			return {
-				...await generatePrefixedEmbeddings(values, "MLE5Large", "query"),
-				warnings: []
-			};
+			return generateEmbeddingsWithWarnings(values, "MLE5Large", "query");
 		}
 	},
 	"multilingual-e5-large-passage": {
@@ -21654,10 +21675,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
 		maxEmbeddingsPerCall: 256,
 		supportsParallelCalls: true,
 		async doEmbed({ values }) {
-			return {
-				...await generatePrefixedEmbeddings(values, "MLE5Large", "passage"),
-				warnings: []
-			};
+			return generateEmbeddingsWithWarnings(values, "MLE5Large", "passage");
 		}
 	}
 } });