@mastra/fastembed 1.3.4-alpha.0 → 1.3.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/fastembed.d.ts +7 -0
- package/dist/fastembed.d.ts.map +1 -1
- package/dist/index.cjs +48 -30
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -31
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -24,6 +24,10 @@ const memory = new Memory({
|
|
|
24
24
|
});
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
+
### Input length
|
|
28
|
+
|
|
29
|
+
FastEmbed models embed at most 512 tokens per input (roughly 2,000 to 2,500 characters of English text). Longer inputs are truncated: text past the limit doesn't affect the embedding. The AI SDK v3 models return a warning when this happens. Chunk long documents to fit the window, for example with `maxSize: 1500` in `MDocument.chunk()`.
|
|
30
|
+
|
|
27
31
|
## Documentation
|
|
28
32
|
|
|
29
33
|
- [Use FastEmbed for local semantic recall](https://mastra.ai/docs/memory/semantic-recall#using-fastembed-local)
|
package/dist/fastembed.d.ts
CHANGED
|
@@ -90,6 +90,13 @@ export declare class FlagEmbedding extends Embedding {
|
|
|
90
90
|
private constructor();
|
|
91
91
|
static init(options: InitStandardOptions): Promise<FlagEmbedding>;
|
|
92
92
|
static init(options: InitCustomOptions): Promise<FlagEmbedding>;
|
|
93
|
+
/**
|
|
94
|
+
* Counts texts longer than the model's token window. `embed()` silently truncates these to `maxLength`.
|
|
95
|
+
*/
|
|
96
|
+
countTruncated(textStrings: string[], batchSize?: number): Promise<{
|
|
97
|
+
count: number;
|
|
98
|
+
maxLength: number;
|
|
99
|
+
}>;
|
|
93
100
|
static retrieveModel(model: EmbeddingModel, cacheDir: PathLike, showDownloadProgress?: boolean): Promise<PathLike>;
|
|
94
101
|
embed(textStrings: string[], batchSize?: number): AsyncGenerator<number[][], void, unknown>;
|
|
95
102
|
passageEmbed(texts: string[], batchSize?: number): AsyncGenerator<number[][], void, unknown>;
|
package/dist/fastembed.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../src/fastembed.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQxC,oBAAY,iBAAiB;IAC3B,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,KAAK,UAAU;IACf,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,oBAAY,cAAc;IACxB,aAAa,0BAA0B;IACvC,SAAS,qBAAqB;IAC9B,YAAY,0BAA0B;IACtC,UAAU,sBAAsB;IAChC,aAAa,2BAA2B;IACxC,UAAU,2BAA2B;IACrC,SAAS,+BAA+B;IACxC,MAAM,WAAW;CAClB;AAcD,oBAAY,oBAAoB;IAC9B,YAAY,+BAA+B;IAC3C,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,UAAU,SAAS;IACjB,KAAK,EAAE,cAAc,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAmBD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC;IAC7B,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAElE,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IAChE,KAAK,EAAE,OAAO,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAClE,oBAAoB,CAAC,EAAE,SAAS,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;IACnC,oBAAoB,EAAE,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,yBAAyB,GAAG,uBAAuB,CAAC;AAEpF,uBAAe,SAAS;IACtB,QAAQ,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/F,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACvD;AAED,uBAAe,eAAe;IAC5B,QAAQ,CAAC,mBAAmB,IAAI,eAAe,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnG,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzG,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3D;AAwED,qBAAa,aAAc,SAAQ,SAAS;IAExC,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IAHf,OAAO,eAMN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACxE,OAAa,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAwCtE;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAelH;IAED,OAAa,aAAa,CACxB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,GAAE,OAAc,GACnC,OAAO,CAAC,QAAQ,CAAC,CA2CnB;IAEM,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAuD1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,6CAGpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGjD;IAED,mBAAmB,IAAI,SAAS,EAAE,CAcjC;CACF;AAED,qBAAa,mBAAoB,SAAQ,eAAe;IAEpD,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAFjB,OAAO,eAKN;IAED,OAAa,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,OAAa,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAyClF,OAAO,CAAC,MAAM,CAAC,aAAa;mBAKP,aAAa;IA6C3B,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAiF1D;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,iDAEpD;IAEK,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGrD;IAED,mBAAmB,IAAI,eAAe,EAAE,CAQvC;CACF"}
|
package/dist/index.cjs
CHANGED
|
@@ -30,6 +30,8 @@ path = __toESM$1(path, 1);
|
|
|
30
30
|
let _anush008_tokenizers = require("@anush008/tokenizers");
|
|
31
31
|
let fs = require("fs");
|
|
32
32
|
fs = __toESM$1(fs, 1);
|
|
33
|
+
let stream = require("stream");
|
|
34
|
+
let stream_promises = require("stream/promises");
|
|
33
35
|
let onnxruntime_node = require("onnxruntime-node");
|
|
34
36
|
onnxruntime_node = __toESM$1(onnxruntime_node, 1);
|
|
35
37
|
let _huggingface_hub = require("@huggingface/hub");
|
|
@@ -21316,6 +21318,25 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
21316
21318
|
});
|
|
21317
21319
|
return new FlagEmbedding(tokenizer, session, model);
|
|
21318
21320
|
}
|
|
21321
|
+
/**
|
|
21322
|
+
* Counts texts longer than the model's token window. `embed()` silently truncates these to `maxLength`.
|
|
21323
|
+
*/
|
|
21324
|
+
async countTruncated(textStrings, batchSize = 256) {
|
|
21325
|
+
let count = 0;
|
|
21326
|
+
let maxLength = 0;
|
|
21327
|
+
for (let i = 0; i < textStrings.length; i += batchSize) {
|
|
21328
|
+
const batch = textStrings.slice(i, i + batchSize);
|
|
21329
|
+
const encodings = await Promise.all(batch.map((textString) => this.tokenizer.encode(textString)));
|
|
21330
|
+
for (const encoding of encodings) if (encoding.getOverflowing().length > 0) {
|
|
21331
|
+
count++;
|
|
21332
|
+
maxLength = encoding.getLength();
|
|
21333
|
+
}
|
|
21334
|
+
}
|
|
21335
|
+
return {
|
|
21336
|
+
count,
|
|
21337
|
+
maxLength
|
|
21338
|
+
};
|
|
21339
|
+
}
|
|
21319
21340
|
static async retrieveModel(model, cacheDir, showDownloadProgress = true) {
|
|
21320
21341
|
if (model === "custom") throw new Error("Custom models must be loaded from modelAbsoluteDirPath");
|
|
21321
21342
|
if (!fs.default.existsSync(cacheDir)) fs.default.mkdirSync(cacheDir, {
|
|
@@ -21326,21 +21347,16 @@ var FlagEmbedding = class FlagEmbedding extends Embedding {
|
|
|
21326
21347
|
if (fs.default.existsSync(modelDir)) return modelDir;
|
|
21327
21348
|
const { repo, files } = HF_MODEL_SOURCES[model];
|
|
21328
21349
|
const tmpDir = fs.default.mkdtempSync(`${modelDir}.tmp-`);
|
|
21329
|
-
const hubCacheDir = path.default.join(tmpDir, ".hf-hub");
|
|
21330
21350
|
try {
|
|
21331
|
-
if (showDownloadProgress) console.
|
|
21351
|
+
if (showDownloadProgress) console.error(`Downloading ${model} from Hugging Face (${repo})...`);
|
|
21332
21352
|
for (const file of [...COMMON_MODEL_FILES, ...files]) {
|
|
21333
|
-
const downloaded = await (0, _huggingface_hub.
|
|
21353
|
+
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
21334
21354
|
repo,
|
|
21335
|
-
path: file
|
|
21336
|
-
cacheDir: hubCacheDir
|
|
21355
|
+
path: file
|
|
21337
21356
|
});
|
|
21338
|
-
|
|
21357
|
+
if (!downloaded) throw new Error(`Model file not found in ${repo}: ${file}`);
|
|
21358
|
+
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(path.default.join(tmpDir, file)));
|
|
21339
21359
|
}
|
|
21340
|
-
fs.default.rmSync(hubCacheDir, {
|
|
21341
|
-
recursive: true,
|
|
21342
|
-
force: true
|
|
21343
|
-
});
|
|
21344
21360
|
try {
|
|
21345
21361
|
fs.default.renameSync(tmpDir, modelDir);
|
|
21346
21362
|
} catch (error) {
|
|
@@ -21478,11 +21494,12 @@ var SparseTextEmbedding = class SparseTextEmbedding extends SparseEmbedding {
|
|
|
21478
21494
|
recursive: true,
|
|
21479
21495
|
mode: 493
|
|
21480
21496
|
});
|
|
21481
|
-
const downloaded = await (0, _huggingface_hub.
|
|
21497
|
+
const downloaded = await (0, _huggingface_hub.downloadFile)({
|
|
21482
21498
|
repo: model,
|
|
21483
21499
|
path: fileName
|
|
21484
21500
|
});
|
|
21485
|
-
if (downloaded
|
|
21501
|
+
if (!downloaded) throw new Error(`Model file not found in ${model}: ${fileName}`);
|
|
21502
|
+
await (0, stream_promises.pipeline)(stream.Readable.fromWeb(downloaded.stream()), fs.default.createWriteStream(outputPath));
|
|
21486
21503
|
}
|
|
21487
21504
|
return modelDir;
|
|
21488
21505
|
}
|
|
@@ -21588,8 +21605,21 @@ async function generateEmbeddings(values, modelType) {
|
|
|
21588
21605
|
if (allResults.length === 0) throw new Error("No embeddings generated");
|
|
21589
21606
|
return { embeddings: allResults };
|
|
21590
21607
|
}
|
|
21591
|
-
|
|
21592
|
-
return
|
|
21608
|
+
function applyPrefix(values, prefix) {
|
|
21609
|
+
return prefix ? values.map((value) => `${prefix}: ${value}`) : values;
|
|
21610
|
+
}
|
|
21611
|
+
async function generateEmbeddingsWithWarnings(values, modelType, prefix) {
|
|
21612
|
+
const inputs = applyPrefix(values, prefix);
|
|
21613
|
+
const result = await generateEmbeddings(inputs, modelType);
|
|
21614
|
+
const { count, maxLength } = await (await getCachedModel(modelType)).countTruncated(inputs);
|
|
21615
|
+
const warnings = count > 0 ? [{
|
|
21616
|
+
type: "other",
|
|
21617
|
+
message: `${count} of ${inputs.length} inputs exceed the model's ${maxLength}-token window and were truncated; text beyond it does not affect the embedding. Use smaller chunks to embed the full text.`
|
|
21618
|
+
}] : [];
|
|
21619
|
+
return {
|
|
21620
|
+
...result,
|
|
21621
|
+
warnings
|
|
21622
|
+
};
|
|
21593
21623
|
}
|
|
21594
21624
|
const fastEmbedLegacyProvider = customProvider$1({ textEmbeddingModels: {
|
|
21595
21625
|
"bge-small-en-v1.5": {
|
|
@@ -21643,10 +21673,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
|
|
|
21643
21673
|
maxEmbeddingsPerCall: 256,
|
|
21644
21674
|
supportsParallelCalls: true,
|
|
21645
21675
|
async doEmbed({ values }) {
|
|
21646
|
-
return
|
|
21647
|
-
...await generateEmbeddings(values, "BGESmallENV15"),
|
|
21648
|
-
warnings: []
|
|
21649
|
-
};
|
|
21676
|
+
return generateEmbeddingsWithWarnings(values, "BGESmallENV15");
|
|
21650
21677
|
}
|
|
21651
21678
|
},
|
|
21652
21679
|
"bge-base-en-v1.5": {
|
|
@@ -21656,10 +21683,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
|
|
|
21656
21683
|
maxEmbeddingsPerCall: 256,
|
|
21657
21684
|
supportsParallelCalls: true,
|
|
21658
21685
|
async doEmbed({ values }) {
|
|
21659
|
-
return
|
|
21660
|
-
...await generateEmbeddings(values, "BGEBaseENV15"),
|
|
21661
|
-
warnings: []
|
|
21662
|
-
};
|
|
21686
|
+
return generateEmbeddingsWithWarnings(values, "BGEBaseENV15");
|
|
21663
21687
|
}
|
|
21664
21688
|
},
|
|
21665
21689
|
"multilingual-e5-large-query": {
|
|
@@ -21669,10 +21693,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
|
|
|
21669
21693
|
maxEmbeddingsPerCall: 256,
|
|
21670
21694
|
supportsParallelCalls: true,
|
|
21671
21695
|
async doEmbed({ values }) {
|
|
21672
|
-
return
|
|
21673
|
-
...await generatePrefixedEmbeddings(values, "MLE5Large", "query"),
|
|
21674
|
-
warnings: []
|
|
21675
|
-
};
|
|
21696
|
+
return generateEmbeddingsWithWarnings(values, "MLE5Large", "query");
|
|
21676
21697
|
}
|
|
21677
21698
|
},
|
|
21678
21699
|
"multilingual-e5-large-passage": {
|
|
@@ -21682,10 +21703,7 @@ const fastEmbedProviderV3 = customProvider({ embeddingModels: {
|
|
|
21682
21703
|
maxEmbeddingsPerCall: 256,
|
|
21683
21704
|
supportsParallelCalls: true,
|
|
21684
21705
|
async doEmbed({ values }) {
|
|
21685
|
-
return
|
|
21686
|
-
...await generatePrefixedEmbeddings(values, "MLE5Large", "passage"),
|
|
21687
|
-
warnings: []
|
|
21688
|
-
};
|
|
21706
|
+
return generateEmbeddingsWithWarnings(values, "MLE5Large", "passage");
|
|
21689
21707
|
}
|
|
21690
21708
|
}
|
|
21691
21709
|
} });
|