@anvia/transformers 0.2.10 → 0.2.12-preview.20260620T171638.d652d47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +18 -14
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { EmbeddingModel, Embedding } from '@anvia/core/embeddings';
|
|
2
2
|
|
|
3
|
-
declare const DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
4
3
|
type TransformersPooling = "mean" | "cls";
|
|
5
4
|
type TransformersFeatureExtractionPipeline = (texts: string[], options: {
|
|
6
5
|
pooling: TransformersPooling;
|
|
@@ -14,6 +13,8 @@ type TransformersEmbeddingModelOptions = {
|
|
|
14
13
|
normalize?: boolean | undefined;
|
|
15
14
|
maxBatchSize?: number | undefined;
|
|
16
15
|
};
|
|
16
|
+
|
|
17
|
+
declare const DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
17
18
|
declare class TransformersEmbeddingModel implements EmbeddingModel {
|
|
18
19
|
private readonly extractor;
|
|
19
20
|
readonly model: string;
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
|
-
// src/
|
|
1
|
+
// src/model.ts
|
|
2
2
|
import { pipeline as transformersPipeline } from "@huggingface/transformers";
|
|
3
|
+
|
|
4
|
+
// src/helpers.ts
|
|
5
|
+
function parseVectors(value, expectedLength) {
|
|
6
|
+
if (!Array.isArray(value) || value.length !== expectedLength) {
|
|
7
|
+
throw new Error(
|
|
8
|
+
`Transformers embedding model returned ${Array.isArray(value) ? value.length : 0} embeddings for ${expectedLength} texts`
|
|
9
|
+
);
|
|
10
|
+
}
|
|
11
|
+
return value.map((vector, index) => {
|
|
12
|
+
if (!Array.isArray(vector) || !vector.every((item) => typeof item === "number")) {
|
|
13
|
+
throw new Error(`Transformers embedding model returned an invalid vector at index ${index}`);
|
|
14
|
+
}
|
|
15
|
+
return vector;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// src/model.ts
|
|
3
20
|
var DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
4
21
|
var TransformersEmbeddingModel = class _TransformersEmbeddingModel {
|
|
5
22
|
constructor(extractor, options = {}) {
|
|
@@ -40,19 +57,6 @@ var TransformersEmbeddingModel = class _TransformersEmbeddingModel {
|
|
|
40
57
|
function createTransformersEmbeddingModel(options = {}) {
|
|
41
58
|
return TransformersEmbeddingModel.create(options);
|
|
42
59
|
}
|
|
43
|
-
function parseVectors(value, expectedLength) {
|
|
44
|
-
if (!Array.isArray(value) || value.length !== expectedLength) {
|
|
45
|
-
throw new Error(
|
|
46
|
-
`Transformers embedding model returned ${Array.isArray(value) ? value.length : 0} embeddings for ${expectedLength} texts`
|
|
47
|
-
);
|
|
48
|
-
}
|
|
49
|
-
return value.map((vector, index) => {
|
|
50
|
-
if (!Array.isArray(vector) || !vector.every((item) => typeof item === "number")) {
|
|
51
|
-
throw new Error(`Transformers embedding model returned an invalid vector at index ${index}`);
|
|
52
|
-
}
|
|
53
|
-
return vector;
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
60
|
export {
|
|
57
61
|
DEFAULT_TRANSFORMERS_EMBEDDING_MODEL,
|
|
58
62
|
TransformersEmbeddingModel,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/
|
|
1
|
+
{"version":3,"sources":["../src/model.ts","../src/helpers.ts"],"sourcesContent":["import type { Embedding, EmbeddingModel } from \"@anvia/core/embeddings\";\nimport { pipeline as transformersPipeline } from \"@huggingface/transformers\";\nimport { parseVectors } from \"./helpers.js\";\nimport type {\n TransformersEmbeddingModelOptions,\n TransformersFeatureExtractionPipeline,\n TransformersPooling,\n} from \"./types.js\";\n\nexport const DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = \"Xenova/all-MiniLM-L6-v2\";\n\nexport class TransformersEmbeddingModel implements EmbeddingModel {\n readonly model: string;\n readonly maxBatchSize: number;\n\n private readonly pooling: TransformersPooling;\n private readonly normalize: boolean;\n\n constructor(\n private readonly extractor: TransformersFeatureExtractionPipeline,\n options: TransformersEmbeddingModelOptions = {},\n ) {\n this.model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;\n this.pooling = options.pooling ?? \"mean\";\n this.normalize = options.normalize ?? true;\n this.maxBatchSize = Math.max(1, Math.trunc(options.maxBatchSize ?? 16));\n }\n\n static async create(\n options: TransformersEmbeddingModelOptions = {},\n ): Promise<TransformersEmbeddingModel> {\n const model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;\n const extractor = (await transformersPipeline(\n \"feature-extraction\",\n model,\n )) as TransformersFeatureExtractionPipeline;\n\n return new TransformersEmbeddingModel(extractor, { ...options, model });\n }\n\n async embedTexts(texts: string[]): Promise<Embedding[]> {\n if (texts.length === 0) {\n return [];\n }\n\n const output = await this.extractor(texts, {\n pooling: this.pooling,\n normalize: this.normalize,\n });\n const vectors = parseVectors(output.tolist(), texts.length);\n\n return texts.map((document, index) => ({\n document,\n vector: vectors[index] as number[],\n }));\n }\n}\n\nexport function createTransformersEmbeddingModel(\n options: TransformersEmbeddingModelOptions = {},\n): Promise<TransformersEmbeddingModel> {\n return TransformersEmbeddingModel.create(options);\n}\n","export function parseVectors(value: unknown, expectedLength: number): number[][] {\n if (!Array.isArray(value) || value.length !== expectedLength) {\n throw new Error(\n `Transformers embedding model returned ${Array.isArray(value) ? value.length : 0} embeddings for ${expectedLength} texts`,\n );\n }\n\n return value.map((vector, index) => {\n if (!Array.isArray(vector) || !vector.every((item) => typeof item === \"number\")) {\n throw new Error(`Transformers embedding model returned an invalid vector at index ${index}`);\n }\n return vector;\n });\n}\n"],"mappings":";AACA,SAAS,YAAY,4BAA4B;;;ACD1C,SAAS,aAAa,OAAgB,gBAAoC;AAC/E,MAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,WAAW,gBAAgB;AAC5D,UAAM,IAAI;AAAA,MACR,yCAAyC,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS,CAAC,mBAAmB,cAAc;AAAA,IACnH;AAAA,EACF;AAEA,SAAO,MAAM,IAAI,CAAC,QAAQ,UAAU;AAClC,QAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,CAAC,OAAO,MAAM,CAAC,SAAS,OAAO,SAAS,QAAQ,GAAG;AAC/E,YAAM,IAAI,MAAM,oEAAoE,KAAK,EAAE;AAAA,IAC7F;AACA,WAAO;AAAA,EACT,CAAC;AACH;;;ADJO,IAAM,uCAAuC;AAE7C,IAAM,6BAAN,MAAM,4BAAqD;AAAA,EAOhE,YACmB,WACjB,UAA6C,CAAC,GAC9C;AAFiB;AAGjB,SAAK,QAAQ,QAAQ,SAAS;AAC9B,SAAK,UAAU,QAAQ,WAAW;AAClC,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,eAAe,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,gBAAgB,EAAE,CAAC;AAAA,EACxE;AAAA,EAPmB;AAAA,EAPV;AAAA,EACA;AAAA,EAEQ;AAAA,EACA;AAAA,EAYjB,aAAa,OACX,UAA6C,CAAC,GACT;AACrC,UAAM,QAAQ,QAAQ,SAAS;AAC/B,UAAM,YAAa,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,WAAO,IAAI,4BAA2B,WAAW,EAAE,GAAG,SAAS,MAAM,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,WAAW,OAAuC;AACtD,QAAI,MAAM,WAAW,GAAG;AACtB,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,SAAS,MAAM,KAAK,UAAU,OAAO;AAAA,MACzC,SAAS,KAAK;AAAA,MACd,WAAW,KAAK;AAAA,IAClB,CAAC;AACD,UAAM,UAAU,aAAa,OAAO,OAAO,GAAG,MAAM,MAAM;AAE1D,WAAO,MAAM,IAAI,CAAC,UAAU,WAAW;AAAA,MACrC;AAAA,MACA,QAAQ,QAAQ,KAAK;AAAA,IACvB,EAAE;AAAA,EACJ;AACF;AAEO,SAAS,iCACd,UAA6C,CAAC,GACT;AACrC,SAAO,2BAA2B,OAAO,OAAO;AAClD;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anvia/transformers",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.12-preview.20260620T171638.d652d47",
|
|
4
4
|
"description": "Transformers.js embedding model adapter for Anvia.",
|
|
5
5
|
"author": "anvia",
|
|
6
6
|
"maintainer": "Indra Zulfi",
|
|
@@ -29,14 +29,14 @@
|
|
|
29
29
|
"@huggingface/transformers": "^4.2.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
|
+
"@anvia/core": "0.7.2-preview.20260620T171638.d652d47",
|
|
32
33
|
"@types/node": "^24.9.1",
|
|
33
34
|
"tsup": "^8.5.0",
|
|
34
35
|
"typescript": "^5.9.3",
|
|
35
|
-
"vitest": "^4.0.8"
|
|
36
|
-
"@anvia/core": "0.7.0"
|
|
36
|
+
"vitest": "^4.0.8"
|
|
37
37
|
},
|
|
38
38
|
"peerDependencies": {
|
|
39
|
-
"@anvia/core": "
|
|
39
|
+
"@anvia/core": "0.7.2-preview.20260620T171638.d652d47"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
42
|
"build": "tsup src/index.ts --format esm --dts --sourcemap --clean",
|