@anvia/transformers 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Indra Zulfi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,91 @@
+# @anvia/transformers
+
+Transformers.js embedding model adapter for Anvia.
+
+Use this package when you want local embedding generation through `@huggingface/transformers`, especially for development or lightweight RAG workflows.
+
+## Installation
+
+```sh
+pnpm add @anvia/transformers @anvia/core @huggingface/transformers
+```
+
+In this monorepo, the package is available through the workspace:
+
+```sh
+pnpm --filter @anvia/transformers build
+```
+
+## Usage
+
+```ts
+import { embedDocuments, InMemoryVectorStore } from "@anvia/core";
+import { createTransformersEmbeddingModel } from "@anvia/transformers";
+
+const embeddingModel = await createTransformersEmbeddingModel();
+
+const documents = await embedDocuments(
+  embeddingModel,
+  [
+    {
+      id: "password-reset",
+      title: "Password reset policy",
+      body: "Password reset links expire after 30 minutes.",
+    },
+    {
+      id: "priority-support",
+      title: "Priority support",
+      body: "Enterprise customers receive priority support.",
+    },
+  ],
+  {
+    id: (document) => document.id,
+    content: (document) => `${document.title}\n${document.body}`,
+  },
+);
+
+const store = InMemoryVectorStore.fromDocuments(documents);
+const index = store.index(embeddingModel);
+
+const results = await index.search({
+  query: "How long does a password reset link last?",
+  topK: 3,
+});
+
+console.log(results);
+```
+
+## Default Model
+
+The default embedding model is:
+
+```ts
+Xenova/all-MiniLM-L6-v2
+```
+
+You can pass another feature-extraction model:
+
+```ts
+const embeddingModel = await createTransformersEmbeddingModel({
+  model: "Xenova/all-MiniLM-L6-v2",
+  pooling: "mean",
+  normalize: true,
+  maxBatchSize: 16,
+});
+```
+
+## Exports
+
+- `TransformersEmbeddingModel`
+- `createTransformersEmbeddingModel`
+- `DEFAULT_TRANSFORMERS_EMBEDDING_MODEL`
+- `TransformersEmbeddingModelOptions`
+- `TransformersPooling`
+
+## Development
+
+```sh
+pnpm --filter @anvia/transformers typecheck
+pnpm --filter @anvia/transformers test
+pnpm --filter @anvia/transformers build
+```

package/dist/index.d.ts

@@ -0,0 +1,29 @@
+import { EmbeddingModel, Embedding } from '@anvia/core';
+
+declare const DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
+type TransformersPooling = "mean" | "cls";
+type TransformersFeatureExtractionPipeline = (texts: string[], options: {
+    pooling: TransformersPooling;
+    normalize: boolean;
+}) => Promise<{
+    tolist(): unknown;
+}>;
+type TransformersEmbeddingModelOptions = {
+    model?: string | undefined;
+    pooling?: TransformersPooling | undefined;
+    normalize?: boolean | undefined;
+    maxBatchSize?: number | undefined;
+};
+declare class TransformersEmbeddingModel implements EmbeddingModel {
+    private readonly extractor;
+    readonly model: string;
+    readonly maxBatchSize: number;
+    private readonly pooling;
+    private readonly normalize;
+    constructor(extractor: TransformersFeatureExtractionPipeline, options?: TransformersEmbeddingModelOptions);
+    static create(options?: TransformersEmbeddingModelOptions): Promise<TransformersEmbeddingModel>;
+    embedTexts(texts: string[]): Promise<Embedding[]>;
+}
+declare function createTransformersEmbeddingModel(options?: TransformersEmbeddingModelOptions): Promise<TransformersEmbeddingModel>;
+
+export { DEFAULT_TRANSFORMERS_EMBEDDING_MODEL, TransformersEmbeddingModel, type TransformersEmbeddingModelOptions, type TransformersFeatureExtractionPipeline, type TransformersPooling, createTransformersEmbeddingModel };

package/dist/index.js

@@ -0,0 +1,61 @@
+// src/index.ts
+import { pipeline as transformersPipeline } from "@huggingface/transformers";
+var DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
+var TransformersEmbeddingModel = class _TransformersEmbeddingModel {
+  constructor(extractor, options = {}) {
+    this.extractor = extractor;
+    this.model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;
+    this.pooling = options.pooling ?? "mean";
+    this.normalize = options.normalize ?? true;
+    this.maxBatchSize = Math.max(1, Math.trunc(options.maxBatchSize ?? 16));
+  }
+  extractor;
+  model;
+  maxBatchSize;
+  pooling;
+  normalize;
+  static async create(options = {}) {
+    const model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;
+    const extractor = await transformersPipeline(
+      "feature-extraction",
+      model
+    );
+    return new _TransformersEmbeddingModel(extractor, { ...options, model });
+  }
+  async embedTexts(texts) {
+    if (texts.length === 0) {
+      return [];
+    }
+    const output = await this.extractor(texts, {
+      pooling: this.pooling,
+      normalize: this.normalize
+    });
+    const vectors = parseVectors(output.tolist(), texts.length);
+    return texts.map((document, index) => ({
+      document,
+      vector: vectors[index]
+    }));
+  }
+};
+function createTransformersEmbeddingModel(options = {}) {
+  return TransformersEmbeddingModel.create(options);
+}
+function parseVectors(value, expectedLength) {
+  if (!Array.isArray(value) || value.length !== expectedLength) {
+    throw new Error(
+      `Transformers embedding model returned ${Array.isArray(value) ? value.length : 0} embeddings for ${expectedLength} texts`
+    );
+  }
+  return value.map((vector, index) => {
+    if (!Array.isArray(vector) || !vector.every((item) => typeof item === "number")) {
+      throw new Error(`Transformers embedding model returned an invalid vector at index ${index}`);
+    }
+    return vector;
+  });
+}
+export {
+  DEFAULT_TRANSFORMERS_EMBEDDING_MODEL,
+  TransformersEmbeddingModel,
+  createTransformersEmbeddingModel
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { Embedding, EmbeddingModel } from \"@anvia/core\";\nimport { pipeline as transformersPipeline } from \"@huggingface/transformers\";\n\nexport const DEFAULT_TRANSFORMERS_EMBEDDING_MODEL = \"Xenova/all-MiniLM-L6-v2\";\n\nexport type TransformersPooling = \"mean\" | \"cls\";\n\nexport type TransformersFeatureExtractionPipeline = (\n  texts: string[],\n  options: { pooling: TransformersPooling; normalize: boolean },\n) => Promise<{ tolist(): unknown }>;\n\nexport type TransformersEmbeddingModelOptions = {\n  model?: string | undefined;\n  pooling?: TransformersPooling | undefined;\n  normalize?: boolean | undefined;\n  maxBatchSize?: number | undefined;\n};\n\nexport class TransformersEmbeddingModel implements EmbeddingModel {\n  readonly model: string;\n  readonly maxBatchSize: number;\n\n  private readonly pooling: TransformersPooling;\n  private readonly normalize: boolean;\n\n  constructor(\n    private readonly extractor: TransformersFeatureExtractionPipeline,\n    options: TransformersEmbeddingModelOptions = {},\n  ) {\n    this.model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;\n    this.pooling = options.pooling ?? \"mean\";\n    this.normalize = options.normalize ?? true;\n    this.maxBatchSize = Math.max(1, Math.trunc(options.maxBatchSize ?? 16));\n  }\n\n  static async create(\n    options: TransformersEmbeddingModelOptions = {},\n  ): Promise<TransformersEmbeddingModel> {\n    const model = options.model ?? DEFAULT_TRANSFORMERS_EMBEDDING_MODEL;\n    const extractor = (await transformersPipeline(\n      \"feature-extraction\",\n      model,\n    )) as TransformersFeatureExtractionPipeline;\n\n    return new TransformersEmbeddingModel(extractor, { ...options, model });\n  }\n\n  async embedTexts(texts: string[]): Promise<Embedding[]> {\n    if (texts.length === 0) {\n      return [];\n    }\n\n    const output = await this.extractor(texts, {\n      pooling: this.pooling,\n      normalize: this.normalize,\n    });\n    const vectors = parseVectors(output.tolist(), texts.length);\n\n    return texts.map((document, index) => ({\n      document,\n      vector: vectors[index] as number[],\n    }));\n  }\n}\n\nexport function createTransformersEmbeddingModel(\n  options: TransformersEmbeddingModelOptions = {},\n): Promise<TransformersEmbeddingModel> {\n  return TransformersEmbeddingModel.create(options);\n}\n\nfunction parseVectors(value: unknown, expectedLength: number): number[][] {\n  if (!Array.isArray(value) || value.length !== expectedLength) {\n    throw new Error(\n      `Transformers embedding model returned ${Array.isArray(value) ? value.length : 0} embeddings for ${expectedLength} texts`,\n    );\n  }\n\n  return value.map((vector, index) => {\n    if (!Array.isArray(vector) || !vector.every((item) => typeof item === \"number\")) {\n      throw new Error(`Transformers embedding model returned an invalid vector at index ${index}`);\n    }\n    return vector;\n  });\n}\n"],"mappings":";AACA,SAAS,YAAY,4BAA4B;AAE1C,IAAM,uCAAuC;AAgB7C,IAAM,6BAAN,MAAM,4BAAqD;AAAA,EAOhE,YACmB,WACjB,UAA6C,CAAC,GAC9C;AAFiB;AAGjB,SAAK,QAAQ,QAAQ,SAAS;AAC9B,SAAK,UAAU,QAAQ,WAAW;AAClC,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,eAAe,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,gBAAgB,EAAE,CAAC;AAAA,EACxE;AAAA,EAPmB;AAAA,EAPV;AAAA,EACA;AAAA,EAEQ;AAAA,EACA;AAAA,EAYjB,aAAa,OACX,UAA6C,CAAC,GACT;AACrC,UAAM,QAAQ,QAAQ,SAAS;AAC/B,UAAM,YAAa,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,WAAO,IAAI,4BAA2B,WAAW,EAAE,GAAG,SAAS,MAAM,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,WAAW,OAAuC;AACtD,QAAI,MAAM,WAAW,GAAG;AACtB,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,SAAS,MAAM,KAAK,UAAU,OAAO;AAAA,MACzC,SAAS,KAAK;AAAA,MACd,WAAW,KAAK;AAAA,IAClB,CAAC;AACD,UAAM,UAAU,aAAa,OAAO,OAAO,GAAG,MAAM,MAAM;AAE1D,WAAO,MAAM,IAAI,CAAC,UAAU,WAAW;AAAA,MACrC;AAAA,MACA,QAAQ,QAAQ,KAAK;AAAA,IACvB,EAAE;AAAA,EACJ;AACF;AAEO,SAAS,iCACd,UAA6C,CAAC,GACT;AACrC,SAAO,2BAA2B,OAAO,OAAO;AAClD;AAEA,SAAS,aAAa,OAAgB,gBAAoC;AACxE,MAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,WAAW,gBAAgB;AAC5D,UAAM,IAAI;AAAA,MACR,yCAAyC,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS,CAAC,mBAAmB,cAAc;AAAA,IACnH;AAAA,EACF;AAEA,SAAO,MAAM,IAAI,CAAC,QAAQ,UAAU;AAClC,QAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,CAAC,OAAO,MAAM,CAAC,SAAS,OAAO,SAAS,QAAQ,GAAG;AAC/E,YAAM,IAAI,MAAM,oEAAoE,KAAK,EAAE;AAAA,IAC7F;AACA,WAAO;AAAA,EACT,CAAC;AACH;","names":[]}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@anvia/transformers",
+  "version": "0.1.0",
+  "description": "Transformers.js embedding model adapter for Anvia.",
+  "author": "anvia",
+  "maintainer": "Indra Zulfi",
+  "license": "MIT",
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.2.0",
+    "@anvia/core": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.9.1",
+    "tsup": "^8.5.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.8"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --sourcemap --clean",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  }
+}