@loreai/core 0.17.1 → 0.18.0

package/dist/node/embedding-worker-types.d.ts

@@ -1,10 +1,11 @@
 /**
  * Shared message types for the embedding worker thread.
  *
- * The embedding worker (`embedding-worker.ts`) runs fastembed/ONNX inference
- * in a separate `node:worker_threads` Worker so the main thread's event loop
- * stays free during inference. This file defines the message protocol between
- * the main thread (`LocalProvider` in `embedding.ts`) and the worker.
+ * The embedding worker (`embedding-worker.ts`) runs ONNX inference via
+ * `@huggingface/transformers` in a separate `node:worker_threads` Worker
+ * so the main thread's event loop stays free during inference. This file
+ * defines the message protocol between the main thread (`LocalProvider`
+ * in `embedding.ts`) and the worker.
  *
  * Imported by both sides — keep this file free of runtime dependencies.
  */
@@ -13,7 +14,7 @@ export interface EmbedRequest {
     type: "embed";
     /** Monotonic request ID for correlating responses. */
     id: number;
-    /** Texts to embed. */
+    /** Texts to embed (already prefixed with task instruction by the caller). */
     texts: string[];
     /** "document" for storage, "query" for search. */
     inputType: "document" | "query";
@@ -52,15 +53,19 @@ export interface InitError {
 export type WorkerOutbound = EmbedResult | EmbedError | InitError;
 /** Passed to the worker via `workerData` at construction time. */
 export interface WorkerInitData {
-    /** fastembed model name, e.g. "BGESmallENV15". */
-    modelName: string;
+    /** HuggingFace model ID, e.g. "nomic-ai/nomic-embed-text-v1.5". */
+    modelId: string;
+    /** Target embedding dimensions. For Nomic v1.5 with Matryoshka,
+     *  this controls how many leading dims to keep (64–768). */
+    dimensions: number;
     /** Vendored model info for binary mode, or null for npm mode.
-     *  Mirrors the `globalThis.__LORE_VENDOR_MODEL__` registration which
-     *  only exists on the main thread — passed explicitly so the worker
-     *  can hand it to `FlagEmbedding.init()`. */
+     *  In binary mode, model files are pre-extracted to a local dir
+     *  and we point transformers.js at that path instead of downloading
+     *  from HuggingFace Hub. */
     vendorModel: {
-        modelAbsoluteDirPath: string;
-        modelName: string;
+        /** Absolute path to the dir containing model files
+         *  (config.json, tokenizer.json, onnx/model_quantized.onnx, …). */
+        localModelPath: string;
     } | null;
 }
 //# sourceMappingURL=embedding-worker-types.d.ts.map

package/dist/node/embedding-worker-types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding-worker-types.d.ts","sourceRoot":"","sources":["../../src/embedding-worker-types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,kCAAkC;AAClC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,kDAAkD;IAClD,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC;IAChC,qEAAqE;IACrE,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;CAC7B;AAED,sCAAsC;AACtC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,eAAe,CAAC;AAM3D,gFAAgF;AAChF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX;gEAC4D;IAC5D,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED;4DAC4D;AAC5D,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,CAAC;IACnB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;AAMlE,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB;;;iDAG6C;IAC7C,WAAW,EAAE;QAAE,oBAAoB,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACzE"}
+{"version":3,"file":"embedding-worker-types.d.ts","sourceRoot":"","sources":["../../src/embedding-worker-types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,kCAAkC;AAClC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,6EAA6E;IAC7E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,kDAAkD;IAClD,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC;IAChC,qEAAqE;IACrE,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;CAC7B;AAED,sCAAsC;AACtC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,eAAe,CAAC;AAM3D,gFAAgF;AAChF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX;gEAC4D;IAC5D,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED;4DAC4D;AAC5D,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,CAAC;IACnB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;AAMlE,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,mEAAmE;IACnE,OAAO,EAAE,MAAM,CAAC;IAChB;gEAC4D;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB;;;gCAG4B;IAC5B,WAAW,EAAE;QACX;2EACmE;QACnE,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;CACV"}

package/dist/node/embedding-worker.d.ts

@@ -1,8 +1,9 @@
 /**
- * Embedding worker thread — runs fastembed/ONNX inference off the main thread.
+ * Embedding worker thread — runs ONNX inference via @huggingface/transformers
+ * off the main thread.
  *
  * This file is the entry point for a `node:worker_threads` Worker spawned by
- * `LocalProvider` in `embedding.ts`. It owns the `FlagEmbedding` ONNX model
+ * `LocalProvider` in `embedding.ts`. It owns the transformers.js pipeline
  * and processes embed requests sequentially from a priority queue. Moving
  * inference here keeps the main thread's event loop free — HTTP requests,
  * SSE streams, and session APIs are no longer blocked during embedding.
@@ -10,6 +11,12 @@
  * Communication uses `parentPort` message passing with structured clone.
  * Float32Array vectors are sent back directly (Bun preserves identity).
  *
+ * The worker applies Nomic's recommended post-processing:
+ *   1. Mean pooling (via pipeline option)
+ *   2. Layer normalization
+ *   3. Matryoshka dimension truncation (if dimensions < full 768)
+ *   4. L2 normalization
+ *
  * @see embedding-worker-types.ts for the message protocol.
  */
 export {};

package/dist/node/embedding-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding-worker.d.ts","sourceRoot":"","sources":["../../src/embedding-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG"}
+{"version":3,"file":"embedding-worker.d.ts","sourceRoot":"","sources":["../../src/embedding-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG"}