@loreai/core 0.17.1 → 0.19.0

package/src/distillation.ts

@@ -14,6 +14,7 @@ import {
 } from "./prompt";
 import { toolStripAnnotation } from "./gradient";
 import { workerSessionIDs } from "./worker";
+import { distillLimiter } from "./session-limiter";
 import type { LLMClient } from "./types";
 
 // Re-export for backwards compat — index.ts and others may still import from here.
@@ -610,8 +611,23 @@ function resetOrphans(projectPath: string, sessionID: string): number {
   return orphans.length;
 }
 
-// Main distillation entry point — called on session.idle or when urgent
+// Main distillation entry point — called on session.idle or when urgent.
+// Serialized per session via p-limit(1) to prevent concurrent runs from
+// reading the same undistilled messages and producing duplicate rows.
 export async function run(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+  force?: boolean;
+  skipMeta?: boolean;
+  urgent?: boolean;
+  callType?: "batch" | "direct";
+}): Promise<{ rounds: number; distilled: number }> {
+  return distillLimiter.get(input.sessionID)(() => runInner(input));
+}
+
+async function runInner(input: {
   llm: LLMClient;
   projectPath: string;
   sessionID: string;
@@ -697,7 +713,8 @@ export async function run(input: {
       gen0Count(input.projectPath, input.sessionID) >=
       cfg.distillation.metaThreshold
     ) {
-      await metaDistill({
+      // Call inner directly — we're already under the per-session limiter.
+      await metaDistillInner({
         llm: input.llm,
         projectPath: input.projectPath,
         sessionID: input.sessionID,
@@ -776,17 +793,29 @@ async function distillSegment(input: {
     return null;
   }
 
-  const distillId = storeDistillation({
-    projectPath: input.projectPath,
-    sessionID: input.sessionID,
-    observations: result.observations,
-    sourceIDs: input.messages.map((m) => m.id),
-    generation: 0,
-    rCompression: rComp,
-    cNorm,
-    callType: input.callType,
-  });
-  temporal.markDistilled(input.messages.map((m) => m.id));
+  // Atomic: store distillation + mark source messages as distilled in one
+  // transaction. Without this, a crash between the two statements would leave
+  // messages undistilled but with an existing distillation row, causing
+  // re-processing on restart and duplicate distillation content.
+  let distillId: string;
+  db().exec("BEGIN IMMEDIATE");
+  try {
+    distillId = storeDistillation({
+      projectPath: input.projectPath,
+      sessionID: input.sessionID,
+      observations: result.observations,
+      sourceIDs: input.messages.map((m) => m.id),
+      generation: 0,
+      rCompression: rComp,
+      cNorm,
+      callType: input.callType,
+    });
+    temporal.markDistilled(input.messages.map((m) => m.id));
+    db().exec("COMMIT");
+  } catch (e) {
+    db().exec("ROLLBACK");
+    throw e;
+  }
 
   log.info(
     `distill segment: ${input.messages.length} msgs, ` +
@@ -811,7 +840,8 @@ async function distillSegment(input: {
 
   // Fire-and-forget: extract decision/preference patterns → knowledge entries
   if (config().knowledge.enabled) {
-    for (const pat of extractPatterns(result.observations)) {
+    const patterns = extractPatterns(result.observations);
+    for (const pat of patterns) {
       try {
         ltm.create({
           projectPath: input.projectPath,
@@ -825,6 +855,9 @@ async function distillSegment(input: {
         // Dedup guard in ltm.create() handles duplicates — swallow errors
       }
     }
+    if (patterns.length > 0) {
+      log.info(`pattern extraction: ${patterns.length} entries from distillation`);
+    }
   }
 
   return result;
@@ -836,7 +869,8 @@ async function distillSegment(input: {
  * via `<previous-meta-summary>` so the LLM updates in place rather than
  * re-deriving from scratch.
  *
- * Exported for tests; `run()` is the production entry point.
+ * Serialized per session via the same p-limit(1) as `run()`. Exported for
+ * the idle handler which calls metaDistill() independently of run().
  */
 export async function metaDistill(input: {
   llm: LLMClient;
@@ -845,6 +879,17 @@ export async function metaDistill(input: {
   model?: { providerID: string; modelID: string };
   urgent?: boolean;
   callType?: "batch" | "direct";
+}): Promise<DistillationResult | null> {
+  return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
+}
+
+async function metaDistillInner(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+  urgent?: boolean;
+  callType?: "batch" | "direct";
 }): Promise<DistillationResult | null> {
   const existing = loadGen0(input.projectPath, input.sessionID);
 
@@ -928,7 +973,8 @@ export async function metaDistill(input: {
 
   // Fire-and-forget: extract decision/preference patterns → knowledge entries
   if (config().knowledge.enabled) {
-    for (const pat of extractPatterns(result.observations)) {
+    const patterns = extractPatterns(result.observations);
+    for (const pat of patterns) {
       try {
         ltm.create({
           projectPath: input.projectPath,
@@ -942,6 +988,9 @@ export async function metaDistill(input: {
         // Dedup guard in ltm.create() handles duplicates — swallow errors
       }
     }
+    if (patterns.length > 0) {
+      log.info(`pattern extraction: ${patterns.length} entries from meta-distillation`);
+    }
   }
 
   return result;

package/src/embedding-vendor.ts

@@ -1,31 +1,19 @@
 /**
- * Vendored bge-small registration for the standalone Lore binary.
+ * Vendored model registration for the standalone Lore binary.
  *
- * The Bun-compiled `lore` binary uses `bun build --compile` to bundle
- * `fastembed` + `onnxruntime-node` + `@anush008/tokenizers-<platform>`
- * directly into the executable — including the platform-specific
- * `.node` addons which Bun embeds and dlopens from `$bunfs` at runtime.
+ * The Bun-compiled `lore` binary bundles `@huggingface/transformers`
+ * (which includes ONNX Runtime) into the executable. The model weights
+ * and tokenizer files (nomic-embed-text-v1.5, INT8 quantized) are
+ * embedded as Bun assets and extracted to a cache dir on first run.
  *
- * Two pieces don't fit into Bun's automatic bundling and need our help:
- *
- *  1. **Side-load shared libraries**. `onnxruntime_binding.node` does a
- *     runtime `dlopen("libonnxruntime.so.1")` (or the .dylib / .dll
- *     equivalent) for the actual ONNX Runtime computation library. Bun
- *     doesn't follow this kind of dependency. The binary's wrapper
- *     pre-loads these libs via `bun:ffi` *before* fastembed evaluates,
- *     so when the addon's dlopen fires it finds the cached handle.
- *
- *  2. **Model weights + tokenizer**. fastembed downloads from the HF
- *     Hub on first use; we want zero network on first run. The wrapper
- *     embeds the bge-small INT8 files as Bun assets, writes them to a
- *     real disk dir on first run, and sets `globalThis.__LORE_VENDOR_MODEL__`
- *     to that path. This module exposes that registration to the
- *     LocalProvider so it can hand the path to fastembed's CUSTOM-mode
- *     init (`modelAbsoluteDirPath` + `modelName`).
+ * The binary's wrapper sets `globalThis.__LORE_VENDOR_MODEL__` to the
+ * extraction path. This module exposes that registration to the
+ * `LocalProvider` so it can configure the transformers.js `env` to
+ * load from the local path instead of downloading from HuggingFace Hub.
  *
  * In npm-mode usage from `@loreai/opencode` / `@loreai/pi` the global
- * is unset and `vendorModelInfo()` returns `null`, so the LocalProvider
- * falls through to fastembed's default Qdrant repo + cache.
+ * is unset and `vendorModelInfo()` returns `null`, so transformers.js
+ * downloads the model from HF Hub on first use and caches it locally.
  */
 
 // ---------------------------------------------------------------------------
@@ -34,13 +22,10 @@
 
 /** What the binary wrapper writes to globalThis after extracting model files. */
 export interface VendorRegistration {
-  /** Absolute path to the dir containing the bge-small files
-   *  (config.json, tokenizer.json, model_quantized.onnx, …). Pass to
-   *  fastembed as `modelAbsoluteDirPath` in CUSTOM init. */
-  modelAbsoluteDirPath: string;
-  /** Filename of the ONNX weights inside that dir. Pass to fastembed
-   *  as `modelName` in CUSTOM init. */
-  modelName: string;
+  /** Absolute path to the dir containing the model files in HF layout
+   *  (config.json, tokenizer.json, onnx/model_quantized.onnx, …).
+   *  Passed to transformers.js as `env.localModelPath`. */
+  localModelPath: string;
   /** Target identifier the binary was built for, e.g. "linux-x64".
    *  Diagnostic only — the runtime doesn't branch on it. */
   target: string;
@@ -69,25 +54,23 @@ export function _setVendorRegistration(reg: VendorRegistration | null): void {
 // Public entry
 // ---------------------------------------------------------------------------
 
-/** Subset of the registration fastembed needs. Stripped of the
- *  diagnostic fields so the LocalProvider has exactly what it should
- *  hand to `FlagEmbedding.init`. */
+/** Subset of the registration the LocalProvider needs. */
 export interface VendorModelInfo {
-  modelAbsoluteDirPath: string;
-  modelName: string;
+  /** Absolute path to the dir containing the model in HF layout.
+   *  Set as `env.localModelPath` in the worker thread. */
+  localModelPath: string;
 }
 
 /**
- * Resolve the bundled-model arguments for fastembed CUSTOM init. Returns
- * `null` when no vendor is registered (npm-mode), so the caller can fall
- * through to fastembed's default cacheDir/HF Hub flow.
+ * Resolve the vendored model path for transformers.js local loading.
+ * Returns `null` when no vendor is registered (npm-mode), so the caller
+ * falls through to transformers.js's default HF Hub download + cache.
  */
 export function vendorModelInfo(): VendorModelInfo | null {
   const reg = getRegistration();
   if (!reg) return null;
   return {
-    modelAbsoluteDirPath: reg.modelAbsoluteDirPath,
-    modelName: reg.modelName,
+    localModelPath: reg.localModelPath,
   };
 }
 

package/src/embedding-worker-types.ts

@@ -1,10 +1,11 @@
 /**
  * Shared message types for the embedding worker thread.
  *
- * The embedding worker (`embedding-worker.ts`) runs fastembed/ONNX inference
- * in a separate `node:worker_threads` Worker so the main thread's event loop
- * stays free during inference. This file defines the message protocol between
- * the main thread (`LocalProvider` in `embedding.ts`) and the worker.
+ * The embedding worker (`embedding-worker.ts`) runs ONNX inference via
+ * `@huggingface/transformers` in a separate `node:worker_threads` Worker
+ * so the main thread's event loop stays free during inference. This file
+ * defines the message protocol between the main thread (`LocalProvider`
+ * in `embedding.ts`) and the worker.
  *
  * Imported by both sides — keep this file free of runtime dependencies.
  */
@@ -18,7 +19,7 @@ export interface EmbedRequest {
   type: "embed";
   /** Monotonic request ID for correlating responses. */
   id: number;
-  /** Texts to embed. */
+  /** Texts to embed (already prefixed with task instruction by the caller). */
   texts: string[];
   /** "document" for storage, "query" for search. */
   inputType: "document" | "query";
@@ -72,11 +73,18 @@ export type WorkerOutbound = EmbedResult | EmbedError | InitError;
 
 /** Passed to the worker via `workerData` at construction time. */
 export interface WorkerInitData {
-  /** fastembed model name, e.g. "BGESmallENV15". */
-  modelName: string;
+  /** HuggingFace model ID, e.g. "nomic-ai/nomic-embed-text-v1.5". */
+  modelId: string;
+  /** Target embedding dimensions. For Nomic v1.5 with Matryoshka,
+   *  this controls how many leading dims to keep (64–768). */
+  dimensions: number;
   /** Vendored model info for binary mode, or null for npm mode.
-   *  Mirrors the `globalThis.__LORE_VENDOR_MODEL__` registration which
-   *  only exists on the main thread — passed explicitly so the worker
-   *  can hand it to `FlagEmbedding.init()`. */
-  vendorModel: { modelAbsoluteDirPath: string; modelName: string } | null;
+   *  In binary mode, model files are pre-extracted to a local dir
+   *  and we point transformers.js at that path instead of downloading
+   *  from HuggingFace Hub. */
+  vendorModel: {
+    /** Absolute path to the dir containing model files
+     *  (config.json, tokenizer.json, onnx/model_quantized.onnx, …). */
+    localModelPath: string;
+  } | null;
 }

package/src/embedding-worker.ts

@@ -1,8 +1,9 @@
 /**
- * Embedding worker thread — runs fastembed/ONNX inference off the main thread.
+ * Embedding worker thread — runs ONNX inference via @huggingface/transformers
+ * off the main thread.
  *
  * This file is the entry point for a `node:worker_threads` Worker spawned by
- * `LocalProvider` in `embedding.ts`. It owns the `FlagEmbedding` ONNX model
+ * `LocalProvider` in `embedding.ts`. It owns the transformers.js pipeline
  * and processes embed requests sequentially from a priority queue. Moving
  * inference here keeps the main thread's event loop free — HTTP requests,
  * SSE streams, and session APIs are no longer blocked during embedding.
@@ -10,6 +11,12 @@
  * Communication uses `parentPort` message passing with structured clone.
  * Float32Array vectors are sent back directly (Bun preserves identity).
  *
+ * The worker applies Nomic's recommended post-processing:
+ *   1. Mean pooling (via pipeline option)
+ *   2. Layer normalization
+ *   3. Matryoshka dimension truncation (if dimensions < full 768)
+ *   4. L2 normalization
+ *
  * @see embedding-worker-types.ts for the message protocol.
  */
 
@@ -25,53 +32,72 @@ import type {
 // workerData
 // ---------------------------------------------------------------------------
 
-const { modelName, vendorModel } = workerData as WorkerInitData;
+const { modelId, dimensions, vendorModel } = workerData as WorkerInitData;
 
 // ---------------------------------------------------------------------------
 // Model lifecycle — lazy init on first embed request
 // ---------------------------------------------------------------------------
 
-/** The fastembed model, typed to the subset of methods we use. */
-type FastembedModel = {
-  queryEmbed(text: string): Promise<number[]>;
-  passageEmbed(texts: string[], batchSize?: number): AsyncGenerator<number[][]>;
+/** The transformers.js pipeline instance, typed loosely since the exact
+ *  return type depends on the pipeline task. */
+type FeatureExtractionPipeline = {
+  (texts: string[], options?: Record<string, unknown>): Promise<{
+    dims: number[];
+    data: Float32Array;
+    tolist(): number[][];
+  }>;
+  dispose?(): Promise<void>;
 };
 
-let model: FastembedModel | null = null;
+let pipe: FeatureExtractionPipeline | null = null;
+let layerNormFn: ((input: unknown, normalized_shape: number[]) => {
+  dims: number[];
+  data: Float32Array;
+  normalize(p: number, dim: number): { tolist(): number[][]; data: Float32Array; dims: number[] };
+  slice(...args: unknown[]): { normalize(p: number, dim: number): { tolist(): number[][]; data: Float32Array; dims: number[] } };
+}) | null = null;
 let initPromise: Promise<void> | null = null;
 let initFailed = false;
 let initError: string | null = null;
 
 /**
- * Ensure the fastembed model is loaded. Lazy — first call triggers the
- * dynamic import + FlagEmbedding.init(), subsequent calls return immediately.
+ * Ensure the transformers.js pipeline is loaded. Lazy — first call triggers
+ * the dynamic import + pipeline creation, subsequent calls return immediately.
  * On failure, marks the worker as permanently broken and posts `init-error`.
  */
-async function ensureModel(): Promise<FastembedModel> {
-  if (model) return model;
-  if (initFailed) throw new Error(initError ?? "fastembed init previously failed");
+async function ensurePipeline(): Promise<void> {
+  if (pipe) return;
+  if (initFailed) throw new Error(initError ?? "pipeline init previously failed");
 
   if (!initPromise) {
     initPromise = (async () => {
-      const fastembed = await import("fastembed");
-      const { EmbeddingModel, FlagEmbedding } = fastembed;
+      const transformers = await import("@huggingface/transformers");
+      const { pipeline, env, layer_norm } = transformers;
+
+      // Configure transformers.js environment
+      env.allowRemoteModels = !vendorModel;
+      env.allowLocalModels = true;
 
-      let m: unknown;
       if (vendorModel) {
-        // Binary mode: use pre-extracted model files.
-        m = await FlagEmbedding.init({
-          model: EmbeddingModel.CUSTOM,
-          modelAbsoluteDirPath: vendorModel.modelAbsoluteDirPath,
-          modelName: vendorModel.modelName,
-        });
-      } else {
-        // npm mode: resolve model name against fastembed's enum.
-        const enumValue = (EmbeddingModel as Record<string, string>)[modelName];
-        m = await FlagEmbedding.init({
-          model: enumValue ?? modelName,
-        } as { model: typeof EmbeddingModel.BGESmallENV15 });
+        // Binary mode: point at pre-extracted model files on disk.
+        env.localModelPath = vendorModel.localModelPath;
+        env.allowRemoteModels = false;
       }
-      model = m as FastembedModel;
+
+      // Create feature-extraction pipeline with ONNX quantized model.
+      // dtype: 'q8' selects the INT8 quantized ONNX variant (model_quantized.onnx)
+      // which is ~137MB for Nomic v1.5 vs ~547MB for the full FP32 model.
+      //
+      // device: "cpu" — in npm mode, transformers.js uses onnxruntime-node
+      // (native CPU). In the compiled binary, onnxruntime-node is redirected
+      // to onnxruntime-web by the build plugin, which handles "cpu" via its
+      // WASM+SIMD backend (API-compatible, ~2x faster on batch workloads).
+      pipe = (await pipeline("feature-extraction", modelId, {
+        dtype: "q8",
+        device: "cpu",
+      })) as unknown as FeatureExtractionPipeline;
+
+      layerNormFn = layer_norm as typeof layerNormFn;
     })().catch((err) => {
       initFailed = true;
       initError = err instanceof Error ? err.message : String(err);
@@ -83,8 +109,7 @@ async function ensureModel(): Promise<FastembedModel> {
   }
 
   await initPromise;
-  if (!model) throw new Error("model init completed but model is null");
-  return model;
+  if (!pipe) throw new Error("pipeline init completed but pipe is null");
 }
 
 // ---------------------------------------------------------------------------
@@ -135,31 +160,70 @@ async function drain(): Promise<void> {
 // Embed processing
 // ---------------------------------------------------------------------------
 
+/**
+ * Detect ONNX runtime out-of-memory errors. The runtime throws opaque
+ * numeric error codes (e.g. "287180544") for allocation failures rather
+ * than a readable message. We match on large numeric-only strings and
+ * known OOM patterns.
+ */
+function isOomError(msg: string): boolean {
+  // Pure numeric error codes ≥ 6 digits are ORT allocation failures
+  if (/^\d{6,}$/.test(msg)) return true;
+  // Explicit OOM messages from various ONNX backends
+  if (/out.of.memory|alloc.*fail|oom/i.test(msg)) return true;
+  return false;
+}
+
 async function processEmbed(req: EmbedRequest): Promise<void> {
   try {
-    const m = await ensureModel();
+    await ensurePipeline();
+
+    // Run feature extraction with mean pooling.
+    // truncation: true caps each text at the model's max length (8192 tokens
+    // for Nomic v1.5), preventing oversized inputs from causing OOM.
+    const output = await pipe!(req.texts, { pooling: "mean", truncation: true });
+
+    // Post-process following Nomic's recipe:
+    //   1. Layer normalization over the full hidden dimension
+    //   2. Matryoshka truncation to target dimensions
+    //   3. L2 normalization
+    const fullDim = output.dims[output.dims.length - 1]; // 768 for Nomic v1.5
+    const truncate = dimensions < fullDim;
+
+    let normalized: { tolist(): number[][]; data: Float32Array; dims: number[] };
+    if (truncate) {
+      // layer_norm → slice → L2 normalize
+      normalized = layerNormFn!(output, [fullDim])
+        .slice(null, [0, dimensions])
+        .normalize(2, -1);
+    } else {
+      // layer_norm → L2 normalize (no truncation)
+      normalized = layerNormFn!(output, [fullDim])
+        .normalize(2, -1);
+    }
 
-    let vectors: Float32Array[];
+    // Extract per-text vectors from the batched tensor.
+    const numTexts = req.texts.length;
+    const vectors: Float32Array[] = [];
+    const dim = truncate ? dimensions : fullDim;
 
-    if (req.inputType === "query" && req.texts.length === 1) {
-      // Single query — use queryEmbed for better quality.
-      const vec = await m.queryEmbed(req.texts[0]);
-      vectors = [new Float32Array(vec)];
-    } else {
-      // Batch document embedding via async generator.
-      vectors = [];
-      for await (const batch of m.passageEmbed(req.texts)) {
-        for (const vec of batch) {
-          vectors.push(new Float32Array(vec));
-        }
-      }
+    for (let i = 0; i < numTexts; i++) {
+      const start = i * dim;
+      const vec = new Float32Array(dim);
+      vec.set(normalized.data.subarray(start, start + dim));
+      vectors.push(vec);
     }
 
     post({ type: "result", id: req.id, vectors });
   } catch (err) {
-    // Don't re-post init-error — it was already sent in ensureModel().
+    // Don't re-post init-error — it was already sent in ensurePipeline().
     if (!initFailed) {
-      const msg = err instanceof Error ? err.message : String(err);
+      const raw = err instanceof Error ? err.message : String(err);
+      const msg = isOomError(raw)
+        ? `ONNX runtime out of memory (batch=${req.texts.length}, ` +
+          `longest≈${Math.max(...req.texts.map((t) => t.length))} chars). ` +
+          `Try reducing batch size. Raw: ${raw}`
+        : raw;
       post({ type: "error", id: req.id, error: msg });
     }
   }