@plur-ai/core 0.7.7 → 0.8.2

package/dist/chunk-UETCDULF.js

@@ -0,0 +1,196 @@
+import {
+  atomicWrite
+} from "./chunk-MY4XVDCE.js";
+
+// src/fts.ts
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "the",
+  "and",
+  "for",
+  "that",
+  "this",
+  "with",
+  "from",
+  "are",
+  "was",
+  "were",
+  "been",
+  "have",
+  "has",
+  "not",
+  "but",
+  "its",
+  "you",
+  "your",
+  "can",
+  "will",
+  "should",
+  "would",
+  "could",
+  "may",
+  "might"
+]);
+function ftsTokenize(text) {
+  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
+}
+function engramSearchText(engram) {
+  const parts = [engram.statement];
+  if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
+  if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
+  if (engram.entities) {
+    for (const e of engram.entities) {
+      parts.push(e.name);
+      if (e.type !== "other") parts.push(e.type);
+    }
+  }
+  if (engram.temporal) {
+    if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
+    if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
+  }
+  if (engram.rationale) parts.push(engram.rationale);
+  return parts.join(" ");
+}
+function computeIdf(engrams, queryTokens) {
+  const N = engrams.length;
+  if (N === 0) return /* @__PURE__ */ new Map();
+  const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
+  const idf = /* @__PURE__ */ new Map();
+  for (const qt of queryTokens) {
+    let df = 0;
+    for (const termSet of engramTermSets) {
+      if (termSet.has(qt) || Array.from(termSet).some((t) => t.includes(qt) || qt.includes(t))) {
+        df++;
+      }
+    }
+    idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
+  }
+  return idf;
+}
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
+  const allTerms = ftsTokenize(engramSearchText(engram));
+  if (queryTokens.length === 0) return 0;
+  const docLen = allTerms.length;
+  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
+  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
+  let score = 0;
+  for (const qt of queryTokens) {
+    let effectiveIdf;
+    if (!idfWeights) {
+      effectiveIdf = 1;
+    } else if (hasNonZeroIdf) {
+      effectiveIdf = idfWeights.get(qt) ?? 0;
+      if (effectiveIdf === 0) continue;
+    } else {
+      effectiveIdf = 1;
+    }
+    let tf = 0;
+    for (const t of allTerms) {
+      if (t.includes(qt) || qt.includes(t)) tf++;
+    }
+    if (tf === 0) continue;
+    const numerator = tf * (BM25_K1 + 1);
+    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
+    score += effectiveIdf * (numerator / denominator);
+  }
+  return score;
+}
+function searchEngrams(engrams, query, limit = 20) {
+  const queryTokens = ftsTokenize(query);
+  if (queryTokens.length === 0) return [];
+  const idfWeights = computeIdf(engrams, queryTokens);
+  const avgDocLength = engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
+  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
+}
+
+// src/embeddings.ts
+import { existsSync, readFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { createHash } from "crypto";
+var embedPipeline = null;
+var transformersUnavailable = false;
+async function getEmbedder() {
+  if (transformersUnavailable) return null;
+  if (!embedPipeline) {
+    try {
+      const { pipeline } = await import("./transformers.node-PH5YK5EA.js");
+      embedPipeline = await pipeline("feature-extraction", "Xenova/bge-small-en-v1.5", {
+        dtype: "fp32"
+      });
+    } catch {
+      transformersUnavailable = true;
+      return null;
+    }
+  }
+  return embedPipeline;
+}
+async function embed(text) {
+  const embedder = await getEmbedder();
+  if (!embedder) return null;
+  const result = await embedder(text, { pooling: "cls", normalize: true });
+  return new Float32Array(result.data);
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+function loadCache(cachePath) {
+  if (!existsSync(cachePath)) return {};
+  try {
+    return JSON.parse(readFileSync(cachePath, "utf8"));
+  } catch {
+    return {};
+  }
+}
+function saveCache(cachePath, cache) {
+  const dir = cachePath.substring(0, cachePath.lastIndexOf("/"));
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  atomicWrite(cachePath, JSON.stringify(cache));
+}
+function hashStatement(statement) {
+  return createHash("sha256").update(statement).digest("hex").slice(0, 16);
+}
+async function embeddingSearch(engrams, query, limit, storagePath) {
+  if (engrams.length === 0) return [];
+  const cachePath = storagePath ? join(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
+  const cache = loadCache(cachePath);
+  const queryEmbedding = await embed(query);
+  if (!queryEmbedding) {
+    return [];
+  }
+  const similarities = [];
+  for (const engram of engrams) {
+    const searchText = engramSearchText(engram);
+    const hash = hashStatement(searchText);
+    let engramEmbedding;
+    if (cache[engram.id]?.hash === hash) {
+      engramEmbedding = new Float32Array(cache[engram.id].embedding);
+    } else {
+      const emb = await embed(searchText);
+      if (!emb) return [];
+      engramEmbedding = emb;
+      cache[engram.id] = {
+        hash,
+        embedding: Array.from(engramEmbedding)
+      };
+    }
+    const score = cosineSimilarity(queryEmbedding, engramEmbedding);
+    similarities.push({ engram, score });
+  }
+  saveCache(cachePath, cache);
+  similarities.sort((a, b) => b.score - a.score);
+  return similarities.slice(0, limit).map((s) => s.engram);
+}
+
+export {
+  ftsTokenize,
+  engramSearchText,
+  computeIdf,
+  ftsScore,
+  searchEngrams,
+  embed,
+  cosineSimilarity,
+  embeddingSearch
+};

package/dist/{embeddings-2IODIQAF.js → embeddings-EX7QPXJS.js}

@@ -2,7 +2,8 @@ import {
   cosineSimilarity,
   embed,
   embeddingSearch
-} from "./chunk-KMVQYBNP.js";
+} from "./chunk-UETCDULF.js";
+import "./chunk-MY4XVDCE.js";
 import "./chunk-2ZDO52B4.js";
 export {
   cosineSimilarity,