@plur-ai/core 0.3.1 → 0.4.1

package/dist/chunk-KMVQYBNP.js

@@ -0,0 +1,397 @@
+// src/fts.ts
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "the",
+  "and",
+  "for",
+  "that",
+  "this",
+  "with",
+  "from",
+  "are",
+  "was",
+  "were",
+  "been",
+  "have",
+  "has",
+  "not",
+  "but",
+  "its",
+  "you",
+  "your",
+  "can",
+  "will",
+  "should",
+  "would",
+  "could",
+  "may",
+  "might"
+]);
+function ftsTokenize(text) {
+  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((w) => w.length > 2).filter((w) => !STOP_WORDS.has(w));
+}
+function engramSearchText(engram) {
+  const parts = [engram.statement];
+  if (engram.domain) parts.push(engram.domain.replace(/\./g, " "));
+  if (engram.tags.length > 0) parts.push(engram.tags.join(" "));
+  if (engram.entities) {
+    for (const e of engram.entities) {
+      parts.push(e.name);
+      if (e.type !== "other") parts.push(e.type);
+    }
+  }
+  if (engram.temporal) {
+    if (engram.temporal.valid_from) parts.push(engram.temporal.valid_from);
+    if (engram.temporal.valid_until) parts.push(engram.temporal.valid_until);
+  }
+  if (engram.rationale) parts.push(engram.rationale);
+  return parts.join(" ");
+}
+function computeIdf(engrams, queryTokens) {
+  const N = engrams.length;
+  if (N === 0) return /* @__PURE__ */ new Map();
+  const engramTermSets = engrams.map((e) => new Set(ftsTokenize(engramSearchText(e))));
+  const idf = /* @__PURE__ */ new Map();
+  for (const qt of queryTokens) {
+    let df = 0;
+    for (const termSet of engramTermSets) {
+      if (termSet.has(qt) || Array.from(termSet).some((t) => t.includes(qt) || qt.includes(t))) {
+        df++;
+      }
+    }
+    idf.set(qt, Math.max(0, Math.log(N / (1 + df))));
+  }
+  return idf;
+}
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+function ftsScore(engram, queryTokens, idfWeights, avgDocLength) {
+  const allTerms = ftsTokenize(engramSearchText(engram));
+  if (queryTokens.length === 0) return 0;
+  const docLen = allTerms.length;
+  const avgdl = avgDocLength && avgDocLength > 0 ? avgDocLength : docLen;
+  const hasNonZeroIdf = idfWeights && Array.from(idfWeights.values()).some((v) => v > 0);
+  let score = 0;
+  for (const qt of queryTokens) {
+    let effectiveIdf;
+    if (!idfWeights) {
+      effectiveIdf = 1;
+    } else if (hasNonZeroIdf) {
+      effectiveIdf = idfWeights.get(qt) ?? 0;
+      if (effectiveIdf === 0) continue;
+    } else {
+      effectiveIdf = 1;
+    }
+    let tf = 0;
+    for (const t of allTerms) {
+      if (t.includes(qt) || qt.includes(t)) tf++;
+    }
+    if (tf === 0) continue;
+    const numerator = tf * (BM25_K1 + 1);
+    const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * docLen / avgdl);
+    score += effectiveIdf * (numerator / denominator);
+  }
+  return score;
+}
+function searchEngrams(engrams, query, limit = 20) {
+  const queryTokens = ftsTokenize(query);
+  if (queryTokens.length === 0) return [];
+  const idfWeights = computeIdf(engrams, queryTokens);
+  const avgDocLength = engrams.length > 0 ? engrams.reduce((sum, e) => sum + ftsTokenize(engramSearchText(e)).length, 0) / engrams.length : 0;
+  return engrams.map((e) => ({ engram: e, score: ftsScore(e, queryTokens, idfWeights, avgDocLength) })).filter((r) => r.score > 0).sort((a, b) => b.score - a.score).slice(0, limit).map((r) => r.engram);
+}
+
+// src/embeddings.ts
+import { existsSync as existsSync2, readFileSync, mkdirSync as mkdirSync2 } from "fs";
+import { join as join2 } from "path";
+import { createHash } from "crypto";
+
+// src/sync.ts
+import { execFileSync } from "child_process";
+import { existsSync, writeFileSync, renameSync, mkdirSync, unlinkSync, statSync } from "fs";
+import { join, dirname } from "path";
+var GITIGNORE = `# PLUR \u2014 derived/cache files (regenerated automatically)
+embeddings/
+.embeddings-cache.json
+*.db
+*.sqlite
+exchange/
+`;
+function git(args, cwd) {
+  return execFileSync("git", args, { cwd, encoding: "utf8", timeout: 3e4 }).trim();
+}
+function gitSafe(args, cwd) {
+  try {
+    return git(args, cwd);
+  } catch {
+    return null;
+  }
+}
+function isGitRepo(root) {
+  return existsSync(join(root, ".git"));
+}
+function hasGitCli() {
+  try {
+    execFileSync("git", ["--version"], { encoding: "utf8", timeout: 5e3 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function getRemote(root) {
+  return gitSafe(["remote", "get-url", "origin"], root);
+}
+function isDirty(root) {
+  const status = gitSafe(["status", "--porcelain"], root);
+  return status !== null && status.length > 0;
+}
+function countDiff(root, direction) {
+  const tracking = gitSafe(["rev-parse", "--abbrev-ref", "@{u}"], root);
+  if (!tracking) return 0;
+  const flag = direction === "ahead" ? "--left-only" : "--right-only";
+  const count = gitSafe(["rev-list", flag, "--count", "HEAD...@{u}"], root);
+  return count ? parseInt(count, 10) : 0;
+}
+function getSyncStatus(root) {
+  if (!isGitRepo(root)) {
+    return { initialized: false, remote: null, dirty: false, branch: null, ahead: 0, behind: 0 };
+  }
+  const branch = gitSafe(["rev-parse", "--abbrev-ref", "HEAD"], root);
+  const remote = getRemote(root);
+  if (remote) gitSafe(["fetch", "origin", "--quiet"], root);
+  return {
+    initialized: true,
+    remote,
+    dirty: isDirty(root),
+    branch,
+    ahead: countDiff(root, "ahead"),
+    behind: countDiff(root, "behind")
+  };
+}
+function initRepo(root) {
+  git(["init"], root);
+  atomicWrite(join(root, ".gitignore"), GITIGNORE);
+  git(["add", "-A"], root);
+  git(["commit", "-m", "Initial PLUR engram store"], root);
+}
+function commitChanges(root) {
+  if (!isDirty(root)) return 0;
+  git(["add", "-A"], root);
+  const diff = gitSafe(["diff", "--cached", "--stat", "--shortstat"], root);
+  const now = (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
+  git(["commit", "-m", `plur sync ${now}`], root);
+  const match = diff?.match(/(\d+) file/);
+  return match ? parseInt(match[1], 10) : 1;
+}
+function hasConflictMarkers(root) {
+  const result = gitSafe(["grep", "-l", "<<<<<<<"], root);
+  return result !== null && result.length > 0;
+}
+function pullRebase(root) {
+  const result = gitSafe(["pull", "--rebase", "origin", "main"], root);
+  if (result !== null) return true;
+  gitSafe(["rebase", "--abort"], root);
+  const mergeResult = gitSafe(["pull", "origin", "main", "--no-edit"], root);
+  if (mergeResult !== null) return true;
+  if (hasConflictMarkers(root)) {
+    gitSafe(["merge", "--abort"], root);
+    throw new Error("Sync conflict: YAML files have merge conflicts that require manual resolution. Your local changes are preserved.");
+  }
+  git(["add", "-A"], root);
+  gitSafe(["commit", "-m", "plur sync: merge conflict resolved (kept both)"], root);
+  return true;
+}
+function sync(root, remote) {
+  if (!hasGitCli()) {
+    throw new Error("git is not installed. Install git to enable sync.");
+  }
+  if (!isGitRepo(root)) {
+    initRepo(root);
+    if (remote) {
+      git(["remote", "add", "origin", remote], root);
+      const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
+      git(["push", "-u", "origin", branch], root);
+      return { action: "initialized", message: `Initialized and pushed to ${remote}`, remote, files_changed: 0 };
+    }
+    return {
+      action: "initialized",
+      message: "Initialized local git repo. Call plur.sync with remote to enable cross-device sync.",
+      remote: null,
+      files_changed: 0
+    };
+  }
+  const existingRemote = getRemote(root);
+  if (remote && !existingRemote) {
+    git(["remote", "add", "origin", remote], root);
+    const filesChanged2 = commitChanges(root);
+    const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], root);
+    git(["push", "-u", "origin", branch], root);
+    return { action: "synced", message: `Remote added and pushed to ${remote}`, remote, files_changed: filesChanged2 };
+  }
+  if (!existingRemote) {
+    const filesChanged2 = commitChanges(root);
+    if (filesChanged2 === 0) {
+      return { action: "up-to-date", message: 'No changes to commit. Add a remote with plur.sync({ remote: "..." }) to enable cross-device sync.', remote: null, files_changed: 0 };
+    }
+    return { action: "committed", message: `Committed ${filesChanged2} file(s) locally.`, remote: null, files_changed: filesChanged2 };
+  }
+  const filesChanged = commitChanges(root);
+  gitSafe(["fetch", "origin", "--quiet"], root);
+  const behind = countDiff(root, "behind");
+  const aheadBefore = countDiff(root, "ahead");
+  if (behind > 0) {
+    pullRebase(root);
+  }
+  const aheadAfter = countDiff(root, "ahead");
+  if (aheadAfter > 0) {
+    gitSafe(["push", "origin"], root);
+  }
+  if (filesChanged === 0 && behind === 0 && aheadBefore === 0) {
+    return { action: "up-to-date", message: "Already in sync.", remote: existingRemote, files_changed: 0 };
+  }
+  const parts = [];
+  if (filesChanged > 0) parts.push(`${filesChanged} file(s) committed`);
+  if (behind > 0) parts.push(`pulled ${behind} remote commit(s)`);
+  if (aheadAfter === 0 && aheadBefore > 0) parts.push("pushed");
+  return {
+    action: "synced",
+    message: `Synced. ${parts.join(", ")}.`,
+    remote: existingRemote,
+    files_changed: filesChanged
+  };
+}
+function withLock(filePath, fn, options) {
+  const lockPath = filePath + ".lock";
+  const maxRetries = options?.maxRetries ?? 5;
+  const baseDelay = options?.baseDelay ?? 100;
+  const staleThreshold = options?.staleThreshold ?? 1e4;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      writeFileSync(lockPath, `${process.pid}`, { flag: "wx" });
+      break;
+    } catch (err) {
+      if (err.code !== "EEXIST") throw err;
+      try {
+        const stat = statSync(lockPath);
+        if (Date.now() - stat.mtimeMs > staleThreshold) {
+          unlinkSync(lockPath);
+          continue;
+        }
+      } catch {
+        continue;
+      }
+      if (attempt === maxRetries) {
+        throw new Error(`Failed to acquire lock on ${filePath} after ${maxRetries} retries`);
+      }
+      const delay = baseDelay * Math.pow(2, attempt);
+      const end = Date.now() + delay;
+      while (Date.now() < end) {
+      }
+    }
+  }
+  try {
+    return fn();
+  } finally {
+    try {
+      unlinkSync(lockPath);
+    } catch {
+    }
+  }
+}
+function atomicWrite(filePath, content) {
+  const dir = dirname(filePath);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const tmp = filePath + ".tmp";
+  writeFileSync(tmp, content);
+  renameSync(tmp, filePath);
+}
+
+// src/embeddings.ts
+var embedPipeline = null;
+var transformersUnavailable = false;
+async function getEmbedder() {
+  if (transformersUnavailable) return null;
+  if (!embedPipeline) {
+    try {
+      const { pipeline } = await import("./transformers.node-PH5YK5EA.js");
+      embedPipeline = await pipeline("feature-extraction", "Xenova/bge-small-en-v1.5", {
+        dtype: "fp32"
+      });
+    } catch {
+      transformersUnavailable = true;
+      return null;
+    }
+  }
+  return embedPipeline;
+}
+async function embed(text) {
+  const embedder = await getEmbedder();
+  if (!embedder) return null;
+  const result = await embedder(text, { pooling: "cls", normalize: true });
+  return new Float32Array(result.data);
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+function loadCache(cachePath) {
+  if (!existsSync2(cachePath)) return {};
+  try {
+    return JSON.parse(readFileSync(cachePath, "utf8"));
+  } catch {
+    return {};
+  }
+}
+function saveCache(cachePath, cache) {
+  const dir = cachePath.substring(0, cachePath.lastIndexOf("/"));
+  if (!existsSync2(dir)) mkdirSync2(dir, { recursive: true });
+  atomicWrite(cachePath, JSON.stringify(cache));
+}
+function hashStatement(statement) {
+  return createHash("sha256").update(statement).digest("hex").slice(0, 16);
+}
+async function embeddingSearch(engrams, query, limit, storagePath) {
+  if (engrams.length === 0) return [];
+  const cachePath = storagePath ? join2(storagePath, ".embeddings-cache.json") : ".embeddings-cache.json";
+  const cache = loadCache(cachePath);
+  const queryEmbedding = await embed(query);
+  if (!queryEmbedding) {
+    return [];
+  }
+  const similarities = [];
+  for (const engram of engrams) {
+    const searchText = engramSearchText(engram);
+    const hash = hashStatement(searchText);
+    let engramEmbedding;
+    if (cache[engram.id]?.hash === hash) {
+      engramEmbedding = new Float32Array(cache[engram.id].embedding);
+    } else {
+      const emb = await embed(searchText);
+      if (!emb) return [];
+      engramEmbedding = emb;
+      cache[engram.id] = {
+        hash,
+        embedding: Array.from(engramEmbedding)
+      };
+    }
+    const score = cosineSimilarity(queryEmbedding, engramEmbedding);
+    similarities.push({ engram, score });
+  }
+  saveCache(cachePath, cache);
+  similarities.sort((a, b) => b.score - a.score);
+  return similarities.slice(0, limit).map((s) => s.engram);
+}
+
+export {
+  getSyncStatus,
+  sync,
+  withLock,
+  atomicWrite,
+  ftsTokenize,
+  engramSearchText,
+  ftsScore,
+  searchEngrams,
+  embed,
+  cosineSimilarity,
+  embeddingSearch
+};

package/dist/embeddings-2IODIQAF.js

@@ -0,0 +1,11 @@
+import {
+  cosineSimilarity,
+  embed,
+  embeddingSearch
+} from "./chunk-KMVQYBNP.js";
+import "./chunk-2ZDO52B4.js";
+export {
+  cosineSimilarity,
+  embed,
+  embeddingSearch
+};