enigma-memory 0.1.18 → 0.1.22

package/packages/rag/models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

package/packages/rag/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@enigma/rag",
+  "version": "0.1.22",
+  "description": "Enigma ZK-RAG: Client-Side Semantic Vector Search & Blinded Context Retrieval",
+  "type": "module",
+  "main": "src/index.js",
+  "types": "./index.d.ts",
+  "files": [
+    "src/",
+    "models/",
+    "index.d.ts"
+  ],
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./embeddings": "./src/embeddings.js",
+    "./vector-store": "./src/vector-store.js",
+    "./blinded-search": "./src/blinded-search.js"
+  },
+  "dependencies": {
+    "@enigma/core": "workspace:*",
+    "@huggingface/transformers": "4.2.0"
+  }
+}

package/packages/rag/src/blinded-search.js

@@ -0,0 +1,109 @@
+// Enigma ZK-RAG: Blinded Vector Retrieval Engine.
+// Allows an AI agent or client to query a remote vector index without leaking
+// the query embedding coordinates using Inner-Product Preserving Orthogonal Transformations.
+import crypto from 'node:crypto';
+import { EMBEDDING_DIM } from './embeddings.js';
+
+/**
+ * Generates a deterministic pseudo-random orthogonal matrix Q using Gram-Schmidt orthogonalization.
+ * Q has property Q^T * Q = I, which preserves inner products: (Q*u) · (Q*v) = u · v.
+ * @param {Buffer|string} secretKey - 32-byte secret key shared between trusted client sessions
+ * @param {number} [dim=384] - Vector dimensionality
+ * @returns {Float32Array[]} Array of d orthogonal basis vectors
+ */
+export function generateOrthogonalBasis(secretKey, dim = EMBEDDING_DIM) {
+  const seed = Buffer.isBuffer(secretKey) ? secretKey : Buffer.from(secretKey, 'utf8');
+  const basis = [];
+
+  // Generate random vectors from seed using HKDF
+  for (let i = 0; i < dim; i++) {
+    const raw = Buffer.from(crypto.hkdfSync('sha256', seed, Buffer.from(`enigma.basis.v1.${i}`, 'utf8'), Buffer.alloc(0), dim * 4));
+    const vec = new Float32Array(dim);
+    for (let j = 0; j < dim; j++) {
+      const u32 = raw.readInt32LE(j * 4);
+      vec[j] = u32 / 2147483648.0; // scale to [-1, 1]
+    }
+    basis.push(vec);
+  }
+
+  // Gram-Schmidt process
+  const ortho = [];
+  for (let i = 0; i < dim; i++) {
+    const v = new Float32Array(basis[i]);
+    for (let j = 0; j < i; j++) {
+      const u = ortho[j];
+      let dot = 0;
+      for (let k = 0; k < dim; k++) dot += basis[i][k] * u[k];
+      for (let k = 0; k < dim; k++) v[k] -= dot * u[k];
+    }
+    // Normalize
+    let normSq = 0;
+    for (let k = 0; k < dim; k++) normSq += v[k] * v[k];
+    const norm = Math.sqrt(normSq);
+    if (norm > 1e-12) {
+      for (let k = 0; k < dim; k++) v[k] /= norm;
+    }
+    ortho.push(v);
+  }
+
+  return ortho;
+}
+
+/**
+ * Transforms a vector through an orthogonal basis matrix: v' = Q * v.
+ * @param {Float32Array} vector - Original embedding vector
+ * @param {Float32Array[]} basis - Orthogonal basis matrix Q
+ * @returns {Float32Array} Transformed vector
+ */
+export function applyOrthogonalTransform(vector, basis) {
+  const dim = vector.length;
+  const out = new Float32Array(dim);
+  for (let i = 0; i < dim; i++) {
+    let sum = 0;
+    const row = basis[i];
+    for (let j = 0; j < dim; j++) {
+      sum += row[j] * vector[j];
+    }
+    out[i] = sum;
+  }
+  return out;
+}
+
+/**
+ * Client: Blinds a query vector with a private random positive scalar alpha and orthogonal rotation.
+ * Query: q_blind = alpha * Q * q.
+ * The server computes (q_blind · v'_i) = alpha * (q · v_i), which strictly preserves relative similarity ranking.
+ * @param {Float32Array} queryVector
+ * @param {Float32Array[]} basis
+ * @param {number} [alpha] - Optional private scaling factor > 0 (randomized if omitted)
+ * @returns {{ blindedVector: Float32Array, alpha: number }}
+ */
+export function blindQueryVector(queryVector, basis, alpha = null) {
+  const scale = alpha !== null ? alpha : (1.0 + (crypto.randomBytes(4).readUInt32LE(0) / 4294967295.0) * 4.0);
+  const rotated = applyOrthogonalTransform(queryVector, basis);
+  const blinded = new Float32Array(rotated.length);
+  for (let i = 0; i < rotated.length; i++) {
+    blinded[i] = rotated[i] * scale;
+  }
+  return { blindedVector: blinded, alpha: scale };
+}
+
+/**
+ * Server: Evaluates dot product ranking on blinded vectors without learning the query vector.
+ * @param {Float32Array} blindedQuery - Blinded query vector from client
+ * @param {Map<string, Float32Array>} transformedDocs - Transformed document vectors (Q * v_i)
+ * @param {number} [topK=5]
+ * @returns {Array<{ id: string, blindedScore: number }>}
+ */
+export function evaluateBlindedRanking(blindedQuery, transformedDocs, topK = 5) {
+  const scores = [];
+  for (const [id, docVec] of transformedDocs.entries()) {
+    let dot = 0;
+    for (let i = 0; i < blindedQuery.length; i++) {
+      dot += blindedQuery[i] * docVec[i];
+    }
+    scores.push({ id, blindedScore: dot });
+  }
+  scores.sort((a, b) => b.blindedScore - a.blindedScore);
+  return scores.slice(0, topK);
+}

package/packages/rag/src/bm25.js

@@ -0,0 +1,169 @@
+// Enigma Hybrid RAG: BM25 Lexical Inverted Index & Scoring.
+// Implements Okapi BM25 ranking for exact keyword matching, technical identifiers,
+// code symbols, and Solana addresses.
+
+export class BM25Index {
+  /**
+   * @param {Object} [options]
+   * @param {number} [options.k1=1.2] - Term frequency saturation parameter
+   * @param {number} [options.b=0.75] - Document length normalization parameter
+   */
+  constructor(options = {}) {
+    this.k1 = typeof options.k1 === 'number' ? options.k1 : 1.2;
+    this.b = typeof options.b === 'number' ? options.b : 0.75;
+
+    /** @type {Map<string, { id: string, text: string, scope: string, terms: string[], length: number }>} */
+    this.docs = new Map();
+    /** @type {Map<string, Set<string>>} term -> Set<docId> */
+    this.invertedIndex = new Map();
+    this.totalDocLength = 0;
+  }
+
+  /**
+   * Tokenizes text into normalized lexical terms.
+   * Preserves alphanumeric identifiers, hex hashes, and technical terms.
+   * @param {string} text
+   * @returns {string[]}
+   */
+  tokenize(text) {
+    if (!text || typeof text !== 'string') return [];
+    return text
+      .toLowerCase()
+      .split(/[^a-zA-Z0-9_\-]/)
+      .filter(t => t.length >= 2);
+  }
+
+  /**
+   * Indexes or updates a document in the BM25 index.
+   * @param {string} id
+   * @param {string} text
+   * @param {string} [scope='enigma.global']
+   */
+  add(id, text, scope = 'enigma.global') {
+    if (this.docs.has(id)) {
+      this.remove(id);
+    }
+
+    const terms = this.tokenize(text);
+    const length = terms.length;
+    this.docs.set(id, { id, text, scope, terms, length });
+    this.totalDocLength += length;
+
+    const termSet = new Set(terms);
+    for (const term of termSet) {
+      if (!this.invertedIndex.has(term)) {
+        this.invertedIndex.set(term, new Set());
+      }
+      this.invertedIndex.get(term).add(id);
+    }
+  }
+
+  /**
+   * Removes a document from the BM25 index.
+   * @param {string} id
+   */
+  remove(id) {
+    const doc = this.docs.get(id);
+    if (!doc) return;
+
+    this.totalDocLength = Math.max(0, this.totalDocLength - doc.length);
+    const termSet = new Set(doc.terms);
+    for (const term of termSet) {
+      const posting = this.invertedIndex.get(term);
+      if (posting) {
+        posting.delete(id);
+        if (posting.size === 0) this.invertedIndex.delete(term);
+      }
+    }
+    this.docs.delete(id);
+  }
+
+  /**
+   * Clears all indexed documents, inverted index tokens, and document length aggregates.
+   */
+  clear() {
+    this.docs.clear();
+    this.invertedIndex.clear();
+    this.totalDocLength = 0;
+  }
+
+  /**
+   * Computes Okapi BM25 score for a query against a document.
+   * @param {string[]} queryTerms
+   * @param {Object} doc
+   * @param {number} avgDocLen
+   * @param {number} N - Total document count
+   * @returns {number}
+   */
+  scoreDoc(queryTerms, doc, avgDocLen, N) {
+    let score = 0;
+    const termFreqs = new Map();
+    for (const t of doc.terms) {
+      termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+    }
+
+    for (const term of queryTerms) {
+      const posting = this.invertedIndex.get(term);
+      const n_t = posting ? posting.size : 0;
+      if (n_t === 0) continue;
+
+      // Robertson-Spärck Jones IDF
+      const idf = Math.log((N - n_t + 0.5) / (n_t + 0.5) + 1.0);
+      const f = termFreqs.get(term) || 0;
+      if (f === 0) continue;
+
+      const num = f * (this.k1 + 1);
+      const denom = f + this.k1 * (1 - this.b + this.b * (doc.length / (avgDocLen || 1)));
+      score += idf * (num / denom);
+    }
+
+    return score;
+  }
+
+  /**
+   * Performs BM25 lexical search.
+   * @param {string} queryText
+   * @param {Object} [options]
+   * @param {number} [options.topK=10]
+   * @param {string} [options.scope]
+   * @returns {Array<{ id: string, text: string, score: number, scope: string }>}
+   */
+  search(queryText, options = {}) {
+    const queryTerms = this.tokenize(queryText);
+    if (queryTerms.length === 0 || this.docs.size === 0) return [];
+
+    const topK = options.topK || 10;
+    const scope = options.scope;
+    const N = this.docs.size;
+    const avgDocLen = this.totalDocLength / N;
+
+    // Collect candidate document IDs that match at least one query term
+    const candidateIds = new Set();
+    for (const term of queryTerms) {
+      const posting = this.invertedIndex.get(term);
+      if (posting) {
+        for (const id of posting) candidateIds.add(id);
+      }
+    }
+
+    const scored = [];
+    for (const id of candidateIds) {
+      const doc = this.docs.get(id);
+      if (!doc) continue;
+      if (scope && doc.scope !== scope) continue;
+
+      const rawScore = this.scoreDoc(queryTerms, doc, avgDocLen, N);
+      if (rawScore > 0) {
+        scored.push({
+          id: doc.id,
+          text: doc.text,
+          score: rawScore,
+          scope: doc.scope,
+        });
+      }
+    }
+
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, topK);
+  }
+}