enigma-memory 0.1.18 → 0.1.22

package/packages/rag/src/research.js

@@ -0,0 +1,107 @@
+// Enigma RAG Research: Experimental Blinded Retrieval Prototype.
+// WARNING: UNSUPPORTED RESEARCH PROTOTYPE — NOT ZERO-KNOWLEDGE HOMOMORPHIC ENCRYPTION.
+// Orthogonal basis projections preserve dot-product ranking, but an adversarial server
+// can observe query norms and cluster geometries. Local-first search via EncryptedVectorStore
+// is the recommended production architecture.
+import crypto from 'node:crypto';
+import { EMBEDDING_DIM } from './embeddings.js';
+
+export const RESEARCH_DISCLAIMER = Object.freeze({
+  status: 'experimental_research_prototype',
+  homomorphicEncryption: false,
+  zeroKnowledgeVerified: false,
+  recommendation: 'Use EncryptedVectorStore for local sovereign retrieval.',
+});
+
+/**
+ * Generates an orthogonal basis matrix Q for research ranking simulations.
+ * @param {Buffer|string} seed
+ * @param {number} [dim=384]
+ * @returns {Float32Array[]}
+ */
+export function generateResearchOrthogonalBasis(seed, dim = EMBEDDING_DIM) {
+  const seedBuf = Buffer.isBuffer(seed) ? seed : Buffer.from(seed, 'utf8');
+  const basis = [];
+
+  for (let i = 0; i < dim; i++) {
+    const raw = Buffer.from(crypto.hkdfSync('sha256', seedBuf, Buffer.from(`enigma.research.basis.${i}`, 'utf8'), Buffer.alloc(0), dim * 4));
+    const vec = new Float32Array(dim);
+    for (let j = 0; j < dim; j++) {
+      vec[j] = raw.readInt32LE(j * 4) / 2147483648.0;
+    }
+    basis.push(vec);
+  }
+
+  // Gram-Schmidt orthogonalization
+  const ortho = [];
+  for (let i = 0; i < dim; i++) {
+    const v = new Float32Array(basis[i]);
+    for (let j = 0; j < i; j++) {
+      const u = ortho[j];
+      let dot = 0;
+      for (let k = 0; k < dim; k++) dot += basis[i][k] * u[k];
+      for (let k = 0; k < dim; k++) v[k] -= dot * u[k];
+    }
+    let normSq = 0;
+    for (let k = 0; k < dim; k++) normSq += v[k] * v[k];
+    const norm = Math.sqrt(normSq);
+    if (norm > 1e-12) {
+      for (let k = 0; k < dim; k++) v[k] /= norm;
+    }
+    ortho.push(v);
+  }
+
+  return ortho;
+}
+
+/**
+ * Applies basis transform: v' = Q * v.
+ * @param {Float32Array} vector
+ * @param {Float32Array[]} basis
+ * @returns {Float32Array}
+ */
+export function applyResearchBasisTransform(vector, basis) {
+  const dim = vector.length;
+  const out = new Float32Array(dim);
+  for (let i = 0; i < dim; i++) {
+    let sum = 0;
+    const row = basis[i];
+    for (let j = 0; j < dim; j++) sum += row[j] * vector[j];
+    out[i] = sum;
+  }
+  return out;
+}
+
+/**
+ * Transforms query with scalar blinding for prototype evaluation.
+ * @param {Float32Array} queryVector
+ * @param {Float32Array[]} basis
+ * @param {number} [alpha=2.0]
+ * @returns {{ blindedVector: Float32Array, alpha: number }}
+ */
+export function createResearchBlindedQuery(queryVector, basis, alpha = 2.0) {
+  const rotated = applyResearchBasisTransform(queryVector, basis);
+  const blinded = new Float32Array(rotated.length);
+  for (let i = 0; i < rotated.length; i++) {
+    blinded[i] = rotated[i] * alpha;
+  }
+  return { blindedVector: blinded, alpha };
+}
+
+/**
+ * Evaluates ranking over basis-transformed documents.
+ * @param {Float32Array} blindedQuery
+ * @param {Map<string, Float32Array>} transformedDocs
+ * @param {number} [topK=5]
+ * @returns {Array<{ id: string, score: number }>}
+ */
+export function evaluateResearchBlindedRanking(blindedQuery, transformedDocs, topK = 5) {
+  const scores = [];
+  for (const [id, docVec] of transformedDocs.entries()) {
+    let dot = 0;
+    for (let i = 0; i < blindedQuery.length; i++) dot += blindedQuery[i] * docVec[i];
+    scores.push({ id, score: dot });
+  }
+  scores.sort((a, b) => b.score - a.score);
+  return scores.slice(0, topK);
+}

package/packages/rag/src/vector-store.js

@@ -0,0 +1,430 @@
+// Enigma Sovereign Encrypted Vector Store & Hybrid RAG Engine.
+// Combines client-side dense vector embeddings, Okapi BM25 lexical inverted indexing,
+// Reciprocal Rank Fusion (RRF), semantic candidate reranking, and AES-256-GCM encryption-at-rest.
+import crypto from 'node:crypto';
+import {
+  createEmbeddingProvider,
+  cosineSimilarity,
+  quantizeVector,
+  dequantizeVector,
+  EMBEDDING_DIM,
+} from './embeddings.js';
+import { BM25Index } from './bm25.js';
+import { reciprocalRankFusion } from './hybrid.js';
+import { semanticReranker } from './reranker.js';
+
+const VECTOR_STORE_AAD = Buffer.from('enigma.encrypted_vectors.v1', 'utf8');
+
+export class EncryptedVectorStore {
+  /**
+   * @param {Object} [options]
+   * @param {Object} [options.embeddingProvider]
+   * @param {number} [options.dimension=384]
+   * @param {boolean} [options.quantize=true]
+   */
+  constructor(options = {}) {
+    this.dimension = options.dimension || EMBEDDING_DIM;
+    this.quantize = options.quantize !== false;
+    this.embedder = options.embeddingProvider || createEmbeddingProvider({ dimension: this.dimension });
+    /** @type {Map<string, Object>} */
+    this.documents = new Map();
+    /** @type {Map<string, Int8Array|Float32Array>} */
+    this.vectors = new Map();
+    /** @type {BM25Index} */
+    this.bm25 = new BM25Index();
+    /** @type {number} */
+    this.epoch = 0;
+  }
+  /**
+   * Asynchronously indexes a memory document into both dense vector store and BM25 index.
+   * @param {string} id
+   * @param {string} text
+   * @param {Object} [metadata={}]
+   * @returns {Promise<Object>}
+   */
+  async add(id, text, metadata = {}, options = {}) {
+    if (!id || typeof id !== 'string') throw new Error('Document ID must be a non-empty string');
+    if (!text || typeof text !== 'string') throw new Error('Document text must be a non-empty string');
+
+    const opEpoch = typeof options.epoch === 'number' ? options.epoch : this.epoch;
+    if (options.signal?.aborted) {
+      return null;
+    }
+
+    const floatVec = await this.embedder.embed(text);
+
+    // Abort guard: If store was cleared or reset while embedding was in flight, do not mutate state
+    if (this.epoch !== opEpoch || options.signal?.aborted) {
+      return null;
+    }
+
+    const storedVec = this.quantize ? quantizeVector(floatVec) : floatVec;
+    const scope = metadata.scope || 'enigma.global';
+
+    const doc = {
+      id,
+      text,
+      scope,
+      importance: typeof metadata.importance === 'number' ? metadata.importance : 1.0,
+      createdAt: metadata.createdAt ?? Date.now(),
+      metadata: { ...metadata },
+    };
+
+    this.documents.set(id, doc);
+    this.vectors.set(id, storedVec);
+    this.bm25.add(id, text, scope);
+
+    return { id, scope: doc.scope, dimension: this.dimension, quantized: this.quantize };
+  }
+
+  /**
+   * Synchronous add when using local synchronous embedder.
+   * @param {string} id
+   * @param {string} text
+   * @param {Object} [metadata={}]
+   * @returns {Object}
+   */
+  addSync(id, text, metadata = {}) {
+    if (!id || typeof id !== 'string') throw new Error('Document ID must be a non-empty string');
+    if (!text || typeof text !== 'string') throw new Error('Document text must be a non-empty string');
+
+    const floatVec = this.embedder.embedSync(text);
+    const storedVec = this.quantize ? quantizeVector(floatVec) : floatVec;
+    const scope = metadata.scope || 'enigma.global';
+
+    const doc = {
+      id,
+      text,
+      scope,
+      importance: typeof metadata.importance === 'number' ? metadata.importance : 1.0,
+      createdAt: metadata.createdAt ?? Date.now(),
+      metadata: { ...metadata },
+    };
+
+    this.documents.set(id, doc);
+    this.vectors.set(id, storedVec);
+    this.bm25.add(id, text, scope);
+
+    return { id, scope: doc.scope, dimension: this.dimension, quantized: this.quantize };
+  }
+
+  /**
+   * Batch insert memory atoms asynchronously.
+   * @param {Array<{ id: string, text: string, metadata?: Object }>} items
+   * @returns {Promise<number>}
+   */
+  async addBatch(items) {
+    if (!Array.isArray(items)) throw new TypeError('items must be an array');
+    let count = 0;
+    for (const item of items) {
+      await this.add(item.id, item.text, item.metadata || {});
+      count++;
+    }
+    return count;
+  }
+
+  /**
+   * Removes a document by ID from dense and BM25 indices.
+   * @param {string} id
+   * @returns {boolean}
+   */
+  remove(id) {
+    this.vectors.delete(id);
+    this.bm25.remove(id);
+    return this.documents.delete(id);
+  }
+
+  delete(id) {
+    return this.remove(id);
+  }
+
+  /**
+   * Updates retrieval metadata without re-embedding non-prose cryptographic payloads.
+   * @param {string} id
+   * @param {Object} metadata
+   * @returns {boolean}
+   */
+  updateMetadata(id, metadata) {
+    const doc = this.documents.get(id);
+    if (!doc) return false;
+    doc.metadata = { ...metadata };
+    if (typeof metadata.importance === 'number') doc.importance = metadata.importance;
+    return true;
+  }
+
+  /**
+   * Resets and zeroes out in-memory vector buffers, documents, and lexical BM25 index maps.
+   */
+  clear() {
+    this.epoch++;
+    for (const vec of this.vectors.values()) {
+      if (ArrayBuffer.isView(vec) && typeof vec.fill === 'function') {
+        vec.fill(0);
+      }
+    }
+    this.vectors.clear();
+    this.documents.clear();
+    this.bm25.clear();
+  }
+
+  /**
+   */
+  get size() {
+    return this.documents.size;
+  }
+
+  /**
+   * Hybrid search combining dense semantic vectors, BM25 keyword matching, and candidate reranking.
+   * @param {string|Float32Array} query
+   * @param {Object} [options]
+   * @param {'hybrid'|'dense'|'bm25'} [options.mode='hybrid']
+   * @param {number} [options.topK=5]
+   * @param {number} [options.threshold=-1.0]
+   * @param {string} [options.scope]
+   * @param {number} [options.denseWeight=1.0]
+   * @param {number} [options.bm25Weight=1.0]
+   * @returns {Promise<Array<{ id: string, text: string, score: number, scope: string, metadata: Object }>>}
+   */
+  async search(query, options = {}) {
+    const searchEpoch = this.epoch;
+    if (options.signal?.aborted) return [];
+    const mode = options.mode || 'hybrid';
+    const topK = Math.max(1, options.topK ?? 5);
+
+    // If query is raw vector or mode is dense-only
+    if (query instanceof Float32Array || mode === 'dense') {
+      const queryVec = query instanceof Float32Array ? query : await this.embedder.embed(query);
+      if (this.epoch !== searchEpoch || options.signal?.aborted) return [];
+      return this.searchByVector(queryVec, options);
+    }
+
+    // If query is string and mode is BM25-only
+    if (mode === 'bm25') {
+      if (this.epoch !== searchEpoch || options.signal?.aborted) return [];
+      return this.bm25.search(query, { topK, scope: options.scope });
+    }
+
+    // Hybrid Mode: Dense + BM25 + Reciprocal Rank Fusion + Reranking
+    const queryVec = await this.embedder.embed(query);
+    if (this.epoch !== searchEpoch || options.signal?.aborted) return [];
+    const denseCandidates = this.searchByVector(queryVec, { ...options, topK: topK * 3 });
+    const bm25Candidates = this.bm25.search(query, { topK: topK * 3, scope: options.scope });
+
+    const fused = reciprocalRankFusion(denseCandidates, bm25Candidates, {
+      k: 60,
+      denseWeight: options.denseWeight ?? 1.0,
+      bm25Weight: options.bm25Weight ?? 1.0,
+      topK: topK * 2,
+    });
+
+    if (this.epoch !== searchEpoch || options.signal?.aborted) return [];
+    return semanticReranker(query, fused, { topK });
+  }
+
+  /**
+   * Synchronous search using local synchronous embedder.
+   * @param {string|Float32Array} query
+   * @param {Object} [options]
+   * @returns {Array<Object>}
+   */
+  searchSync(query, options = {}) {
+    const topK = Math.max(1, options.topK || 5);
+    if (query instanceof Float32Array || options.mode === 'dense') {
+      const queryVec = query instanceof Float32Array ? query : this.embedder.embedSync(query);
+      return this.searchByVector(queryVec, options);
+    }
+
+    if (options.mode === 'bm25') {
+      return this.bm25.search(query, { topK, scope: options.scope });
+    }
+
+    const queryVec = this.embedder.embedSync(query);
+    const denseCandidates = this.searchByVector(queryVec, { ...options, topK: topK * 3 });
+    const bm25Candidates = this.bm25.search(query, { topK: topK * 3, scope: options.scope });
+
+    const fused = reciprocalRankFusion(denseCandidates, bm25Candidates, {
+      k: 60,
+      denseWeight: options.denseWeight ?? 1.0,
+      bm25Weight: options.bm25Weight ?? 1.0,
+      topK: topK * 2,
+    });
+
+    return semanticReranker(query, fused, { topK });
+  }
+
+  /**
+   * Core vector similarity scoring loop.
+   * @param {Float32Array} queryVec
+   * @param {Object} [options]
+   * @returns {Array<Object>}
+   */
+  searchByVector(queryVec, options = {}) {
+    const topK = Math.max(1, options.topK || 5);
+    const threshold = options.threshold !== undefined ? options.threshold : -1.0;
+    const scopeFilter = options.scope;
+
+    const candidates = [];
+
+    for (const [id, doc] of this.documents.entries()) {
+      if (scopeFilter && doc.scope !== scopeFilter && doc.scope !== 'enigma.global') {
+        continue;
+      }
+
+      const rawVec = this.vectors.get(id);
+      const docFloatVec = this.quantize ? dequantizeVector(rawVec) : rawVec;
+      const sim = cosineSimilarity(queryVec, docFloatVec);
+
+      if (sim >= threshold) {
+        candidates.push({
+          id: doc.id,
+          text: doc.text,
+          score: Math.round(sim * 10000) / 10000,
+          cosine: Math.round(sim * 10000) / 10000,
+          scope: doc.scope,
+          importance: doc.importance,
+          createdAt: doc.createdAt,
+          metadata: doc.metadata,
+        });
+      }
+    }
+
+    candidates.sort((a, b) => b.score - a.score);
+    return candidates.slice(0, topK);
+  }
+
+  /**
+   * Compiles a token-budgeted prompt context pack.
+   * @param {string} query
+   * @param {Object} [options]
+   * @returns {Promise<{ contextText: string, items: Array, totalChars: number, estimatedTokens: number }>}
+   */
+  async compileContextPack(query, options = {}) {
+    const packEpoch = this.epoch;
+    const maxTokens = options.maxTokens ?? 2000;
+    if (typeof maxTokens !== 'number' || !Number.isFinite(maxTokens) || maxTokens < 0) {
+      throw new TypeError('maxTokens must be a finite non-negative number');
+    }
+    const maxChars = maxTokens * 4;
+    const results = await this.search(query, options);
+    if (this.epoch !== packEpoch || options.signal?.aborted) {
+      return { contextText: '', items: [], totalChars: 0, estimatedTokens: 0 };
+    }
+
+    const packedItems = [];
+    let currentChars = 0;
+
+    for (const res of results) {
+      const itemLen = res.text.length;
+      if (currentChars + itemLen > maxChars) break;
+      packedItems.push(res);
+      currentChars += itemLen;
+    }
+
+    const contextText = packedItems.map((item, idx) => `[Context #${idx + 1} (${item.scope})]: ${item.text}`).join('\n\n');
+
+    return {
+      contextText,
+      items: packedItems,
+      totalChars: contextText.length,
+      estimatedTokens: Math.ceil(contextText.length / 4),
+    };
+  }
+
+  /**
+   * Encrypts the entire vector collection at rest using AES-256-GCM.
+   * @param {Buffer|string} vaultKey
+   * @returns {{ format: string, ciphertext: string, iv: string, authTag: string, count: number }}
+   */
+  encryptForVault(vaultKey) {
+    const keyBuf = Buffer.isBuffer(vaultKey) ? vaultKey : Buffer.from(vaultKey, 'hex');
+    if (keyBuf.length !== 32) throw new Error('vaultKey must be exactly 32 bytes');
+
+    const serializedData = JSON.stringify({
+      version: 3,
+      dimension: this.dimension,
+      quantized: this.quantize,
+      embeddingProviderType: this.embedder.providerType || 'custom',
+      documents: Array.from(this.documents.values()),
+      vectors: Object.fromEntries(Array.from(this.vectors.entries()).map(([k, v]) => [k, Array.from(v)])),
+    });
+
+    const iv = crypto.randomBytes(12);
+    const cipher = crypto.createCipheriv('aes-256-gcm', keyBuf, iv);
+    cipher.setAAD(VECTOR_STORE_AAD);
+
+    const ciphertext = Buffer.concat([cipher.update(Buffer.from(serializedData, 'utf8')), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+
+    return {
+      format: 'enigma.encrypted_vectors.v1',
+      ciphertext: ciphertext.toString('base64'),
+      iv: iv.toString('hex'),
+      authTag: authTag.toString('hex'),
+      count: this.documents.size,
+    };
+  }
+
+  /**
+   * Decrypts and restores an encrypted vector store bundle from vault storage.
+   * @param {Object} encryptedBundle
+   * @param {Buffer|string} vaultKey
+   * @param {{ embeddingProvider?: Object }} [options]
+   * @returns {EncryptedVectorStore}
+   */
+  static decryptFromVault(encryptedBundle, vaultKey, options = {}) {
+    if (!encryptedBundle || encryptedBundle.format !== 'enigma.encrypted_vectors.v1') {
+      throw new Error('Invalid encrypted vector store bundle format');
+    }
+    const keyBuf = Buffer.isBuffer(vaultKey) ? vaultKey : Buffer.from(vaultKey, 'hex');
+    if (keyBuf.length !== 32) throw new Error('vaultKey must be exactly 32 bytes');
+
+    const iv = Buffer.from(encryptedBundle.iv, 'hex');
+    const authTag = Buffer.from(encryptedBundle.authTag, 'hex');
+    const ciphertext = Buffer.from(encryptedBundle.ciphertext, 'base64');
+
+    const decipher = crypto.createDecipheriv('aes-256-gcm', keyBuf, iv);
+    decipher.setAAD(VECTOR_STORE_AAD);
+    decipher.setAuthTag(authTag);
+
+    const decryptedBuf = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    const parsed = JSON.parse(decryptedBuf.toString('utf8'));
+
+    let embeddingProvider = options.embeddingProvider;
+    if (!embeddingProvider) {
+      if (parsed.embeddingProviderType === 'local_minilm') {
+        embeddingProvider = createEmbeddingProvider({ modelType: 'neural' });
+      } else if (parsed.embeddingProviderType === 'lexical_hash') {
+        embeddingProvider = createEmbeddingProvider({ modelType: 'lexical' });
+      } else {
+        throw new Error(
+          `Encrypted vector bundle requires an explicit embeddingProvider for provider type "${parsed.embeddingProviderType || 'unrecorded'}"`
+        );
+      }
+    } else if (
+      parsed.embeddingProviderType
+      && parsed.embeddingProviderType !== 'custom'
+      && embeddingProvider.providerType !== parsed.embeddingProviderType
+    ) {
+      throw new Error(
+        `Embedding provider type mismatch: bundle requires "${parsed.embeddingProviderType}", received "${embeddingProvider.providerType || 'custom'}"`
+      );
+    }
+
+    const store = new EncryptedVectorStore({
+      dimension: parsed.dimension || EMBEDDING_DIM,
+      quantize: parsed.quantized !== false,
+      embeddingProvider,
+    });
+
+    for (const doc of parsed.documents || []) {
+      store.documents.set(doc.id, doc);
+      store.bm25.add(doc.id, doc.text, doc.scope);
+      if (parsed.vectors && parsed.vectors[doc.id]) {
+        const raw = parsed.vectors[doc.id];
+        store.vectors.set(doc.id, parsed.quantized ? new Int8Array(raw) : new Float32Array(raw));
+      }
+    }
+
+    return store;
+  }
+}

package/packages/rag/src/verify-model-artifacts.mjs

@@ -0,0 +1,4 @@
+import { verifyBundledModel } from './embeddings.js';
+
+await verifyBundledModel();
+process.stdout.write('Bundled MiniLM SHA-256 manifest verified.\n');